bpo-25643: Refactor the C tokenizer

Author: pablogsal

Lib/test/test_eof.py

@@ -55,13 +55,13 @@ def test_line_continuation_EOF_from_file_bpo2180(self):
             file_name = script_helper.make_script(temp_dir, 'foo', '\\')
             rc, out, err = script_helper.assert_python_failure(file_name)
             self.assertIn(b'unexpected EOF while parsing', err)
-            self.assertIn(b'line 2', err)
+            self.assertIn(b'line 1', err)
             self.assertIn(b'\\', err)
 
             file_name = script_helper.make_script(temp_dir, 'foo', 'y = 6\\')
             rc, out, err = script_helper.assert_python_failure(file_name)
             self.assertIn(b'unexpected EOF while parsing', err)
-            self.assertIn(b'line 2', err)
+            self.assertIn(b'line 1', err)
             self.assertIn(b'y = 6\\', err)
 
 if __name__ == "__main__":

Lib/test/test_source_encoding.py

@@ -205,6 +205,23 @@ def test_utf8_bom_and_utf8_coding_line(self):
                b'print(ascii("\xc3\xa4"))\n')
         self.check_script_output(src, br"'\xe4'")
 
+    def test_crlf(self):
+        src = (b'print(ascii("""\r\n"""))\n')
+        out = self.check_script_output(src, br"'\n'")
+
+    def test_crcrlf(self):
+        src = (b'print(ascii("""\r\r\n"""))\n')
+        out = self.check_script_output(src, br"'\n\n'")
+
+    def test_crcrcrlf(self):
+        src = (b'print(ascii("""\r\r\r\n"""))\n')
+        out = self.check_script_output(src, br"'\n\n\n'")
+
+    def test_crcrcrlf2(self):
+        src = (b'#coding:iso-8859-1\n'
+               b'print(ascii("""\r\r\r\n"""))\n')
+        out = self.check_script_output(src, br"'\n\n\n'")
+
 
 class BytesSourceEncodingTest(AbstractSourceEncodingTest, unittest.TestCase):