From c90a4b348f89d39dff07b6c6db1c79c613b04366 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 25 Jun 2019 11:54:18 +0300 Subject: [PATCH 1/3] bpo-24214: Fixed the UTF-8 and UTF-16 incremental decoders. (GH-14304) * The UTF-8 incremental decoders fails now fast if encounter a sequence that can't be handled by the error handler. * The UTF-16 incremental decoders with the surrogatepass error handler decodes now a lone low surrogate with final=False. (cherry picked from commit 894263ba80af4b7733c2df95b527e96953922656) Co-authored-by: Serhiy Storchaka --- Lib/test/test_codecs.py | 24 +++++++++++++++++++ .../2019-06-22-12-45-20.bpo-24214.hIiHeD.rst | 2 ++ Objects/stringlib/codecs.h | 6 ++--- Objects/unicodeobject.c | 10 +++++--- 4 files changed, 36 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2019-06-22-12-45-20.bpo-24214.hIiHeD.rst diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index 248ef6fe07dd28..7dfcac0c4bf77b 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -404,11 +404,18 @@ def test_lone_surrogates(self): def test_incremental_surrogatepass(self): # Test incremental decoder for surrogatepass handler: # see issue #24214 + # High surrogate data = '\uD901'.encode(self.encoding, 'surrogatepass') for i in range(1, len(data)): dec = codecs.getincrementaldecoder(self.encoding)('surrogatepass') self.assertEqual(dec.decode(data[:i]), '') self.assertEqual(dec.decode(data[i:], True), '\uD901') + # Low surrogate + data = '\uDC02'.encode(self.encoding, 'surrogatepass') + for i in range(1, len(data)): + dec = codecs.getincrementaldecoder(self.encoding)('surrogatepass') + self.assertEqual(dec.decode(data[:i]), '') + self.assertEqual(dec.decode(data[i:]), '\uDC02') class UTF32Test(ReadTest, unittest.TestCase): @@ -849,6 +856,23 @@ def test_surrogatepass_handler(self): with self.assertRaises(UnicodeDecodeError): b"abc\xed\xa0z".decode(self.encoding, "surrogatepass") + def test_incremental_errors(self): + # Test that the incremental decoder can fail with final=False. + # See issue #24214 + cases = [b'\x80', b'\xBF', b'\xC0', b'\xC1', b'\xF5', b'\xF6', b'\xFF'] + for prefix in (b'\xC2', b'\xDF', b'\xE0', b'\xE0\xA0', b'\xEF', + b'\xEF\xBF', b'\xF0', b'\xF0\x90', b'\xF0\x90\x80', + b'\xF4', b'\xF4\x8F', b'\xF4\x8F\xBF'): + for suffix in b'\x7F', b'\xC0': + cases.append(prefix + suffix) + cases.extend((b'\xE0\x80', b'\xE0\x9F', b'\xED\xA0\x80', + b'\xED\xBF\xBF', b'\xF0\x80', b'\xF0\x8F', b'\xF4\x90')) + + for data in cases: + with self.subTest(data=data): + dec = codecs.getincrementaldecoder(self.encoding)() + self.assertRaises(UnicodeDecodeError, dec.decode, data) + @unittest.skipUnless(sys.platform == 'win32', 'cp65001 is a Windows-only codec') diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-06-22-12-45-20.bpo-24214.hIiHeD.rst b/Misc/NEWS.d/next/Core and Builtins/2019-06-22-12-45-20.bpo-24214.hIiHeD.rst new file mode 100644 index 00000000000000..2d70ce05deaedf --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2019-06-22-12-45-20.bpo-24214.hIiHeD.rst @@ -0,0 +1,2 @@ +Improved support of the surrogatepass error handler in the UTF-8 and UTF-16 +incremental decoders. diff --git a/Objects/stringlib/codecs.h b/Objects/stringlib/codecs.h index f019d9a96bfba3..efa6cf3a7c2434 100644 --- a/Objects/stringlib/codecs.h +++ b/Objects/stringlib/codecs.h @@ -207,7 +207,7 @@ STRINGLIB(utf8_decode)(const char **inptr, const char *end, goto InvalidContinuation1; } else if (ch == 0xF4 && ch2 >= 0x90) { /* invalid sequence - \xF4\x90\x80\80- -- 110000- overflow */ + \xF4\x90\x80\x80- -- 110000- overflow */ goto InvalidContinuation1; } if (!IS_CONTINUATION_BYTE(ch3)) { @@ -573,10 +573,10 @@ STRINGLIB(utf16_decode)(const unsigned char **inptr, const unsigned char *e, } /* UTF-16 code pair: */ - if (q >= e) - goto UnexpectedEnd; if (!Py_UNICODE_IS_HIGH_SURROGATE(ch)) goto IllegalEncoding; + if (q >= e) + goto UnexpectedEnd; ch2 = (q[ihi] << 8) | q[ilo]; q += 2; if (!Py_UNICODE_IS_LOW_SURROGATE(ch2)) diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index ed1e4a4dd55604..c0e9c2d112ab64 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -4888,11 +4888,15 @@ PyUnicode_DecodeUTF8Stateful(const char *s, endinpos = startinpos + 1; break; case 2: - case 3: - case 4: - if (s == end || consumed) { + if (consumed && (unsigned char)s[0] == 0xED && end - s == 2 + && (unsigned char)s[1] >= 0xA0 && (unsigned char)s[1] <= 0xBF) + { + /* Truncated surrogate code in range D800-DFFF */ goto End; } + /* fall through */ + case 3: + case 4: errmsg = "invalid continuation byte"; startinpos = s - starts; endinpos = startinpos + ch - 1; From f6909d080ff215ca3a7ca5c53e3ac29befd727c5 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 25 Jun 2019 12:21:11 +0300 Subject: [PATCH 2/3] Fix tests for cp65001. --- Lib/test/test_codecs.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index 7dfcac0c4bf77b..d4e7b9a071e3e8 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -415,7 +415,8 @@ def test_incremental_surrogatepass(self): for i in range(1, len(data)): dec = codecs.getincrementaldecoder(self.encoding)('surrogatepass') self.assertEqual(dec.decode(data[:i]), '') - self.assertEqual(dec.decode(data[i:]), '\uDC02') + final = encoding == "cp65001" + self.assertEqual(dec.decode(data[i:], final), '\uDC02') class UTF32Test(ReadTest, unittest.TestCase): From 63c882dcd314a72ef7a82e349087db5402484aab Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 25 Jun 2019 12:22:24 +0300 Subject: [PATCH 3/3] Update test_codecs.py --- Lib/test/test_codecs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index d4e7b9a071e3e8..b39ea54e4b2a39 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -415,7 +415,7 @@ def test_incremental_surrogatepass(self): for i in range(1, len(data)): dec = codecs.getincrementaldecoder(self.encoding)('surrogatepass') self.assertEqual(dec.decode(data[:i]), '') - final = encoding == "cp65001" + final = self.encoding == "cp65001" self.assertEqual(dec.decode(data[i:], final), '\uDC02')