From b48e28f641598ffaa0efc81fb4914c837756d060 Mon Sep 17 00:00:00 2001 From: john feng Date: Mon, 9 Sep 2024 15:15:10 -0700 Subject: [PATCH 1/5] modify the utest to accept 3 bytes unicode in py27 --- src/core/tests/Test_TelemetryWriter.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/core/tests/Test_TelemetryWriter.py b/src/core/tests/Test_TelemetryWriter.py index eb716f21..3cd752ef 100644 --- a/src/core/tests/Test_TelemetryWriter.py +++ b/src/core/tests/Test_TelemetryWriter.py @@ -17,6 +17,7 @@ import json import os import re +import sys import time import unittest from core.src.bootstrap.Constants import Constants @@ -102,9 +103,9 @@ def test_write_event_msg_size_limit(self): f.close() def test_write_event_msg_size_limit_char_more_than_1_bytes(self): - """ Perform 1 byte truncation on char that is more than 1 byte, use decode('utf-8', errors='replace') to replace bad unicode with a good 1 byte char (�) """ + """ Perform 1 byte truncation on char that is more than 1 byte, use decode('utf-8', errors='replace') to replace bad unicode with a good 1 byte char (\uFFFD) """ - message = "a€bc"*3074 # €(\xe2\x82\xac) is 3 bytes char can be written in windows console w/o encoding + message = "a\u20acbc" * 3074 # (\xe2\x82\xac) is 3 bytes char can be written in windows console w/o encoding self.runtime.telemetry_writer.write_event(message, Constants.TelemetryEventLevel.Error, "Test Task") latest_event_file = [pos_json for pos_json in os.listdir(self.runtime.telemetry_writer.events_folder_path) if re.search('^[0-9]+.json$', pos_json)][-1] with open(os.path.join(self.runtime.telemetry_writer.events_folder_path, latest_event_file), 'r+') as f: @@ -112,9 +113,9 @@ def test_write_event_msg_size_limit_char_more_than_1_bytes(self): self.assertTrue(events is not None) self.assertEqual(events[-1]["TaskName"], "Test Task") self.assertTrue(len(events[-1]["Message"]) < len(message.encode('utf-8'))) - chars_dropped = len(message.encode('utf-8')) - Constants.TELEMETRY_MSG_SIZE_LIMIT_IN_CHARS + Constants.TELEMETRY_BUFFER_FOR_DROPPED_COUNT_MSG_IN_CHARS + Constants.TELEMETRY_EVENT_COUNTER_MSG_SIZE_LIMIT_IN_CHARS - self.assertTrue("a€bc" in events[-1]["Message"]) - self.assertTrue("a€bc" * (len(message) + 1 - chars_dropped) + ". [{0} chars dropped]".format(chars_dropped) in events[-1]["Message"]) # len(message) + 1 due to bad unicode will be replaced by � + self.assertTrue("a\u20acbc" in events[-1]["Message"]) + self.assertTrue(len(events[-1]["Message"]) < Constants.TELEMETRY_MSG_SIZE_LIMIT_IN_CHARS) + f.close() # TODO: The following 3 tests cause widespread test suite failures (on master), so leaving it out. And tracking in: Task 10912099: [Bug] Bug in telemetry writer - overwriting prior events in fast execution From 2ff6ed30a057f8a81af9baaec7d50cbdab791a7e Mon Sep 17 00:00:00 2001 From: john feng Date: Mon, 9 Sep 2024 15:29:22 -0700 Subject: [PATCH 2/5] remove unused import --- src/core/tests/Test_TelemetryWriter.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/core/tests/Test_TelemetryWriter.py b/src/core/tests/Test_TelemetryWriter.py index 3cd752ef..afbac00d 100644 --- a/src/core/tests/Test_TelemetryWriter.py +++ b/src/core/tests/Test_TelemetryWriter.py @@ -17,7 +17,6 @@ import json import os import re -import sys import time import unittest from core.src.bootstrap.Constants import Constants From d24788ef41e057804279150b095264314b228f18 Mon Sep 17 00:00:00 2001 From: john feng Date: Wed, 11 Sep 2024 10:26:08 -0700 Subject: [PATCH 3/5] add assertion for lower bound min msg limit --- src/core/tests/Test_TelemetryWriter.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/core/tests/Test_TelemetryWriter.py b/src/core/tests/Test_TelemetryWriter.py index afbac00d..4821746a 100644 --- a/src/core/tests/Test_TelemetryWriter.py +++ b/src/core/tests/Test_TelemetryWriter.py @@ -17,6 +17,7 @@ import json import os import re +import sys import time import unittest from core.src.bootstrap.Constants import Constants @@ -97,6 +98,7 @@ def test_write_event_msg_size_limit(self): self.assertTrue(events is not None) self.assertEqual(events[-1]["TaskName"], "Test Task") self.assertTrue(len(events[-1]["Message"]) < len(message.encode('utf-8'))) + self.assertEqual(len(events[-1]["Message"]), Constants.TELEMETRY_MSG_SIZE_LIMIT_IN_CHARS - 1) chars_dropped = len(message.encode('utf-8')) - Constants.TELEMETRY_MSG_SIZE_LIMIT_IN_CHARS + Constants.TELEMETRY_BUFFER_FOR_DROPPED_COUNT_MSG_IN_CHARS + Constants.TELEMETRY_EVENT_COUNTER_MSG_SIZE_LIMIT_IN_CHARS self.assertTrue("a"*(len(message.encode('utf-8')) - chars_dropped) + ". [{0} chars dropped]".format(chars_dropped) in events[-1]["Message"]) f.close() @@ -105,6 +107,8 @@ def test_write_event_msg_size_limit_char_more_than_1_bytes(self): """ Perform 1 byte truncation on char that is more than 1 byte, use decode('utf-8', errors='replace') to replace bad unicode with a good 1 byte char (\uFFFD) """ message = "a\u20acbc" * 3074 # (\xe2\x82\xac) is 3 bytes char can be written in windows console w/o encoding + min_msg_limit_in_bytes = Constants.TELEMETRY_MSG_SIZE_LIMIT_IN_CHARS - Constants.TELEMETRY_BUFFER_FOR_DROPPED_COUNT_MSG_IN_CHARS - Constants.TELEMETRY_EVENT_COUNTER_MSG_SIZE_LIMIT_IN_CHARS + max_msg_limit_in_bytes = Constants.TELEMETRY_MSG_SIZE_LIMIT_IN_CHARS self.runtime.telemetry_writer.write_event(message, Constants.TelemetryEventLevel.Error, "Test Task") latest_event_file = [pos_json for pos_json in os.listdir(self.runtime.telemetry_writer.events_folder_path) if re.search('^[0-9]+.json$', pos_json)][-1] with open(os.path.join(self.runtime.telemetry_writer.events_folder_path, latest_event_file), 'r+') as f: @@ -113,7 +117,8 @@ def test_write_event_msg_size_limit_char_more_than_1_bytes(self): self.assertEqual(events[-1]["TaskName"], "Test Task") self.assertTrue(len(events[-1]["Message"]) < len(message.encode('utf-8'))) self.assertTrue("a\u20acbc" in events[-1]["Message"]) - self.assertTrue(len(events[-1]["Message"]) < Constants.TELEMETRY_MSG_SIZE_LIMIT_IN_CHARS) + self.assertTrue(len(events[-1]["Message"].encode('utf-8')) > min_msg_limit_in_bytes) + self.assertTrue(len(events[-1]["Message"].encode('utf-8')) < max_msg_limit_in_bytes) f.close() From 94d81f8a4465350fc06d86a6428f6459c43bd965 Mon Sep 17 00:00:00 2001 From: john feng Date: Wed, 11 Sep 2024 10:27:39 -0700 Subject: [PATCH 4/5] remove assertion for checking equal message length --- src/core/tests/Test_TelemetryWriter.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/core/tests/Test_TelemetryWriter.py b/src/core/tests/Test_TelemetryWriter.py index 4821746a..96b60aee 100644 --- a/src/core/tests/Test_TelemetryWriter.py +++ b/src/core/tests/Test_TelemetryWriter.py @@ -17,7 +17,6 @@ import json import os import re -import sys import time import unittest from core.src.bootstrap.Constants import Constants @@ -98,7 +97,6 @@ def test_write_event_msg_size_limit(self): self.assertTrue(events is not None) self.assertEqual(events[-1]["TaskName"], "Test Task") self.assertTrue(len(events[-1]["Message"]) < len(message.encode('utf-8'))) - self.assertEqual(len(events[-1]["Message"]), Constants.TELEMETRY_MSG_SIZE_LIMIT_IN_CHARS - 1) chars_dropped = len(message.encode('utf-8')) - Constants.TELEMETRY_MSG_SIZE_LIMIT_IN_CHARS + Constants.TELEMETRY_BUFFER_FOR_DROPPED_COUNT_MSG_IN_CHARS + Constants.TELEMETRY_EVENT_COUNTER_MSG_SIZE_LIMIT_IN_CHARS self.assertTrue("a"*(len(message.encode('utf-8')) - chars_dropped) + ". [{0} chars dropped]".format(chars_dropped) in events[-1]["Message"]) f.close() From f61101634c247dbbeb32770bac4082cab86bc3c3 Mon Sep 17 00:00:00 2001 From: john feng Date: Wed, 18 Sep 2024 10:48:22 -0700 Subject: [PATCH 5/5] =?UTF-8?q?remove=20=E2=82=AC=20from=20test=5Fstatusha?= =?UTF-8?q?ndlertruncation.py?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/core/tests/Test_StatusHandlerTruncation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/tests/Test_StatusHandlerTruncation.py b/src/core/tests/Test_StatusHandlerTruncation.py index 73b44f90..ba2ff8e8 100644 --- a/src/core/tests/Test_StatusHandlerTruncation.py +++ b/src/core/tests/Test_StatusHandlerTruncation.py @@ -937,9 +937,9 @@ def __set_up_patches_func(self, val, random_char=None): test_patches_list.append('python-samba' + str(i)) if random_char is not None: - test_patches_version_list.append('2:4.4.5+dfsg-2ubuntu€' + random_char) + test_patches_version_list.append('2:4.4.5+dfsg-2ubuntu\u20ac' + random_char) else: - test_patches_version_list.append('2:4.4.5+dfsg-2ubuntu€') + test_patches_version_list.append('2:4.4.5+dfsg-2ubuntu\u20ac') return test_patches_list, test_patches_version_list