From f09f912bc2feab8317c28daa208a5a6b0408dbec Mon Sep 17 00:00:00 2001 From: Pilmo Date: Sat, 5 Aug 2023 14:29:16 +0900 Subject: [PATCH 01/11] implement StartEncryptionEvent --- pymysqlreplication/binlogstream.py | 5 +++-- pymysqlreplication/event.py | 22 ++++++++++++++++++++++ pymysqlreplication/packet.py | 2 +- 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/pymysqlreplication/binlogstream.py b/pymysqlreplication/binlogstream.py index fa65aa22..7a90772d 100644 --- a/pymysqlreplication/binlogstream.py +++ b/pymysqlreplication/binlogstream.py @@ -14,7 +14,7 @@ QueryEvent, RotateEvent, FormatDescriptionEvent, XidEvent, GtidEvent, StopEvent, XAPrepareEvent, BeginLoadQueryEvent, ExecuteLoadQueryEvent, - HeartbeatLogEvent, NotImplementedEvent, MariadbGtidEvent) + HeartbeatLogEvent, NotImplementedEvent, MariadbGtidEvent, StartEncryptionEvent) from .exceptions import BinLogNotEnabled from .row_event import ( UpdateRowsEvent, WriteRowsEvent, DeleteRowsEvent, TableMapEvent) @@ -600,7 +600,8 @@ def _allowed_event_list(self, only_events, ignored_events, TableMapEvent, HeartbeatLogEvent, NotImplementedEvent, - MariadbGtidEvent + MariadbGtidEvent, + StartEncryptionEvent )) if ignored_events is not None: for e in ignored_events: diff --git a/pymysqlreplication/event.py b/pymysqlreplication/event.py index d75c9db8..1c1942a8 100644 --- a/pymysqlreplication/event.py +++ b/pymysqlreplication/event.py @@ -437,6 +437,28 @@ def _dump(self): print("Value: %d" % (self.value)) +class StartEncryptionEvent(BinLogEvent): + """ + Attributes: + schema + key_version + nonce + """ + + def __init__(self, from_packet, event_size, table_map, ctl_connection, **kwargs): + super(StartEncryptionEvent, self).__init__(from_packet, event_size, table_map, ctl_connection, **kwargs) + + self.schema = self.packet.read_uint8() + self.key_version = self.packet.read_uint32() + self.nonce = self.packet.read(12) + + def _dump(self): + print("Schema: %d" % self.schema) + print("Key version: %d" % self.key_version) + print(f"Nonce: {self.nonce}") + + + class NotImplementedEvent(BinLogEvent): def __init__(self, from_packet, event_size, table_map, ctl_connection, **kwargs): super(NotImplementedEvent, self).__init__( diff --git a/pymysqlreplication/packet.py b/pymysqlreplication/packet.py index 94baefdf..5ae17b18 100644 --- a/pymysqlreplication/packet.py +++ b/pymysqlreplication/packet.py @@ -87,7 +87,7 @@ class BinLogPacketWrapper(object): constants.MARIADB_BINLOG_CHECKPOINT_EVENT: event.NotImplementedEvent, constants.MARIADB_GTID_EVENT: event.MariadbGtidEvent, constants.MARIADB_GTID_GTID_LIST_EVENT: event.NotImplementedEvent, - constants.MARIADB_START_ENCRYPTION_EVENT: event.NotImplementedEvent + constants.MARIADB_START_ENCRYPTION_EVENT: event.StartEncryptionEvent } def __init__(self, from_packet, table_map, From 399077fa42b99428cc59fa151bc3b3a1c9f3dccb Mon Sep 17 00:00:00 2001 From: "cucuridas@gamil.com" <3310223@naver.com> Date: Sat, 5 Aug 2023 14:48:28 +0900 Subject: [PATCH 02/11] Completed setup of MariaDB test environment. --- .mariadb/my.cnf | 23 +++++++++++++++++ .mariadb/no_encryption_key.key | 1 + docker-compose.yml | 44 +++++++++++++++++++++++--------- pymysqlreplication/tests/base.py | 26 +++++++++++++++++++ 4 files changed, 82 insertions(+), 12 deletions(-) create mode 100644 .mariadb/my.cnf create mode 100755 .mariadb/no_encryption_key.key diff --git a/.mariadb/my.cnf b/.mariadb/my.cnf new file mode 100644 index 00000000..c530c80c --- /dev/null +++ b/.mariadb/my.cnf @@ -0,0 +1,23 @@ +[client-server] +# Port or socket location where to connect +# port = 3306 +socket = /run/mysqld/mysqld.sock + +# Import all .cnf files from configuration directory + +!includedir /etc/mysql/mariadb.conf.d/ +!includedir /etc/mysql/conf.d/ + + +[mariadb] +plugin_load_add = file_key_management +# Key files that are not encrypted +loose_file_key_management_filename = /opt/key_file/no_encryption_key.key + +# Encrypted key file +# loose_file_key_management_filename=/opt/key_file/keyfile.enc +# loose_file_key_management_filekey=FILE:/opt/key_file/no_encryption_key.key +# file_key_management_encryption_algorithm=aes_ctr + +# Set encrypt_binlog +encrypt_binlog=ON \ No newline at end of file diff --git a/.mariadb/no_encryption_key.key b/.mariadb/no_encryption_key.key new file mode 100755 index 00000000..476ede79 --- /dev/null +++ b/.mariadb/no_encryption_key.key @@ -0,0 +1 @@ +1;dda0ccb18a28b0b4c2448b5f0217a134 \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index d6449d2c..b11e8556 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,17 +1,37 @@ version: '3.2' services: - percona-5.7: - image: percona:5.7 - environment: - MYSQL_ALLOW_EMPTY_PASSWORD: true - ports: - - 3306:3306 - command: mysqld --log-bin=mysql-bin.log --server-id 1 --binlog-format=row --gtid_mode=on --enforce-gtid-consistency=on --log_slave_updates + # percona-5.7: + # image: percona:5.7 + # environment: + # MYSQL_ALLOW_EMPTY_PASSWORD: true + # ports: + # - 3306:3306 + # command: mysqld --log-bin=mysql-bin.log --server-id 1 --binlog-format=row --gtid_mode=on --enforce-gtid-consistency=on --log_slave_updates + + # percona-5.7-ctl: + # image: percona:5.7 + # environment: + # MYSQL_ALLOW_EMPTY_PASSWORD: true + # ports: + # - 3307:3307 + # command: mysqld --log-bin=mysql-bin.log --server-id 1 --binlog-format=row --gtid_mode=on --enforce-gtid-consistency=on --log_slave_updates -P 3307 - percona-5.7-ctl: - image: percona:5.7 + mariadb-10.6: + image: mariadb:10.6 environment: - MYSQL_ALLOW_EMPTY_PASSWORD: true + MARIADB_ALLOW_EMPTY_ROOT_PASSWORD: 1 ports: - - 3307:3307 - command: mysqld --log-bin=mysql-bin.log --server-id 1 --binlog-format=row --gtid_mode=on --enforce-gtid-consistency=on --log_slave_updates -P 3307 + - "3308:3306" + command: | + --server-id=1 + --default-authentication-plugin=mysql_native_password + --log-bin=master-bin + --binlog-format=row + --log-slave-updates=on + volumes: + - type: bind + source: ./.mariadb + target: /opt/key_file + - type: bind + source: ./.mariadb/my.cnf + target: /etc/mysql/my.cnf \ No newline at end of file diff --git a/pymysqlreplication/tests/base.py b/pymysqlreplication/tests/base.py index a7975714..c8cfc821 100644 --- a/pymysqlreplication/tests/base.py +++ b/pymysqlreplication/tests/base.py @@ -121,3 +121,29 @@ def bin_log_basename(self): bin_log_basename = cursor.fetchone()[0] bin_log_basename = bin_log_basename.split("/")[-1] return bin_log_basename + + +class PyMySQLReplicationMariaDbTestCase(PyMySQLReplicationTestCase): + def setUp(self): + # default + self.database = { + "host": "localhost", + "user": "root", + "passwd": "", + "port": 3308, + "use_unicode": True, + "charset": "utf8", + "db": "pymysqlreplication_test" + } + + self.conn_control = None + db = copy.copy(self.database) + db["db"] = None + self.connect_conn_control(db) + self.execute("DROP DATABASE IF EXISTS pymysqlreplication_test") + self.execute("CREATE DATABASE pymysqlreplication_test") + db = copy.copy(self.database) + self.connect_conn_control(db) + self.stream = None + self.resetBinLog() + \ No newline at end of file From 591278bd109f9e21b64cd28f57daa8f2988e7d02 Mon Sep 17 00:00:00 2001 From: Pilmo Date: Sat, 5 Aug 2023 15:25:39 +0900 Subject: [PATCH 03/11] Fix: Correct assertion in test_basic:_allowed_event_list --- pymysqlreplication/tests/test_basic.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pymysqlreplication/tests/test_basic.py b/pymysqlreplication/tests/test_basic.py index 0db8a264..65f55971 100644 --- a/pymysqlreplication/tests/test_basic.py +++ b/pymysqlreplication/tests/test_basic.py @@ -25,8 +25,8 @@ def ignoredEvents(self): return [GtidEvent] def test_allowed_event_list(self): - self.assertEqual(len(self.stream._allowed_event_list(None, None, False)), 16) - self.assertEqual(len(self.stream._allowed_event_list(None, None, True)), 15) + self.assertEqual(len(self.stream._allowed_event_list(None, None, False)), 17) + self.assertEqual(len(self.stream._allowed_event_list(None, None, True)), 16) self.assertEqual(len(self.stream._allowed_event_list(None, [RotateEvent], False)), 15) self.assertEqual(len(self.stream._allowed_event_list([RotateEvent], None, False)), 1) From 7cefc3ea7f9c78e38c8861ac864af84f2a6a0d2f Mon Sep 17 00:00:00 2001 From: "cucuridas@gamil.com" <3310223@naver.com> Date: Sat, 5 Aug 2023 15:32:37 +0900 Subject: [PATCH 04/11] Fixed docker-compose.yaml --- docker-compose.yml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index b11e8556..0c018fe8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,20 +1,20 @@ version: '3.2' services: - # percona-5.7: - # image: percona:5.7 - # environment: - # MYSQL_ALLOW_EMPTY_PASSWORD: true - # ports: - # - 3306:3306 - # command: mysqld --log-bin=mysql-bin.log --server-id 1 --binlog-format=row --gtid_mode=on --enforce-gtid-consistency=on --log_slave_updates + percona-5.7: + image: percona:5.7 + environment: + MYSQL_ALLOW_EMPTY_PASSWORD: true + ports: + - 3306:3306 + command: mysqld --log-bin=mysql-bin.log --server-id 1 --binlog-format=row --gtid_mode=on --enforce-gtid-consistency=on --log_slave_updates - # percona-5.7-ctl: - # image: percona:5.7 - # environment: - # MYSQL_ALLOW_EMPTY_PASSWORD: true - # ports: - # - 3307:3307 - # command: mysqld --log-bin=mysql-bin.log --server-id 1 --binlog-format=row --gtid_mode=on --enforce-gtid-consistency=on --log_slave_updates -P 3307 + percona-5.7-ctl: + image: percona:5.7 + environment: + MYSQL_ALLOW_EMPTY_PASSWORD: true + ports: + - 3307:3307 + command: mysqld --log-bin=mysql-bin.log --server-id 1 --binlog-format=row --gtid_mode=on --enforce-gtid-consistency=on --log_slave_updates -P 3307 mariadb-10.6: image: mariadb:10.6 From b9026e08671fcddda3982c6cd5bc5aa601bebc5a Mon Sep 17 00:00:00 2001 From: Pilmo Date: Sat, 5 Aug 2023 15:37:03 +0900 Subject: [PATCH 05/11] Fix: Correct assertion in test_basic:_allowed_event_list --- pymysqlreplication/tests/test_basic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymysqlreplication/tests/test_basic.py b/pymysqlreplication/tests/test_basic.py index 65f55971..396f8cbc 100644 --- a/pymysqlreplication/tests/test_basic.py +++ b/pymysqlreplication/tests/test_basic.py @@ -27,7 +27,7 @@ def ignoredEvents(self): def test_allowed_event_list(self): self.assertEqual(len(self.stream._allowed_event_list(None, None, False)), 17) self.assertEqual(len(self.stream._allowed_event_list(None, None, True)), 16) - self.assertEqual(len(self.stream._allowed_event_list(None, [RotateEvent], False)), 15) + self.assertEqual(len(self.stream._allowed_event_list(None, [RotateEvent], False)), 16) self.assertEqual(len(self.stream._allowed_event_list([RotateEvent], None, False)), 1) def test_read_query_event(self): From 73d1d520b27dc0f6f8f51c81b51c21ec6f238385 Mon Sep 17 00:00:00 2001 From: BeautterLife Date: Sat, 5 Aug 2023 19:31:29 +0900 Subject: [PATCH 06/11] Add unit tests for MariadbStartEncryptionEvent class --- pymysqlreplication/tests/test_basic.py | 33 ++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/pymysqlreplication/tests/test_basic.py b/pymysqlreplication/tests/test_basic.py index 396f8cbc..e1d4ee82 100644 --- a/pymysqlreplication/tests/test_basic.py +++ b/pymysqlreplication/tests/test_basic.py @@ -1002,6 +1002,39 @@ def test_parsing(self): gtid = Gtid("57b70f4e-20d3-11e5-a393-4a63946f7eac:1-:1") gtid = Gtid("57b70f4e-20d3-11e5-a393-4a63946f7eac::1") +class TestMariadbBinlogStreamReader(base.PyMySQLReplicationMariaDbTestCase): + + def test_start_encryption_event(self): + query = "CREATE TABLE test (id INT NOT NULL AUTO_INCREMENT, data VARCHAR (50) NOT NULL, PRIMARY KEY (id))" + self.execute(query) + query = "INSERT INTO test (data) VALUES('Hello World')" + self.execute(query) + self.execute("COMMIT") + + self.assertIsInstance(self.stream.fetchone(), RotateEvent) + self.assertIsInstance(self.stream.fetchone(), FormatDescriptionEvent) + + start_encryption_event = self.stream.fetchone() + self.assertIsInstance(start_encryption_event, StartEncryptionEvent) + + schema = start_encryption_event.schema + key_version = start_encryption_event.key_version + nonce = start_encryption_event.nonce + + try: + with open("../../.mariadb/no_encryption_key.key", "r") as key_file: + first_line = key_file.readline() + key_version_from_key_file = int(first_line.split(";")[0]) + except Exception as e: + self.fail("raised unexpected exception: {exception}".format(exception=e)) + finally: + self.resetBinLog() + + # schema is always 1 + self.assertEqual(schema, 1) + self.assertEqual(key_version, key_version_from_key_file) + self.assertEqual(type(nonce), bytes) + self.assertEqual(len(nonce), 12) if __name__ == "__main__": import unittest From 6e5fb823bb5df899356fd7d70d768763928bafea Mon Sep 17 00:00:00 2001 From: BeautterLife Date: Sat, 5 Aug 2023 21:39:30 +0900 Subject: [PATCH 07/11] Fix:find file from current directroy --- pymysqlreplication/tests/test_basic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymysqlreplication/tests/test_basic.py b/pymysqlreplication/tests/test_basic.py index e1d4ee82..cfbfb6dd 100644 --- a/pymysqlreplication/tests/test_basic.py +++ b/pymysqlreplication/tests/test_basic.py @@ -1022,7 +1022,7 @@ def test_start_encryption_event(self): nonce = start_encryption_event.nonce try: - with open("../../.mariadb/no_encryption_key.key", "r") as key_file: + with open("./../../.mariadb/no_encryption_key.key", "r") as key_file: first_line = key_file.readline() key_version_from_key_file = int(first_line.split(";")[0]) except Exception as e: From 780d7717f7fcc533f74734eb99d94814642b47ec Mon Sep 17 00:00:00 2001 From: BeautterLife Date: Tue, 8 Aug 2023 23:23:02 +0900 Subject: [PATCH 08/11] Fix: find the file path --- pymysqlreplication/tests/test_basic.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pymysqlreplication/tests/test_basic.py b/pymysqlreplication/tests/test_basic.py index cfbfb6dd..c7ebc476 100644 --- a/pymysqlreplication/tests/test_basic.py +++ b/pymysqlreplication/tests/test_basic.py @@ -17,6 +17,8 @@ from pymysqlreplication.constants.BINLOG import * from pymysqlreplication.row_event import * +from pathlib import Path + __all__ = ["TestBasicBinLogStreamReader", "TestMultipleRowBinLogStreamReader", "TestCTLConnectionSettings", "TestGtidBinLogStreamReader"] @@ -1021,8 +1023,12 @@ def test_start_encryption_event(self): key_version = start_encryption_event.key_version nonce = start_encryption_event.nonce + from pathlib import Path + + encryption_key_file_path = Path(__file__).parent.parent.parent + try: - with open("./../../.mariadb/no_encryption_key.key", "r") as key_file: + with open(f"{encryption_key_file_path}/.mariadb/no_encryption_key.key", "r") as key_file: first_line = key_file.readline() key_version_from_key_file = int(first_line.split(";")[0]) except Exception as e: From 16586a6bf42a9965438075b56d725246d88956e4 Mon Sep 17 00:00:00 2001 From: Pilmo Date: Thu, 10 Aug 2023 02:42:09 +0900 Subject: [PATCH 09/11] Rename StartEncryptionEvent to MariadbStartEncryptionEvent --- pymysqlreplication/binlogstream.py | 4 ++-- pymysqlreplication/event.py | 5 ++--- pymysqlreplication/packet.py | 2 +- pymysqlreplication/tests/test_basic.py | 2 +- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/pymysqlreplication/binlogstream.py b/pymysqlreplication/binlogstream.py index 7a90772d..df307d9e 100644 --- a/pymysqlreplication/binlogstream.py +++ b/pymysqlreplication/binlogstream.py @@ -14,7 +14,7 @@ QueryEvent, RotateEvent, FormatDescriptionEvent, XidEvent, GtidEvent, StopEvent, XAPrepareEvent, BeginLoadQueryEvent, ExecuteLoadQueryEvent, - HeartbeatLogEvent, NotImplementedEvent, MariadbGtidEvent, StartEncryptionEvent) + HeartbeatLogEvent, NotImplementedEvent, MariadbGtidEvent, MariadbStartEncryptionEvent) from .exceptions import BinLogNotEnabled from .row_event import ( UpdateRowsEvent, WriteRowsEvent, DeleteRowsEvent, TableMapEvent) @@ -601,7 +601,7 @@ def _allowed_event_list(self, only_events, ignored_events, HeartbeatLogEvent, NotImplementedEvent, MariadbGtidEvent, - StartEncryptionEvent + MariadbStartEncryptionEvent )) if ignored_events is not None: for e in ignored_events: diff --git a/pymysqlreplication/event.py b/pymysqlreplication/event.py index 1c1942a8..9ab18169 100644 --- a/pymysqlreplication/event.py +++ b/pymysqlreplication/event.py @@ -437,7 +437,7 @@ def _dump(self): print("Value: %d" % (self.value)) -class StartEncryptionEvent(BinLogEvent): +class MariadbStartEncryptionEvent(BinLogEvent): """ Attributes: schema @@ -446,7 +446,7 @@ class StartEncryptionEvent(BinLogEvent): """ def __init__(self, from_packet, event_size, table_map, ctl_connection, **kwargs): - super(StartEncryptionEvent, self).__init__(from_packet, event_size, table_map, ctl_connection, **kwargs) + super(MariadbStartEncryptionEvent, self).__init__(from_packet, event_size, table_map, ctl_connection, **kwargs) self.schema = self.packet.read_uint8() self.key_version = self.packet.read_uint32() @@ -458,7 +458,6 @@ def _dump(self): print(f"Nonce: {self.nonce}") - class NotImplementedEvent(BinLogEvent): def __init__(self, from_packet, event_size, table_map, ctl_connection, **kwargs): super(NotImplementedEvent, self).__init__( diff --git a/pymysqlreplication/packet.py b/pymysqlreplication/packet.py index 5ae17b18..6c60b3e1 100644 --- a/pymysqlreplication/packet.py +++ b/pymysqlreplication/packet.py @@ -87,7 +87,7 @@ class BinLogPacketWrapper(object): constants.MARIADB_BINLOG_CHECKPOINT_EVENT: event.NotImplementedEvent, constants.MARIADB_GTID_EVENT: event.MariadbGtidEvent, constants.MARIADB_GTID_GTID_LIST_EVENT: event.NotImplementedEvent, - constants.MARIADB_START_ENCRYPTION_EVENT: event.StartEncryptionEvent + constants.MARIADB_START_ENCRYPTION_EVENT: event.MariadbStartEncryptionEvent } def __init__(self, from_packet, table_map, diff --git a/pymysqlreplication/tests/test_basic.py b/pymysqlreplication/tests/test_basic.py index c7ebc476..67b73eac 100644 --- a/pymysqlreplication/tests/test_basic.py +++ b/pymysqlreplication/tests/test_basic.py @@ -1017,7 +1017,7 @@ def test_start_encryption_event(self): self.assertIsInstance(self.stream.fetchone(), FormatDescriptionEvent) start_encryption_event = self.stream.fetchone() - self.assertIsInstance(start_encryption_event, StartEncryptionEvent) + self.assertIsInstance(start_encryption_event, MariadbStartEncryptionEvent) schema = start_encryption_event.schema key_version = start_encryption_event.key_version From 22c9caebccbc44597fd1777a6a39e48d3ac29813 Mon Sep 17 00:00:00 2001 From: "cucuridas@gamil.com" <3310223@naver.com> Date: Thu, 10 Aug 2023 14:27:55 +0900 Subject: [PATCH 10/11] add description for MariadbStartEncruptionEvent --- pymysqlreplication/event.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/pymysqlreplication/event.py b/pymysqlreplication/event.py index 9ab18169..13d654bb 100644 --- a/pymysqlreplication/event.py +++ b/pymysqlreplication/event.py @@ -439,10 +439,18 @@ def _dump(self): class MariadbStartEncryptionEvent(BinLogEvent): """ + Since MariaDB 10.1.7, + the START_ENCRYPTION event is written to every binary log file + if encrypt_binlog is set to ON. Prior to enabling this setting, + additional configuration steps are required in MariaDB. + (Link: **https://mariadb.com/kb/en/encrypting-binary-logs/**) + + This event is written just once, after the Format Description event + Attributes: - schema - key_version - nonce + schema: The Encryption scheme, always set to 1 for system files. + key_version: The Encryption key version. + nonce: Nonce (12 random bytes) of current binlog file. """ def __init__(self, from_packet, event_size, table_map, ctl_connection, **kwargs): From c8341c3daf9144c23835d7f2a2c4bd7e4a3af106 Mon Sep 17 00:00:00 2001 From: "cucuridas@gamil.com" <3310223@naver.com> Date: Thu, 10 Aug 2023 19:07:21 +0900 Subject: [PATCH 11/11] fix description for MariadbStartEncryptionEvent --- pymysqlreplication/event.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymysqlreplication/event.py b/pymysqlreplication/event.py index 13d654bb..0094882c 100644 --- a/pymysqlreplication/event.py +++ b/pymysqlreplication/event.py @@ -443,7 +443,7 @@ class MariadbStartEncryptionEvent(BinLogEvent): the START_ENCRYPTION event is written to every binary log file if encrypt_binlog is set to ON. Prior to enabling this setting, additional configuration steps are required in MariaDB. - (Link: **https://mariadb.com/kb/en/encrypting-binary-logs/**) + (Link: https://mariadb.com/kb/en/encrypting-binary-logs/) This event is written just once, after the Format Description event