From 1f2e8b8b6278b8a20b0b7ec0ad8976b2e27e25db Mon Sep 17 00:00:00 2001 From: mirageoasis Date: Fri, 20 Oct 2023 10:51:25 +0900 Subject: [PATCH 1/5] refactor : eradicate f-string --- pymysqlreplication/binlogstream.py | 12 +++++------ pymysqlreplication/event.py | 2 +- pymysqlreplication/tests/test_basic.py | 3 +-- pymysqlreplication/tests/test_data_type.py | 25 ++++++++-------------- 4 files changed, 17 insertions(+), 25 deletions(-) diff --git a/pymysqlreplication/binlogstream.py b/pymysqlreplication/binlogstream.py index 98876b39..62c71dc7 100644 --- a/pymysqlreplication/binlogstream.py +++ b/pymysqlreplication/binlogstream.py @@ -81,11 +81,11 @@ def __init__(self, value): self.hostname = value def __repr__(self): - return "" % ( - self.hostname, - self.username, - self.password, - self.port, + return ( + f"" ) def encoded(self, server_id, master_id=0): @@ -367,7 +367,7 @@ def __connect_to_stream(self): # master_heartbeat_period is nanoseconds heartbeat = int(heartbeat * 1000000000) cur = self._stream_connection.cursor() - cur.execute("SET @master_heartbeat_period= %d" % heartbeat) + cur.execute(f"SET @master_heartbeat_period= {heartbeat}") cur.close() # When replicating from Mariadb 10.6.12 using binlog coordinates, a slave capability < 4 triggers a bug in diff --git a/pymysqlreplication/event.py b/pymysqlreplication/event.py index 82b89974..2d649bb6 100644 --- a/pymysqlreplication/event.py +++ b/pymysqlreplication/event.py @@ -472,7 +472,7 @@ def __init__(self, from_packet, event_size, table_map, ctl_connection, **kwargs) def _dump(self): super()._dump() - print(f"Schema: {self.schema}" % (self.schema)) + print(f"Schema: {self.schema}") print(f"Execution time: {self.execution_time}") print(f"Query: {self.query}") diff --git a/pymysqlreplication/tests/test_basic.py b/pymysqlreplication/tests/test_basic.py index ee6bfe62..edfbde47 100644 --- a/pymysqlreplication/tests/test_basic.py +++ b/pymysqlreplication/tests/test_basic.py @@ -13,7 +13,6 @@ from pymysql.protocol import MysqlPacket import pytest - __all__ = [ "TestBasicBinLogStreamReader", "TestMultipleRowBinLogStreamReader", @@ -750,7 +749,7 @@ def test_ignore_decode_errors(self): self.stream.close() self.execute("CREATE TABLE test (data VARCHAR(50) CHARACTER SET utf8mb4)") self.execute_with_args( - "INSERT INTO test (data) VALUES (%s)", (problematic_unicode_string) + f"INSERT INTO test (data) VALUES ({problematic_unicode_string})" ) self.execute("COMMIT") diff --git a/pymysqlreplication/tests/test_data_type.py b/pymysqlreplication/tests/test_data_type.py index 4c03ab1c..9483f818 100644 --- a/pymysqlreplication/tests/test_data_type.py +++ b/pymysqlreplication/tests/test_data_type.py @@ -607,12 +607,10 @@ def test_json_array(self): def test_json_large(self): data = dict( - [("foooo%i" % i, "baaaaar%i" % i) for i in range(2560)] + [(f"foooo{i}", f"baaaaar{i}") for i in range(2560)] ) # Make it large enough to reach 2^16 length create_query = "CREATE TABLE test (id int, value json);" - insert_query = ( - """INSERT INTO test (id, value) VALUES (1, '%s');""" % json.dumps(data) - ) + insert_query = f"INSERT INTO test (id, value) VALUES (1, '{json.dumps(data)}');" event = self.create_and_insert_value(create_query, insert_query) if event.table_map[event.table_id].column_name_flag: self.assertEqual(event.rows[0]["values"]["value"], to_binary_dict(data)) @@ -621,8 +619,8 @@ def test_json_large_array(self): "Test json array larger than 64k bytes" create_query = "CREATE TABLE test (id int, value json);" large_array = dict(my_key=[i for i in range(100000)]) - insert_query = "INSERT INTO test (id, value) VALUES (1, '%s');" % ( - json.dumps(large_array), + insert_query = ( + f"INSERT INTO test (id, value) VALUES (1, '{json.dumps(large_array)}');" ) event = self.create_and_insert_value(create_query, insert_query) if event.table_map[event.table_id].column_name_flag: @@ -632,12 +630,10 @@ def test_json_large_array(self): def test_json_large_with_literal(self): data = dict( - [("foooo%i" % i, "baaaaar%i" % i) for i in range(2560)], literal=True + [(f"foooo{i}", f"baaaaar{i}") for i in range(2560)], literal=True ) # Make it large with literal create_query = "CREATE TABLE test (id int, value json);" - insert_query = ( - """INSERT INTO test (id, value) VALUES (1, '%s');""" % json.dumps(data) - ) + insert_query = f"INSERT INTO test (id, value) VALUES (1, '{json.dumps(data)}');" event = self.create_and_insert_value(create_query, insert_query) if event.table_map[event.table_id].column_name_flag: self.assertEqual(event.rows[0]["values"]["value"], to_binary_dict(data)) @@ -661,7 +657,7 @@ def test_json_types(self): data = {"foo": t} create_query = "CREATE TABLE test (id int, value json);" insert_query = ( - """INSERT INTO test (id, value) VALUES (1, '%s');""" % json.dumps(data) + f"INSERT INTO test (id, value) VALUES (1, '{json.dumps(data)}');" ) event = self.create_and_insert_value(create_query, insert_query) if event.table_map[event.table_id].column_name_flag: @@ -687,7 +683,7 @@ def test_json_basic(self): for data in types: create_query = "CREATE TABLE test (id int, value json);" insert_query = ( - """INSERT INTO test (id, value) VALUES (1, '%s');""" % json.dumps(data) + f"INSERT INTO test (id, value) VALUES (1, '{json.dumps(data)}');" ) event = self.create_and_insert_value(create_query, insert_query) if event.table_map[event.table_id].column_name_flag: @@ -709,10 +705,7 @@ def test_json_long_string(self): create_query = "CREATE TABLE test (id int, value json);" # The string length needs to be larger than what can fit in a single byte. string_value = "super_long_string" * 100 - insert_query = ( - 'INSERT INTO test (id, value) VALUES (1, \'{"my_key": "%s"}\');' - % (string_value,) - ) + insert_query = f'INSERT INTO test (id, value) VALUES (1, \'{{"my_key": "{string_value}"}}\')' event = self.create_and_insert_value(create_query, insert_query) if event.table_map[event.table_id].column_name_flag: self.assertEqual( From bd1c47536e3c7ccc51d3899a11d0a7c744f465be Mon Sep 17 00:00:00 2001 From: mirageoasis Date: Fri, 20 Oct 2023 11:04:37 +0900 Subject: [PATCH 2/5] fixed : test_ignore_decode_errors --- pymysqlreplication/tests/test_basic.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pymysqlreplication/tests/test_basic.py b/pymysqlreplication/tests/test_basic.py index edfbde47..95fa25ea 100644 --- a/pymysqlreplication/tests/test_basic.py +++ b/pymysqlreplication/tests/test_basic.py @@ -748,9 +748,7 @@ def test_ignore_decode_errors(self): ) self.stream.close() self.execute("CREATE TABLE test (data VARCHAR(50) CHARACTER SET utf8mb4)") - self.execute_with_args( - f"INSERT INTO test (data) VALUES ({problematic_unicode_string})" - ) + self.execute(f"INSERT INTO test (data) VALUES ({problematic_unicode_string})") self.execute("COMMIT") # Initialize with ignore_decode_errors=False From 98c8389b38dfb100fde2f15fbbcf5051cf6e4c52 Mon Sep 17 00:00:00 2001 From: mirageoasis Date: Fri, 20 Oct 2023 11:15:25 +0900 Subject: [PATCH 3/5] fixed : test_ignore_decode_errors --- pymysqlreplication/tests/test_basic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymysqlreplication/tests/test_basic.py b/pymysqlreplication/tests/test_basic.py index 95fa25ea..674eba25 100644 --- a/pymysqlreplication/tests/test_basic.py +++ b/pymysqlreplication/tests/test_basic.py @@ -748,7 +748,7 @@ def test_ignore_decode_errors(self): ) self.stream.close() self.execute("CREATE TABLE test (data VARCHAR(50) CHARACTER SET utf8mb4)") - self.execute(f"INSERT INTO test (data) VALUES ({problematic_unicode_string})") + self.execute(f"INSERT INTO test (data) VALUES ('{problematic_unicode_string}')") self.execute("COMMIT") # Initialize with ignore_decode_errors=False From aa62a03625847310047e309b93f0572c59c5b19d Mon Sep 17 00:00:00 2001 From: mirageoasis Date: Fri, 20 Oct 2023 11:18:48 +0900 Subject: [PATCH 4/5] fixed : temporary on test_ignore_decode_errors --- pymysqlreplication/tests/test_basic.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pymysqlreplication/tests/test_basic.py b/pymysqlreplication/tests/test_basic.py index 674eba25..26deaf21 100644 --- a/pymysqlreplication/tests/test_basic.py +++ b/pymysqlreplication/tests/test_basic.py @@ -748,7 +748,9 @@ def test_ignore_decode_errors(self): ) self.stream.close() self.execute("CREATE TABLE test (data VARCHAR(50) CHARACTER SET utf8mb4)") - self.execute(f"INSERT INTO test (data) VALUES ('{problematic_unicode_string}')") + self.execute_with_args( + "INSERT INTO test (data) VALUES (%s)", (problematic_unicode_string) + ) self.execute("COMMIT") # Initialize with ignore_decode_errors=False From e609b2da5160ab4e9e7f1aa5cd1d2bb7ad7a8e58 Mon Sep 17 00:00:00 2001 From: mirageoasis Date: Sat, 21 Oct 2023 18:50:46 +0900 Subject: [PATCH 5/5] fixed : fixed string to exeute_with_args to improve security --- pymysqlreplication/binlogstream.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymysqlreplication/binlogstream.py b/pymysqlreplication/binlogstream.py index 62c71dc7..feaeff3f 100644 --- a/pymysqlreplication/binlogstream.py +++ b/pymysqlreplication/binlogstream.py @@ -367,7 +367,7 @@ def __connect_to_stream(self): # master_heartbeat_period is nanoseconds heartbeat = int(heartbeat * 1000000000) cur = self._stream_connection.cursor() - cur.execute(f"SET @master_heartbeat_period= {heartbeat}") + cur.execute_with_args("SET @master_heartbeat_period= %d", (heartbeat,)) cur.close() # When replicating from Mariadb 10.6.12 using binlog coordinates, a slave capability < 4 triggers a bug in