From 0238a7dc0b757029bcdc4bdbbe4eb855a849f25c Mon Sep 17 00:00:00 2001 From: ksiamro Date: Tue, 23 Jul 2024 10:28:15 +0200 Subject: [PATCH 1/5] Fix issue with too large DB Load DB data in chunks for Scan and Update --- update-encryption.php | 199 +++++++++++++++++++++++++----------------- 1 file changed, 118 insertions(+), 81 deletions(-) diff --git a/update-encryption.php b/update-encryption.php index 324c655..fc0486b 100644 --- a/update-encryption.php +++ b/update-encryption.php @@ -1,26 +1,26 @@ /** - * This code is licensed under the MIT License. - * - * MIT License - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ +* This code is licensed under the MIT License. +* +* MIT License +* +* Permission is hereby granted, free of charge, to any person obtaining a copy +* of this software and associated documentation files (the "Software"), to deal +* in the Software without restriction, including without limitation the rights +* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +* copies of the Software, and to permit persons to whom the Software is +* furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in all +* copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +* SOFTWARE. +*/ query("SELECT * FROM $table")->fetchAll(PDO::FETCH_ASSOC); - if (!$data) - continue; - foreach ($data as $row) { - $idField = ''; - $idValue = ''; - $isCoreConfigData = false; - if (preg_match("%core_config_data%", $table)) { - $idField = 'config_id'; - $idValue = $row['config_id']; - $isCoreConfigData = true; + + $offset = 0; + $limit = 1000; // Adjust the chunk size as needed + $moreRowsAvailable = true; + + while ($moreRowsAvailable) { + $query = $db->prepare("SELECT * FROM $table LIMIT :offset, :limit"); + $query->bindParam(':offset', $offset, PDO::PARAM_INT); + $query->bindParam(':limit', $limit, PDO::PARAM_INT); + $query->execute(); + $data = $query->fetchAll(PDO::FETCH_ASSOC); + + if (!$data) { + $moreRowsAvailable = false; } else { - foreach ($row as $fieldName => $value) { - if (preg_match('%_id$%', $fieldName)) { - $idField = $fieldName; - $idValue = $value; + foreach ($data as $row) { + $idField = ''; + $idValue = ''; + $isCoreConfigData = false; + if (preg_match("%core_config_data%", $table)) { + $idField = 'config_id'; + $idValue = $row['config_id']; + $isCoreConfigData = true; + } else { + foreach ($row as $fieldName => $value) { + if (preg_match('%_id$%', $fieldName)) { + $idField = $fieldName; + $idValue = $value; + } + } } - } - } - foreach ($row as $fieldName => $value) { - if (($value !== null) && preg_match("%^\d\:\d\:%", $value)) { - $chunks = explode(':', $value); - $decrypted = 'N/A'; - $reEncrypted = 'N/A'; - $path = $isCoreConfigData ? $row['path'] : 'N/A'; - if ($params['decrypt'] && isset($params['key'])) { - $decrypted = $crypt->decrypt(base64_decode($chunks[2])); - } - if ($params['re-encrypt'] && isset($params['key'])) { - $reEncrypted = sprintf("%d:3:%s", $params['key-number'], base64_encode($cryptNew->encrypt($decrypted))); - } - $update = [ - $table, $idField, $idValue, $path, $fieldName, $value, $decrypted, $reEncrypted - ]; - fputcsv($f, $update); - $encryptedField = sprintf("$table::$fieldName"); - if (!in_array($encryptedField, $encryptedFields)) { - $encryptedFields[] = $encryptedField; + foreach ($row as $fieldName => $value) { + if (($value !== null) && preg_match("%^\d\:\d\:%", $value)) { + $chunks = explode(':', $value); + $decrypted = 'N/A'; + $reEncrypted = 'N/A'; + $path = $isCoreConfigData ? $row['path'] : 'N/A'; + if ($params['decrypt'] && isset($params['key'])) { + $decrypted = $crypt->decrypt(base64_decode($chunks[2])); + } + if ($params['re-encrypt'] && isset($params['key'])) { + $reEncrypted = sprintf("%d:3:%s", $params['key-number'], + base64_encode($cryptNew->encrypt($decrypted))); + } + $update = [ + $table, $idField, $idValue, $path, $fieldName, $value, $decrypted, $reEncrypted + ]; + fputcsv($f, $update); + $encryptedField = sprintf("$table::$fieldName"); + if (!in_array($encryptedField, $encryptedFields)) { + $encryptedFields[] = $encryptedField; + } + } } + $offset += $limit; + } } + } - } + fclose($f); print_r($encryptedFields); } else if ($command == 'update-table' || $command == 'update-record') { if (!isset($params['table'])) @@ -180,7 +197,10 @@ $recordFilter = sprintf(" AND `%s`='%d'", $idField, $id); $query = sprintf("SELECT * FROM `%s` WHERE `%s` LIKE '%d:3%%' %s", $table, $field, $keyNumber, $recordFilter); echo $query . "\n"; - $data = $db->query($query)->fetchAll(PDO::FETCH_ASSOC); + + $offset = 0; + $limit = 1000; // Adjust the chunk size as needed + $moreRowsAvailable = true; $fileHandler = null; $backupHandler = null; @@ -188,25 +208,42 @@ $fileHandler = fopen($params['dump'], 'a'); $backupHandler = fopen('backup-' . $params['dump'], 'a'); } - foreach ($data as $row) { - $value = $row[$field]; - $chunks = explode(':', $value); - $decrypted = $crypt->decrypt(base64_decode($chunks[2])); - $reEncrypted = sprintf("%d:3:%s", $params['key-number'], base64_encode($cryptNew->encrypt($decrypted))); - - echo "UPDATING row $idField=" . $idField . ", $field=" . $row[$field] . "; New value = " . $reEncrypted . "\n"; - $updateQuery = sprintf("UPDATE `%s` SET `%s`='%s' WHERE `%s`='%d' LIMIT 1;", $table, $field, $reEncrypted, $idField, $row[$idField]); - $backupQuery = sprintf("UPDATE `%s` SET `%s`='%s' WHERE `%s`='%d' LIMIT 1;", $table, $field, $value, $idField, $row[$idField]); - - if ($params['dump']) { - fwrite($fileHandler, $updateQuery . "\n"); - fwrite($backupHandler, $backupQuery . "\n"); - } - echo " " . $updateQuery . "\n"; - if (!isset($params['dry-run']) || !$params['dry-run']) { - echo "UPDATING !\n"; - $db->query($updateQuery); + while ($moreRowsAvailable) { + $query = sprintf("SELECT * FROM `%s` WHERE `%s` LIKE '%d:3%%' %s LIMIT :offset, :limit", $table, $field, $keyNumber, $recordFilter); + $stmt = $db->prepare($query); + $stmt->bindParam(':offset', $offset, PDO::PARAM_INT); + $stmt->bindParam(':limit', $limit, PDO::PARAM_INT); + $stmt->execute(); + $data = $stmt->fetchAll(PDO::FETCH_ASSOC); + + foreach ($data as $row) { + $value = $row[$field]; + $chunks = explode(':', $value); + $decrypted = $crypt->decrypt(base64_decode($chunks[2])); + $reEncrypted = sprintf("%d:3:%s", $params['key-number'], base64_encode($cryptNew->encrypt($decrypted))); + + echo "UPDATING row $idField=" . $idField . ", $field=" . $row[$field] . "; New value = " . $reEncrypted . "\n"; + $updateQuery = + sprintf("UPDATE `%s` SET `%s`='%s' WHERE `%s`='%d' LIMIT 1;", $table, $field, $reEncrypted, $idField, + $row[$idField]); + $backupQuery = + sprintf("UPDATE `%s` SET `%s`='%s' WHERE `%s`='%d' LIMIT 1;", $table, $field, $value, $idField, + $row[$idField]); + + if ($params['dump']) { + fwrite($fileHandler, $updateQuery . "\n"); + fwrite($backupHandler, $backupQuery . "\n"); + } + echo " " . $updateQuery . "\n"; + if (!isset($params['dry-run']) || !$params['dry-run']) { + echo "UPDATING !\n"; + $db->query($updateQuery); + } } + $offset += $limit; + } + if ($params['dump']) { + fclose($fileHandler); + fclose($backupHandler); } } - From 466ad7a4e8624b34d335da1bc5b5bb93d5fe0940 Mon Sep 17 00:00:00 2001 From: ksiamro Date: Tue, 23 Jul 2024 10:29:48 +0200 Subject: [PATCH 2/5] Revert changes for PHPdocs --- update-encryption.php | 44 +++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/update-encryption.php b/update-encryption.php index fc0486b..9fea3d3 100644 --- a/update-encryption.php +++ b/update-encryption.php @@ -1,26 +1,26 @@ /** -* This code is licensed under the MIT License. -* -* MIT License -* -* Permission is hereby granted, free of charge, to any person obtaining a copy -* of this software and associated documentation files (the "Software"), to deal -* in the Software without restriction, including without limitation the rights -* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -* copies of the Software, and to permit persons to whom the Software is -* furnished to do so, subject to the following conditions: -* -* The above copyright notice and this permission notice shall be included in all -* copies or substantial portions of the Software. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -* SOFTWARE. -*/ + * This code is licensed under the MIT License. + * + * MIT License + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ Date: Tue, 23 Jul 2024 13:21:10 +0200 Subject: [PATCH 3/5] Fix issue for never ending loop for update --- update-encryption.php | 51 ++++++++++++++++++++----------------------- 1 file changed, 24 insertions(+), 27 deletions(-) diff --git a/update-encryption.php b/update-encryption.php index 9fea3d3..a9beba4 100644 --- a/update-encryption.php +++ b/update-encryption.php @@ -153,12 +153,9 @@ $decrypted = $crypt->decrypt(base64_decode($chunks[2])); } if ($params['re-encrypt'] && isset($params['key'])) { - $reEncrypted = sprintf("%d:3:%s", $params['key-number'], - base64_encode($cryptNew->encrypt($decrypted))); + $reEncrypted = sprintf("%d:3:%s", $params['key-number'], base64_encode($cryptNew->encrypt($decrypted))); } - $update = [ - $table, $idField, $idValue, $path, $fieldName, $value, $decrypted, $reEncrypted - ]; + $update = [$table, $idField, $idValue, $path, $fieldName, $value, $decrypted, $reEncrypted]; fputcsv($f, $update); $encryptedField = sprintf("$table::$fieldName"); if (!in_array($encryptedField, $encryptedFields)) { @@ -216,28 +213,28 @@ $stmt->execute(); $data = $stmt->fetchAll(PDO::FETCH_ASSOC); - foreach ($data as $row) { - $value = $row[$field]; - $chunks = explode(':', $value); - $decrypted = $crypt->decrypt(base64_decode($chunks[2])); - $reEncrypted = sprintf("%d:3:%s", $params['key-number'], base64_encode($cryptNew->encrypt($decrypted))); - - echo "UPDATING row $idField=" . $idField . ", $field=" . $row[$field] . "; New value = " . $reEncrypted . "\n"; - $updateQuery = - sprintf("UPDATE `%s` SET `%s`='%s' WHERE `%s`='%d' LIMIT 1;", $table, $field, $reEncrypted, $idField, - $row[$idField]); - $backupQuery = - sprintf("UPDATE `%s` SET `%s`='%s' WHERE `%s`='%d' LIMIT 1;", $table, $field, $value, $idField, - $row[$idField]); - - if ($params['dump']) { - fwrite($fileHandler, $updateQuery . "\n"); - fwrite($backupHandler, $backupQuery . "\n"); - } - echo " " . $updateQuery . "\n"; - if (!isset($params['dry-run']) || !$params['dry-run']) { - echo "UPDATING !\n"; - $db->query($updateQuery); + if (!$data) { + $moreRowsAvailable = false; + } else { + foreach ($data as $row) { + $value = $row[$field]; + $chunks = explode(':', $value); + $decrypted = $crypt->decrypt(base64_decode($chunks[2])); + $reEncrypted = sprintf("%d:3:%s", $params['key-number'], base64_encode($cryptNew->encrypt($decrypted))); + + echo "UPDATING row $idField=" . $idField . ", $field=" . $row[$field] . "; New value = " . $reEncrypted . "\n"; + $updateQuery = sprintf("UPDATE `%s` SET `%s`='%s' WHERE `%s`='%d' LIMIT 1;", $table, $field, $reEncrypted, $idField, $row[$idField]); + $backupQuery = sprintf("UPDATE `%s` SET `%s`='%s' WHERE `%s`='%d' LIMIT 1;", $table, $field, $value, $idField, $row[$idField]); + + if ($params['dump']) { + fwrite($fileHandler, $updateQuery . "\n"); + fwrite($backupHandler, $backupQuery . "\n"); + } + echo " " . $updateQuery . "\n"; + if (!isset($params['dry-run']) || !$params['dry-run']) { + echo "UPDATING !\n"; + $db->query($updateQuery); + } } } $offset += $limit; From 838fac17a2237fdb81b0338c8a82cf2f0a5940a4 Mon Sep 17 00:00:00 2001 From: ksiamro Date: Tue, 23 Jul 2024 14:28:22 +0200 Subject: [PATCH 4/5] Change key-index to proper key-number in Readme --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 877f984..9317004 100644 --- a/README.md +++ b/README.md @@ -56,8 +56,8 @@ This command DOES CHANGE the database!!! Be careful!! Only run it when old enc ``` update-encryption.php update-table --table=core_config_data\ - --id-field=config_id --field=value --key=NEW_KEY --key-index=1\ - --old-key-index=[0/1] + --id-field=config_id --field=value --key=NEW_KEY --key-number=1\ + --old-key-number=[0/1] --dump=rotation.sql --dry-run ``` From a7e6fb9b8fbac16ba17219ec7c576c9e440745b0 Mon Sep 17 00:00:00 2001 From: ksiamro Date: Tue, 23 Jul 2024 14:31:17 +0200 Subject: [PATCH 5/5] Allow to run update command without defined dump param --- update-encryption.php | 1 + 1 file changed, 1 insertion(+) diff --git a/update-encryption.php b/update-encryption.php index a9beba4..fdcdeb8 100644 --- a/update-encryption.php +++ b/update-encryption.php @@ -38,6 +38,7 @@ 'dry-run' => false, 'key-number' => 1, 'old-key-number' => 0, + 'dump' => false, 'output' => 'encrypted-values.csv' ];