From df6c154a771060d8c68663ba6ad9628856f4acbd Mon Sep 17 00:00:00 2001 From: Fotis Vasilopoulos Date: Thu, 3 Aug 2023 21:31:04 +0300 Subject: [PATCH] feat(migration-scripts) add sql-blueprinting script --- sql-blueprinting/.env.example | 8 ++ sql-blueprinting/.gitignore | 3 + sql-blueprinting/README.md | 26 +++++++ sql-blueprinting/index.js | 136 ++++++++++++++++++++++++++++++++++ sql-blueprinting/package.json | 22 ++++++ 5 files changed, 195 insertions(+) create mode 100644 sql-blueprinting/.env.example create mode 100644 sql-blueprinting/.gitignore create mode 100644 sql-blueprinting/README.md create mode 100644 sql-blueprinting/index.js create mode 100644 sql-blueprinting/package.json diff --git a/sql-blueprinting/.env.example b/sql-blueprinting/.env.example new file mode 100644 index 0000000..e322d5b --- /dev/null +++ b/sql-blueprinting/.env.example @@ -0,0 +1,8 @@ +DATABASE_BLUEPRINT_NAME=clean_db +DATABASE_TARGET_NAME=existing_db + +DATABASE_HOST= +DATABASE_PORT= +DATABASE_USER= +DATABASE_PASSWORD= +DATABASE_NAME= \ No newline at end of file diff --git a/sql-blueprinting/.gitignore b/sql-blueprinting/.gitignore new file mode 100644 index 0000000..6fd6d65 --- /dev/null +++ b/sql-blueprinting/.gitignore @@ -0,0 +1,3 @@ +node_modules +*.env +package-lock.json \ No newline at end of file diff --git a/sql-blueprinting/README.md b/sql-blueprinting/README.md new file mode 100644 index 0000000..1a06a0e --- /dev/null +++ b/sql-blueprinting/README.md @@ -0,0 +1,26 @@ +# SQL Blueprinting + +Compare two databases and drop the tables that are not common in both. +We run our strapi project in a new sql db so it creates its clean structure. +This db can be used as a blueprint since it is created by our strapi current state and doesnt have old entries etc. + +## Usage example + +- DB1 is a blueprint db that contains only a schema, we will use this db as a structure referance. +- DB2 is a production db that contains the data and a schema. +- We want to drop from the DB2 (prod) the tables that does not appear in the structure of DB1 +- After cleaning our prod db according to blueprint we can migrate it to v4 + +## Description + +Since we have to cleanup by order keys, columns and finally the tables, the db sets foreign key checks to 0 and after running back to 1. + +## Run + +- npm i +- npm run start + +## Important Notes + +- Please use this script on clone of your production db. +- This script drops all columns, collections and tables that does not exist in blueprint database, so use it carefully. diff --git a/sql-blueprinting/index.js b/sql-blueprinting/index.js new file mode 100644 index 0000000..6088594 --- /dev/null +++ b/sql-blueprinting/index.js @@ -0,0 +1,136 @@ +import chalk from 'chalk'; +import { config } from 'dotenv'; +import mysql from 'mysql2/promise'; + +config(); + +const db1 = process.env.DATABASE_BLUEPRINT_NAME; //reference database +const db2 = process.env.DATABASE_TARGET_NAME; // target database + +const connection = await mysql.createConnection({ + host: process.env.DATABASE_HOST, + port: process.env.DATABASE_PORT, + user: process.env.DATABASE_USER, + password: process.env.DATABASE_PASSWORD, + database: process.env.DATABASE_NAME, +}); + +connection.connect((err) => { + if (err) throw err; + console.log(chalk.bold.greenBright('Connected to the database!')); +}); + +const getTables = async (db) => { + const [tables] = await connection.query( + 'SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = ?', + [db] + ); + return tables; +}; + +const dropTable = async (db, table) => { + await connection.query(`DROP TABLE IF EXISTS ??.??`, [db, table]); + return `The table ${chalk.bold.redBright(table)} does not exists in both databases. Dropping...`; +}; + +const getColumns = async (db, table) => { + const [columns] = await connection.query( + 'SELECT * FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ?', + [db, table] + ); + return columns; +}; + +const dropColumn = async (db, table, column) => { + await connection.query(`ALTER TABLE ??.?? DROP COLUMN ??`, [db, table, column]); + return `The column ${chalk.bold.redBright(column)} does not exists in both ${chalk.bold.redBright( + table + )} tables. Dropping...`; +}; + +const dropRow = async (db, table, column, value) => { + await connection.query(`DELETE FROM ??.?? WHERE ?? = ?`, [db, table, column, value]); + return `The row ${chalk.bold.redBright(value)} does not exists in both ${chalk.bold.redBright( + table + )} tables. Dropping...`; +}; + +const toggleForeignKeyCheck = async (state) => { + await connection.query(`SET FOREIGN_KEY_CHECKS = ${state}`); + return 'Foreign Key Check is set to ' + state + '!'; +}; + +const getCoreStore = async (db) => { + const [coreStore] = await connection.query('SELECT * FROM ??.core_store', [db]); + return coreStore; +}; + +(async () => { + try { + let foreignKeyCheckState = 0; + toggleForeignKeyCheck(foreignKeyCheckState).then((res) => + console.log(chalk.bold.yellowBright(res)) + ); + const tableNames_db1 = await getTables(db1); + const tableNames_db2 = await getTables(db2); + + for (const tableName_db2 of tableNames_db2) { + let tableExistanceFlag = false; + let targetTableName = tableName_db2.TABLE_NAME; + tableNames_db1.forEach((table_db1) => { + if (targetTableName === table_db1.TABLE_NAME) { + tableExistanceFlag = true; + } + }); + if (tableExistanceFlag && targetTableName !== 'core_store') { + console.log( + `The table ${chalk.bold.greenBright(targetTableName)} exists in both databases.` + ); + const columns_db1 = await getColumns(db1, targetTableName); + const columns_db2 = await getColumns(db2, targetTableName); + + for (const column_db2 of columns_db2) { + let columnExistanceFlag = false; + let columnNameDB2 = column_db2.COLUMN_NAME; + columns_db1.forEach((column_db1) => { + if (columnNameDB2 === column_db1.COLUMN_NAME) { + columnExistanceFlag = true; + } + }); + if (!columnExistanceFlag) { + const dropColumnMsg = await dropColumn(db2, targetTableName, columnNameDB2); + console.log(dropColumnMsg); + } + } + } else if (targetTableName === 'core_store') { + const coreStore1 = await getCoreStore(db1); + const coreStore2 = await getCoreStore(db2); + for (const coreStore2Item of coreStore2) { + let coreStoreExistanceFlag = false; + let coreStore2ItemKey = coreStore2Item.key; + coreStore1.forEach((coreStore1Item) => { + if (coreStore2ItemKey === coreStore1Item.key) { + coreStoreExistanceFlag = true; + } + }); + if (!coreStoreExistanceFlag) { + const dropRowMsg = await dropRow(db2, targetTableName, 'key', coreStore2ItemKey); + console.log(dropRowMsg); + } + } + } else { + const dropTableMsg = await dropTable(db2, targetTableName); + console.log(dropTableMsg); + } + } + foreignKeyCheckState = 1; + toggleForeignKeyCheck(foreignKeyCheckState) + .then((res) => console.log(chalk.bold.yellowBright(res))) + .then(() => { + console.log('Database cleanup is done, closing connection...'); + connection.end(); + }); + } catch (err) { + console.log(err); + } +})(); diff --git a/sql-blueprinting/package.json b/sql-blueprinting/package.json new file mode 100644 index 0000000..d248f8b --- /dev/null +++ b/sql-blueprinting/package.json @@ -0,0 +1,22 @@ +{ + "name": "sql-blueprinting", + "version": "0.1.0", + "description": "", + "main": "index.js", + "author": "FotisVasilopoulos", + "scripts": { + "start": "node index.js" + }, + "dependencies": { + "chalk": "5.2.0", + "mysql": "2.18.1", + "mysql2": "3.2.4", + "ora": "6.2.0", + "dotenv": "16.0.0" + }, + "type": "module", + "engines": { + "npm": ">=6.0.0" + }, + "license": "UNLICENSED" +}