Skip to content

Commit

Permalink
https://github.com/rahulmalik87/pstress/issues/34
Browse files Browse the repository at this point in the history 
Improve the grammar SQL by adding DATE, DATETIME and TIMESTAMP
  • Loading branch information
@rahulmalik87
rahulmalik87 committed Feb 15, 2024
1 parent 5b23503 commit db29364
Show file tree
Hide file tree
Showing 3 changed files with 207 additions and 131 deletions.
11 changes: 8 additions & 3 deletions src/grammar.sql
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,9 @@
SELECT * FROM T1 JOIN T2 ON T1_INT_1 = T2_INT_1 ORDER BY T2_VARCHAR_1 DESC, T1_VARCHAR_1
SELECT @@VERSION
SELECT count(*) FROM T1 JOIN T2 ON T1_INT_1 = T2_INT_1 ORDER BY T2_VARCHAR_1 DESC, T1_VARCHAR_1
SELECT count(*) FROM T1
SELECT sum(T1_INT_1) FROM T1
SELECT avg(T1_INT_1) FROM T1
SELECT max(T1_INT_1) FROM T1
SELECT min(T1_INT_1) FROM T1
SELECT * FROM T1
SELECT NOW()
SELECT DATE_ADD(T1_DATE_1, INTERVAL 1 DAY) FROM T1
SELECT DATE_SUB(T1_DATE_1, INTERVAL 1 DAY) FROM T1
250 changes: 122 additions & 128 deletions src/random_test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3279,157 +3279,151 @@ void alter_tablespace_rename(Thd1 *thd) {
}
}

/* load special sql from a file */
static std::vector<std::string> load_grammar_sql_from() {
/* load special sql from a file and return */
static std::vector<grammar_tables> load_grammar_sql_from() {
std::vector<std::string> array;
auto grammar_file = opt_string(GRAMMAR_FILE);
std::string sql, file;
if (grammar_file == "grammar.sql")
file = std::string(binary_fullpath) + "/" + std::string(grammar_file);
else
file = grammar_file;

std::ifstream myfile(file);
if (myfile.is_open()) {
while (!myfile.eof()) {
getline(myfile, sql);
/* do not process any blank lines */
if (sql.find_first_not_of("\t\n ") != std::string::npos)
array.push_back(sql);
}
myfile.close();
} else
throw std::runtime_error("unable to open file " + file);
return array;
{
std::string sql, file;
if (grammar_file == "grammar.sql")
file = std::string(binary_fullpath) + "/" + std::string(grammar_file);
else
file = grammar_file;

std::ifstream myfile(file);
if (myfile.is_open()) {
while (!myfile.eof()) {
getline(myfile, sql);
/* do not process any blank lines */
if (sql.find_first_not_of("\t\n ") != std::string::npos)
array.push_back(sql);
}
myfile.close();
} else
throw std::runtime_error("unable to open file " + file);
}
std::vector<grammar_tables> tables;

for (auto &sql : array) {
/* Parse the grammar SQL and create a table object */
std::vector<grammar_table> sql_tables;
int tab_sql = 1; // start with 1
do { // search for table
std::smatch match;
std::string tab_p = "T" + std::to_string(tab_sql++); // table pattern

if (regex_search(sql, match, std::regex(tab_p))) {

auto add_columns = [&](grammar_table::sql_col_types type) {
int col_sql = 1;
do {
std::string col_p = tab_p + "_" +
grammar_table::get_col_type(type) + "_" +
std::to_string(col_sql);
if (regex_search(sql, match, std::regex(col_p))) {
sql_tables.back().column_count.at(type)++;
col_sql++;
} else {
break;
}
} while (true);
};

sql_tables.emplace_back(tab_p);

for (auto &type : grammar_table::get_vector_of_col_type()) {
add_columns(type);
}
} else
// if no more table found,
break;
} while (true);
tables.emplace_back(sql, sql_tables);
}
return tables;
}

/* return preformatted sql */
static void grammar_sql(std::vector<Table *> *all_tables, Thd1 *thd) {

static std::vector<std::string> all_sql = load_grammar_sql_from();
enum sql_col_types { INT, VARCHAR };
static auto all_tables_from_grammar = load_grammar_sql_from();

if (all_sql.size() == 0)
if (all_tables_from_grammar.size() == 0)
return;

struct table {
table(std::string n, std::vector<std::string> i, std::vector<std::string> v)
: name(n), int_col(i), varchar_col(v){};
std::string name;
std::vector<std::string> int_col;
std::vector<std::string> varchar_col;
};

auto sql = all_sql[rand_int(all_sql.size() - 1)];

/* parse SQL in table */
std::vector<std::vector<int>> sql_tables;

int tab_sql = 1; // number of tables in sql
bool table_found;

do { // search for table
std::smatch match;
std::string tab_p = "T" + std::to_string(tab_sql); // table pattern

if (regex_search(sql, match, std::regex(tab_p))) {
table_found = true;
sql_tables.push_back({0, 0});
auto currrent_table =
all_tables_from_grammar.at(rand_int(all_tables_from_grammar.size() - 1));

int col_sql = 1;
bool column_found;
auto sql = currrent_table.sql;
auto &sql_tables = currrent_table.tables;

do { // search of int column
std::string col_p = tab_p + "_INT_" + std::to_string(col_sql);
if (regex_search(sql, match, std::regex(col_p))) {
column_found = true;
sql_tables.at(tab_sql - 1).at(INT)++;
col_sql++;
} else
column_found = false;
} while (column_found);
// Find the real table and columns
for (auto &table : sql_tables) {
int table_check = 100; // try to find table
do {
auto working_table = all_tables->at(rand_int(all_tables->size() - 1));
working_table->table_mutex.lock();
table.found_name = working_table->name_;

col_sql = 1;
auto columns = working_table->columns_;
int column_check = 20; // max number of times to find column
do {
std::string col_p = tab_p + "_VARCHAR_" + std::to_string(col_sql);
if (regex_search(sql, match, std::regex(col_p))) {
column_found = true;
sql_tables.at(tab_sql - 1).at(VARCHAR)++;
col_sql++;
} else
column_found = false;
} while (column_found);
} else
table_found = false;
tab_sql++;
} while (table_found);

std::vector<table> final_tables;
auto col = columns->at(rand_int(columns->size() - 1));
auto col_type =
grammar_table::get_col_type(col->col_type_to_string(col->type_));

/* try at max 100 times */
int table_check = 100;

while (sql_tables.size() > 0 && table_check-- > 0) {
// if a valid column is not found in the table
if (col_type == grammar_table::MAX)
continue;

auto int_columns = sql_tables.back().at(INT);
auto varchar_columns = sql_tables.back().at(VARCHAR);
std::vector<std::string> int_cols_str, var_cols_str;
int column_check = 20;
auto table = all_tables->at(rand_int(all_tables->size() - 1));
table->table_mutex.lock();
auto columns = table->columns_;
if (table.column_count.at(col_type) > 0 &&
table.column_count.at(col_type) !=
(int)table.columns.at(col_type).size()) {
table.columns.at(col_type).emplace_back(col->name_);
}
} while (column_check-- > 0 &&
table.total_column_count() != table.total_column_written());

// find columns in table //
do {
auto col = columns->at(rand_int(columns->size() - 1));
working_table->table_mutex.unlock();

if (int_columns > 0 && col->type_ == Column::INT) {
int_cols_str.push_back(col->name_);
int_columns--;
}
if (varchar_columns > 0 && col->type_ == Column::VARCHAR) {
var_cols_str.push_back(col->name_);
varchar_columns--;
}
if (table.total_column_count() != table.total_column_written())
table.reset_columns();

if (int_columns == 0 && varchar_columns == 0) {
final_tables.emplace_back(table->name_, int_cols_str, var_cols_str);
sql_tables.pop_back();
}
} while (!(int_columns == 0 && varchar_columns == 0) && column_check-- > 0);
} while (table.total_column_count() != table.total_column_written() &&
table_check-- > 0);

table->table_mutex.unlock();
if (table.total_column_count() != table.total_column_written()) {
std::cout << "NOT ABLE TO FIND any table for SQL in 100 iteration " << sql
<< std::endl;
return;
}
}

if (sql_tables.size() == 0) {

for (size_t i = 0; i < final_tables.size(); i++) {
auto table = final_tables.at(i);
auto table_name = "T" + std::to_string(i + 1);

/* replace int column */
for (size_t j = 0; j < table.int_col.size(); j++)
sql = std::regex_replace(
sql, std::regex(table_name + "_INT_" + std::to_string(j + 1)),
table_name + "." + table.int_col.at(j));

/* replace varchar column */
for (size_t j = 0; j < table.varchar_col.size(); j++)
sql = std::regex_replace(
sql, std::regex(table_name + "_VARCHAR_" + std::to_string(j + 1)),
table_name + "." + table.varchar_col.at(j));

/* replace table "T1 " => tt_N T1 */
sql = std::regex_replace(sql, std::regex(table_name + " "),
table.name + " " + table_name + " ");
/* replace table "T1$" => tt_N T1*/
sql = std::regex_replace(sql, std::regex(table_name + "$"),
table.name + " " + table_name + "");
/* replace the found column and table */
for (const auto &table : sql_tables) {
auto table_name = table.name;
for (size_t i = 0; i < table.columns.size(); i++) {
auto col = table.columns.at(i);
for (size_t j = 0; j < col.size(); j++) {
sql =
std::regex_replace(sql,
std::regex(table_name + "_" +
grammar_table::get_col_type(
(grammar_table::sql_col_types)i) +
"_" + std::to_string(j + 1)),
table_name + "." + col.at(j));
}
}

execute_sql(sql, thd);
} else
std::cout << "NOT ABLE TO FIND any SQL in special SQL" << std::endl;
/* replace table "T1 " => tt_N T1 */
sql = std::regex_replace(sql, std::regex(table_name + " "),
table.found_name + " " + table.name + " ");
/* replace table "T1$" => tt_N T1*/
sql = std::regex_replace(sql, std::regex(table_name + "$"),
table.found_name + " " + table.name + "");
}
if (!execute_sql(sql, thd)) {
print_and_log("Grammar SQL failed " + sql, thd);
}
}

/* save metadata to a file */
Expand Down
77 changes: 77 additions & 0 deletions src/random_test.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -473,4 +473,81 @@ void alter_database_encryption(Thd1 *thd);
void create_in_memory_data();
void generate_metadata_for_tables();
void create_database_tablespace(Thd1 *thd);
/* Grammar table class used for parsing the grammar file */
struct grammar_table {
grammar_table(std::string n)
: name(n), column_count(grammar_table::MAX, 0),
columns(grammar_table::MAX) {}
std::string name;
std::string found_name;
std::vector<int> column_count;
std::vector<std::vector<std::string>> columns;
bool table_found = false;
enum sql_col_types { INT, VARCHAR, DATETIME, DATE, TIMESTAMP, MAX };
static sql_col_types get_col_type(std::string type) {
if (type == "INT")
return INT;
if (type == "VARCHAR")
return VARCHAR;
if (type == "DATETIME")
return DATETIME;
if (type == "DATE")
return DATE;
if (type == "TIMESTAMP")
return TIMESTAMP;
return MAX;
}
static std::string get_col_type(sql_col_types type) {
switch (type) {
case INT:
return "INT";
case VARCHAR:
return "VARCHAR";
case DATETIME:
return "DATETIME";
case DATE:
return "DATE";
case TIMESTAMP:
return "TIMESTAMP";
case MAX:
break;
}
return "";
}
int total_column_count() {
int total = 0;
for (auto &i : column_count)
total += i;
return total;
}
int total_column_written() {
int total = 0;
for (auto &i : columns)
total += i.size();
return total;
}
void reset_columns() {
for (auto &i : columns) {
i.clear();
}
found_name = "";
}

static std::vector<sql_col_types> get_vector_of_col_type() {
std::vector<sql_col_types> v;
v.push_back(INT);
v.push_back(VARCHAR);
v.push_back(DATETIME);
v.push_back(DATE);
v.push_back(TIMESTAMP);
return v;
}
};
struct grammar_tables {
/* use tables move constructor */
grammar_tables(std::string sql_, std::vector<grammar_table> tables_)
: sql(sql_), tables(tables_){};
std::string sql;
std::vector<grammar_table> tables;
};
#endif

0 comments on commit db29364

Please sign in to comment.