Skip to content

Commit 5f4ecea

Browse files
MytherinMytherin
authored
Merge pull request #134 from Mytherin/issue116
Add `sqlite_query` method, and use this method when binding views that cannot be run inside DuckDB
2 parents 754bed3 + 03318ee commit 5f4ecea

File tree

12 files changed

+234
-19
lines changed

12 files changed

+234
-19
lines changed

‎data/db/unsupported_view.db‎

8 KB
Binary file not shown.

‎src/include/sqlite_scanner.hpp‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ struct SqliteBindData : public TableFunctionData {
2121

2222
vector<string> names;
2323
vector<LogicalType> types;
24+
string sql;
2425

2526
RowIdInfo row_id_info;
2627
bool all_varchar = false;
@@ -41,4 +42,9 @@ class SqliteAttachFunction : public TableFunction {
4142
SqliteAttachFunction();
4243
};
4344

45+
class SQLiteQueryFunction : public TableFunction {
46+
public:
47+
SQLiteQueryFunction();
48+
};
49+
4450
} // namespace duckdb

‎src/include/sqlite_stmt.hpp‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
//===----------------------------------------------------------------------===//
22
// DuckDB
33
//
4-
// sqlite_utils.hpp
4+
// sqlite_stmt.hpp
55
//
66
//
77
//===----------------------------------------------------------------------===//
@@ -44,6 +44,8 @@ class SQLiteStatement {
4444
void BindBlob(idx_t col, const string_t &value);
4545
void BindValue(Vector &col, idx_t c, idx_t r);
4646
int GetType(idx_t col);
47+
string GetName(idx_t col);
48+
idx_t GetColumnCount();
4749
bool IsOpen();
4850
void Close();
4951
void CheckTypeMatches(const SqliteBindData &bind_data, sqlite3_value *val, int sqlite_column_type,

‎src/sqlite_extension.cpp‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,9 @@ static void LoadInternal(DatabaseInstance &db) {
2727
SqliteAttachFunction attach_func;
2828
ExtensionUtil::RegisterFunction(db, attach_func);
2929

30+
SQLiteQueryFunction query_func;
31+
ExtensionUtil::RegisterFunction(db, query_func);
32+
3033
auto &config = DBConfig::GetConfig(db);
3134
config.AddExtensionOption("sqlite_all_varchar", "Load all SQLite columns as VARCHAR columns", LogicalType::BOOLEAN);
3235

‎src/sqlite_scanner.cpp‎

Lines changed: 20 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -99,23 +99,27 @@ static void SqliteInitInternal(ClientContext &context, const SqliteBindData &bin
9999
local_state.owned_db = SQLiteDB::Open(bind_data.file_name.c_str(), options);
100100
local_state.db = &local_state.owned_db;
101101
}
102-
103-
auto col_names = StringUtil::Join(
104-
local_state.column_ids.data(), local_state.column_ids.size(), ", ", [&](const idx_t column_id) {
105-
return column_id == (column_t)-1 ? "ROWID"
106-
: '"' + SQLiteUtils::SanitizeIdentifier(bind_data.names[column_id]) + '"';
107-
});
108-
109-
auto sql =
110-
StringUtil::Format("SELECT %s FROM \"%s\"", col_names, SQLiteUtils::SanitizeIdentifier(bind_data.table_name));
111-
if (bind_data.rows_per_group.IsValid()) {
112-
// we are scanning a subset of the rows - generate a WHERE clause based on
113-
// the rowid
114-
auto where_clause = StringUtil::Format(" WHERE ROWID BETWEEN %d AND %d", rowid_min, rowid_max);
115-
sql += where_clause;
102+
string sql;
103+
if (bind_data.sql.empty()) {
104+
auto col_names = StringUtil::Join(
105+
local_state.column_ids.data(), local_state.column_ids.size(), ", ", [&](const idx_t column_id) {
106+
return column_id == (column_t)-1 ? "ROWID"
107+
: '"' + SQLiteUtils::SanitizeIdentifier(bind_data.names[column_id]) + '"';
108+
});
109+
110+
sql =
111+
StringUtil::Format("SELECT %s FROM \"%s\"", col_names, SQLiteUtils::SanitizeIdentifier(bind_data.table_name));
112+
if (bind_data.rows_per_group.IsValid()) {
113+
// we are scanning a subset of the rows - generate a WHERE clause based on
114+
// the rowid
115+
auto where_clause = StringUtil::Format(" WHERE ROWID BETWEEN %d AND %d", rowid_min, rowid_max);
116+
sql += where_clause;
117+
} else {
118+
// we are scanning the entire table - no need for a WHERE clause
119+
D_ASSERT(rowid_min == 0);
120+
}
116121
} else {
117-
// we are scanning the entire table - no need for a WHERE clause
118-
D_ASSERT(rowid_min == 0);
122+
sql = bind_data.sql;
119123
}
120124
local_state.stmt = local_state.db->Prepare(sql.c_str());
121125
}

‎src/sqlite_stmt.cpp‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,16 @@ int SQLiteStatement::GetType(idx_t col) {
4343
return sqlite3_column_type(stmt, col);
4444
}
4545

46+
string SQLiteStatement::GetName(idx_t col) {
47+
D_ASSERT(stmt);
48+
return sqlite3_column_name(stmt, col);
49+
}
50+
51+
idx_t SQLiteStatement::GetColumnCount() {
52+
D_ASSERT(stmt);
53+
return sqlite3_column_count(stmt);
54+
}
55+
4656
bool SQLiteStatement::IsOpen() {
4757
return stmt;
4858
}

‎src/storage/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ add_library(
55
sqlite_index.cpp
66
sqlite_index_entry.cpp
77
sqlite_insert.cpp
8+
sqlite_query.cpp
89
sqlite_table_entry.cpp
910
sqlite_schema_entry.cpp
1011
sqlite_transaction.cpp

‎src/storage/sqlite_query.cpp‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
#include "duckdb.hpp"
2+
3+
#include "duckdb/parser/parsed_data/create_table_function_info.hpp"
4+
#include "sqlite_scanner.hpp"
5+
#include "duckdb/main/database_manager.hpp"
6+
#include "duckdb/main/attached_database.hpp"
7+
#include "storage/sqlite_catalog.hpp"
8+
#include "storage/sqlite_table_entry.hpp"
9+
#include "storage/sqlite_transaction.hpp"
10+
#include "sqlite_db.hpp"
11+
#include "sqlite_stmt.hpp"
12+
#include "sqlite_utils.hpp"
13+
14+
namespace duckdb {
15+
16+
static unique_ptr<FunctionData> SQLiteQueryBind(ClientContext &context, TableFunctionBindInput &input,
17+
vector<LogicalType> &return_types, vector<string> &names) {
18+
auto result = make_uniq<SqliteBindData>();
19+
20+
if (input.inputs[0].IsNull() || input.inputs[1].IsNull()) {
21+
throw BinderException("Parameters to sqlite_query cannot be NULL");
22+
}
23+
24+
// look up the database to query
25+
auto db_name = input.inputs[0].GetValue<string>();
26+
auto &db_manager = DatabaseManager::Get(context);
27+
auto db = db_manager.GetDatabase(context, db_name);
28+
if (!db) {
29+
throw BinderException("Failed to find attached database \"%s\" referenced in sqlite_query", db_name);
30+
}
31+
auto &catalog = db->GetCatalog();
32+
if (catalog.GetCatalogType() != "sqlite") {
33+
throw BinderException("Attached database \"%s\" does not refer to a SQLite database", db_name);
34+
}
35+
auto &sqlite_catalog = catalog.Cast<SQLiteCatalog>();
36+
auto &transaction = SQLiteTransaction::Get(context, catalog);
37+
auto sql = input.inputs[1].GetValue<string>();
38+
// strip any trailing semicolons
39+
StringUtil::RTrim(sql);
40+
while (!sql.empty() && sql.back() == ';') {
41+
sql = sql.substr(0, sql.size() - 1);
42+
StringUtil::RTrim(sql);
43+
}
44+
45+
auto &con = transaction.GetDB();
46+
auto stmt = con.Prepare(sql);
47+
if (!stmt.stmt) {
48+
throw BinderException("Failed to prepare query \"%s\"", sql);
49+
}
50+
for(idx_t c = 0; c < stmt.GetColumnCount(); c++) {
51+
return_types.emplace_back(LogicalType::VARCHAR);
52+
names.emplace_back(stmt.GetName(c));
53+
}
54+
stmt.Close();
55+
if (names.empty()) {
56+
throw BinderException("Failed to execute query \"%s\" - query must return data", sql);
57+
}
58+
result->rows_per_group = optional_idx();
59+
result->sql = std::move(sql);
60+
result->all_varchar = true;
61+
result->file_name = sqlite_catalog.GetDBPath();
62+
result->global_db = &con;
63+
return std::move(result);
64+
}
65+
66+
SQLiteQueryFunction::SQLiteQueryFunction()
67+
: TableFunction("sqlite_query", {LogicalType::VARCHAR, LogicalType::VARCHAR}, nullptr, SQLiteQueryBind) {
68+
SqliteScanFunction scan_function;
69+
init_global = scan_function.init_global;
70+
init_local = scan_function.init_local;
71+
function = scan_function.function;
72+
global_initialization = TableFunctionInitialization::INITIALIZE_ON_SCHEDULE;
73+
}
74+
} // namespace duckdb

‎src/storage/sqlite_transaction.cpp‎

Lines changed: 30 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,9 @@
77
#include "duckdb/parser/parsed_data/create_view_info.hpp"
88
#include "duckdb/catalog/catalog_entry/index_catalog_entry.hpp"
99
#include "duckdb/catalog/catalog_entry/view_catalog_entry.hpp"
10-
#include "duckdb/parser/parser.hpp"
10+
#include "duckdb/parser/parsed_data/create_view_info.hpp"
1111
#include "duckdb/parser/statement/create_statement.hpp"
12+
#include "duckdb/parser/parser.hpp"
1213
#include "duckdb/parser/parsed_expression_iterator.hpp"
1314
#include "duckdb/parser/expression/columnref_expression.hpp"
1415

@@ -48,6 +49,23 @@ SQLiteTransaction &SQLiteTransaction::Get(ClientContext &context, Catalog &catal
4849
return Transaction::Get(context, catalog).Cast<SQLiteTransaction>();
4950
}
5051

52+
string ExtractSelectStatement(const string &create_view) {
53+
Parser parser;
54+
parser.ParseQuery(create_view);
55+
if (parser.statements.size() != 1 || parser.statements[0]->type != StatementType::CREATE_STATEMENT) {
56+
throw BinderException(
57+
"Failed to create view from SQL string - \"%s\" - statement did not contain a single CREATE VIEW statement",
58+
create_view);
59+
}
60+
auto &create_statement = parser.statements[0]->Cast<CreateStatement>();
61+
if (create_statement.info->type != CatalogType::VIEW_ENTRY) {
62+
throw BinderException(
63+
"Failed to create view from SQL string - \"%s\" - view did not contain a CREATE VIEW statement", create_view);
64+
}
65+
auto &view_info = create_statement.info->Cast<CreateViewInfo>();
66+
return view_info.query->ToString();
67+
}
68+
5169
void ExtractColumnIds(const ParsedExpression &expr, TableCatalogEntry &table, CreateIndexInfo &info) {
5270
if (expr.GetExpressionType() == ExpressionType::COLUMN_REF) {
5371
auto &colref = expr.Cast<ColumnRefExpression>();
@@ -117,7 +135,17 @@ optional_ptr<CatalogEntry> SQLiteTransaction::GetCatalogEntry(const string &entr
117135
string sql;
118136
db->GetViewInfo(entry_name, sql);
119137

120-
auto view_info = CreateViewInfo::FromCreateView(*context.lock(), sql);
138+
139+
unique_ptr<CreateViewInfo> view_info;
140+
try {
141+
view_info = CreateViewInfo::FromCreateView(*context.lock(), sql);
142+
} catch(std::exception &ex) {
143+
auto view_sql = ExtractSelectStatement(sql);
144+
auto catalog_name = StringUtil::Replace(sqlite_catalog.GetName(), "\"", "\"\"");
145+
auto escaped_view_sql = StringUtil::Replace(view_sql, "'", "''");
146+
auto view_def = StringUtil::Format("CREATE VIEW %s AS FROM sqlite_query(\"%s\", '%s')", entry_name, catalog_name, escaped_view_sql);
147+
view_info = CreateViewInfo::FromCreateView(*context.lock(), view_def);
148+
}
121149
view_info->internal = false;
122150
result = make_uniq<ViewCatalogEntry>(sqlite_catalog, sqlite_catalog.GetMainSchema(), *view_info);
123151
break;

‎test/sql/storage/attach_sqlite_query.test‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
# name: test/sql/storage/attach_sqlite_query.test
2+
# description:
3+
# group: [sqlite_storage]
4+
5+
require sqlite_scanner
6+
7+
statement ok
8+
ATTACH '__TEST_DIR__/sqlite_query_test.db' AS s (TYPE SQLITE)
9+
10+
query I
11+
SELECT * FROM sqlite_query(s, 'SELECT 42 a')
12+
----
13+
42
14+
15+
query I
16+
SELECT * FROM sqlite_query(s, 'SELECT unixepoch(''1992-01-01'') a')
17+
----
18+
694224000
19+
20+
statement ok
21+
CREATE OR REPLACE TABLE s.tbl AS SELECT * FROM range(10000) t(r);
22+
23+
query I
24+
SELECT SUM(r) FROM s.tbl;
25+
----
26+
49995000
27+
28+
query I
29+
SELECT SUM(r::INT) FROM sqlite_query(s, 'SELECT * FROM tbl')
30+
----
31+
49995000
32+
33+
statement ok
34+
ATTACH 'data/db/sakila.db' AS sakila (TYPE SQLITE, READ_ONLY)
35+
36+
query II
37+
SELECT last_name, first_name FROM sqlite_query(sakila, 'SELECT * FROM actor WHERE last_name=''GUINESS''') ORDER BY ALL
38+
----
39+
GUINESS ED
40+
GUINESS PENELOPE
41+
GUINESS SEAN
42+
43+
statement error
44+
SELECT * FROM sqlite_query(s, '')
45+
----
46+
Failed to prepare
47+
48+
statement error
49+
SELECT * FROM sqlite_query(s, 'SELEC 42')
50+
----
51+
Failed to prepare
52+
53+
statement error
54+
SELECT * FROM sqlite_query(s, 'CREATE TABLE my_table(a,b,c)')
55+
----
56+
query must return data

0 commit comments

Comments
 (0)