Skip to content

Commit 64f3b12

Browse files
pdetpdet
authored
Merge pull request #570 from pdet/upd_m
V1.4 -> Main
2 parents 9849f1b + 6f71a86 commit 64f3b12

File tree

13 files changed

+110
-105
lines changed

13 files changed

+110
-105
lines changed

‎src/common/ducklake_types.cpp‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ struct DefaultType {
1212
LogicalTypeId id;
1313
};
1414

15-
using ducklake_type_array = std::array<DefaultType, 28>;
15+
using ducklake_type_array = std::array<DefaultType, 31>;
1616

1717
static constexpr const ducklake_type_array DUCKLAKE_TYPES {{{"boolean", LogicalTypeId::BOOLEAN},
1818
{"int8", LogicalTypeId::TINYINT},
@@ -41,6 +41,9 @@ static constexpr const ducklake_type_array DUCKLAKE_TYPES {{{"boolean", LogicalT
4141
{"varchar", LogicalTypeId::VARCHAR},
4242
{"blob", LogicalTypeId::BLOB},
4343
{"uuid", LogicalTypeId::UUID},
44+
{"struct", LogicalTypeId::STRUCT},
45+
{"map", LogicalTypeId::MAP},
46+
{"list", LogicalTypeId::LIST},
4447
{"unknown", LogicalTypeId::UNKNOWN}}};
4548

4649
static LogicalType ParseBaseType(const string &str) {

‎src/functions/ducklake_compaction_functions.cpp‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -538,9 +538,11 @@ unique_ptr<LogicalOperator> BindCompaction(ClientContext &context, TableFunction
538538
auto schemas = ducklake_catalog.GetSchemas(context);
539539
for (auto &cur_schema : schemas) {
540540
cur_schema.get().Scan(context, CatalogType::TABLE_ENTRY, [&](CatalogEntry &entry) {
541-
auto &cur_table = entry.Cast<DuckLakeTableEntry>();
542-
GenerateCompaction(context, transaction, ducklake_catalog, input, cur_table, type, delete_threshold,
543-
compactions);
541+
if (entry.type == CatalogType::TABLE_ENTRY) {
542+
auto &cur_table = entry.Cast<DuckLakeTableEntry>();
543+
GenerateCompaction(context, transaction, ducklake_catalog, input, cur_table, type,
544+
delete_threshold, compactions);
545+
}
544546
});
545547
}
546548
return GenerateCompactionOperator(input, bind_index, compactions);
@@ -549,9 +551,11 @@ unique_ptr<LogicalOperator> BindCompaction(ClientContext &context, TableFunction
549551
auto schema_entry = catalog.GetSchema(context, catalog.GetName(), schema, OnEntryNotFound::THROW_EXCEPTION);
550552
auto &ducklake_schema = schema_entry->Cast<DuckLakeSchemaEntry>();
551553
ducklake_schema.Scan(context, CatalogType::TABLE_ENTRY, [&](CatalogEntry &entry) {
552-
auto &cur_table = entry.Cast<DuckLakeTableEntry>();
553-
GenerateCompaction(context, transaction, ducklake_catalog, input, cur_table, type, delete_threshold,
554-
compactions);
554+
if (entry.type == CatalogType::TABLE_ENTRY) {
555+
auto &cur_table = entry.Cast<DuckLakeTableEntry>();
556+
GenerateCompaction(context, transaction, ducklake_catalog, input, cur_table, type, delete_threshold,
557+
compactions);
558+
}
555559
});
556560
return GenerateCompactionOperator(input, bind_index, compactions);
557561
}

‎src/functions/ducklake_set_option.cpp‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@ struct DuckLakeSetOptionData : public TableFunctionData {
1818
static unique_ptr<FunctionData> DuckLakeSetOptionBind(ClientContext &context, TableFunctionBindInput &input,
1919
vector<LogicalType> &return_types, vector<string> &names) {
2020
auto &catalog = BaseMetadataFunction::GetCatalog(context, input.inputs[0]);
21-
auto &ducklake_catalog = catalog.Cast<DuckLakeCatalog>();
2221
DuckLakeConfigOption config_option;
2322
auto &option = config_option.option.key;
2423
auto &value = config_option.option.value;
@@ -68,10 +67,6 @@ static unique_ptr<FunctionData> DuckLakeSetOptionBind(ClientContext &context, Ta
6867
auto target_file_size_bytes = DBConfig::ParseMemoryLimit(val.ToString());
6968
value = to_string(target_file_size_bytes);
7069
} else if (option == "data_inlining_row_limit") {
71-
auto &metadata_catalog = Catalog::GetCatalog(context, ducklake_catalog.MetadataDatabaseName());
72-
if (!metadata_catalog.IsDuckCatalog()) {
73-
throw NotImplementedException("Data Inlining is currently only implemented for DuckDB as a DBMS");
74-
}
7570
auto data_inlining_row_limit = val.DefaultCastAs(LogicalType::UBIGINT).GetValue<idx_t>();
7671
value = to_string(data_inlining_row_limit);
7772
} else if (option == "require_commit_message") {

‎src/include/metadata_manager/postgres_metadata_manager.hpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@ class PostgresMetadataManager : public DuckLakeMetadataManager {
1616
public:
1717
explicit PostgresMetadataManager(DuckLakeTransaction &transaction);
1818

19+
bool TypeIsNativelySupported(const LogicalType &type) override;
20+
1921
protected:
2022
string GetLatestSnapshotQuery() const override;
2123
};

‎src/include/storage/ducklake_metadata_manager.hpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,8 @@ class DuckLakeMetadataManager {
9797

9898
static unique_ptr<DuckLakeMetadataManager> Create(DuckLakeTransaction &transaction);
9999

100+
virtual bool TypeIsNativelySupported(const LogicalType &type);
101+
100102
DuckLakeMetadataManager &Get(DuckLakeTransaction &transaction);
101103

102104
//! Initialize a new DuckLake

‎src/metadata_manager/postgres_metadata_manager.cpp‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,18 @@ PostgresMetadataManager::PostgresMetadataManager(DuckLakeTransaction &transactio
66
: DuckLakeMetadataManager(transaction) {
77
}
88

9+
bool PostgresMetadataManager::TypeIsNativelySupported(const LogicalType &type) {
10+
switch (type.id()) {
11+
// Unnamed composite types are not supported.
12+
case LogicalTypeId::STRUCT:
13+
case LogicalTypeId::MAP:
14+
case LogicalTypeId::LIST:
15+
return false;
16+
default:
17+
return true;
18+
}
19+
}
20+
921
string PostgresMetadataManager::GetLatestSnapshotQuery() const {
1022
return R"(
1123
SELECT * FROM postgres_query({METADATA_CATALOG_NAME_LITERAL},

‎src/storage/ducklake_initializer.cpp‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -67,12 +67,6 @@ void DuckLakeInitializer::Initialize() {
6767
// if the schema is not explicitly set by the user - set it to the default schema in the catalog
6868
options.metadata_schema = transaction.GetDefaultSchemaName();
6969
}
70-
if (catalog.DataInliningRowLimit(SchemaIndex(), TableIndex()) > 0) {
71-
auto &metadata_catalog = Catalog::GetCatalog(*transaction.GetConnection().context, options.metadata_database);
72-
if (!metadata_catalog.IsDuckCatalog()) {
73-
throw NotImplementedException("Data inlining is currently only supported on DuckDB catalogs");
74-
}
75-
}
7670
// after the metadata database is attached initialize the ducklake
7771
// check if we are loading an existing DuckLake or creating a new one
7872
// FIXME: verify that all tables are in the correct format instead

‎src/storage/ducklake_inlined_data_reader.cpp‎

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
#include "storage/ducklake_metadata_manager.hpp"
55
#include "duckdb/storage/table/column_segment.hpp"
66
#include "duckdb/planner/table_filter_state.hpp"
7+
#include "storage/ducklake_catalog.hpp"
78
#include "storage/ducklake_delete_filter.hpp"
89

910
namespace duckdb {
@@ -38,7 +39,7 @@ bool DuckLakeInlinedDataReader::TryInitializeScan(ClientContext &context, Global
3839
// scanning data from a table - read it from the metadata catalog
3940
auto transaction = read_info.GetTransaction();
4041
auto &metadata_manager = transaction->GetMetadataManager();
41-
42+
auto &ducklake_catalog = transaction->GetCatalog();
4243
// push the projections directly into the read
4344
vector<string> columns_to_read;
4445
for (auto &column_id : column_indexes) {
@@ -64,11 +65,16 @@ bool DuckLakeInlinedDataReader::TryInitializeScan(ClientContext &context, Global
6465
break;
6566
}
6667
if (!virtual_column.empty()) {
67-
columns_to_read.push_back(virtual_column);
68+
columns_to_read.push_back(KeywordHelper::WriteOptionallyQuoted(virtual_column));
6869
continue;
6970
}
7071
}
71-
columns_to_read.push_back(columns[index].name);
72+
string projected_column = KeywordHelper::WriteOptionallyQuoted(columns[index].name);
73+
if (!ducklake_catalog.IsDuckCatalog()) {
74+
// If it's not a duckdb catalog, we add a cast.
75+
projected_column += "::" + columns[index].type.ToString();
76+
}
77+
columns_to_read.push_back(projected_column);
7278
}
7379
if (deletion_filter) {
7480
// we have a deletion filter - the deletions are on row-ids, not on ordinals
@@ -78,12 +84,12 @@ bool DuckLakeInlinedDataReader::TryInitializeScan(ClientContext &context, Global
7884
scan_column_ids.push_back(i);
7985
virtual_columns.push_back(InlinedVirtualColumn::NONE);
8086
}
81-
columns_to_read.push_back("row_id");
87+
columns_to_read.push_back(KeywordHelper::WriteOptionallyQuoted("row_id"));
8288
virtual_columns.emplace_back(InlinedVirtualColumn::COLUMN_EMPTY);
8389
}
8490
if (columns_to_read.empty()) {
8591
// COUNT(*) - read row_id but don't emit
86-
columns_to_read.push_back("row_id");
92+
columns_to_read.push_back(KeywordHelper::WriteOptionallyQuoted("row_id"));
8793
virtual_columns.emplace_back(InlinedVirtualColumn::COLUMN_EMPTY);
8894
}
8995
switch (read_info.scan_type) {
@@ -159,7 +165,6 @@ AsyncResult DuckLakeInlinedDataReader::Scan(ClientContext &context, GlobalTableF
159165
if (!virtual_columns.empty()) {
160166
scan_chunk.Reset();
161167
data->data->Scan(state, scan_chunk);
162-
163168
idx_t source_idx = 0;
164169
for (idx_t c = 0; c < virtual_columns.size(); c++) {
165170
switch (virtual_columns[c]) {

‎src/storage/ducklake_metadata_manager.cpp‎

Lines changed: 20 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,10 @@ DuckLakeMetadataManager &DuckLakeMetadataManager::Get(DuckLakeTransaction &trans
3939
return transaction.GetMetadataManager();
4040
}
4141

42+
bool DuckLakeMetadataManager::TypeIsNativelySupported(const LogicalType &type) {
43+
return true;
44+
}
45+
4246
FileSystem &DuckLakeMetadataManager::GetFileSystem() {
4347
return FileSystem::GetFileSystem(transaction.GetCatalog().GetDatabase());
4448
}
@@ -1014,7 +1018,6 @@ FilterSQLResult DuckLakeMetadataManager::ConvertFilterPushdownToSQL(const Filter
10141018
string
10151019
DuckLakeMetadataManager::GenerateCTESectionFromRequirements(const unordered_map<idx_t, CTERequirement> &requirements,
10161020
TableIndex table_id) {
1017-
10181021
if (requirements.empty()) {
10191022
return "";
10201023
}
@@ -1050,7 +1053,6 @@ DuckLakeMetadataManager::GenerateCTESectionFromRequirements(const unordered_map<
10501053

10511054
FilterPushdownQueryComponents
10521055
DuckLakeMetadataManager::GenerateFilterPushdownComponents(const FilterPushdownInfo &filter_info, TableIndex table_id) {
1053-
10541056
FilterPushdownQueryComponents result;
10551057

10561058
if (filter_info.column_filters.empty()) {
@@ -1522,10 +1524,12 @@ static void ColumnToSQLRecursive(const DuckLakeColumnInfo &column, TableIndex ta
15221524
}
15231525

15241526
string DuckLakeMetadataManager::GetColumnType(const DuckLakeColumnInfo &col) {
1525-
if (col.children.empty()) {
1526-
return DuckLakeTypes::FromString(col.type).ToString();
1527+
auto column_type = DuckLakeTypes::FromString(col.type);
1528+
if (!TypeIsNativelySupported(column_type)) {
1529+
return "VARCHAR";
15271530
}
1528-
if (col.type == "struct") {
1531+
switch (column_type.id()) {
1532+
case LogicalTypeId::STRUCT: {
15291533
string result;
15301534
for (auto &child : col.children) {
15311535
if (!result.empty()) {
@@ -1535,13 +1539,20 @@ string DuckLakeMetadataManager::GetColumnType(const DuckLakeColumnInfo &col) {
15351539
}
15361540
return "STRUCT(" + result + ")";
15371541
}
1538-
if (col.type == "list") {
1542+
case LogicalTypeId::LIST: {
15391543
return GetColumnType(col.children[0]) + "[]";
15401544
}
1541-
if (col.type == "map") {
1545+
case LogicalTypeId::MAP: {
15421546
return StringUtil::Format("MAP(%s, %s)", GetColumnType(col.children[0]), GetColumnType(col.children[1]));
15431547
}
1544-
throw InternalException("Unsupported nested type %s in DuckLakeMetadataManager::GetColumnType", col.type);
1548+
default:
1549+
if (!col.children.empty()) {
1550+
// This is a nested structure that we currently do not support.
1551+
throw NotImplementedException("Unsupported nested type %s in DuckLakeMetadataManager::GetColumnType",
1552+
col.type);
1553+
}
1554+
return column_type.ToString();
1555+
}
15451556
}
15461557

15471558
string DuckLakeMetadataManager::GetInlinedTableQuery(const DuckLakeTableInfo &table, const string &table_name) {
@@ -1891,7 +1902,7 @@ static string GetProjection(const vector<string> &columns_to_read) {
18911902
if (!result.empty()) {
18921903
result += ", ";
18931904
}
1894-
result += KeywordHelper::WriteOptionallyQuoted(entry);
1905+
result += entry;
18951906
}
18961907
return result;
18971908
}

‎test/configs/postgres.json‎

Lines changed: 5 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -15,43 +15,14 @@
1515
"env_value": "postgres:dbname=ducklakedb"
1616
}],
1717
"skip_tests": [
18-
{
19-
"reason": "Data Inlining Not Supported",
20-
"paths": [
21-
"test/sql/data_inlining/basic_data_inlining.test",
22-
"test/sql/concurrent/concurrent_insert_conflict.test",
23-
"test/sql/concurrent/concurrent_insert_data_inlining.test",
24-
"test/sql/data_inlining/data_inlining_alter.test",
25-
"test/sql/data_inlining/data_inlining_constraints.test",
26-
"test/sql/data_inlining/data_inlining_delete.test",
27-
"test/sql/data_inlining/data_inlining_encryption.test",
28-
"test/sql/data_inlining/data_inlining_flush.test",
29-
"test/sql/data_inlining/data_inlining_flush_schema.test",
30-
"test/sql/data_inlining/data_inlining_large.test",
31-
"test/sql/data_inlining/data_inlining_option.test",
32-
"test/sql/data_inlining/data_inlining_option_transaction_local.test",
33-
"test/sql/data_inlining/data_inlining_partitions.test",
34-
"test/sql/data_inlining/data_inlining_table_changes.test",
35-
"test/sql/data_inlining/data_inlining_transaction_local_alter.test",
36-
"test/sql/data_inlining/data_inlining_transaction_local_delete.test",
37-
"test/sql/data_inlining/data_inlining_types.test",
38-
"test/sql/data_inlining/data_inlining_update.test",
39-
"test/sql/transaction/transaction_conflict_inlining.test",
40-
"test/sql/data_inlining/inlining_global_options.test",
41-
"test/sql/checkpoint/checkpoint_updates_interleaved.test",
42-
"test/sql/checkpoint/checkpoint_ducklake.test",
43-
"test/sql/checkpoint/many_inserts_transaction.test",
44-
"test/sql/data_inlining/insert_inlining_concurrent.test",
45-
"test/sql/transaction/transaction_inlining.test",
46-
"test/sql/data_inlining/data_inlining_issue504.test"
47-
]
48-
},
4918
{
5019
"reason": "Tests only for DuckDB as a catalog",
5120
"paths": [
5221
"test/sql/general/missing_parquet.test",
5322
"test/sql/general/paths.test",
54-
"test/sql/general/default_path.test"
23+
"test/sql/general/default_path.test",
24+
"test/sql/autoloading/autoload_data_path.test",
25+
"test/sql/general/metadata_parameters.test"
5526
]
5627
},
5728
{
@@ -62,10 +33,9 @@
6233
]
6334
},
6435
{
65-
"reason": "DuckDB as a catalog specific",
36+
"reason": "Postgres Locking (Not a bug, rather PG behavior)",
6637
"paths": [
67-
"test/sql/autoloading/autoload_data_path.test",
68-
"test/sql/general/metadata_parameters.test"
38+
"test/sql/transaction/transaction_conflict_inlining.test"
6939
]
7040
}
7141
]

0 commit comments

Comments
 (0)