Merge pull request duckdb#115 from Mytherin/noexperimentalpushdown

Enable filter pushdown by default - and add more extensive testing for filtering on all types

Author: Mytherin

src/mysql_extension.cpp

@@ -110,7 +110,7 @@ static void LoadInternal(DatabaseInstance &db) {
 
 	config.AddExtensionOption("mysql_experimental_filter_pushdown",
 	                          "Whether or not to use filter pushdown (currently experimental)", LogicalType::BOOLEAN,
-	                          Value::BOOLEAN(false));
+	                          Value::BOOLEAN(true));
 	config.AddExtensionOption("mysql_debug_show_queries", "DEBUG SETTING: print all queries sent to MySQL to stdout",
 	                          LogicalType::BOOLEAN, Value::BOOLEAN(false), SetMySQLDebugQueryPrint);
 	config.AddExtensionOption("mysql_tinyint1_as_boolean", "Whether or not to convert TINYINT(1) columns to BOOLEAN",

src/mysql_filter_pushdown.cpp

@@ -1,5 +1,7 @@
 #include "mysql_filter_pushdown.hpp"
 #include "mysql_utils.hpp"
+#include "duckdb/planner/filter/optional_filter.hpp"
+#include "duckdb/planner/filter/in_filter.hpp"
 
 namespace duckdb {
 
@@ -30,12 +32,26 @@ string MySQLFilterPushdown::TransformComparison(ExpressionType type) {
 	}
 }
 
+
+static string TransformBlobToMySQL(const string &val) {
+	char const HEX_DIGITS[] = "0123456789ABCDEF";
+
+	string result = "x'";
+	for(idx_t i = 0; i < val.size(); i++) {
+		uint8_t byte_val = static_cast<uint8_t>(val[i]);
+		result += HEX_DIGITS[(byte_val >> 4) & 0xf];
+		result += HEX_DIGITS[byte_val & 0xf];
+	}
+	result += "'";
+	return result;
+}
+
 string MySQLFilterPushdown::TransformConstant(const Value &val) {
 	if (val.type().IsNumeric()) {
 		return val.ToSQLString();
 	}
 	if (val.type().id() == LogicalTypeId::BLOB) {
-		throw NotImplementedException("Unsupported type for filter pushdown: BLOB");
+		return TransformBlobToMySQL(StringValue::Get(val));
 	}
 	if (val.type().id() == LogicalTypeId::TIMESTAMP_TZ) {
 		return val.DefaultCastAs(LogicalType::TIMESTAMP).DefaultCastAs(LogicalType::VARCHAR).ToSQLString();
@@ -63,6 +79,21 @@ string MySQLFilterPushdown::TransformFilter(string &column_name, TableFilter &fi
 		auto operator_string = TransformComparison(constant_filter.comparison_type);
 		return StringUtil::Format("%s %s %s", column_name, operator_string, constant_string);
 	}
+	case TableFilterType::OPTIONAL_FILTER: {
+		auto &optional_filter = filter.Cast<OptionalFilter>();
+		return TransformFilter(column_name, *optional_filter.child_filter);
+	}
+	case TableFilterType::IN_FILTER: {
+		auto &in_filter = filter.Cast<InFilter>();
+		string in_list;
+		for(auto &val : in_filter.values) {
+			if (!in_list.empty()) {
+				in_list += ", ";
+			}
+			in_list += TransformConstant(val);
+		}
+		return column_name + " IN (" + in_list + ")";
+		}
 	default:
 		throw InternalException("Unsupported table filter type");
 	}

src/storage/mysql_execute_query.cpp

@@ -111,7 +111,29 @@ string ExtractFilters(PhysicalOperator &child, const string &statement) {
 		if (!table_scan.table_filters) {
 			return string();
 		}
-		throw NotImplementedException("Pushed down table filters not supported currently");
+		string result;
+		for(auto &entry : table_scan.table_filters->filters) {
+			auto column_index = entry.first;
+			auto &filter = entry.second;
+			string column_name;
+			if (column_index < table_scan.names.size()) {
+				const auto col_id = table_scan.column_ids[column_index].GetPrimaryIndex();
+				if (col_id == COLUMN_IDENTIFIER_ROW_ID) {
+					column_name = "rowid";
+				} else {
+					column_name = table_scan.names[col_id];
+				}
+			}
+			BoundReferenceExpression bound_ref(std::move(column_name), LogicalTypeId::INVALID, 0);
+			auto filter_expr = filter->ToExpression(bound_ref);
+			auto filter_str = filter_expr->ToString();
+			if (result.empty()) {
+				result = std::move(filter_str);
+			} else {
+				result += " AND " + filter_str;
+			}
+		}
+		return result;
 	} else {
 		throw NotImplementedException("Unsupported operator type %s in %s statement - only simple deletes "
 		                              "(e.g. %s "

test/sql/attach_filter_pushdown.test

@@ -63,7 +63,7 @@ SELECT COUNT(*) FROM s1.text_tbl WHERE v= '🦆'
 1
 
 # blob pushdown
-statement error
+query I
 SELECT COUNT(*) FROM s1.blob_tbl WHERE bl= BLOB '\x80'
 ----
-Unsupported
+1

test/sql/attach_types.test

@@ -66,3 +66,21 @@ SELECT COLUMNS(*)::VARCHAR FROM s.all_types
 false	-128	-32768	-2147483648	-9223372036854775808	0	0	0	2000-01-01	00:00:00	2000-01-01 01:02:03	2000-01-01 01:02:03	2000-01-01 01:02:03	2000-01-01 01:02:03	00:00:00+15:00	2000-01-01 01:02:03	-999.9	-99999.9999	-999999999999.999999	-9999999999999999999999999999.9999999999	00000000-0000-0000-0000-000000000000	00:00:00	🦆🦆🦆🦆🦆🦆	thisisalongblob\x00withnullbytes	0010001001011100010101011010111	DUCK_DUCK_ENUM	enum_0	enum_0
 true	127	32767	2147483647	9223372036854775807	255	65535	4294967295	2000-01-01	24:00:00	2000-01-01 01:02:03	2000-01-01 01:02:03	2000-01-01 01:02:03	2000-01-01 01:02:03	00:00:00+15:00	2000-01-01 01:02:03	999.9	99999.9999	999999999999.999999	9999999999999999999999999999.9999999999	ffffffff-ffff-ffff-ffff-ffffffffffff	83 years 3 months 999 days 00:16:39.999999	goo\0se	\x00\x00\x00a	10101	GOOSE	enum_299	enum_69999
 NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL
+
+# filter pushdown
+foreach column_name bool tinyint smallint int bigint utinyint usmallint uint date time timestamp timestamp_s timestamp_ms timestamp_ns time_tz timestamp_tz dec_4_1 dec_9_4 dec_18_6 dec38_10 uuid interval varchar blob bit small_enum medium_enum large_enum
+
+statement ok
+SET VARIABLE minimum_value=(SELECT MIN(${column_name}) min_val FROM s.all_types);
+
+query I
+SELECT ANY_VALUE(${column_name})=getvariable('minimum_value') FROM s.all_types WHERE ${column_name}=getvariable('minimum_value')
+----
+true
+
+query I
+SELECT ANY_VALUE(${column_name})=getvariable('minimum_value') FROM s.all_types WHERE ${column_name} IN (getvariable('minimum_value'), getvariable('minimum_value'))
+----
+true
+
+endloop