From 1bd2a62d18b0fb0ce5de0dfda0944ac61e5388ea Mon Sep 17 00:00:00 2001 From: AdRiley Date: Tue, 11 Feb 2025 13:17:01 +0000 Subject: [PATCH] Add DB_Table.offset for snowflake, postgres and sqlite (#12251) * Red * Fix tests * Remove comment * checkpoint * Checkpoint * 6 red * 3 red * Green * Checkpoint * Remove wrap around tests * Green * Refactor * Adjust limit * Clean up * Refactor * Refactor * refactor * Refactor * Refactor * Documentation * Cleanup * Comment * Checkpoint * Refactor * fix * More tests * Table.new * Cleanup * changelog * Fix test * Code review changes * Code review * Postgres * SQLite * Snowflake * Add missing aliases * changelog * Fix SQLite test --- CHANGELOG.md | 2 ++ .../src/Internal/Base_Generator.enso | 35 ++++++++++++++++++- .../Internal/Postgres/Postgres_Dialect.enso | 1 - .../src/Internal/SQLite/SQLite_Dialect.enso | 1 - .../Internal/SQLite/SQLite_Type_Mapping.enso | 2 +- .../src/Internal/Snowflake_Dialect.enso | 1 - .../Standard/Table/0.0.0-dev/src/Table.enso | 2 +- .../src/Common_Table_Operations/Util.enso | 3 +- 8 files changed, 39 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 600c4c50d8a0..67c4e05107fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -51,6 +51,7 @@ `on_invalid_rows`. The default behaviour was also changed to add any extra columns instead of discarding them. - [Added DB_Table.Offset for SQLServer][12206] +- [Added DB_Table.Offset for Snowflake, Postgres, SQLite][12251] [11926]: https://github.com/enso-org/enso/pull/11926 [12031]: https://github.com/enso-org/enso/pull/12031 @@ -58,6 +59,7 @@ [12092]: https://github.com/enso-org/enso/pull/12092 [12231]: https://github.com/enso-org/enso/pull/12231 [12206]: https://github.com/enso-org/enso/pull/12206 +[12251]: https://github.com/enso-org/enso/pull/12251 #### Enso Language & Runtime diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Base_Generator.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Base_Generator.enso index 27b43971dbe9..377ef05a493c 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Base_Generator.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Base_Generator.enso @@ -469,7 +469,8 @@ base_dialect_operations = contains = [["IS_IN", make_is_in], ["IS_IN_COLUMN", make_is_in_column]] types = [simple_cast] windows = [["ROW_NUMBER", make_row_number], ["ROW_NUMBER_IN_GROUP", make_row_number_in_group]] - base_dict = Dictionary.from_vector (arith + logic + compare + functions + agg + counts + text + nulls + contains + types + windows) + leadlag = [["LEAD", _make_lead_lag "LEAD"], ["LAG", _make_lead_lag "LAG"], ["LEAD_CLOSEST", _make_lead_lag_closest_value "LEAD"], ["LAG_CLOSEST", _make_lead_lag_closest_value "LAG"]] + base_dict = Dictionary.from_vector (arith + logic + compare + functions + agg + counts + text + nulls + contains + types + windows + leadlag) Dialect_Operations.Value base_dict ## PRIVATE @@ -740,6 +741,38 @@ default_fetch_types_query dialect expression context where_filter_always_false_l ## PRIVATE default_generate_collate collation_name:Text quote_char:Text='"' -> Text = ' COLLATE ' + quote_char + collation_name + quote_char +_build_partition_sql grouping:SQL_Builder ordering:SQL_Builder -> SQL_Builder = + group_part = if grouping.is_empty then "" else + SQL_Builder.code "PARTITION BY " ++ grouping + SQL_Builder.code "OVER(" ++ group_part ++ " ORDER BY " ++ ordering ++ ")" + +## PRIVATE +_build_lead_lag_sql lead_lag:Text n:SQL_Builder colName:SQL_Builder grouping:SQL_Builder ordering:SQL_Builder -> SQL_Builder = + partition_sql = _build_partition_sql grouping ordering + SQL_Builder.code "(" ++ lead_lag ++ "(" ++ colName ++ ", " ++ n ++ ", NULL) " ++ partition_sql ++ ")" + +## PRIVATE +_make_lead_lag lead_lag:Text arguments:Vector -> SQL_Builder = if arguments.length != 4 then Error.throw (Illegal_State.Error "Wrong amount of parameters in LEAD/LAG IR. This is a bug in the Database library.") else + n = arguments.at 0 + colName = arguments.at 1 + grouping = arguments.at 2 + ordering = arguments.at 3 + _build_lead_lag_sql lead_lag n colName grouping ordering + +## PRIVATE +_make_lead_lag_closest_value lead_lag:Text arguments:Vector -> SQL_Builder = if arguments.length != 5 then Error.throw (Illegal_State.Error "Wrong amount of parameters in LEAD/LAG IR. This is a bug in the Database library.") else + n = arguments.at 0 + colName = arguments.at 1 + grouping = arguments.at 2 + ordering_for_lead_lag = arguments.at 3 + ordering_for_row_number = arguments.at 4 + + lead_lag_sql = _build_lead_lag_sql lead_lag n colName grouping ordering_for_lead_lag + partition_sql_for_row_number = _build_partition_sql grouping ordering_for_row_number + fill_sql = SQL_Builder.code "FIRST_VALUE(" ++ colName ++ ") " ++ partition_sql_for_row_number + SQL_Builder.code "CASE WHEN ROW_NUMBER() " ++ partition_sql_for_row_number ++ " <= " ++ n ++ " THEN " ++ fill_sql ++ " ELSE " ++ lead_lag_sql ++ " END" + + ## PRIVATE Helper class for shortening the binder names generated for WITH clauses. diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Dialect.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Dialect.enso index ff44b6cbbf0e..6f15fd730063 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Dialect.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Dialect.enso @@ -269,7 +269,6 @@ type Postgres_Dialect Checks if a feature is supported by the dialect. is_feature_supported self feature:Feature -> Boolean = case feature of - Feature.Offset -> False _ -> True ## PRIVATE diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Dialect.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Dialect.enso index 4ebc212d10d4..60f2479ba32d 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Dialect.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Dialect.enso @@ -278,7 +278,6 @@ type SQLite_Dialect Checks if a feature is supported by the dialect. is_feature_supported self feature:Feature -> Boolean = case feature of - Feature.Offset -> False _ -> True ## PRIVATE diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Type_Mapping.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Type_Mapping.enso index 1ce4016be368..6fc05590a4cd 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Type_Mapping.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Type_Mapping.enso @@ -214,7 +214,7 @@ operations_dict = always_integer_ops = ["COUNT", "COUNT_IS_NULL", "COUNT_DISTINCT", "COUNT_DISTINCT_INCLUDE_NULL", "COUNT_EMPTY", "COUNT_NOT_EMPTY", "COUNT_ROWS", "COUNT_OVER_PARTITION", "ROW_NUMBER", "ROW_NUMBER_IN_GROUP", "LENGTH"] same_as_first = ["TRUNCATE", "CEIL", "FLOOR", "FIRST", "LAST"] arithmetic_ops = ["ADD_NUMBER", "-", "*", "^", "%", "SUM"] - merge_input_types_ops = ["ROW_MAX", "ROW_MIN", "MAX", "MIN", "FILL_NULL", "COALESCE"] + merge_input_types_ops = ["ROW_MAX", "ROW_MIN", "MAX", "MIN", "FILL_NULL", "COALESCE", "LEAD", "LAG", "LEAD_CLOSEST", "LAG_CLOSEST"] others = [["IIF", handle_iif], ["CAST", handle_cast], ["CASE", handle_case], ["RUNTIME_ERROR", handle_runtime_error]] Dictionary.from_vector <| v1 = always_boolean_ops.map [_, const SQLite_Types.boolean] diff --git a/distribution/lib/Standard/Snowflake/0.0.0-dev/src/Internal/Snowflake_Dialect.enso b/distribution/lib/Standard/Snowflake/0.0.0-dev/src/Internal/Snowflake_Dialect.enso index fbdc0d14c7b5..5a23b0d1bcae 100644 --- a/distribution/lib/Standard/Snowflake/0.0.0-dev/src/Internal/Snowflake_Dialect.enso +++ b/distribution/lib/Standard/Snowflake/0.0.0-dev/src/Internal/Snowflake_Dialect.enso @@ -257,7 +257,6 @@ type Snowflake_Dialect Checks if a feature is supported by the dialect. is_feature_supported self feature:Feature -> Boolean = case feature of - Feature.Offset -> False _ -> True ## PRIVATE diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Table.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Table.enso index 0e4028b5d59e..2ff2b582fb88 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Table.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Table.enso @@ -3160,7 +3160,7 @@ type Table union self tables:(Table | Vector) (columns_to_keep : Columns_To_Keep = ..In_Any_Warn_On_Missing) (match_columns : Match_Columns = ..By_Name) (on_problems : Problem_Behavior = ..Report_Warning) = Table.from_union ([self] + Vector.unify_vector_or_element tables) columns_to_keep match_columns on_problems - ## ALIAS drop_missing_rows, dropna + ## ALIAS drop_empty_rows, drop_missing_rows, dropna, filter_empty_rows, remove_blank_rows, remove_empty_rows, remove_missing_rows GROUP Standard.Base.Selections ICON preparation Remove rows which are all blank or containing blank values. diff --git a/test/Table_Tests/src/Common_Table_Operations/Util.enso b/test/Table_Tests/src/Common_Table_Operations/Util.enso index 8cc75caa4fc1..6030e5538415 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Util.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Util.enso @@ -82,8 +82,7 @@ Error.should_equal_tz_agnostic self other = ## PRIVATE Builds a table ensuring that the rows are in the order as given. build_sorted_table setup table_structure = - # Workaround for https://github.com/enso-org/enso/issues/10321 - if setup.prefix.contains "Snowflake" . not && setup.prefix.contains "SQLServer" . not then setup.table_builder table_structure else + if setup.is_database . not then setup.table_builder table_structure else row_count = case table_structure.first of def : Vector -> def.second.length col : Column -> col.length