Introduce Value_Type.Null, fix edge cases for read_many (#11737)

- Closes #6281

Author: radeusgd

distribution/lib/Standard/Base/0.0.0-dev/src/Data.enso

@@ -5,10 +5,12 @@ import project.Data.Read.Many_Files_List.Many_Files_List
 import project.Data.Read.Return_As.Return_As
 import project.Data.Text.Encoding.Encoding
 import project.Data.Text.Text
+import project.Data.Vector.No_Wrap
 import project.Data.Vector.Vector
 import project.Enso_Cloud.Data_Link.Data_Link
 import project.Enso_Cloud.Data_Link_Helpers
 import project.Error.Error
+import project.Errors.Common.Failed_To_Load
 import project.Errors.Common.Missing_Argument
 import project.Errors.File_Error.File_Error
 import project.Errors.Illegal_Argument.Illegal_Argument
@@ -145,9 +147,11 @@ read path=(Missing_Argument.throw "path") format=Auto_Detect (on_problems : Prob
 read_many : Many_Files_List -> File_Format -> Return_As -> Problem_Behavior -> Any ! File_Error
 read_many (paths : Many_Files_List = Missing_Argument.throw "paths") format=Auto_Detect return=..As_Merged_Table (on_problems : Problem_Behavior = ..Report_Warning) =
     return_as = Return_As.resolve return
-    loaded_objects = paths.paths_to_load.map on_problems=on_problems path->
-        Data.read path format on_problems
-    return_as.make_return paths loaded_objects on_problems
+    if paths.paths_to_load.contains Nothing then Error.throw (Illegal_Argument.Error Illegal_Argument.Error "`paths` cannot contain `Nothing`. Use `filter` with `..Not_Nothing` to remove.") else
+        loaded_objects = paths.paths_to_load.map on_problems=No_Wrap.Value path->
+            Data.read path format on_problems . catch Any error->
+                Failed_To_Load.Warning path error
+        return_as.make_return paths loaded_objects on_problems
 
 ## ALIAS load text, open text
    GROUP Input

distribution/lib/Standard/Base/0.0.0-dev/src/Data/Read/Many_Files_List.enso

@@ -7,6 +7,15 @@ import project.Data.Vector.Vector
    used in `Data.read_many`.
 type Many_Files_List
     ## PRIVATE
+       Arguments:
+       - original_value: The original value that represents a list of files.
+         Some return modes may use it as it contains more information than just
+         list of files (e.g. it can be a source table).
+       - paths_to_load: A vector of paths to load. This is the fallback that can
+         be used by any return mode if it does not recognize the original value.
+         The vector is expected to contain values that can be passed into
+         `Data.read` (so it can be Text, File, URI or any other kind of file -
+         e.g. S3_File).
     Value original_value paths_to_load:Vector
 
     ## PRIVATE

distribution/lib/Standard/Base/0.0.0-dev/src/Data/Read/Return_As.enso

@@ -3,10 +3,12 @@ import project.Data.Read.Many_Files_List.Many_Files_List
 import project.Data.Text.Text
 import project.Data.Vector.Vector
 import project.Error.Error
+import project.Errors.Common.Failed_To_Load
 import project.Errors.Common.Type_Error
 import project.Errors.Illegal_Argument.Illegal_Argument
 import project.Errors.Problem_Behavior.Problem_Behavior
 import project.Function.Function
+import project.Meta
 import project.Metadata.Display
 import project.Metadata.Widget
 import project.Nothing.Nothing
@@ -33,7 +35,7 @@ type Return_As
     to_display_text self -> Text = self.underlying.to_display_text
 
     ## PRIVATE
-    make_return self (input : Many_Files_List) (objects : Vector Any) (on_problems : Problem_Behavior) =
+    make_return self (input : Many_Files_List) (objects : Vector (Any | Failed_To_Load)) (on_problems : Problem_Behavior) =
         self.underlying.make_return input objects on_problems
 
     ## PRIVATE
@@ -77,9 +79,17 @@ type Return_As_Base
         Panic.catch Type_Error (value:Return_As_Base) _->Nothing
 
     ## PRIVATE
-    make_return self (input : Many_Files_List) (objects : Vector Any) (on_problems : Problem_Behavior) =
-        _ = [input, on_problems]
-        objects
+    make_return self (input : Many_Files_List) (objects : Vector (Any | Failed_To_Load)) (on_problems : Problem_Behavior) =
+        _ = input
+        replace_with_nothing_and_propagate objects on_problems
+
+## PRIVATE
+   A helper method that takes a Vector and replaces `Failed_To_Load` with `Nothing`, raising them as warnings.
+replace_with_nothing_and_propagate (vector : Vector (Any | Failed_To_Load)) (on_problems : Problem_Behavior) =
+    failed = vector.filter o-> o.is_a Failed_To_Load
+    if failed.is_empty then vector else
+        on_problems.attach_problems_before failed <|
+            vector.map o-> if o.is_a Failed_To_Load then Nothing else o
 
 ## PRIVATE
 Return_As.from (that : Return_As_Base) =

distribution/lib/Standard/Base/0.0.0-dev/src/Errors/Common.enso

@@ -607,3 +607,13 @@ type Floating_Point_Equality
             "(Error  (location = "+location+"))"
         Floating_Point_Equality.Used_As_Dictionary_Key value ->
             "(Used_As_Dictionary_Key (value = "+value.to_text+"))"
+
+## A warning indicating that a file failed to be loaded.
+type Failed_To_Load
+    ## PRIVATE
+    Warning path cause
+
+    ## PRIVATE
+       Create a human-readable version of the error.
+    to_display_text : Text
+    to_display_text self = "Failed to load file at path: "+self.path.to_display_text+": "+self.cause.to_display_text

distribution/lib/Standard/Database/0.0.0-dev/src/DB_Column.enso

@@ -597,12 +597,13 @@ type DB_Column
          cause a hard error, the value may be truncated or wrap-around etc.
     + : DB_Column | Any -> DB_Column
     + self other =
-        op = case Value_Type_Helpers.resolve_addition_kind self other of
-            Value_Type_Helpers.Addition_Kind.Numeric_Add -> "ADD_NUMBER"
-            Value_Type_Helpers.Addition_Kind.Text_Concat -> "ADD_TEXT"
-        op.if_not_error <|
-            new_name = self.naming_helper.binary_operation_name "+" self other
-            self.make_binary_op op other new_name
+        new_name = self.naming_helper.binary_operation_name "+" self other
+        case Value_Type_Helpers.resolve_addition_kind self other of
+            Value_Type_Helpers.Addition_Kind.Numeric_Add ->
+                self.make_binary_op "ADD_NUMBER" other new_name
+            Value_Type_Helpers.Addition_Kind.Text_Concat ->
+                self.make_binary_op "ADD_TEXT" other new_name
+            Nothing -> self.const Nothing
 
     ## ALIAS minus, subtract, time difference
        GROUP Standard.Base.Operators
@@ -632,6 +633,7 @@ type DB_Column
                 self.make_binary_op "-" other
             Value_Type_Helpers.Subtraction_Kind.Date_Time_Difference ->
                 Error.throw (Unsupported_Database_Operation.Error "Subtracting date/time value")
+            Nothing -> self.const Nothing
 
     ## ALIAS multiply, product, times
        GROUP Standard.Base.Operators

distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Type_Mapping.enso

@@ -64,6 +64,8 @@ type Postgres_Type_Mapping
                    but we may also consider using the standard SQL `bit(n)` and `bit varying(n)` types.
                    See: https://www.postgresql.org/docs/current/datatype-bit.html
                 SQL_Type.Value Types.BINARY "bytea" precision=max_precision
+            Value_Type.Null ->
+                Error.throw (Unsupported_Database_Type.Error "Null" "Postgres")
             Value_Type.Mixed ->
                 Error.throw (Unsupported_Database_Type.Error "Mixed" "Postgres")
             Value_Type.Unsupported_Data_Type type_name underlying_type ->
@@ -146,7 +148,7 @@ type Postgres_Type_Mapping
 simple_types_map = Dictionary.from_vector <|
     ints = [[Types.SMALLINT, Value_Type.Integer Bits.Bits_16], [Types.BIGINT, Value_Type.Integer Bits.Bits_64], [Types.INTEGER, Value_Type.Integer Bits.Bits_32]]
     floats = [[Types.DOUBLE, Value_Type.Float Bits.Bits_64], [Types.REAL, Value_Type.Float Bits.Bits_32]]
-    other = [[Types.DATE, Value_Type.Date], [Types.TIME, Value_Type.Time]]
+    other = [[Types.DATE, Value_Type.Date], [Types.TIME, Value_Type.Time], [Types.NULL, Value_Type.Null]]
     ints + floats + other
 
 ## PRIVATE

distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Dialect.enso

@@ -192,14 +192,12 @@ type SQLite_Dialect
     adapt_unified_column : Internal_Column -> Value_Type -> (SQL_Expression -> SQL_Type_Reference) -> Internal_Column
     adapt_unified_column self column approximate_result_type infer_result_type_from_database_callback =
         _ = infer_result_type_from_database_callback
-        # TODO [RW] This may be revisited with #6281.
-        case approximate_result_type of
-            Nothing -> column
-            _ ->
-                sql_type = self.get_type_mapping.value_type_to_sql approximate_result_type Problem_Behavior.Ignore
-                new_expression = self.make_cast_expression column sql_type
-                new_sql_type_reference = SQL_Type_Reference.from_constant sql_type
-                Internal_Column.Value column.name new_sql_type_reference new_expression
+        needs_cast = approximate_result_type != Value_Type.Null
+        if needs_cast.not then column else
+            sql_type = self.get_type_mapping.value_type_to_sql approximate_result_type Problem_Behavior.Ignore
+            new_expression = self.make_cast_expression column sql_type
+            new_sql_type_reference = SQL_Type_Reference.from_constant sql_type
+            Internal_Column.Value column.name new_sql_type_reference new_expression
 
     ## PRIVATE
        Add an extra cast to adjust the output type of certain operations with
@@ -637,6 +635,9 @@ make_custom_cast column target_value_type type_mapping =
     result = Ref.new Nothing
     column_type =
         type_mapping.sql_type_to_value_type column.sql_type_reference.get
+    if target_value_type == Value_Type.Null then
+        result.put column.expression
+
     if target_value_type.is_text && (column_type == Value_Type.Boolean) then
         expr = SQL_Expression.Operation "IIF" [column.expression, SQL_Expression.Literal "'true'", SQL_Expression.Literal "'false'"]
         result.put expr

distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Type_Mapping.enso

@@ -60,6 +60,7 @@ type SQLite_Type_Mapping
             Value_Type.Date -> unsupported_date_time
             Value_Type.Date_Time _ -> unsupported_date_time
             Value_Type.Binary _ _ -> SQLite_Types.blob
+            Value_Type.Null -> SQLite_Types.null
             Value_Type.Mixed ->
                 ## The best we could do would be to store mixed values and
                    report invalid type or coerce the values to Text. Both seem
@@ -151,7 +152,7 @@ simple_types_map = Dictionary.from_vector <|
     numerics = [Types.DECIMAL, Types.NUMERIC] . map x-> [x, default_float]
     strings = [Types.CHAR, Types.VARCHAR] . map x-> [x, default_text]
     blobs = [Types.BINARY, Types.BLOB, Types.CLOB] . map x-> [x, Value_Type.Binary]
-    special_types = [[Types.BOOLEAN, Value_Type.Boolean]]
+    special_types = [[Types.BOOLEAN, Value_Type.Boolean], [Types.NULL, Value_Type.Null]]
     ints + floats + numerics + strings + blobs + special_types
 
 ## PRIVATE
@@ -246,6 +247,9 @@ type SQLite_Types
        The artificial 6th affinity that is used to distinguish boolean columns.
     boolean = SQL_Type.Value Types.BOOLEAN "BOOLEAN"
 
+    ## PRIVATE
+    null = SQL_Type.Value Types.NULL "NULL"
+
 ## PRIVATE
 default_text = Value_Type.Char size=Nothing variable_length=True
 

distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Upload/Helpers/SQL_Helpers.enso

@@ -2,6 +2,7 @@ private
 
 from Standard.Base import all
 import Standard.Base.Data.Vector.No_Wrap
+import Standard.Base.Errors.Illegal_Argument.Illegal_Argument
 
 import project.Column_Description.Column_Description
 import project.Connection.Connection.Connection
@@ -10,6 +11,7 @@ import project.Internal.IR.Query.Query
 import project.Internal.IR.SQL_Expression.SQL_Expression
 import project.SQL_Query.SQL_Query
 import project.SQL_Statement.SQL_Statement
+from project.Errors import Unsupported_Database_Type
 
 ## PRIVATE
 make_batched_insert_template : Connection -> Text -> Vector (Vector Text) -> SQL_Query
@@ -32,6 +34,9 @@ prepare_create_table_statement connection table_name columns primary_key tempora
     type_mapping = connection.dialect.get_type_mapping
     column_descriptors = columns.map on_problems=No_Wrap.Value def->
         sql_type = type_mapping.value_type_to_sql def.value_type on_problems
+            . catch Unsupported_Database_Type error->
+                Error.throw (Illegal_Argument.Error "Definition for column ["+def.name+"] is invalid: "+error.to_display_text)
+
         sql_type_text = type_mapping.sql_type_to_text sql_type
         Create_Column_Descriptor.Value def.name sql_type_text def.constraints
     connection.dialect.generate_sql <|

distribution/lib/Standard/Microsoft/0.0.0-dev/src/Internal/SQLServer_Type_Mapping.enso

@@ -62,6 +62,7 @@ type SQLServer_Type_Mapping
                         True  -> SQL_Type.Value Types.BINARY "VARBINARY" size
                         False -> SQL_Type.Value Types.BINARY "BINARY" size
             Value_Type.Mixed -> Error.throw (Unsupported_Database_Type.Error "Mixed" "SQLServer")
+            Value_Type.Null -> Error.throw (Unsupported_Database_Type.Error "Null" "SQLServer")
             Value_Type.Unsupported_Data_Type type_name underlying_type ->
                 underlying_type.if_nothing <| Error.throw <| Illegal_Argument.Error <|
                     "An unsupported SQL type ["+type_name.to_text+"] cannot be converted into an SQL type because it did not contain the SQL metadata needed to reconstruct it."
@@ -101,6 +102,7 @@ type SQLServer_Type_Mapping
                 "varbinary" -> Value_Type.Binary size=sql_type.precision variable_length=True
                 "binary" -> Value_Type.Binary size=sql_type.precision variable_length=False
                 _ -> on_unknown_type sql_type
+            Types.NULL -> Value_Type.Null
             _ -> case sql_type.name of
                 "datetimeoffset" -> Value_Type.Date_Time with_timezone=True
                 _ -> on_unknown_type sql_type