fix(ddl): fix modify column data type null to not null failed (#17650)

* fix(ddl): fix modify column data type null to not null failed

* add tests

* fix tests

Author: b41sh

src/query/expression/src/values.rs

@@ -1436,10 +1436,23 @@ impl Column {
                 })
                 .take(len),
             )),
-            DataType::Nullable(ty) => NullableColumn::new_column(
-                Column::random(ty, len, options),
-                Bitmap::from((0..len).map(|_| rng.gen_bool(0.5)).collect::<Vec<bool>>()),
-            ),
+            DataType::Nullable(ty) => {
+                let column = Column::random(ty, len, options);
+                let bitmap =
+                    Bitmap::from((0..len).map(|_| rng.gen_bool(0.5)).collect::<Vec<bool>>());
+
+                // If the value is NULL, insert default value in underlying column
+                let mut builder = ColumnBuilder::with_capacity(ty, len);
+                for (valid, value) in bitmap.iter().zip(column.iter()) {
+                    if valid {
+                        builder.push(value);
+                    } else {
+                        builder.push_default();
+                    }
+                }
+                let new_column = builder.build();
+                NullableColumn::new_column(new_column, bitmap)
+            }
             DataType::Array(inner_ty) => {
                 let mut inner_len = 0;
                 let mut offsets: Vec<u64> = Vec::with_capacity(len + 1);

src/query/service/src/interpreters/interpreter_table_modify_column.rs

@@ -267,7 +267,7 @@ impl ModifyTableColumnInterpreter {
                 // 2. default expr and computed expr not changed. Otherwise, we need fill value for
                 //    new added column.
                 if ((table.storage_format_as_parquet() && is_alter_column_string_to_binary)
-                    || old_field.data_type.remove_nullable() == field.data_type.remove_nullable())
+                    || old_field.data_type == field.data_type)
                     && old_field.default_expr == field.default_expr
                     && old_field.computed_expr == field.computed_expr
                 {
@@ -301,6 +301,8 @@ impl ModifyTableColumnInterpreter {
             .map(|(index, field)| {
                 if modified_field_indices.contains(&index) {
                     let old_field = schema.field_with_name(&field.name).unwrap();
+                    let need_remove_nullable =
+                        old_field.data_type.is_nullable() && !field.data_type.is_nullable();
                     // If the column type is Tuple or Array(Tuple), the difference in the number of leaf columns may cause
                     // the auto cast to fail.
                     // We read the leaf column data, and then use build function to construct a new Tuple or Array(Tuple).
@@ -421,7 +423,11 @@ impl ModifyTableColumnInterpreter {
                             )
                         }
                         (_, _) => {
-                            format!("`{}`", field.name)
+                            if need_remove_nullable {
+                                format!("remove_nullable(`{}`)", field.name)
+                            } else {
+                                format!("`{}`", field.name)
+                            }
                         }
                     }
                 } else {

tests/suites/0_stateless/17_altertable/17_0005_alter_table_modify_column_type.result

@@ -41,3 +41,19 @@ Error: APIError: QueryFailed: [1006]null value in column `a` of table `f` violat
 	2
 a	VARCHAR	NO	''	
 b	INT	NO	0	
+begin test modify column NULL to not NULL
+4
+a	1
+b	NULL
+NULL	3
+d	4
+a	1
+b	NULL
+	3
+d	4
+a	1
+b	0
+	3
+d	4
+a	VARCHAR	NO	''	
+b	INT	NO	0	

tests/suites/0_stateless/17_altertable/17_0005_alter_table_modify_column_type.sh

@@ -59,4 +59,14 @@ echo "INSERT INTO test_modify_column_type.f (a,b) values('',2)"  | $BENDSQL_CLIE
 echo "SELECT a,b from test_modify_column_type.f order by b"  | $BENDSQL_CLIENT_CONNECT
 echo "DESC test_modify_column_type.f"  | $BENDSQL_CLIENT_CONNECT
 
+echo "begin test modify column NULL to not NULL"
+echo "CREATE TABLE test_modify_column_type.g(a STRING NULL, b INT NULL) STORAGE_FORMAT='native'"  | $BENDSQL_CLIENT_CONNECT
+echo "INSERT INTO test_modify_column_type.g VALUES('a',1),('b',NULL),(NULL,3),('d',4)"  | $BENDSQL_CLIENT_CONNECT
+echo "SELECT a,b from test_modify_column_type.g" | $BENDSQL_CLIENT_CONNECT
+echo "ALTER TABLE test_modify_column_type.g MODIFY COLUMN a STRING NOT NULL"  | $BENDSQL_CLIENT_CONNECT
+echo "SELECT a,b from test_modify_column_type.g" | $BENDSQL_CLIENT_CONNECT
+echo "ALTER TABLE test_modify_column_type.g MODIFY COLUMN b INT NOT NULL"  | $BENDSQL_CLIENT_CONNECT
+echo "SELECT a,b from test_modify_column_type.g" | $BENDSQL_CLIENT_CONNECT
+echo "DESC test_modify_column_type.g"  | $BENDSQL_CLIENT_CONNECT
+
 echo "DROP DATABASE IF EXISTS test_modify_column_type" | $BENDSQL_CLIENT_CONNECT