feat: impl Create/Drop Index for Ngram Index (#17789)

* feat: impl `Create/Drop `Index for Ngram Index

Signed-off-by: Kould <[email protected]>

* chore: codefmt

Signed-off-by: Kould <[email protected]>

---------

Signed-off-by: Kould <[email protected]>

Author: KKould

Cargo.lock

@@ -194,6 +194,7 @@ databend-enterprise-fail-safe = { path = "src/query/ee_features/fail_safe" }
 databend-enterprise-hilbert-clustering = { path = "src/query/ee_features/hilbert_clustering" }
 databend-enterprise-inverted-index = { path = "src/query/ee_features/inverted_index" }
 databend-enterprise-meta = { path = "src/meta/ee" }
+databend-enterprise-ngram-index = { path = "src/query/ee_features/ngram_index" }
 databend-enterprise-query = { path = "src/query/ee" }
 databend-enterprise-resources-management = { path = "src/query/ee_features/resources_management" }
 databend-enterprise-storage-encryption = { path = "src/query/ee_features/storage_encryption" }

Cargo.toml

@@ -75,6 +75,8 @@ pub enum Feature {
     SystemManagement,
     #[serde(alias = "hilbert_clustering", alias = "HILBERT_CLUSTERING")]
     HilbertClustering,
+    #[serde(alias = "ngram_index", alias = "NGRAM_INDEX")]
+    NgramIndex,
     #[serde(other)]
     Unknown,
 }
@@ -122,6 +124,7 @@ impl fmt::Display for Feature {
             Feature::AmendTable => write!(f, "amend_table"),
             Feature::SystemManagement => write!(f, "system_management"),
             Feature::HilbertClustering => write!(f, "hilbert_clustering"),
+            Feature::NgramIndex => write!(f, "ngram_index"),
             Feature::Unknown => write!(f, "unknown"),
         }
     }
@@ -169,7 +172,8 @@ impl Feature {
             | (Feature::VirtualColumn, Feature::VirtualColumn)
             | (Feature::AttacheTable, Feature::AttacheTable)
             | (Feature::StorageEncryption, Feature::StorageEncryption)
-            | (Feature::HilbertClustering, Feature::HilbertClustering) => Ok(true),
+            | (Feature::HilbertClustering, Feature::HilbertClustering)
+            | (Feature::NgramIndex, Feature::NgramIndex) => Ok(true),
             (_, _) => Ok(false),
         }
     }
@@ -338,6 +342,11 @@ mod tests {
             serde_json::from_str::<Feature>("\"hilbert_clustering\"").unwrap()
         );
 
+        assert_eq!(
+            Feature::NgramIndex,
+            serde_json::from_str::<Feature>("\"NgramIndex\"").unwrap()
+        );
+
         assert_eq!(
             Feature::Unknown,
             serde_json::from_str::<Feature>("\"ssss\"").unwrap()
@@ -371,11 +380,12 @@ mod tests {
                 }),
                 Feature::AmendTable,
                 Feature::HilbertClustering,
+                Feature::NgramIndex,
             ]),
         };
 
         assert_eq!(
-            "LicenseInfo{ type: enterprise, org: databend, tenants: [databend_tenant,foo], features: [aggregate_index,amend_table,attach_table,compute_quota(threads_num: 1, memory_usage: 1),computed_column,data_mask,hilbert_clustering,inverted_index,license_info,storage_encryption,storage_quota(storage_usage: 1),stream,vacuum,virtual_column] }",
+            "LicenseInfo{ type: enterprise, org: databend, tenants: [databend_tenant,foo], features: [aggregate_index,amend_table,attach_table,compute_quota(threads_num: 1, memory_usage: 1),computed_column,data_mask,hilbert_clustering,inverted_index,license_info,ngram_index,storage_encryption,storage_quota(storage_usage: 1),stream,vacuum,virtual_column] }",
             license_info.to_string()
         );
     }

src/common/license/src/license.rs

@@ -2590,6 +2590,7 @@ impl<KV: kvapi::KVApi<Error = MetaError> + ?Sized> SchemaApi for KV {
             let version = old_version.unwrap_or(Uuid::new_v4().simple().to_string());
 
             let index = TableIndex {
+                index_type: req.index_type.clone(),
                 name: req.name.clone(),
                 column_ids: req.column_ids.clone(),
                 sync_creation: req.sync_creation,

src/meta/api/src/schema_api_impl.rs

@@ -117,6 +117,7 @@ use databend_common_meta_app::schema::TableIdHistoryIdent;
 use databend_common_meta_app::schema::TableIdList;
 use databend_common_meta_app::schema::TableIdToName;
 use databend_common_meta_app::schema::TableIdent;
+use databend_common_meta_app::schema::TableIndexType;
 use databend_common_meta_app::schema::TableInfo;
 use databend_common_meta_app::schema::TableMeta;
 use databend_common_meta_app::schema::TableNameIdent;
@@ -6171,6 +6172,7 @@ impl SchemaApiTestSuite {
             info!("--- create table index 1");
             let req = CreateTableIndexReq {
                 create_option: CreateOption::Create,
+                index_type: TableIndexType::Inverted,
                 tenant: tenant.clone(),
                 table_id,
                 name: index_name_1.clone(),
@@ -6196,6 +6198,7 @@ impl SchemaApiTestSuite {
                 column_ids: index_column_ids_1.clone(),
                 sync_creation: true,
                 options: BTreeMap::new(),
+                index_type: TableIndexType::Inverted,
             };
             let res = mt.create_table_index(req).await;
             assert!(res.is_err());
@@ -6209,6 +6212,7 @@ impl SchemaApiTestSuite {
                 column_ids: index_column_ids_2.clone(),
                 sync_creation: true,
                 options: BTreeMap::new(),
+                index_type: TableIndexType::Inverted,
             };
             let res = mt.create_table_index(req).await;
             assert!(res.is_ok());
@@ -6230,6 +6234,7 @@ impl SchemaApiTestSuite {
                 column_ids: index_column_ids_1.clone(),
                 sync_creation: true,
                 options: BTreeMap::new(),
+                index_type: TableIndexType::Inverted,
             };
 
             let res = mt.create_table_index(req).await;
@@ -6250,6 +6255,7 @@ impl SchemaApiTestSuite {
                 column_ids: index_column_ids_1.clone(),
                 sync_creation: true,
                 options: BTreeMap::new(),
+                index_type: TableIndexType::Inverted,
             };
 
             let res = mt.create_table_index(req).await;
@@ -6266,6 +6272,7 @@ impl SchemaApiTestSuite {
                 column_ids: index_column_ids_3.clone(),
                 sync_creation: true,
                 options: BTreeMap::new(),
+                index_type: TableIndexType::Inverted,
             };
             let res = mt.create_table_index(req).await;
             assert!(res.is_err());
@@ -6291,6 +6298,7 @@ impl SchemaApiTestSuite {
         {
             info!("--- drop table index");
             let req = DropTableIndexReq {
+                index_type: TableIndexType::Inverted,
                 tenant: tenant.clone(),
                 if_exists: false,
                 table_id,
@@ -6302,6 +6310,7 @@ impl SchemaApiTestSuite {
             assert!(res.is_ok());
 
             let req = DropTableIndexReq {
+                index_type: TableIndexType::Inverted,
                 tenant: tenant.clone(),
                 if_exists: false,
                 table_id,
@@ -6311,6 +6320,7 @@ impl SchemaApiTestSuite {
             assert!(res.is_err());
 
             let req = DropTableIndexReq {
+                index_type: TableIndexType::Inverted,
                 tenant: tenant.clone(),
                 if_exists: true,
                 table_id,
@@ -6369,6 +6379,7 @@ impl SchemaApiTestSuite {
                 column_ids: index_column_ids_1.clone(),
                 sync_creation: true,
                 options: BTreeMap::new(),
+                index_type: TableIndexType::Inverted,
             };
             index2_drop_start_time = Utc::now();
             let res = mt.create_table_index(req).await;

src/meta/api/src/schema_api_test_suite.rs

@@ -120,6 +120,7 @@ pub use table::TableIdList;
 pub use table::TableIdToName;
 pub use table::TableIdent;
 pub use table::TableIndex;
+pub use table::TableIndexType;
 pub use table::TableInfo;
 pub use table::TableMeta;
 pub use table::TableNameIdent;

src/meta/app/src/schema/mod.rs

@@ -283,8 +283,17 @@ pub struct TableMeta {
     pub indexes: BTreeMap<String, TableIndex>,
 }
 
+#[derive(
+    serde::Serialize, serde::Deserialize, Clone, Debug, Eq, PartialEq, num_derive::FromPrimitive,
+)]
+pub enum TableIndexType {
+    Inverted = 0,
+    Ngram = 1,
+}
+
 #[derive(serde::Serialize, serde::Deserialize, Clone, Debug, Eq, PartialEq)]
 pub struct TableIndex {
+    pub index_type: TableIndexType,
     pub name: String,
     pub column_ids: Vec<u32>,
     // if true, index will create after data written to databend,
@@ -471,6 +480,19 @@ impl Display for TableInfo {
     }
 }
 
+impl Display for TableIndexType {
+    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
+        match self {
+            TableIndexType::Inverted => {
+                write!(f, "INVERTED")
+            }
+            TableIndexType::Ngram => {
+                write!(f, "NGRAM")
+            }
+        }
+    }
+}
+
 /// Save table name id list history.
 #[derive(serde::Serialize, serde::Deserialize, Clone, Debug, Default, Eq, PartialEq)]
 pub struct TableIdList {
@@ -846,6 +868,7 @@ pub struct UpdateTableMetaReply {}
 #[derive(Clone, Debug, PartialEq, Eq)]
 pub struct CreateTableIndexReq {
     pub create_option: CreateOption,
+    pub index_type: TableIndexType,
     pub tenant: Tenant,
     pub table_id: u64,
     pub name: String,
@@ -872,6 +895,7 @@ impl Display for CreateTableIndexReq {
 
 #[derive(Clone, Debug, PartialEq, Eq)]
 pub struct DropTableIndexReq {
+    pub index_type: TableIndexType,
     pub tenant: Tenant,
     pub if_exists: bool,
     pub table_id: u64,

src/meta/app/src/schema/table.rs

@@ -28,6 +28,7 @@ use databend_common_meta_app::storage::StorageParams;
 use databend_common_meta_app::tenant::Tenant;
 use databend_common_meta_app_types::non_empty::NonEmptyString;
 use databend_common_protos::pb;
+use num::FromPrimitive;
 
 use crate::reader_check_msg;
 use crate::FromToProto;
@@ -355,6 +356,8 @@ impl FromToProto for mt::TableIndex {
         reader_check_msg(p.ver, p.min_reader_ver)?;
 
         let v = Self {
+            index_type: FromPrimitive::from_i32(p.index_type)
+                .ok_or_else(|| Incompatible::new(format!("invalid IndexType: {}", p.index_type)))?,
             name: p.name,
             column_ids: p.column_ids,
             sync_creation: p.sync_creation,
@@ -373,6 +376,7 @@ impl FromToProto for mt::TableIndex {
             sync_creation: self.sync_creation,
             version: self.version.clone(),
             options: self.options.clone(),
+            index_type: self.index_type.clone() as i32,
         };
         Ok(p)
     }

src/meta/proto-conv/src/table_from_to_protobuf_impl.rs

@@ -154,6 +154,7 @@ const META_CHANGE_LOG: &[(u64, &str)] = &[
     (122, "2025-03-11: Add: table_meta and virtual_data_schema"),
     (123, "2025-03-27: Add: add compression in user.proto/ParquetFileFormatParam"),
     (124, "2025-04-01: Add: add headers in udf.proto/UDFServer"),
+    (125, "2025-04-16: Add: add index_type in table.proto/TableIndex"),
     // Dear developer:
     //      If you're gonna add a new metadata version, you'll have to add a test for it.
     //      You could just copy an existing test file(e.g., `../tests/it/v024_table_meta.rs`)

src/meta/proto-conv/src/util.rs

@@ -119,3 +119,4 @@ mod v121_avro_format_params;
 mod v122_virtual_schema;
 mod v123_parquet_format_params;
 mod v124_udf_server_headers;
+mod v125_table_index;