chore: backport set cache capacity to rel/v1.2.636-rc8.2 (#17343)

* chore: bring back function `set_cache_capacity` (#17196)

* chore:bring back func `set_cache_capacity`

* test cases

* revert config file

* tweak test case

* fix: use usize as bytes capacity

* tweak lock scope

* rename vars/methods of cache mgr

* resovle pick conflicts

* cargo fmt

* tweak audit.toml, bypass RUSTSEC-2025-0001

* pin patched opendal to tag, instead of commit hash

* taplo fmt

* tweak golden file of ut test_caches_table

Author: dantengsky

.cargo/audit.toml

@@ -53,4 +53,6 @@ ignore = [
     "RUSTSEC-2024-0399",
     # `idna` accepts Punycode labels that do not produce any non-ASCII when decoded
     "RUSTSEC-2024-0421",
+    # gix-worktree-state nonexclusive checkout sets executable files world-writable
+    "RUSTSEC-2025-0001",
 ]

Cargo.lock

@@ -417,7 +417,7 @@ deltalake = { git = "https://github.com/delta-io/delta-rs", rev = "57795da" }
 ethnum = { git = "https://github.com/ariesdevil/ethnum-rs", rev = "4cb05f1" }
 openai_api_rust = { git = "https://github.com/datafuse-extras/openai-api", rev = "819a0ed" }
 # patched opendal which categories the XML dersierialization errors as recoverable
-opendal = { git = "https://github.com/datafuse-extras/opendal-for-release-v1.2.636", rev = "6c490a0" }
+opendal = { git = "https://github.com/datafuse-extras/opendal-for-release-v1.2.636", tag = "v0.49.0-xml-deser-retry" }
 
 # till new version(> 0.7) of opensrv released, locked on commit hash
 opensrv-mysql = { git = "https://github.com/databendlabs/opensrv", rev = "6cbb806" }

Cargo.toml

@@ -72,6 +72,10 @@ pub trait Cache<K: Eq + Hash + MemSized, V: MemSized> {
 
     fn items_capacity(&self) -> u64;
 
+    fn set_bytes_capacity(&mut self, capacity: usize);
+
+    fn set_items_capacity(&mut self, capacity: usize);
+
     /// Returns the bytes size of all the key-value pairs in the cache.
     fn bytes_size(&self) -> u64;
 

src/common/cache/src/cache.rs

@@ -301,6 +301,20 @@ impl<K: Eq + Hash + MemSized, V: MemSized> Cache<K, V> for LruCache<K, V> {
         self.max_items as u64
     }
 
+    fn set_bytes_capacity(&mut self, max_bytes: usize) {
+        while self.bytes > max_bytes || self.map.len() > self.max_items {
+            self.pop_by_policy();
+        }
+        self.max_bytes = max_bytes;
+    }
+
+    fn set_items_capacity(&mut self, max_items: usize) {
+        while self.bytes > self.max_bytes || self.map.len() > max_items {
+            self.pop_by_policy();
+        }
+        self.max_items = max_items;
+    }
+
     /// Returns the bytes size of all the key-value pairs in the cache.
     fn bytes_size(&self) -> u64 {
         self.bytes as u64

src/common/cache/src/cache/lru.rs

@@ -84,7 +84,7 @@ const SYSTEM_TABLES_ALLOW_LIST: [&str; 19] = [
 ];
 
 // table functions that need `Super` privilege
-const SYSTEM_TABLE_FUNCTIONS: [&str; 1] = ["fuse_amend"];
+const SYSTEM_TABLE_FUNCTIONS: [&str; 2] = ["fuse_amend", "set_cache_capacity"];
 
 impl PrivilegeAccess {
     pub fn create(ctx: Arc<QueryContext>) -> Box<dyn AccessChecker> {

src/query/service/src/interpreters/access/privilege_access.rs

@@ -26,6 +26,7 @@ use databend_common_storages_fuse::table_functions::FuseColumnFunc;
 use databend_common_storages_fuse::table_functions::FuseEncodingFunc;
 use databend_common_storages_fuse::table_functions::FuseStatisticsFunc;
 use databend_common_storages_fuse::table_functions::FuseVacuumTemporaryTable;
+use databend_common_storages_fuse::table_functions::SetCacheCapacity;
 use databend_common_storages_fuse::table_functions::TableFunctionTemplate;
 use databend_common_storages_stream::stream_status_table_func::StreamStatusTable;
 use databend_storages_common_table_meta::table_id_ranges::SYS_TBL_FUC_ID_END;
@@ -136,6 +137,14 @@ impl TableFunctionFactory {
             ),
         );
 
+        creators.insert(
+            "set_cache_capacity".to_string(),
+            (
+                next_id(),
+                Arc::new(TableFunctionTemplate::<SetCacheCapacity>::create),
+            ),
+        );
+
         creators.insert(
             "fuse_segment".to_string(),
             (

src/query/service/src/table_functions/table_function_factory.rs

@@ -9,7 +9,7 @@ DB.Table: 'system'.'caches', Table: caches-table_id:1, ver:0, Engine: SystemCach
 | 'test-node' | 'memory_cache_compact_segment_info'          | 0        | 0        | 1073741824 | 'bytes'  | 0        | 0        | 0        |
 | 'test-node' | 'memory_cache_inverted_index_file'           | 0        | 0        | 2147483648 | 'bytes'  | 0        | 0        | 0        |
 | 'test-node' | 'memory_cache_inverted_index_file_meta_data' | 0        | 0        | 3000       | 'count'  | 0        | 0        | 0        |
-| 'test-node' | 'memory_cache_parquet_file_meta'             | 0        | 0        | 3000       | 'count'  | 0        | 0        | 0        |
+| 'test-node' | 'memory_cache_parquet_meta_data'             | 0        | 0        | 3000       | 'count'  | 0        | 0        | 0        |
 | 'test-node' | 'memory_cache_prune_partitions'              | 0        | 0        | 256        | 'count'  | 0        | 0        | 0        |
 | 'test-node' | 'memory_cache_table_snapshot'                | 0        | 0        | 256        | 'count'  | 0        | 0        | 0        |
 | 'test-node' | 'memory_cache_table_statistics'              | 0        | 0        | 256        | 'count'  | 0        | 0        | 0        |