feat: improve data struct in store for saving storage space

Cargo.toml

@@ -17,7 +17,7 @@ strip = true
 opt-level = 's'
 
 [workspace.package]
-version = "0.7.4"
+version = "0.8.0"
 edition = "2021"
 repository = "https://github.com/ldclabs/ic-oss"
 keywords = ["file", "storage", "oss", "s3", "icp"]

examples/ai_canister/Cargo.toml

@@ -19,5 +19,5 @@ serde = { workspace = true }
 serde_bytes = { workspace = true }
 getrandom = { workspace = true }
 rand = { version = "0.8", features = ["getrandom"] }
-ic-oss-types = { path = "../../src/ic_oss_types", version = "0.7" }
-ic-oss-can = { path = "../../src/ic_oss_can", version = "0.7" }
+ic-oss-types = { path = "../../src/ic_oss_types", version = "0.8" }
+ic-oss-can = { path = "../../src/ic_oss_can", version = "0.8" }

src/ic_oss/Cargo.toml

@@ -20,4 +20,4 @@ tokio-stream = { workspace = true }
 futures = { workspace = true }
 sha3 = { workspace = true }
 ic-agent = "0.36"
-ic-oss-types = { path = "../ic_oss_types", version = "0.7" }
+ic-oss-types = { path = "../ic_oss_types", version = "0.8" }

src/ic_oss_bucket/Cargo.toml

@@ -28,4 +28,4 @@ ic-http-certification = { workspace = true }
 getrandom = { workspace = true }
 lazy_static = "1.4"
 hyperx = { git = "https://github.com/ldclabs/hyperx", rev = "4b9bd373b8c4d29a32e59912bf598ba69273c032" }
-ic-oss-types = { path = "../ic_oss_types", version = "0.7" }
+ic-oss-types = { path = "../ic_oss_types", version = "0.8" }

src/ic_oss_bucket/src/store.rs

@@ -35,22 +35,36 @@ static ZERO_HASH: [u8; 32] = [0; 32];
 
 #[derive(Clone, Deserialize, Serialize)]
 pub struct Bucket {
+    #[serde(rename = "n", alias = "name")]
     pub name: String,
+    #[serde(rename = "fi", alias = "file_id")]
     pub file_id: u32,
+    #[serde(rename = "fo", alias = "folder_id")]
     pub folder_id: u32,
+    #[serde(rename = "fz", alias = "max_file_size")]
     pub max_file_size: u64,
+    #[serde(rename = "fd", alias = "max_folder_depth")]
     pub max_folder_depth: u8,
+    #[serde(rename = "mc", alias = "max_children")]
     pub max_children: u16,
+    #[serde(rename = "cds", alias = "max_custom_data_size")]
     pub max_custom_data_size: u16,
+    #[serde(rename = "h", alias = "enable_hash_index")]
     pub enable_hash_index: bool,
-    pub status: i8,     // -1: archived; 0: readable and writable; 1: readonly
+    #[serde(rename = "s", alias = "status")]
+    pub status: i8, // -1: archived; 0: readable and writable; 1: readonly
+    #[serde(rename = "v", alias = "visibility")]
     pub visibility: u8, // 0: private; 1: public
+    #[serde(rename = "m", alias = "managers")]
     pub managers: BTreeSet<Principal>, // managers can read and write
     // auditors can read and list even if the bucket is private
+    #[serde(rename = "a", alias = "auditors")]
     pub auditors: BTreeSet<Principal>,
     // used to verify the request token signed with SECP256K1
+    #[serde(rename = "ec", alias = "trusted_ecdsa_pub_keys")]
     pub trusted_ecdsa_pub_keys: Vec<ByteBuf>,
     // used to verify the request token signed with ED25519
+    #[serde(rename = "ed", alias = "trusted_eddsa_pub_keys")]
     pub trusted_eddsa_pub_keys: Vec<ByteArray<32>>,
 }
 
@@ -224,18 +238,31 @@ impl Storable for FileId {
 
 #[derive(Clone, Default, Deserialize, Serialize)]
 pub struct FileMetadata {
+    #[serde(rename = "p", alias = "parent")]
     pub parent: u32, // 0: root
+    #[serde(rename = "n", alias = "name")]
     pub name: String,
+    #[serde(rename = "t", alias = "content_type")]
     pub content_type: String, // MIME types
+    #[serde(rename = "i", alias = "size")]
     pub size: u64,
+    #[serde(rename = "f", alias = "filled")]
     pub filled: u64,
+    #[serde(rename = "ca", alias = "created_at")]
     pub created_at: u64, // unix timestamp in milliseconds
+    #[serde(rename = "ua", alias = "updated_at")]
     pub updated_at: u64, // unix timestamp in milliseconds
+    #[serde(rename = "c", alias = "chunks")]
     pub chunks: u32,
+    #[serde(rename = "s", alias = "status")]
     pub status: i8, // -1: archived; 0: readable and writable; 1: readonly
+    #[serde(rename = "h", alias = "hash")]
     pub hash: Option<ByteArray<32>>, // recommend sha3 256
+    #[serde(rename = "k", alias = "dek")]
     pub dek: Option<ByteBuf>, // // Data Encryption Key that encrypted by BYOK or vetKey in COSE_Encrypt0
+    #[serde(rename = "cu", alias = "custom")]
     pub custom: Option<MapValue>, // custom metadata
+    #[serde(rename = "e", alias = "ex")]
     pub ex: Option<MapValue>, // External Resource, ER indicates that the file is an external resource.
 }
 
@@ -295,13 +322,20 @@ impl Storable for Chunk {
 // folder
 #[derive(Clone, Debug, Default, Deserialize, Serialize)]
 pub struct FolderMetadata {
+    #[serde(rename = "p", alias = "parent")]
     pub parent: u32, // 0: root
+    #[serde(rename = "n", alias = "name")]
     pub name: String,
-    pub files: BTreeSet<u32>,   // length <= max_children
+    #[serde(rename = "fi", alias = "files")]
+    pub files: BTreeSet<u32>, // length <= max_children
+    #[serde(rename = "fo", alias = "folders")]
     pub folders: BTreeSet<u32>, // length <= max_children
-    pub created_at: u64,        // unix timestamp in milliseconds
-    pub updated_at: u64,        // unix timestamp in milliseconds
-    pub status: i8,             // -1: archived; 0: readable and writable; 1: readonly
+    #[serde(rename = "ca", alias = "created_at")]
+    pub created_at: u64, // unix timestamp in milliseconds
+    #[serde(rename = "ua", alias = "updated_at")]
+    pub updated_at: u64, // unix timestamp in milliseconds
+    #[serde(rename = "s", alias = "status")]
+    pub status: i8, // -1: archived; 0: readable and writable; 1: readonly
 }
 
 impl FolderMetadata {

src/ic_oss_can/Cargo.toml

@@ -16,4 +16,4 @@ serde_bytes = { workspace = true }
 ciborium = { workspace = true }
 ic-cdk = { workspace = true }
 ic-stable-structures = { workspace = true }
-ic-oss-types = { path = "../ic_oss_types", version = "0.7" }
+ic-oss-types = { path = "../ic_oss_types", version = "0.8" }

src/ic_oss_cli/Cargo.toml

@@ -15,8 +15,8 @@ serde_bytes = { workspace = true }
 tokio = { workspace = true }
 sha3 = { workspace = true }
 hex = { workspace = true }
-ic-oss = { path = "../ic_oss", version = "0.7" }
-ic-oss-types = { path = "../ic_oss_types", version = "0.7" }
+ic-oss = { path = "../ic_oss", version = "0.8" }
+ic-oss-types = { path = "../ic_oss_types", version = "0.8" }
 ic-agent = "0.36"
 anyhow = "1"
 clap = { version = "=4.5", features = ["derive"] }

src/ic_oss_cluster/Cargo.toml

@@ -25,4 +25,4 @@ hex = { workspace = true }
 serde = { workspace = true }
 serde_bytes = { workspace = true }
 getrandom = { workspace = true }
-ic-oss-types = { path = "../ic_oss_types", version = "0.7" }
+ic-oss-types = { path = "../ic_oss_types", version = "0.8" }

src/ic_oss_cluster/src/store.rs

@@ -24,23 +24,27 @@ type Memory = VirtualMemory<DefaultMemoryImpl>;
 
 #[derive(Clone, Default, Deserialize, Serialize)]
 pub struct State {
-    #[serde(default)]
+    #[serde(default, rename = "n", alias = "name")]
     pub name: String,
+    #[serde(rename = "k", alias = "ecdsa_key_name")]
     pub ecdsa_key_name: String,
+    #[serde(rename = "t", alias = "ecdsa_token_public_key")]
     pub ecdsa_token_public_key: String,
+    #[serde(rename = "e", alias = "token_expiration")]
     pub token_expiration: u64, // in seconds
+    #[serde(rename = "m", alias = "managers")]
     pub managers: BTreeSet<Principal>,
-    #[serde(default)]
+    #[serde(default, rename = "lv", alias = "bucket_latest_version")]
     pub bucket_latest_version: ByteArray<32>,
-    #[serde(default)]
+    #[serde(default, rename = "p", alias = "bucket_upgrade_path")]
     pub bucket_upgrade_path: HashMap<ByteArray<32>, ByteArray<32>>,
-    #[serde(default)]
+    #[serde(default, rename = "dl", alias = "bucket_deployed_list")]
     pub bucket_deployed_list: BTreeMap<Principal, (u64, ByteArray<32>)>,
-    #[serde(default)]
+    #[serde(default, rename = "up", alias = "bucket_upgrade_process")]
     pub bucket_upgrade_process: Option<ByteBuf>,
-    #[serde(default)]
+    #[serde(default, rename = "tt", alias = "bucket_topup_threshold")]
     pub bucket_topup_threshold: u128,
-    #[serde(default)]
+    #[serde(default, rename = "ta", alias = "bucket_topup_amount")]
     pub bucket_topup_amount: u128,
 }
 
@@ -98,9 +102,13 @@ impl Storable for PoliciesTable {
 
 #[derive(Clone, Deserialize, Serialize)]
 pub struct Wasm {
+    #[serde(rename = "a", alias = "created_at")]
     pub created_at: u64, // in milliseconds
+    #[serde(rename = "b", alias = "created_by")]
     pub created_by: Principal,
+    #[serde(rename = "d", alias = "description")]
     pub description: String,
+    #[serde(rename = "w", alias = "wasm")]
     pub wasm: ByteBuf,
 }
 
@@ -120,11 +128,17 @@ impl Storable for Wasm {
 
 #[derive(Clone, Deserialize, Serialize)]
 pub struct DeployLog {
+    #[serde(rename = "d", alias = "deploy_at")]
     pub deploy_at: u64, // in milliseconds
+    #[serde(rename = "c", alias = "canister")]
     pub canister: Principal,
+    #[serde(rename = "p", alias = "prev_hash")]
     pub prev_hash: ByteArray<32>,
+    #[serde(rename = "w", alias = "wasm_hash")]
     pub wasm_hash: ByteArray<32>,
+    #[serde(rename = "a", alias = "args")]
     pub args: ByteBuf,
+    #[serde(rename = "e", alias = "error")]
     pub error: Option<String>,
 }
 

src/ic_oss_types/src/lib.rs

@@ -14,9 +14,6 @@ pub mod file;
 pub mod folder;
 pub mod permission;
 
-mod bytes;
-pub use bytes::*;
-
 // should update to ICRC3Map
 pub type MapValue =
     BTreeMap<String, icrc_ledger_types::icrc::generic_metadata_value::MetadataValue>;