From edd9f1fe1e4e051ca83bdb9c419a87ddbb8b5c61 Mon Sep 17 00:00:00 2001 From: 0xZensh Date: Thu, 29 Aug 2024 08:13:08 +0800 Subject: [PATCH] feat: improve data struct in store for saving storage space --- Cargo.lock | 12 ++-- Cargo.toml | 2 +- examples/ai_canister/Cargo.toml | 4 +- src/ic_oss/Cargo.toml | 2 +- src/ic_oss_bucket/Cargo.toml | 2 +- src/ic_oss_bucket/src/store.rs | 44 +++++++++++-- src/ic_oss_can/Cargo.toml | 2 +- src/ic_oss_cli/Cargo.toml | 4 +- src/ic_oss_cluster/Cargo.toml | 2 +- src/ic_oss_cluster/src/store.rs | 28 ++++++--- src/ic_oss_types/src/bytes.rs | 105 -------------------------------- src/ic_oss_types/src/lib.rs | 3 - 12 files changed, 75 insertions(+), 135 deletions(-) delete mode 100644 src/ic_oss_types/src/bytes.rs diff --git a/Cargo.lock b/Cargo.lock index c8fc8cb..30b9df4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1367,7 +1367,7 @@ dependencies = [ [[package]] name = "ic-oss" -version = "0.7.4" +version = "0.8.0" dependencies = [ "bytes", "candid", @@ -1384,7 +1384,7 @@ dependencies = [ [[package]] name = "ic-oss-can" -version = "0.7.4" +version = "0.8.0" dependencies = [ "candid", "ciborium", @@ -1397,7 +1397,7 @@ dependencies = [ [[package]] name = "ic-oss-cli" -version = "0.7.4" +version = "0.8.0" dependencies = [ "anyhow", "candid", @@ -1418,7 +1418,7 @@ dependencies = [ [[package]] name = "ic-oss-types" -version = "0.7.4" +version = "0.8.0" dependencies = [ "base64 0.21.7", "candid", @@ -1492,7 +1492,7 @@ checksum = "8de254dd67bbd58073e23dc1c8553ba12fa1dc610a19de94ad2bbcd0460c067f" [[package]] name = "ic_oss_bucket" -version = "0.7.4" +version = "0.8.0" dependencies = [ "base64 0.21.7", "candid", @@ -1512,7 +1512,7 @@ dependencies = [ [[package]] name = "ic_oss_cluster" -version = "0.7.4" +version = "0.8.0" dependencies = [ "candid", "ciborium", diff --git a/Cargo.toml b/Cargo.toml index efa7498..74abcaa 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ strip = true opt-level = 's' [workspace.package] -version = "0.7.4" +version = "0.8.0" edition = "2021" repository = "https://github.com/ldclabs/ic-oss" keywords = ["file", "storage", "oss", "s3", "icp"] diff --git a/examples/ai_canister/Cargo.toml b/examples/ai_canister/Cargo.toml index adc4d17..0d8a6e7 100644 --- a/examples/ai_canister/Cargo.toml +++ b/examples/ai_canister/Cargo.toml @@ -19,5 +19,5 @@ serde = { workspace = true } serde_bytes = { workspace = true } getrandom = { workspace = true } rand = { version = "0.8", features = ["getrandom"] } -ic-oss-types = { path = "../../src/ic_oss_types", version = "0.7" } -ic-oss-can = { path = "../../src/ic_oss_can", version = "0.7" } +ic-oss-types = { path = "../../src/ic_oss_types", version = "0.8" } +ic-oss-can = { path = "../../src/ic_oss_can", version = "0.8" } diff --git a/src/ic_oss/Cargo.toml b/src/ic_oss/Cargo.toml index a4875c6..0fc92c3 100644 --- a/src/ic_oss/Cargo.toml +++ b/src/ic_oss/Cargo.toml @@ -20,4 +20,4 @@ tokio-stream = { workspace = true } futures = { workspace = true } sha3 = { workspace = true } ic-agent = "0.36" -ic-oss-types = { path = "../ic_oss_types", version = "0.7" } +ic-oss-types = { path = "../ic_oss_types", version = "0.8" } diff --git a/src/ic_oss_bucket/Cargo.toml b/src/ic_oss_bucket/Cargo.toml index 59d57aa..b759d34 100644 --- a/src/ic_oss_bucket/Cargo.toml +++ b/src/ic_oss_bucket/Cargo.toml @@ -28,4 +28,4 @@ ic-http-certification = { workspace = true } getrandom = { workspace = true } lazy_static = "1.4" hyperx = { git = "https://github.com/ldclabs/hyperx", rev = "4b9bd373b8c4d29a32e59912bf598ba69273c032" } -ic-oss-types = { path = "../ic_oss_types", version = "0.7" } +ic-oss-types = { path = "../ic_oss_types", version = "0.8" } diff --git a/src/ic_oss_bucket/src/store.rs b/src/ic_oss_bucket/src/store.rs index 84e741b..f6e9fac 100644 --- a/src/ic_oss_bucket/src/store.rs +++ b/src/ic_oss_bucket/src/store.rs @@ -35,22 +35,36 @@ static ZERO_HASH: [u8; 32] = [0; 32]; #[derive(Clone, Deserialize, Serialize)] pub struct Bucket { + #[serde(rename = "n", alias = "name")] pub name: String, + #[serde(rename = "fi", alias = "file_id")] pub file_id: u32, + #[serde(rename = "fo", alias = "folder_id")] pub folder_id: u32, + #[serde(rename = "fz", alias = "max_file_size")] pub max_file_size: u64, + #[serde(rename = "fd", alias = "max_folder_depth")] pub max_folder_depth: u8, + #[serde(rename = "mc", alias = "max_children")] pub max_children: u16, + #[serde(rename = "cds", alias = "max_custom_data_size")] pub max_custom_data_size: u16, + #[serde(rename = "h", alias = "enable_hash_index")] pub enable_hash_index: bool, - pub status: i8, // -1: archived; 0: readable and writable; 1: readonly + #[serde(rename = "s", alias = "status")] + pub status: i8, // -1: archived; 0: readable and writable; 1: readonly + #[serde(rename = "v", alias = "visibility")] pub visibility: u8, // 0: private; 1: public + #[serde(rename = "m", alias = "managers")] pub managers: BTreeSet, // managers can read and write // auditors can read and list even if the bucket is private + #[serde(rename = "a", alias = "auditors")] pub auditors: BTreeSet, // used to verify the request token signed with SECP256K1 + #[serde(rename = "ec", alias = "trusted_ecdsa_pub_keys")] pub trusted_ecdsa_pub_keys: Vec, // used to verify the request token signed with ED25519 + #[serde(rename = "ed", alias = "trusted_eddsa_pub_keys")] pub trusted_eddsa_pub_keys: Vec>, } @@ -224,18 +238,31 @@ impl Storable for FileId { #[derive(Clone, Default, Deserialize, Serialize)] pub struct FileMetadata { + #[serde(rename = "p", alias = "parent")] pub parent: u32, // 0: root + #[serde(rename = "n", alias = "name")] pub name: String, + #[serde(rename = "t", alias = "content_type")] pub content_type: String, // MIME types + #[serde(rename = "i", alias = "size")] pub size: u64, + #[serde(rename = "f", alias = "filled")] pub filled: u64, + #[serde(rename = "ca", alias = "created_at")] pub created_at: u64, // unix timestamp in milliseconds + #[serde(rename = "ua", alias = "updated_at")] pub updated_at: u64, // unix timestamp in milliseconds + #[serde(rename = "c", alias = "chunks")] pub chunks: u32, + #[serde(rename = "s", alias = "status")] pub status: i8, // -1: archived; 0: readable and writable; 1: readonly + #[serde(rename = "h", alias = "hash")] pub hash: Option>, // recommend sha3 256 + #[serde(rename = "k", alias = "dek")] pub dek: Option, // // Data Encryption Key that encrypted by BYOK or vetKey in COSE_Encrypt0 + #[serde(rename = "cu", alias = "custom")] pub custom: Option, // custom metadata + #[serde(rename = "e", alias = "ex")] pub ex: Option, // External Resource, ER indicates that the file is an external resource. } @@ -295,13 +322,20 @@ impl Storable for Chunk { // folder #[derive(Clone, Debug, Default, Deserialize, Serialize)] pub struct FolderMetadata { + #[serde(rename = "p", alias = "parent")] pub parent: u32, // 0: root + #[serde(rename = "n", alias = "name")] pub name: String, - pub files: BTreeSet, // length <= max_children + #[serde(rename = "fi", alias = "files")] + pub files: BTreeSet, // length <= max_children + #[serde(rename = "fo", alias = "folders")] pub folders: BTreeSet, // length <= max_children - pub created_at: u64, // unix timestamp in milliseconds - pub updated_at: u64, // unix timestamp in milliseconds - pub status: i8, // -1: archived; 0: readable and writable; 1: readonly + #[serde(rename = "ca", alias = "created_at")] + pub created_at: u64, // unix timestamp in milliseconds + #[serde(rename = "ua", alias = "updated_at")] + pub updated_at: u64, // unix timestamp in milliseconds + #[serde(rename = "s", alias = "status")] + pub status: i8, // -1: archived; 0: readable and writable; 1: readonly } impl FolderMetadata { diff --git a/src/ic_oss_can/Cargo.toml b/src/ic_oss_can/Cargo.toml index 40179ec..0549248 100644 --- a/src/ic_oss_can/Cargo.toml +++ b/src/ic_oss_can/Cargo.toml @@ -16,4 +16,4 @@ serde_bytes = { workspace = true } ciborium = { workspace = true } ic-cdk = { workspace = true } ic-stable-structures = { workspace = true } -ic-oss-types = { path = "../ic_oss_types", version = "0.7" } +ic-oss-types = { path = "../ic_oss_types", version = "0.8" } diff --git a/src/ic_oss_cli/Cargo.toml b/src/ic_oss_cli/Cargo.toml index bd09960..a3a36f7 100644 --- a/src/ic_oss_cli/Cargo.toml +++ b/src/ic_oss_cli/Cargo.toml @@ -15,8 +15,8 @@ serde_bytes = { workspace = true } tokio = { workspace = true } sha3 = { workspace = true } hex = { workspace = true } -ic-oss = { path = "../ic_oss", version = "0.7" } -ic-oss-types = { path = "../ic_oss_types", version = "0.7" } +ic-oss = { path = "../ic_oss", version = "0.8" } +ic-oss-types = { path = "../ic_oss_types", version = "0.8" } ic-agent = "0.36" anyhow = "1" clap = { version = "=4.5", features = ["derive"] } diff --git a/src/ic_oss_cluster/Cargo.toml b/src/ic_oss_cluster/Cargo.toml index 33b1dcc..4c49e2b 100644 --- a/src/ic_oss_cluster/Cargo.toml +++ b/src/ic_oss_cluster/Cargo.toml @@ -25,4 +25,4 @@ hex = { workspace = true } serde = { workspace = true } serde_bytes = { workspace = true } getrandom = { workspace = true } -ic-oss-types = { path = "../ic_oss_types", version = "0.7" } +ic-oss-types = { path = "../ic_oss_types", version = "0.8" } diff --git a/src/ic_oss_cluster/src/store.rs b/src/ic_oss_cluster/src/store.rs index 3293f33..7418330 100644 --- a/src/ic_oss_cluster/src/store.rs +++ b/src/ic_oss_cluster/src/store.rs @@ -24,23 +24,27 @@ type Memory = VirtualMemory; #[derive(Clone, Default, Deserialize, Serialize)] pub struct State { - #[serde(default)] + #[serde(default, rename = "n", alias = "name")] pub name: String, + #[serde(rename = "k", alias = "ecdsa_key_name")] pub ecdsa_key_name: String, + #[serde(rename = "t", alias = "ecdsa_token_public_key")] pub ecdsa_token_public_key: String, + #[serde(rename = "e", alias = "token_expiration")] pub token_expiration: u64, // in seconds + #[serde(rename = "m", alias = "managers")] pub managers: BTreeSet, - #[serde(default)] + #[serde(default, rename = "lv", alias = "bucket_latest_version")] pub bucket_latest_version: ByteArray<32>, - #[serde(default)] + #[serde(default, rename = "p", alias = "bucket_upgrade_path")] pub bucket_upgrade_path: HashMap, ByteArray<32>>, - #[serde(default)] + #[serde(default, rename = "dl", alias = "bucket_deployed_list")] pub bucket_deployed_list: BTreeMap)>, - #[serde(default)] + #[serde(default, rename = "up", alias = "bucket_upgrade_process")] pub bucket_upgrade_process: Option, - #[serde(default)] + #[serde(default, rename = "tt", alias = "bucket_topup_threshold")] pub bucket_topup_threshold: u128, - #[serde(default)] + #[serde(default, rename = "ta", alias = "bucket_topup_amount")] pub bucket_topup_amount: u128, } @@ -98,9 +102,13 @@ impl Storable for PoliciesTable { #[derive(Clone, Deserialize, Serialize)] pub struct Wasm { + #[serde(rename = "a", alias = "created_at")] pub created_at: u64, // in milliseconds + #[serde(rename = "b", alias = "created_by")] pub created_by: Principal, + #[serde(rename = "d", alias = "description")] pub description: String, + #[serde(rename = "w", alias = "wasm")] pub wasm: ByteBuf, } @@ -120,11 +128,17 @@ impl Storable for Wasm { #[derive(Clone, Deserialize, Serialize)] pub struct DeployLog { + #[serde(rename = "d", alias = "deploy_at")] pub deploy_at: u64, // in milliseconds + #[serde(rename = "c", alias = "canister")] pub canister: Principal, + #[serde(rename = "p", alias = "prev_hash")] pub prev_hash: ByteArray<32>, + #[serde(rename = "w", alias = "wasm_hash")] pub wasm_hash: ByteArray<32>, + #[serde(rename = "a", alias = "args")] pub args: ByteBuf, + #[serde(rename = "e", alias = "error")] pub error: Option, } diff --git a/src/ic_oss_types/src/bytes.rs b/src/ic_oss_types/src/bytes.rs deleted file mode 100644 index a3ea64e..0000000 --- a/src/ic_oss_types/src/bytes.rs +++ /dev/null @@ -1,105 +0,0 @@ -use candid::CandidType; -use serde::{Deserialize, Serialize}; -use serde_bytes::ByteArray; -use std::ops::Deref; - -/// ByteN is a wrapper around ByteArray to provide CandidType implementation -#[derive( - Clone, Copy, Default, Debug, Deserialize, Serialize, PartialEq, Eq, PartialOrd, Ord, Hash, -)] -pub struct ByteN(pub ByteArray); - -impl ByteN { - pub fn from_hex(val: &str) -> Result { - let data = hex::decode(val).map_err(|_| format!("failed to decode hex: {}", val))?; - Self::try_from(data.as_slice()) - } -} - -impl CandidType for ByteN { - fn _ty() -> candid::types::internal::Type { - candid::types::internal::TypeInner::Vec(candid::types::internal::TypeInner::Nat8.into()) - .into() - } - fn idl_serialize(&self, serializer: S) -> Result<(), S::Error> - where - S: candid::types::Serializer, - { - serializer.serialize_blob(self.0.as_slice()) - } -} - -impl Deref for ByteN { - type Target = [u8; N]; - - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -impl AsRef<[u8; N]> for ByteN { - fn as_ref(&self) -> &[u8; N] { - &self.0 - } -} - -impl From<[u8; N]> for ByteN { - fn from(val: [u8; N]) -> Self { - Self(ByteArray::new(val)) - } -} - -impl TryFrom<&[u8]> for ByteN { - type Error = String; - - fn try_from(value: &[u8]) -> Result { - if value.len() != N { - return Err(format!("expected {} bytes, got {}", N, value.len())); - } - let mut bytes = [0u8; N]; - bytes.copy_from_slice(value); - Ok(Self(ByteArray::new(bytes))) - } -} - -impl From> for ByteN { - fn from(val: ByteArray) -> Self { - Self(val) - } -} - -impl From> for ByteArray { - fn from(val: ByteN) -> Self { - val.0 - } -} - -impl From> for Vec { - fn from(val: ByteN) -> Self { - Vec::from(val.0.into_array()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::to_cbor_bytes; - use ciborium::from_reader; - - #[test] - fn byte_n_works() { - let b4: ByteN<4> = [1, 2, 3, 4].into(); - let data = to_cbor_bytes(&b4); - assert_eq!(&data, &[68, 1, 2, 3, 4]); - let v: ByteN<4> = from_reader(&data[..]).unwrap(); - assert_eq!(v, b4); - let res: Result, _> = from_reader([69, 1, 2, 3, 4, 0].as_slice()); - // println!("{:?}", res.err()); - assert!( - res.is_err(), - "invalid length 5, expected a byte array of length 4" - ); - let res: ByteN<5> = from_reader([69, 1, 2, 3, 4, 0].as_slice()).unwrap(); - assert_eq!(*res, [1, 2, 3, 4, 0]); - } -} diff --git a/src/ic_oss_types/src/lib.rs b/src/ic_oss_types/src/lib.rs index 70ffb5e..c837ef3 100644 --- a/src/ic_oss_types/src/lib.rs +++ b/src/ic_oss_types/src/lib.rs @@ -14,9 +14,6 @@ pub mod file; pub mod folder; pub mod permission; -mod bytes; -pub use bytes::*; - // should update to ICRC3Map pub type MapValue = BTreeMap;