Compute server storage account hashes in db upgrade.

Author: tmpfs

Cargo.lock

@@ -1,14 +1,9 @@
 use sos_core::{commit::CommitHash, VaultId};
-use std::path::PathBuf;
 use thiserror::Error;
 
 /// Errors generated by the database library.
 #[derive(Debug, Error)]
 pub enum Error {
-    /// Database file already exists.
-    #[error("database '{0}' already exists")]
-    DatabaseExists(PathBuf),
-
     /// Error generated when a folder could not be found in the database.
     #[cfg(feature = "sqlite")]
     #[error("folder '{0}' not found in the database")]

crates/database/src/error.rs

@@ -13,13 +13,16 @@ rustdoc-args = ["--cfg", "docsrs"]
 
 [dependencies]
 sos-core.workspace = true
+sos-backend.workspace = true
 sos-external-files.workspace = true
 sos-client-storage = { workspace = true, features = ["full"] }
 sos-database = { workspace = true, features = ["full"] }
 sos-filesystem = { workspace = true, features = ["full"] }
 sos-audit.workspace = true
 sos-preferences.workspace = true
 sos-system-messages.workspace = true
+sos-server-storage = { workspace = true, features = ["full"] }
+sos-sync.workspace = true
 sos-vault.workspace = true
 sos-vfs.workspace = true
 

crates/database_upgrader/Cargo.toml

@@ -1,4 +1,3 @@
-use sos_core::{commit::CommitHash, VaultId};
 use std::path::PathBuf;
 use thiserror::Error;
 
@@ -9,25 +8,6 @@ pub enum Error {
     #[error("database '{0}' already exists")]
     DatabaseExists(PathBuf),
 
-    /// Error generated when a folder could not be found in the database.
-    #[error("folder '{0}' not found in the database")]
-    DatabaseFolderNotFound(VaultId),
-
-    /// Error generated when a target commit hash could not be found.
-    #[error("commit '{0}' could not be found")]
-    CommitNotFound(CommitHash),
-
-    /// Error generated when replacing events in an event log
-    /// does not compute the same root hash as the expected
-    /// checkpoint.
-    #[error("checkpoint verification failed, expected root hash '{checkpoint}' but computed '{computed}')")]
-    CheckpointVerification {
-        /// Checkpoint root hash.
-        checkpoint: CommitHash,
-        /// Computed root hash.
-        computed: CommitHash,
-    },
-
     /// Errors generated by the core library.
     #[error(transparent)]
     Core(#[from] sos_core::Error),
@@ -44,21 +24,17 @@ pub enum Error {
     #[error(transparent)]
     FileSystem(#[from] sos_filesystem::Error),
 
-    /// Errors generated by the filesystem archive module.
-    #[error(transparent)]
-    BackupArchive(#[from] sos_filesystem::archive::Error),
-
-    /// Error generated converting to fixed length slice.
+    /// Errors generated by the client storage library.
     #[error(transparent)]
-    TryFromSlice(#[from] std::array::TryFromSliceError),
+    ClientStorage(#[from] sos_client_storage::Error),
 
-    /// Error generated converting integers.
+    /// Errors generated by the server storage library.
     #[error(transparent)]
-    TryFromInt(#[from] std::num::TryFromIntError),
+    ServerStorage(#[from] sos_server_storage::Error),
 
-    /// Errors generated by the UUID library.
+    /// Errors generated by the filesystem archive module.
     #[error(transparent)]
-    Uuid(#[from] uuid::Error),
+    BackupArchive(#[from] sos_filesystem::archive::Error),
 
     /// Errors generated by the IO module.
     #[error(transparent)]
@@ -68,25 +44,11 @@ pub enum Error {
     #[error(transparent)]
     Json(#[from] serde_json::Error),
 
-    /// Errors generated by the URL library.
-    #[error(transparent)]
-    UrlParse(#[from] url::ParseError),
-
     /// Errors generated by the async sqlite library.
     #[error(transparent)]
     AsyncSqlite(#[from] sos_database::async_sqlite::Error),
 
     /// Errors generated by the rusqlite library.
     #[error(transparent)]
     Rusqlite(#[from] sos_database::async_sqlite::rusqlite::Error),
-    /*
-    /// Errors generated by refinery migration library.
-    #[error(transparent)]
-    Refinery(#[from] refinery::Error),
-
-
-    /// Errors generated by the URN library.
-    #[error(transparent)]
-    Urn(#[from] urn::Error),
-    */
 }

crates/database_upgrader/src/error.rs

@@ -1,10 +1,16 @@
 //! Import filesystem backed accounts into a database.
 use crate::{Error, Result};
 use serde::{Deserialize, Serialize};
+use sos_backend::BackendTarget;
+use sos_client_storage::ClientStorage;
 use sos_core::{Paths, PublicIdentity};
-use sos_database::{migrations::migrate_client, open_file, open_memory};
+use sos_database::{
+    async_sqlite::Client, migrations::migrate_client, open_file, open_memory,
+};
 use sos_external_files::list_external_files;
 use sos_filesystem::archive::AccountBackup;
+use sos_server_storage::ServerStorage;
+use sos_sync::{StorageEventLogs, SyncStatus, SyncStorage};
 use sos_vault::list_accounts;
 use sos_vfs::{self as vfs, File};
 use std::path::{Path, PathBuf};
@@ -82,7 +88,7 @@ impl UpgradeResult {
 /// Import all accounts found on disc.
 async fn import_accounts(
     options: &UpgradeOptions,
-) -> Result<Vec<PublicIdentity>> {
+) -> Result<(Vec<PublicIdentity>, Vec<(SyncStatus, SyncStatus)>)> {
     let mut client = if !options.dry_run {
         let db_file = options
             .db_file
@@ -98,23 +104,78 @@ async fn import_accounts(
     db_import::import_globals(&mut client, &options.paths).await?;
 
     let accounts = list_accounts(Some(&options.paths)).await?;
+    let mut sync_status = Vec::new();
     for account in &accounts {
         let account_paths =
             options.paths.with_account_id(account.account_id());
-        let account_status =
-            compute_account_status(&account_paths, account).await?;
+        let fs_status =
+            compute_fs_account_status(&account_paths, account).await?;
+
         db_import::import_account(&mut client, &account_paths, &account)
             .await?;
+        let db_status =
+            compute_db_account_status(&account_paths, account, &client)
+                .await?;
+
+        sync_status.push((fs_status, db_status));
     }
-    Ok(accounts)
+    Ok((accounts, sync_status))
 }
 
 /// Compute the account status for the input file system account.
-async fn compute_account_status(
+async fn compute_fs_account_status(
     account_paths: &Paths,
     account: &PublicIdentity,
-) -> Result<()> {
-    Ok(())
+) -> Result<SyncStatus> {
+    let sync_status = if account_paths.is_server() {
+        let storage = ServerStorage::new(
+            account_paths.documents_dir(),
+            account.account_id(),
+            BackendTarget::FileSystem(account_paths.clone()),
+        )
+        .await?;
+        storage.load_all_event_commits().await?;
+        storage.sync_status().await?
+    } else {
+        let storage = ClientStorage::new_unauthenticated(
+            *account.account_id(),
+            BackendTarget::FileSystem(account_paths.clone()),
+        )
+        .await?;
+        storage.load_all_event_commits().await?;
+        storage.sync_status().await?
+    };
+    Ok(sync_status)
+}
+
+/// Compute the account status for the output database account.
+async fn compute_db_account_status(
+    account_paths: &Paths,
+    account: &PublicIdentity,
+    client: &Client,
+) -> Result<SyncStatus> {
+    let sync_status = if account_paths.is_server() {
+        let storage = ServerStorage::new(
+            account_paths.documents_dir(),
+            account.account_id(),
+            BackendTarget::Database(client.clone()),
+        )
+        .await?;
+        storage.sync_status().await?
+    } else {
+        todo!("compute for client accounts");
+        /*
+        let storage = ClientFileSystemStorage::new_unauthenticated(
+            *account.account_id(),
+            account_paths.clone(),
+        )
+        .await?;
+
+        storage.load_all_event_commits().await?;
+        storage.sync_status().await?
+        */
+    };
+    Ok(sync_status)
 }
 
 /// Upgrade all accounts found on disc.
@@ -146,7 +207,33 @@ pub async fn upgrade_accounts(
 
     tracing::debug!("upgrade_accounts::import_accounts");
 
-    let accounts = import_accounts(&options).await?;
+    let (accounts, sync_status) = import_accounts(&options).await?;
+
+    /*
+    for status in &sync_status {
+        println!("{} <-> {}", status.0.root, status.1.root);
+
+        if status.0.identity != status.1.identity {
+            println!("{:#?} != {:#?}", status.0.identity, status.1.identity);
+        }
+
+        if status.0.account != status.1.account {
+            println!("{:#?} != {:#?}", status.0.account, status.1.account);
+        }
+
+        if status.0.device != status.1.device {
+            println!("{:#?} != {:#?}", status.0.device, status.1.device);
+        }
+
+        if status.0.files != status.1.files {
+            println!("{:#?} != {:#?}", status.0.files, status.1.files);
+        }
+
+        if status.0.folders != status.1.folders {
+            println!("{:#?} != {:#?}", status.0.folders, status.1.folders);
+        }
+    }
+    */
 
     if options.copy_file_blobs {
         tracing::debug!("upgrade_accounts::copy_file_blobs");

crates/database_upgrader/src/upgrader/mod.rs

@@ -1,4 +1,5 @@
 use super::{Error, Result};
+use sos_backend::BackendTarget;
 use sos_core::{device::DevicePublicKey, AccountId, Paths};
 use sos_server_storage::{ServerAccountStorage, ServerStorage};
 use sos_signer::ed25519::{self, Verifier, VerifyingKey};
@@ -79,9 +80,12 @@ impl Backend {
                             "server_backend::read_dir",
                         );
 
-                        let account = ServerStorage::new_fs(
+                        let account = ServerStorage::new(
                             &self.directory,
                             &account_id,
+                            BackendTarget::FileSystem(
+                                Paths::new_global_server(&self.directory),
+                            ),
                         )
                         .await?;
 

crates/server/src/backend.rs

@@ -115,8 +115,9 @@ impl ServerDatabaseStorage {
         let (device_log, devices) =
             Self::initialize_device_log(&client, &account_id).await?;
 
-        let file_log =
+        let mut file_log =
             FileEventLog::new_db_file(client.clone(), account_id).await?;
+        file_log.load_tree().await?;
 
         Ok(Self {
             account_id,

crates/storage/server/src/database/mod.rs

@@ -61,8 +61,8 @@ impl ServerFileStorage {
     /// Create folder storage for server-side access.
     pub async fn new(
         account_id: AccountId,
-        data_dir: Option<PathBuf>,
         identity_log: Arc<RwLock<FolderEventLog>>,
+        data_dir: Option<PathBuf>,
     ) -> Result<Self> {
         let data_dir = if let Some(data_dir) = data_dir {
             data_dir

crates/storage/server/src/filesystem/mod.rs

@@ -5,7 +5,8 @@ use crate::{
 use async_trait::async_trait;
 use indexmap::IndexSet;
 use sos_backend::{
-    AccountEventLog, DeviceEventLog, FileEventLog, FolderEventLog,
+    AccountEventLog, BackendTarget, DeviceEventLog, FileEventLog,
+    FolderEventLog,
 };
 use sos_core::{
     commit::{CommitState, Comparison},
@@ -39,8 +40,24 @@ pub enum ServerStorage {
 }
 
 impl ServerStorage {
+    /// Create new server storage.
+    pub async fn new(
+        directory: impl AsRef<Path>,
+        account_id: &AccountId,
+        target: BackendTarget,
+    ) -> Result<Self> {
+        match target {
+            BackendTarget::FileSystem(_) => {
+                Self::new_fs(directory, account_id).await
+            }
+            BackendTarget::Database(client) => {
+                Self::new_db(directory, account_id, client).await
+            }
+        }
+    }
+
     /// Create new file system storage.
-    pub async fn new_fs(
+    async fn new_fs(
         directory: impl AsRef<Path>,
         account_id: &AccountId,
     ) -> Result<Self> {
@@ -56,8 +73,8 @@ impl ServerStorage {
         Ok(Self::FileSystem(
             ServerFileStorage::new(
                 *account_id,
-                Some(directory.as_ref().to_owned()),
                 identity_log,
+                Some(directory.as_ref().to_owned()),
             )
             .await?,
         ))
@@ -81,8 +98,8 @@ impl ServerStorage {
 
         let mut storage = ServerFileStorage::new(
             *account_id,
-            Some(directory.as_ref().to_owned()),
             Arc::new(RwLock::new(identity_log)),
+            Some(directory.as_ref().to_owned()),
         )
         .await?;
         storage.import_account(&account_data).await?;
@@ -91,10 +108,10 @@ impl ServerStorage {
     }
 
     /// Create new database storage.
-    pub async fn new_db(
-        client: Client,
-        account_id: &AccountId,
+    async fn new_db(
         directory: impl AsRef<Path>,
+        account_id: &AccountId,
+        client: Client,
     ) -> Result<Self> {
         let folder_account_id = *account_id;