Refactor external file manager.

Author: tmpfs

crates/account/src/account.rs

@@ -904,12 +904,12 @@ impl LocalAccount {
     /// compatible sign_in() and also the newer sign_in_with_options().
     async fn login(&mut self, key: &AccessKey) -> Result<Vec<Summary>> {
         let account_id = &self.account_id;
-        let data_dir = self.paths().documents_dir().clone();
+        // let data_dir = self.paths().documents_dir().clone();
 
         tracing::debug!(account_id = %account_id, "sign_in");
 
         // Ensure all paths before sign_in
-        let paths = Paths::new(&data_dir, account_id.to_string());
+        let paths = self.paths().with_account_id(account_id);
         paths.ensure().await?;
 
         tracing::debug!(data_dir = ?paths.documents_dir(), "sign_in");
@@ -925,7 +925,7 @@ impl LocalAccount {
         #[allow(unused_mut)]
         let mut storage = ClientStorage::new_authenticated(
             *account_id,
-            Some(data_dir),
+            BackendTarget::FileSystem(paths),
             identity_log,
             user.identity()?.devices()?.current_device(None),
         )
@@ -941,7 +941,7 @@ impl LocalAccount {
 
         Self::initialize_account_log(
             &self.paths,
-            Arc::clone(&storage.account_log),
+            storage.account_log().await?,
         )
         .await?;
 
@@ -1268,6 +1268,7 @@ impl LocalAccount {
         {
             let mut writer = self.storage.write().await;
             let mut move_file_events = writer
+                .external_file_manager_mut()
                 .move_files(
                     &move_secret_data,
                     from.id(),
@@ -1279,6 +1280,7 @@ impl LocalAccount {
                 )
                 .await?;
             writer
+                .external_file_manager_mut()
                 .append_file_mutation_events(&move_file_events)
                 .await?;
             file_events.append(&mut move_file_events);
@@ -1460,7 +1462,7 @@ impl LocalAccount {
         let storage = Arc::new(RwLock::new(
             ClientStorage::new_unauthenticated(
                 account_id,
-                Arc::new(paths.clone()),
+                BackendTarget::FileSystem(paths.clone()),
             )
             .await?,
         ));
@@ -1516,10 +1518,12 @@ impl LocalAccount {
         let account_builder = builder(AccountBuilder::new(
             account_name,
             passphrase.clone(),
-            BackendTarget::FileSystem(paths),
+            BackendTarget::FileSystem(paths.clone()),
         ));
         let new_account = account_builder.finish().await?;
 
+        let paths = paths.with_account_id(&new_account.account_id);
+
         tracing::debug!(
           account_id = %new_account.account_id,
           "new_account::created",
@@ -1530,7 +1534,7 @@ impl LocalAccount {
 
         let mut storage = ClientStorage::new_authenticated(
             account_id,
-            data_dir,
+            BackendTarget::FileSystem(paths),
             identity_log,
             new_account.user.identity()?.devices()?.current_device(None),
         )
@@ -1584,12 +1588,15 @@ impl Account for LocalAccount {
         let account_id = *self.account_id();
         let paths = self.paths();
 
-        let mut storage =
-            ClientStorage::new_unauthenticated(account_id, paths.clone())
-                .await?;
+        let mut storage = ClientStorage::new_unauthenticated(
+            account_id,
+            BackendTarget::FileSystem((&*paths).clone()),
+        )
+        .await?;
 
         {
-            let mut identity_log = storage.identity_log.write().await;
+            let identity_log = storage.identity_log().await?;
+            let mut identity_log = identity_log.write().await;
             let records: Vec<EventRecord> = events.identity.into();
             identity_log.apply_records(records).await?;
             let vault = FolderReducer::new()
@@ -1607,7 +1614,8 @@ impl Account for LocalAccount {
         }
 
         {
-            let mut account_log = storage.account_log.write().await;
+            let account_log = storage.account_log().await?;
+            let mut account_log = account_log.write().await;
             let records: Vec<EventRecord> = events.account.into();
             account_log.apply_records(records).await?;
 
@@ -1617,7 +1625,8 @@ impl Account for LocalAccount {
         }
 
         {
-            let mut device_log = storage.device_log.write().await;
+            let device_log = storage.device_log().await?;
+            let mut device_log = device_log.write().await;
             let records: Vec<EventRecord> = events.device.into();
             device_log.apply_records(records).await?;
             tracing::info!(
@@ -1630,7 +1639,8 @@ impl Account for LocalAccount {
         #[cfg(feature = "files")]
         {
             tracing::info!("import_account_events::files");
-            let mut file_log = storage.file_log.write().await;
+            let file_log = storage.file_log().await?;
+            let mut file_log = file_log.write().await;
             let records: Vec<EventRecord> = events.files.into();
             file_log.apply_records(records).await?;
             tracing::info!(
@@ -2266,8 +2276,10 @@ impl Account for LocalAccount {
         file_name: &str,
     ) -> Result<Vec<u8>> {
         let reader = self.storage.read().await;
-        let buffer =
-            reader.download_file(vault_id, secret_id, file_name).await?;
+        let buffer = reader
+            .external_file_manager()
+            .download_file(vault_id, secret_id, file_name)
+            .await?;
 
         #[cfg(feature = "audit")]
         {

crates/account/src/sync.rs

@@ -8,7 +8,9 @@ use indexmap::IndexSet;
 use sos_backend::{
     AccountEventLog, DeviceEventLog, FolderEventLog, StorageError,
 };
-use sos_client_storage::{ClientAccountStorage, ClientFolderStorage};
+use sos_client_storage::{
+    ClientAccountStorage, ClientDeviceStorage, ClientFolderStorage,
+};
 use sos_core::{decode, events::EventLog};
 use sos_core::{
     events::{
@@ -271,7 +273,8 @@ impl Merge for LocalAccount {
 
         let checked_patch = {
             let storage = self.storage.read().await;
-            let mut event_log = storage.device_log.write().await;
+            let device_log = storage.device_log().await?;
+            let mut event_log = device_log.write().await;
             event_log
                 .patch_checked(&diff.checkpoint, &diff.patch)
                 .await?
@@ -280,13 +283,14 @@ impl Merge for LocalAccount {
         if let CheckedPatch::Success(_) = &checked_patch {
             let devices = {
                 let storage = self.storage.read().await;
-                let event_log = storage.device_log.read().await;
+                let device_log = storage.device_log().await?;
+                let event_log = device_log.read().await;
                 let reducer = DeviceReducer::new(&*event_log);
                 reducer.reduce().await?
             };
 
             let mut storage = self.storage.write().await;
-            storage.devices = devices;
+            storage.set_devices(devices);
 
             outcome.changes += diff.patch.len() as u64;
             outcome.tracked.device =
@@ -310,7 +314,8 @@ impl Merge for LocalAccount {
         );
 
         let storage = self.storage.read().await;
-        let mut event_log = storage.file_log.write().await;
+        let file_log = storage.file_log().await?;
+        let mut event_log = file_log.write().await;
 
         // File events may not have a root commit
         let is_init_diff = diff.last_commit.is_none();
@@ -356,10 +361,7 @@ impl Merge for LocalAccount {
 
             #[cfg(feature = "search")]
             let search = {
-                let index = storage
-                    .index
-                    .as_ref()
-                    .ok_or(sos_client_storage::Error::NoSearchIndex)?;
+                let index = storage.index()?;
                 index.search()
             };
 

crates/ipc/src/web_service/web_accounts.rs

@@ -10,7 +10,7 @@ use sos_core::{
     events::{AccountEvent, EventLog, WriteEvent},
     AccountId, ErrorExt, Paths, VaultId,
 };
-use sos_sync::SyncStorage;
+use sos_sync::{StorageEventLogs, SyncStorage};
 use sos_vault::SecretAccess;
 use std::{collections::HashMap, sync::Arc};
 use tokio::sync::{broadcast, RwLock};
@@ -574,7 +574,8 @@ where
     let storage = account.storage().await;
     let storage = storage.read().await;
 
-    let mut event_log = storage.account_log.write().await;
+    let account_log = storage.account_log().await?;
+    let mut event_log = account_log.write().await;
     let commit = event_log.tree().last_commit();
 
     let patch = event_log.diff_events(commit.as_ref()).await?;

crates/storage/client/src/files/file_manager.rs

@@ -2,12 +2,13 @@
 //! as secrets are created, updated and moved.
 
 use crate::{
-    files::FileStorage, ClientAccountStorage, ClientSecretStorage,
-    ClientStorage, Error, Result,
+    files::FileStorage, filesystem::ClientFileSystemStorage,
+    ClientAccountStorage, ClientSecretStorage, Error, Result,
 };
 use hex;
+use sos_backend::FileEventLog;
 use sos_core::events::FileEvent;
-use sos_core::{basename, SecretId, SecretPath, VaultId};
+use sos_core::{basename, Paths, SecretId, SecretPath, VaultId};
 use sos_external_files::{
     list_folder_files, EncryptedFile, FileMutationEvent, FileProgress,
     FileSource, FileStorageDiff, FileStorageResult,
@@ -20,10 +21,29 @@ use sos_sdk::{
     },
     vfs,
 };
-use std::{collections::HashMap, path::Path};
-use tokio::sync::mpsc;
+use std::{collections::HashMap, path::Path, sync::Arc};
+use tokio::sync::{mpsc, RwLock};
+
+/// Manages external files.
+pub struct ExternalFileManager {
+    paths: Arc<Paths>,
+    file_log: Arc<RwLock<FileEventLog>>,
+    pub(crate) file_password: Option<secrecy::SecretString>,
+}
+
+impl ExternalFileManager {
+    /// Create new external file manager.
+    pub fn new(
+        paths: Arc<Paths>,
+        file_log: Arc<RwLock<FileEventLog>>,
+    ) -> Self {
+        Self {
+            paths,
+            file_log,
+            file_password: None,
+        }
+    }
 
-impl ClientStorage {
     /// Append file mutation events to the file event log.
     pub async fn append_file_mutation_events(
         &mut self,
@@ -127,7 +147,7 @@ impl ClientStorage {
         summary: &Summary,
         secret_data: SecretRow,
         file_progress: &mut Option<mpsc::Sender<FileProgress>>,
-    ) -> Result<Vec<FileMutationEvent>> {
+    ) -> Result<(Vec<FileMutationEvent>, Option<(SecretId, SecretRow)>)> {
         self.write_update_checksum(summary, secret_data, None, file_progress)
             .await
     }
@@ -140,7 +160,7 @@ impl ClientStorage {
         old_secret: &SecretRow,
         new_secret: SecretRow,
         file_progress: &mut Option<mpsc::Sender<FileProgress>>,
-    ) -> Result<Vec<FileMutationEvent>> {
+    ) -> Result<(Vec<FileMutationEvent>, Option<(SecretId, SecretRow)>)> {
         let mut results = Vec::new();
 
         let old_secret_id = old_secret.id();
@@ -186,8 +206,8 @@ impl ClientStorage {
         }
 
         // Write changed files to the new location
-        if !changed_files.is_empty() {
-            let written = self
+        let write_update = if !changed_files.is_empty() {
+            let (written, write_update) = self
                 .write_update_checksum(
                     new_summary,
                     new_secret,
@@ -196,9 +216,12 @@ impl ClientStorage {
                 )
                 .await?;
             results.extend_from_slice(&written);
-        }
+            write_update
+        } else {
+            None
+        };
 
-        Ok(results)
+        Ok((results, write_update))
     }
 
     /// Delete a collection of files from the external storage.
@@ -255,7 +278,7 @@ impl ClientStorage {
         secret_id: &SecretId,
         file_name: &str,
     ) -> Result<FileEvent> {
-        let vault_path = self.paths().files_dir().join(vault_id.to_string());
+        let vault_path = self.paths.files_dir().join(vault_id.to_string());
         let secret_path = vault_path.join(secret_id.to_string());
         let path = secret_path.join(file_name);
 
@@ -335,12 +358,12 @@ impl ClientStorage {
         file_name: &str,
     ) -> Result<FileMutationEvent> {
         let old_vault_path =
-            self.paths().files_dir().join(old_vault_id.to_string());
+            self.paths.files_dir().join(old_vault_id.to_string());
         let old_secret_path = old_vault_path.join(old_secret_id.to_string());
         let old_path = old_secret_path.join(file_name);
 
         let new_path = self
-            .paths()
+            .paths
             .files_dir()
             .join(new_vault_id.to_string())
             .join(new_secret_id.to_string())
@@ -382,7 +405,7 @@ impl ClientStorage {
         mut secret_data: SecretRow,
         sources: Option<Vec<FileSource>>,
         file_progress: &mut Option<mpsc::Sender<FileProgress>>,
-    ) -> Result<Vec<FileMutationEvent>> {
+    ) -> Result<(Vec<FileMutationEvent>, Option<(SecretId, SecretRow)>)> {
         tracing::debug!(folder = ?summary.id(), "write_update_checksum");
 
         let mut results = Vec::new();
@@ -507,12 +530,16 @@ impl ClientStorage {
             (secret, false)
         };
 
-        if changed {
+        let write_update = if changed {
             let secret_data = SecretRow::new(id, new_meta, new_secret);
 
             // Update with new checksum(s)
-            self.write_secret(&id, secret_data, false).await?;
-        }
+            // self.write_secret(&id, secret_data, false).await?;
+
+            Some((id, secret_data))
+        } else {
+            None
+        };
 
         let events = results
             .into_iter()
@@ -521,7 +548,7 @@ impl ClientStorage {
                 event: data.1,
             })
             .collect::<Vec<_>>();
-        Ok(events)
+        Ok((events, write_update))
     }
 }
 

crates/storage/client/src/files/mod.rs

@@ -2,3 +2,4 @@
 mod external_files;
 mod file_manager;
 pub use external_files::FileStorage;
+pub use file_manager::ExternalFileManager;