tests: add write cache test (#948)

Author: roman-khimov

neofs-testlib/neofs_testlib/cli/neofs_cli/shards.py

@@ -1,4 +1,4 @@
-from typing import Optional, List
+from typing import List, Optional
 
 from neofs_testlib.cli.cli_command import CliCommand
 from neofs_testlib.shell import CommandResult
@@ -8,18 +8,18 @@ class NeofsCliShards(CliCommand):
     def flush_cache(
         self,
         endpoint: str,
-        wallet: str,
-        shards_id: Optional[list[str]],
+        wallet: Optional[str] = None,
+        shards_id: Optional[list[str]] = None,
         address: Optional[str] = None,
-        all_shards: bool = False,
+        all: bool = False,
     ) -> CommandResult:
         """
         Flush objects from the write-cache to the main storage.
 
         Args:
             address: Address of wallet account.
             shards_id: List of shard IDs in base58 encoding.
-            all_shards: Process all shards.
+            all: Process all shards.
             endpoint: Remote node address (as 'multiaddr' or '<host>:<port>').
             wallet: WIF (NEP-2) string or path to the wallet or binary key.
 

neofs-testlib/neofs_testlib/cli/neofs_lens/cli.py

@@ -1,11 +1,17 @@
 from typing import Optional
 
 from neofs_testlib.cli.neofs_lens.objects import NeofsLensObject
+from neofs_testlib.cli.neofs_lens.storage import NeofsLensStorage
+from neofs_testlib.cli.neofs_lens.write_cache import NeofsLensWriteCache
 from neofs_testlib.shell import Shell
 
 
 class NeofsLens:
     object: Optional[NeofsLensObject] = None
+    write_cache: Optional[NeofsLensWriteCache] = None
+    storage: Optional[NeofsLensStorage] = None
 
     def __init__(self, shell: Shell, neofs_lens_exec_path: str):
         self.object = NeofsLensObject(shell, neofs_lens_exec_path)
+        self.write_cache = NeofsLensWriteCache(shell, neofs_lens_exec_path)
+        self.storage = NeofsLensStorage(shell, neofs_lens_exec_path)

neofs-testlib/neofs_testlib/cli/neofs_lens/storage.py

@@ -0,0 +1,40 @@
+from neofs_testlib.cli.cli_command import CliCommand
+from neofs_testlib.shell import CommandResult
+
+
+class NeofsLensStorage(CliCommand):
+    def list(
+        self,
+        config: str,
+    ) -> CommandResult:
+        """
+        List all objects stored in a blobstor (as registered in metabase).
+
+        Args:
+                config: Path to file with storage node config
+        Returns:
+                Command's result.
+        """
+        return self._execute(
+            "storage list",
+            **{param: value for param, value in locals().items() if param not in ["self"]},
+        )
+
+    def get(
+        self,
+        address: str,
+        config: str,
+    ) -> CommandResult:
+        """
+        Get object from the NeoFS node's storage snapshot
+
+        Args:
+                address: Object address
+                config: Path to file with storage node config
+        Returns:
+                Command's result.
+        """
+        return self._execute(
+            "storage get",
+            **{param: value for param, value in locals().items() if param not in ["self"]},
+        )

neofs-testlib/neofs_testlib/cli/neofs_lens/write_cache.py

@@ -0,0 +1,42 @@
+from typing import Optional
+
+from neofs_testlib.cli.cli_command import CliCommand
+from neofs_testlib.shell import CommandResult
+
+
+class NeofsLensWriteCache(CliCommand):
+    def list(
+        self,
+        path: str,
+    ) -> CommandResult:
+        """
+        List all objects stored in a write-cache.
+
+        Args:
+                path: Path to storage engine component
+        Returns:
+                Command's result.
+        """
+        return self._execute(
+            "write-cache list",
+            **{param: value for param, value in locals().items() if param not in ["self"]},
+        )
+
+    def get(
+        self,
+        address: str,
+        path: str,
+    ) -> CommandResult:
+        """
+        Get specific object from a write-cache.
+
+        Args:
+                address: Object address
+                path: Path to storage engine component
+        Returns:
+                Command's result.
+        """
+        return self._execute(
+            "write-cache get",
+            **{param: value for param, value in locals().items() if param not in ["self"]},
+        )

neofs-testlib/neofs_testlib/env/env.py

@@ -182,14 +182,16 @@ def deploy_inner_ring_nodes(self, count=1, with_main_chain=False):
                     raise e
 
     @allure.step("Deploy storage node")
-    def deploy_storage_nodes(self, count=1, node_attrs: Optional[dict] = None):
+    def deploy_storage_nodes(self, count=1, node_attrs: Optional[dict] = None, writecache=False):
         logger.info(f"Going to deploy {count} storage nodes")
         deploy_threads = []
         for idx in range(count):
             node_attrs_list = None
             if node_attrs:
                 node_attrs_list = node_attrs.get(idx, None)
-            new_storage_node = StorageNode(self, len(self.storage_nodes) + 1, node_attrs=node_attrs_list)
+            new_storage_node = StorageNode(
+                self, len(self.storage_nodes) + 1, writecache=writecache, node_attrs=node_attrs_list
+            )
             self.storage_nodes.append(new_storage_node)
             deploy_threads.append(threading.Thread(target=new_storage_node.start))
         for t in deploy_threads:
@@ -642,6 +644,13 @@ def download_binary(repo: str, version: str, file: str, target: str):
             current_perm = os.stat(target)
             os.chmod(target, current_perm.st_mode | stat.S_IEXEC)
 
+    def get_binary_version(self, binary_path: str) -> str:
+        raw_version_output = self._run_single_command(binary_path, "--version")
+        for line in raw_version_output.splitlines():
+            if "Version:" in line:
+                return line.split("Version:")[1].strip()
+        return ""
+
     def _generate_temp_file(self, base_dir: str, extension: str = "", prefix: str = "tmp_file") -> str:
         file_path = f"{base_dir}/{prefix}_{''.join(random.choices(string.ascii_lowercase, k=10))}"
         if extension:
@@ -899,12 +908,17 @@ def __init__(self, neofs_env: NeoFSEnv, sn_dir: str):
         self.blobovnicza_path = neofs_env._generate_temp_file(sn_dir, prefix="shard_blobovnicza")
         self.fstree_path = neofs_env._generate_temp_dir(prefix="shards/shard_fstree")
         self.pilorama_path = neofs_env._generate_temp_file(sn_dir, prefix="shard_pilorama")
-        self.wc_path = neofs_env._generate_temp_file(sn_dir, prefix="shard_wc")
+        self.wc_path = neofs_env._generate_temp_dir(prefix="shards/shard_wc")
 
 
 class StorageNode:
     def __init__(
-        self, neofs_env: NeoFSEnv, sn_number: int, node_attrs: Optional[list] = None, attrs: Optional[dict] = None
+        self,
+        neofs_env: NeoFSEnv,
+        sn_number: int,
+        writecache=False,
+        node_attrs: Optional[list] = None,
+        attrs: Optional[dict] = None,
     ):
         self.neofs_env = neofs_env
         self.sn_dir = self.neofs_env._generate_temp_dir(prefix=f"sn_{sn_number}")
@@ -931,6 +945,7 @@ def __init__(
         self.process = None
         self.attrs = {}
         self.node_attrs = node_attrs
+        self.writecache = writecache
         if attrs:
             self.attrs.update(attrs)
 
@@ -966,6 +981,7 @@ def start(self, fresh=True):
                 custom=Path(sn_config_template).is_file(),
                 fschain_endpoint=self.neofs_env.fschain_rpc,
                 shards=self.shards,
+                writecache=self.writecache,
                 wallet=self.wallet,
                 state_file=self.state_file,
                 pprof_address=self.pprof_address,
@@ -1010,7 +1026,7 @@ def delete_data(self):
             os.remove(shard.blobovnicza_path)
             os.rmdir(shard.fstree_path)
             os.remove(shard.pilorama_path)
-            os.remove(shard.wc_path)
+            shutil.rmtree(shard.wc_path, ignore_errors=True)
         os.remove(self.state_file)
         self.shards = [Shard(), Shard()]
 
@@ -1022,6 +1038,7 @@ def delete_data(self):
             custom=Path(sn_config_template).is_file(),
             fschain_endpoint=self.neofs_env.fschain_rpc,
             shards=self.shards,
+            writecache=self.writecache,
             wallet=self.wallet,
             state_file=self.state_file,
             pprof_address=self.pprof_address,
@@ -1045,6 +1062,7 @@ def delete_metadata(self):
             custom=Path(sn_config_template).is_file(),
             fschain_endpoint=self.neofs_env.fschain_rpc,
             shards=self.shards,
+            writecache=self.writecache,
             wallet=self.wallet,
             state_file=self.state_file,
             pprof_address=self.pprof_address,
@@ -1097,13 +1115,6 @@ def _wait_until_not_ready(self):
         assert "Health status: READY" not in result.stdout, "Health is ready"
         assert "Network status: ONLINE" not in result.stdout, "Network is online"
 
-    def _get_version(self) -> str:
-        raw_version_output = self.neofs_env._run_single_command(self.neofs_env.neofs_node_path, "--version")
-        for line in raw_version_output.splitlines():
-            if "Version:" in line:
-                return line.split("Version:")[1].strip()
-        return ""
-
 
 class S3_GW:
     def __init__(self, neofs_env: NeoFSEnv):
@@ -1200,18 +1211,11 @@ def _generate_config(self):
             peers=peers,
             tree_service_endpoint=self.neofs_env.storage_nodes[0].endpoint,
             listen_domain=self.neofs_env.domain,
-            s3_gw_version=self._get_version(),
+            s3_gw_version=self.neofs_env.get_binary_version(self.neofs_env.neofs_s3_gw_path),
             pprof_address=self.pprof_address,
             prometheus_address=self.prometheus_address,
         )
 
-    def _get_version(self) -> str:
-        raw_version_output = self.neofs_env._run_single_command(self.neofs_env.neofs_s3_gw_path, "--version")
-        for line in raw_version_output.splitlines():
-            if "Version:" in line:
-                return line.split("Version:")[1].strip()
-        return ""
-
     def _launch_process(self):
         self.stdout = self.neofs_env._generate_temp_file(self.s3_gw_dir, prefix="s3gw_stdout")
         self.stderr = self.neofs_env._generate_temp_file(self.s3_gw_dir, prefix="s3gw_stderr")

neofs-testlib/neofs_testlib/env/templates/sn.yaml

@@ -44,7 +44,7 @@ storage:
   shard:
     0:
       writecache:
-        enabled: false
+        enabled: {{ writecache }}
         path: {{ shards[0].wc_path }}  # Write-cache root directory
 
       metabase:
@@ -62,7 +62,7 @@ storage:
 
     1:
       writecache:
-        enabled: false
+        enabled: {{ writecache }}
         path: {{ shards[1].wc_path }}  # Write-cache root directory
 
       metabase:

pytest_tests/tests/acl/test_eacl.py

@@ -344,7 +344,7 @@ def test_extended_acl_operations_enforcement(self, wallets, eacl_container_with_
     def test_extended_acl_deny_all_operations_exclude_extended_role(
         self, wallets, eacl_container_with_objects, role: EACLRoleExtendedType
     ):
-        if self.neofs_env.storage_nodes[0]._get_version() > "0.44.2":
+        if self.neofs_env.get_binary_version(self.neofs_env.neofs_node_path) > "0.44.2":
             pytest.skip("This test runs on 0.44.2 and below neofs-node versions")
         user_wallet = wallets.get_wallet()
         other_wallet, other_wallet_allow = wallets.get_wallets_list(EACLRole.OTHERS)[0:2]

pytest_tests/tests/conftest.py

@@ -60,7 +60,7 @@ def neofs_env(temp_directory, artifacts_directory, request):
                 os.remove(shard.blobovnicza_path)
                 shutil.rmtree(shard.fstree_path, ignore_errors=True)
                 os.remove(shard.pilorama_path)
-                os.remove(shard.wc_path)
+                shutil.rmtree(shard.wc_path, ignore_errors=True)
 
         shutil.make_archive(
             os.path.join(get_assets_dir_path(), f"neofs_env_{neofs_env._id}"), "zip", neofs_env._env_dir

pytest_tests/tests/container/test_container.py

@@ -213,7 +213,7 @@ def test_container_deletion_while_sn_down(
             object_should_be_gc_marked(self.neofs_env, node_to_stop, cid, oid)
 
     def test_container_global_name(self, default_wallet, simple_object_size):
-        if self.neofs_env.storage_nodes[0]._get_version() <= "0.43.0":
+        if self.neofs_env.get_binary_version(self.neofs_env.neofs_node_path) <= "0.43.0":
             pytest.skip("This test runs only on post 0.43.0 node version")
 
         with allure.step("Create container"):

pytest_tests/tests/metrics/test_sn_metrics.py

@@ -178,7 +178,7 @@ def test_sn_metrics(single_noded_env: NeoFSEnv, default_wallet: NodeWallet):
             f"invalid value for {metric}"
         )
 
-    node_version = single_noded_env.storage_nodes[0]._get_version()
+    node_version = single_noded_env.get_binary_version(single_noded_env.neofs_node_path)
     assert after_metrics["neofs_node_version"][0]["params"]["version"] == node_version, (
         "invalid value for neofs_node_version"
     )