diff --git a/dynamic_env_pytest_tests/lib/http_gw/http_utils.py b/dynamic_env_pytest_tests/lib/http_gw/http_utils.py new file mode 100644 index 000000000..afdd4a017 --- /dev/null +++ b/dynamic_env_pytest_tests/lib/http_gw/http_utils.py @@ -0,0 +1,78 @@ +import random + +import allure +import neofs_verbs +from grpc_responses import OBJECT_NOT_FOUND, error_matches_status +from neofs_testlib.env.env import StorageNode +from neofs_testlib.shell import Shell +from python_keywords.http_gate import assert_hashes_are_equal, get_via_http_gate +from python_keywords.neofs_verbs import get_object + + +def get_object_and_verify_hashes( + oid: str, + file_name: str, + wallet: str, + cid: str, + shell: Shell, + nodes: list[StorageNode], + endpoint: str, + object_getter=None, +) -> None: + nodes_list = get_nodes_without_object( + wallet=wallet, + cid=cid, + oid=oid, + shell=shell, + nodes=nodes, + ) + # for some reason we can face with case when nodes_list is empty due to object resides in all nodes + if nodes_list: + random_node = random.choice(nodes_list) + else: + random_node = random.choice(nodes) + + object_getter = object_getter or get_via_http_gate + + got_file_path = get_object( + wallet=wallet, + cid=cid, + oid=oid, + shell=shell, + endpoint=random_node.endpoint, + ) + got_file_path_http = object_getter(cid=cid, oid=oid, endpoint=endpoint) + + assert_hashes_are_equal(file_name, got_file_path, got_file_path_http) + + +@allure.step("Get Nodes Without Object") +def get_nodes_without_object( + wallet: str, cid: str, oid: str, shell: Shell, nodes: list[StorageNode] +) -> list[StorageNode]: + """ + The function returns list of nodes which do not store + the given object. + Args: + wallet (str): the path to the wallet on whose behalf + we request the nodes + cid (str): ID of the container which store the object + oid (str): object ID + shell: executor for cli command + Returns: + (list): nodes which do not store the object + """ + nodes_list = [] + for node in nodes: + try: + res = neofs_verbs.head_object( + wallet, cid, oid, shell=shell, endpoint=node.endpoint, is_direct=True + ) + if res is None: + nodes_list.append(node) + except Exception as err: + if error_matches_status(err, OBJECT_NOT_FOUND): + nodes_list.append(node) + else: + raise Exception(f"Got error {err} on head object command") from err + return nodes_list diff --git a/dynamic_env_pytest_tests/pytest.ini b/dynamic_env_pytest_tests/pytest.ini index b6276362a..b0bfed1b3 100644 --- a/dynamic_env_pytest_tests/pytest.ini +++ b/dynamic_env_pytest_tests/pytest.ini @@ -1,8 +1,8 @@ [pytest] log_cli = 1 log_cli_level = debug -log_cli_format = %(asctime)s [%(levelname)4s] %(message)s -log_format = %(asctime)s [%(levelname)4s] %(message)s +log_cli_format = [%(threadName)s] %(asctime)s [%(levelname)4s] %(message)s +log_format = [%(threadName)s] %(asctime)s [%(levelname)4s] %(message)s [%(threadName)s] log_cli_date_format = %Y-%m-%d %H:%M:%S log_date_format = %H:%M:%S markers = diff --git a/dynamic_env_pytest_tests/tests/services/http_gate/test_http_bearer.py b/dynamic_env_pytest_tests/tests/services/http_gate/test_http_bearer.py new file mode 100644 index 000000000..32a9ca5b8 --- /dev/null +++ b/dynamic_env_pytest_tests/tests/services/http_gate/test_http_bearer.py @@ -0,0 +1,140 @@ +import logging + +import allure +import pytest +from container import create_container +from file_helper import generate_file +from http_gate import upload_via_http_gate_curl +from http_gw.http_utils import get_object_and_verify_hashes +from neofs_env.neofs_env_test_base import NeofsEnvTestBase +from python_keywords.acl import ( + EACLAccess, + EACLOperation, + EACLRole, + EACLRule, + bearer_token_base64_from_file, + create_eacl, + form_bearertoken_file, + set_eacl, + sign_bearer, + wait_for_cache_expired, +) +from wellknown_acl import PUBLIC_ACL + +logger = logging.getLogger("NeoLogger") + + +@pytest.mark.sanity +@pytest.mark.http_gate +class Test_http_bearer(NeofsEnvTestBase): + PLACEMENT_RULE = "REP 2 IN X CBF 1 SELECT 2 FROM * AS X" + + @pytest.fixture(scope="class", autouse=True) + @allure.title("[Class/Autouse]: Prepare wallet and deposit") + def prepare_wallet(self, default_wallet): + Test_http_bearer.wallet = default_wallet + + @pytest.fixture(scope="class") + def user_container(self) -> str: + return create_container( + wallet=self.wallet.path, + shell=self.shell, + endpoint=self.neofs_env.sn_rpc, + rule=self.PLACEMENT_RULE, + basic_acl=PUBLIC_ACL, + ) + + @pytest.fixture(scope="class") + def eacl_deny_for_others(self, user_container: str) -> None: + with allure.step(f"Set deny all operations for {EACLRole.OTHERS} via eACL"): + eacl = EACLRule( + access=EACLAccess.DENY, role=EACLRole.OTHERS, operation=EACLOperation.PUT + ) + set_eacl( + self.wallet.path, + user_container, + create_eacl(user_container, eacl, shell=self.shell), + shell=self.shell, + endpoint=self.neofs_env.sn_rpc, + ) + wait_for_cache_expired() + + @pytest.fixture(scope="class") + def bearer_token_no_limit_for_others(self, user_container: str) -> str: + with allure.step(f"Create bearer token for {EACLRole.OTHERS} with all operations allowed"): + bearer = form_bearertoken_file( + self.wallet.path, + user_container, + [ + EACLRule(operation=op, access=EACLAccess.ALLOW, role=EACLRole.OTHERS) + for op in EACLOperation + ], + shell=self.shell, + endpoint=self.neofs_env.sn_rpc, + sign=False, + ) + bearer_signed = f"{bearer}_signed" + sign_bearer( + shell=self.shell, + wallet_path=self.wallet.path, + eacl_rules_file_from=bearer, + eacl_rules_file_to=bearer_signed, + json=False, + ) + return bearer_token_base64_from_file(bearer_signed) + + @allure.title(f"[negative] Put object without bearer token for {EACLRole.OTHERS}") + def test_unable_put_without_bearer_token( + self, simple_object_size: int, user_container: str, eacl_deny_for_others + ): + eacl_deny_for_others + upload_via_http_gate_curl( + cid=user_container, + filepath=generate_file(simple_object_size), + endpoint=f"http://{self.neofs_env.http_gw.address}", + error_pattern="access to object operation denied", + ) + + @pytest.mark.parametrize("bearer_type", ("header", "cookie")) + @pytest.mark.parametrize( + "object_size", + [pytest.lazy_fixture("simple_object_size"), pytest.lazy_fixture("complex_object_size")], + ids=["simple object", "complex object"], + ) + def test_put_with_bearer_when_eacl_restrict( + self, + object_size: int, + bearer_type: str, + user_container: str, + eacl_deny_for_others, + bearer_token_no_limit_for_others: str, + ): + eacl_deny_for_others + bearer = bearer_token_no_limit_for_others + file_path = generate_file(object_size) + with allure.step( + f"Put object with bearer token for {EACLRole.OTHERS}, then get and verify hashes" + ): + headers = None + cookies = None + if bearer_type == "header": + headers = [f" -H 'Authorization: Bearer {bearer}'"] + if bearer_type == "cookie": + cookies = {"Bearer": bearer} + + oid = upload_via_http_gate_curl( + cid=user_container, + filepath=file_path, + endpoint=f"http://{self.neofs_env.http_gw.address}", + headers=headers, + cookies=cookies, + ) + get_object_and_verify_hashes( + oid=oid, + file_name=file_path, + wallet=self.wallet.path, + cid=user_container, + shell=self.shell, + nodes=self.neofs_env.storage_nodes, + endpoint=f"http://{self.neofs_env.http_gw.address}", + ) diff --git a/dynamic_env_pytest_tests/tests/services/http_gate/test_http_gate.py b/dynamic_env_pytest_tests/tests/services/http_gate/test_http_gate.py new file mode 100644 index 000000000..1bb188b17 --- /dev/null +++ b/dynamic_env_pytest_tests/tests/services/http_gate/test_http_gate.py @@ -0,0 +1,553 @@ +import json +import logging +import os + +import allure +import neofs_env.neofs_epoch as neofs_epoch +import pytest +from file_helper import generate_file, generate_file_with_content, get_file_hash +from http_gw.http_utils import get_object_and_verify_hashes +from neofs_env.neofs_env_test_base import NeofsEnvTestBase +from python_keywords.container import create_container +from python_keywords.http_gate import ( + attr_into_header, + get_object_by_attr_and_verify_hashes, + get_via_http_curl, + get_via_http_gate, + get_via_zip_http_gate, + try_to_get_object_and_expect_error, + upload_via_http_gate, + upload_via_http_gate_curl, +) +from python_keywords.neofs_verbs import put_object_to_random_node +from utility import wait_for_gc_pass_on_storage_nodes +from wellknown_acl import PUBLIC_ACL + +logger = logging.getLogger("NeoLogger") +OBJECT_NOT_FOUND_ERROR = "not found" + + +@allure.link( + "https://github.com/nspcc-dev/neofs-http-gw#neofs-http-gateway", name="neofs-http-gateway" +) +@allure.link("https://github.com/nspcc-dev/neofs-http-gw#uploading", name="uploading") +@allure.link("https://github.com/nspcc-dev/neofs-http-gw#downloading", name="downloading") +@pytest.mark.sanity +@pytest.mark.http_gate +class TestHttpGate(NeofsEnvTestBase): + PLACEMENT_RULE_1 = "REP 1 IN X CBF 1 SELECT 1 FROM * AS X" + PLACEMENT_RULE_2 = "REP 2 IN X CBF 2 SELECT 2 FROM * AS X" + + @pytest.fixture(scope="class", autouse=True) + @allure.title("[Class/Autouse]: Prepare wallet and deposit") + def prepare_wallet(self, default_wallet): + TestHttpGate.wallet = default_wallet + + @allure.title("Test Put over gRPC, Get over HTTP") + def test_put_grpc_get_http(self, complex_object_size, simple_object_size): + """ + Test that object can be put using gRPC interface and get using HTTP. + + Steps: + 1. Create simple and large objects. + 2. Put objects using gRPC (neofs-cli). + 3. Download objects using HTTP gate (https://github.com/nspcc-dev/neofs-http-gw#downloading). + 4. Get objects using gRPC (neofs-cli). + 5. Compare hashes for got objects. + 6. Compare hashes for got and original objects. + + Expected result: + Hashes must be the same. + """ + cid = create_container( + self.wallet.path, + shell=self.shell, + endpoint=self.neofs_env.sn_rpc, + rule=self.PLACEMENT_RULE_1, + basic_acl=PUBLIC_ACL, + ) + file_path_simple, file_path_large = generate_file(simple_object_size), generate_file( + complex_object_size + ) + + with allure.step("Put objects using gRPC"): + oid_simple = put_object_to_random_node( + wallet=self.wallet.path, + path=file_path_simple, + cid=cid, + shell=self.shell, + neofs_env=self.neofs_env, + ) + oid_large = put_object_to_random_node( + wallet=self.wallet.path, + path=file_path_large, + cid=cid, + shell=self.shell, + neofs_env=self.neofs_env, + ) + + for oid, file_path in ((oid_simple, file_path_simple), (oid_large, file_path_large)): + get_object_and_verify_hashes( + oid=oid, + file_name=file_path, + wallet=self.wallet.path, + cid=cid, + shell=self.shell, + nodes=self.neofs_env.storage_nodes, + endpoint=f"http://{self.neofs_env.http_gw.address}", + ) + + @allure.title("Verify Content-Disposition header") + def test_put_http_get_http_content_disposition(self, simple_object_size): + cid = create_container( + self.wallet.path, + shell=self.shell, + endpoint=self.neofs_env.sn_rpc, + rule=self.PLACEMENT_RULE_2, + basic_acl=PUBLIC_ACL, + ) + + with allure.step("Verify Content-Disposition"): + file_path = generate_file(simple_object_size) + + oid = upload_via_http_gate( + cid=cid, + path=file_path, + endpoint=f"http://{self.neofs_env.http_gw.address}", + ) + resp = get_via_http_gate( + cid=cid, + oid=oid, + endpoint=f"http://{self.neofs_env.http_gw.address}", + return_response=True, + ) + content_disposition_type, filename = resp.headers["Content-Disposition"].split(";") + assert content_disposition_type.strip() == "inline" + assert filename.strip().split("=")[1] == file_path.split("/")[-1] + + with allure.step("Verify Content-Disposition with download=true"): + file_path = generate_file(simple_object_size) + + oid = upload_via_http_gate( + cid=cid, + path=file_path, + endpoint=f"http://{self.neofs_env.http_gw.address}", + ) + resp = get_via_http_gate( + cid=cid, + oid=oid, + endpoint=f"http://{self.neofs_env.http_gw.address}", + return_response=True, + download=True, + ) + content_disposition_type, filename = resp.headers["Content-Disposition"].split(";") + assert content_disposition_type.strip() == "attachment" + assert filename.strip().split("=")[1] == file_path.split("/")[-1] + + @allure.title("Verify Content-Type if uploaded without any Content-Type specified") + def test_put_http_get_http_without_content_type(self, simple_object_size): + cid = create_container( + self.wallet.path, + shell=self.shell, + endpoint=self.neofs_env.sn_rpc, + rule=self.PLACEMENT_RULE_2, + basic_acl=PUBLIC_ACL, + ) + + with allure.step("Upload binary object"): + file_path = generate_file(simple_object_size) + + oid = upload_via_http_gate( + cid=cid, + path=file_path, + endpoint=f"http://{self.neofs_env.http_gw.address}", + ) + + resp = get_via_http_gate( + cid=cid, + oid=oid, + endpoint=f"http://{self.neofs_env.http_gw.address}", + return_response=True, + ) + assert resp.headers["Content-Type"] == "application/octet-stream" + + with allure.step("Upload text object"): + file_path = generate_file_with_content(simple_object_size, content="123") + + oid = upload_via_http_gate( + cid=cid, + path=file_path, + endpoint=f"http://{self.neofs_env.http_gw.address}", + ) + + resp = get_via_http_gate( + cid=cid, + oid=oid, + endpoint=f"http://{self.neofs_env.http_gw.address}", + return_response=True, + ) + assert resp.headers["Content-Type"] == "text/plain; charset=utf-8" + + @allure.title("Verify Content-Type if uploaded with X-Attribute-Content-Type") + def test_put_http_get_http_with_x_atribute_content_type(self, simple_object_size): + cid = create_container( + self.wallet.path, + shell=self.shell, + endpoint=self.neofs_env.sn_rpc, + rule=self.PLACEMENT_RULE_2, + basic_acl=PUBLIC_ACL, + ) + + with allure.step("Upload object with X-Attribute-Content-Type"): + file_path = generate_file(simple_object_size) + + headers = {"X-Attribute-Content-Type": "CoolContentType"} + oid = upload_via_http_gate( + cid=cid, + path=file_path, + headers=headers, + endpoint=f"http://{self.neofs_env.http_gw.address}", + ) + + resp = get_via_http_gate( + cid=cid, + oid=oid, + endpoint=f"http://{self.neofs_env.http_gw.address}", + return_response=True, + ) + assert resp.headers["Content-Type"] == "CoolContentType" + + @allure.title("Verify Content-Type if uploaded with multipart Content-Type") + def test_put_http_get_http_with_multipart_content_type(self): + cid = create_container( + self.wallet.path, + shell=self.shell, + endpoint=self.neofs_env.sn_rpc, + rule=self.PLACEMENT_RULE_2, + basic_acl=PUBLIC_ACL, + ) + + with allure.step("Upload object with multipart content type"): + file_path = generate_file_with_content(0, content="123") + + oid = upload_via_http_gate( + cid=cid, + path=file_path, + endpoint=f"http://{self.neofs_env.http_gw.address}", + file_content_type="application/json", + ) + + resp = get_via_http_gate( + cid=cid, + oid=oid, + endpoint=f"http://{self.neofs_env.http_gw.address}", + return_response=True, + ) + assert resp.headers["Content-Type"] == "application/json" + + @allure.title("Verify special HTTP headers") + def test_put_http_get_http_special_attributes(self, simple_object_size): + cid = create_container( + self.wallet.path, + shell=self.shell, + endpoint=self.neofs_env.sn_rpc, + rule=self.PLACEMENT_RULE_2, + basic_acl=PUBLIC_ACL, + ) + + file_path = generate_file(simple_object_size) + + oid = upload_via_http_gate( + cid=cid, + path=file_path, + endpoint=f"http://{self.neofs_env.http_gw.address}", + ) + resp = get_via_http_gate( + cid=cid, + oid=oid, + endpoint=f"http://{self.neofs_env.http_gw.address}", + return_response=True, + ) + with open(self.neofs_env.http_gw.wallet.path) as wallet_file: + wallet_json = json.load(wallet_file) + + assert resp.headers["X-Owner-Id"] == wallet_json["accounts"][0]["address"] + assert resp.headers["X-Object-Id"] == oid + assert resp.headers["X-Container-Id"] == cid + + @allure.link("https://github.com/nspcc-dev/neofs-http-gw#uploading", name="uploading") + @allure.link("https://github.com/nspcc-dev/neofs-http-gw#downloading", name="downloading") + @allure.title("Test Put over HTTP, Get over HTTP") + @pytest.mark.smoke + def test_put_http_get_http(self, complex_object_size, simple_object_size): + """ + Test that object can be put and get using HTTP interface. + + Steps: + 1. Create simple and large objects. + 2. Upload objects using HTTP (https://github.com/nspcc-dev/neofs-http-gw#uploading). + 3. Download objects using HTTP gate (https://github.com/nspcc-dev/neofs-http-gw#downloading). + 4. Compare hashes for got and original objects. + + Expected result: + Hashes must be the same. + """ + cid = create_container( + self.wallet.path, + shell=self.shell, + endpoint=self.neofs_env.sn_rpc, + rule=self.PLACEMENT_RULE_2, + basic_acl=PUBLIC_ACL, + ) + file_path_simple, file_path_large = generate_file(simple_object_size), generate_file( + complex_object_size + ) + + with allure.step("Put objects using HTTP"): + oid_simple = upload_via_http_gate( + cid=cid, path=file_path_simple, endpoint=f"http://{self.neofs_env.http_gw.address}" + ) + oid_large = upload_via_http_gate( + cid=cid, path=file_path_large, endpoint=f"http://{self.neofs_env.http_gw.address}" + ) + + for oid, file_path in ((oid_simple, file_path_simple), (oid_large, file_path_large)): + get_object_and_verify_hashes( + oid=oid, + file_name=file_path, + wallet=self.wallet.path, + cid=cid, + shell=self.shell, + nodes=self.neofs_env.storage_nodes, + endpoint=f"http://{self.neofs_env.http_gw.address}", + ) + + @allure.link( + "https://github.com/nspcc-dev/neofs-http-gw#by-attributes", name="download by attributes" + ) + @allure.title("Test Put over HTTP, Get over HTTP with headers") + @pytest.mark.parametrize( + "attributes", + [ + {"fileName": "simple_obj_filename"}, + {"file-Name": "simple obj filename"}, + {"cat%jpeg": "cat%jpeg"}, + ], + ids=["simple", "hyphen", "percent"], + ) + def test_put_http_get_http_with_headers(self, attributes: dict, simple_object_size): + """ + Test that object can be downloaded using different attributes in HTTP header. + + Steps: + 1. Create simple and large objects. + 2. Upload objects using HTTP with particular attributes in the header. + 3. Download objects by attributes using HTTP gate (https://github.com/nspcc-dev/neofs-http-gw#by-attributes). + 4. Compare hashes for got and original objects. + + Expected result: + Hashes must be the same. + """ + cid = create_container( + self.wallet.path, + shell=self.shell, + endpoint=self.neofs_env.sn_rpc, + rule=self.PLACEMENT_RULE_2, + basic_acl=PUBLIC_ACL, + ) + file_path = generate_file(simple_object_size) + + with allure.step("Put objects using HTTP with attribute"): + headers = attr_into_header(attributes) + oid = upload_via_http_gate( + cid=cid, + path=file_path, + headers=headers, + endpoint=f"http://{self.neofs_env.http_gw.address}", + ) + + get_object_by_attr_and_verify_hashes( + oid=oid, + file_name=file_path, + cid=cid, + attrs=attributes, + endpoint=f"http://{self.neofs_env.http_gw.address}", + ) + + @allure.title("Test Expiration-Epoch in HTTP header") + def test_expiration_epoch_in_http(self, simple_object_size): + endpoint = self.neofs_env.sn_rpc + http_endpoint = f"http://{self.neofs_env.http_gw.address}" + + cid = create_container( + self.wallet.path, + shell=self.shell, + endpoint=endpoint, + rule=self.PLACEMENT_RULE_2, + basic_acl=PUBLIC_ACL, + ) + file_path = generate_file(simple_object_size) + oids = [] + + curr_epoch = neofs_epoch.get_epoch(self.neofs_env) + epochs = (curr_epoch, curr_epoch + 1, curr_epoch + 2, curr_epoch + 100) + + for epoch in epochs: + headers = {"X-Attribute-Neofs-Expiration-Epoch": str(epoch)} + + with allure.step("Put objects using HTTP with attribute Expiration-Epoch"): + oids.append( + upload_via_http_gate( + cid=cid, path=file_path, headers=headers, endpoint=http_endpoint + ) + ) + + assert len(oids) == len(epochs), "Expected all objects have been put successfully" + + with allure.step("All objects can be get"): + for oid in oids: + get_via_http_gate(cid=cid, oid=oid, endpoint=http_endpoint) + + for expired_objects, not_expired_objects in [(oids[:1], oids[1:]), (oids[:2], oids[2:])]: + self.tick_epochs_and_wait(1) + + # Wait for GC, because object with expiration is counted as alive until GC removes it + wait_for_gc_pass_on_storage_nodes() + + for oid in expired_objects: + try_to_get_object_and_expect_error( + cid=cid, + oid=oid, + error_pattern=OBJECT_NOT_FOUND_ERROR, + endpoint=f"http://{self.neofs_env.http_gw.address}", + ) + + with allure.step("Other objects can be get"): + for oid in not_expired_objects: + get_via_http_gate(cid=cid, oid=oid, endpoint=http_endpoint) + + @allure.title("Test Zip in HTTP header") + def test_zip_in_http(self, complex_object_size, simple_object_size): + cid = create_container( + self.wallet.path, + shell=self.shell, + endpoint=self.neofs_env.sn_rpc, + rule=self.PLACEMENT_RULE_2, + basic_acl=PUBLIC_ACL, + ) + file_path_simple, file_path_large = generate_file(simple_object_size), generate_file( + complex_object_size + ) + common_prefix = "my_files" + + headers1 = {"X-Attribute-FilePath": f"{common_prefix}/file1"} + headers2 = {"X-Attribute-FilePath": f"{common_prefix}/file2"} + + upload_via_http_gate( + cid=cid, + path=file_path_simple, + headers=headers1, + endpoint=f"http://{self.neofs_env.http_gw.address}", + ) + upload_via_http_gate( + cid=cid, + path=file_path_large, + headers=headers2, + endpoint=f"http://{self.neofs_env.http_gw.address}", + ) + + dir_path = get_via_zip_http_gate( + cid=cid, prefix=common_prefix, endpoint=f"http://{self.neofs_env.http_gw.address}" + ) + + with allure.step("Verify hashes"): + assert get_file_hash(f"{dir_path}/file1") == get_file_hash(file_path_simple) + assert get_file_hash(f"{dir_path}/file2") == get_file_hash(file_path_large) + + @pytest.mark.long + @allure.title("Test Put over HTTP/Curl, Get over HTTP/Curl for large object") + def test_put_http_get_http_large_file(self, complex_object_size): + """ + This test checks upload and download using curl with 'large' object. + Large is object with size up to 20Mb. + """ + cid = create_container( + self.wallet.path, + shell=self.shell, + endpoint=self.neofs_env.sn_rpc, + rule=self.PLACEMENT_RULE_2, + basic_acl=PUBLIC_ACL, + ) + + obj_size = int(os.getenv("BIG_OBJ_SIZE", complex_object_size)) + file_path = generate_file(obj_size) + + with allure.step("Put objects using HTTP"): + oid_gate = upload_via_http_gate( + cid=cid, path=file_path, endpoint=f"http://{self.neofs_env.http_gw.address}" + ) + oid_curl = upload_via_http_gate_curl( + cid=cid, + filepath=file_path, + endpoint=f"http://{self.neofs_env.http_gw.address}", + ) + + get_object_and_verify_hashes( + oid=oid_gate, + file_name=file_path, + wallet=self.wallet.path, + cid=cid, + shell=self.shell, + nodes=self.neofs_env.storage_nodes, + endpoint=f"http://{self.neofs_env.http_gw.address}", + ) + get_object_and_verify_hashes( + oid=oid_curl, + file_name=file_path, + wallet=self.wallet.path, + cid=cid, + shell=self.shell, + nodes=self.neofs_env.storage_nodes, + endpoint=f"http://{self.neofs_env.http_gw.address}", + object_getter=get_via_http_curl, + ) + + @allure.title("Test Put/Get over HTTP using Curl utility") + def test_put_http_get_http_curl(self, complex_object_size, simple_object_size): + """ + Test checks upload and download over HTTP using curl utility. + """ + cid = create_container( + self.wallet.path, + shell=self.shell, + endpoint=self.neofs_env.sn_rpc, + rule=self.PLACEMENT_RULE_2, + basic_acl=PUBLIC_ACL, + ) + file_path_simple, file_path_large = generate_file(simple_object_size), generate_file( + complex_object_size + ) + + with allure.step("Put objects using curl utility"): + oid_simple = upload_via_http_gate_curl( + cid=cid, + filepath=file_path_simple, + endpoint=f"http://{self.neofs_env.http_gw.address}", + ) + oid_large = upload_via_http_gate_curl( + cid=cid, + filepath=file_path_large, + endpoint=f"http://{self.neofs_env.http_gw.address}", + ) + + for oid, file_path in ((oid_simple, file_path_simple), (oid_large, file_path_large)): + get_object_and_verify_hashes( + oid=oid, + file_name=file_path, + wallet=self.wallet.path, + cid=cid, + shell=self.shell, + nodes=self.neofs_env.storage_nodes, + endpoint=f"http://{self.neofs_env.http_gw.address}", + object_getter=get_via_http_curl, + ) diff --git a/dynamic_env_pytest_tests/tests/services/http_gate/test_http_headers.py b/dynamic_env_pytest_tests/tests/services/http_gate/test_http_headers.py new file mode 100644 index 000000000..b9d6778bb --- /dev/null +++ b/dynamic_env_pytest_tests/tests/services/http_gate/test_http_headers.py @@ -0,0 +1,228 @@ +import logging +import os + +import allure +import pytest +from container import ( + create_container, + delete_container, + list_containers, + wait_for_container_deletion, +) +from file_helper import generate_file +from http_gate import ( + attr_into_str_header_curl, + get_object_by_attr_and_verify_hashes, + try_to_get_object_and_expect_error, + try_to_get_object_via_passed_request_and_expect_error, + upload_via_http_gate_curl, +) +from neofs_env.neofs_env_test_base import NeofsEnvTestBase +from pytest import FixtureRequest +from python_keywords.neofs_verbs import delete_object +from wellknown_acl import PUBLIC_ACL + +from helpers.storage_object_info import StorageObjectInfo + +logger = logging.getLogger("NeoLogger") + + +@pytest.mark.sanity +@pytest.mark.http_gate +class Test_http_headers(NeofsEnvTestBase): + PLACEMENT_RULE = "REP 2 IN X CBF 1 SELECT 4 FROM * AS X" + obj1_keys = ["Writer", "Chapter1", "Chapter2"] + obj2_keys = ["Writer", "Ch@pter1", "chapter2"] + values = ["Leo Tolstoy", "peace", "w@r"] + OBJECT_ATTRIBUTES = [ + {obj1_keys[0]: values[0], obj1_keys[1]: values[1], obj1_keys[2]: values[2]}, + {obj2_keys[0]: values[0], obj2_keys[1]: values[1], obj2_keys[2]: values[2]}, + ] + + @pytest.fixture(scope="class", autouse=True) + @allure.title("[Class/Autouse]: Prepare wallet and deposit") + def prepare_wallet(self, default_wallet): + Test_http_headers.wallet = default_wallet + + @pytest.fixture( + params=[ + pytest.lazy_fixture("simple_object_size"), + pytest.lazy_fixture("complex_object_size"), + ], + ids=["simple object", "complex object"], + scope="class", + ) + def storage_objects_with_attributes(self, request: FixtureRequest) -> list[StorageObjectInfo]: + storage_objects = [] + wallet = self.wallet.path + cid = create_container( + wallet=self.wallet.path, + shell=self.shell, + endpoint=self.neofs_env.sn_rpc, + rule=self.PLACEMENT_RULE, + basic_acl=PUBLIC_ACL, + ) + file_path = generate_file(request.param) + for attributes in self.OBJECT_ATTRIBUTES: + storage_object_id = upload_via_http_gate_curl( + cid=cid, + filepath=file_path, + endpoint=f"http://{self.neofs_env.http_gw.address}", + headers=attr_into_str_header_curl(attributes), + ) + storage_object = StorageObjectInfo(cid, storage_object_id) + storage_object.size = os.path.getsize(file_path) + storage_object.wallet_file_path = wallet + storage_object.file_path = file_path + storage_object.attributes = attributes + + storage_objects.append(storage_object) + + yield storage_objects + + @allure.title("Get object1 by attribute") + def test_object1_can_be_get_by_attr( + self, storage_objects_with_attributes: list[StorageObjectInfo] + ): + """ + Test to get object#1 by attribute and comapre hashes + + Steps: + 1. Download object#1 with attributes [Chapter2=w@r] and compare hashes + """ + + storage_object_1 = storage_objects_with_attributes[0] + + with allure.step( + f'Download object#1 via wget with attributes Chapter2: {storage_object_1.attributes["Chapter2"]} and compare hashes' + ): + get_object_by_attr_and_verify_hashes( + oid=storage_object_1.oid, + file_name=storage_object_1.file_path, + cid=storage_object_1.cid, + attrs={"Chapter2": storage_object_1.attributes["Chapter2"]}, + endpoint=f"http://{self.neofs_env.http_gw.address}", + ) + + @allure.title("Test get object2 with different attributes, then delete object2 and get object1") + def test_object2_can_be_get_by_attr( + self, storage_objects_with_attributes: list[StorageObjectInfo] + ): + """ + Test to get object2 with different attributes, then delete object2 and get object1 using 1st attribute. Note: obj1 and obj2 have the same attribute#1, + and when obj2 is deleted you can get obj1 by 1st attribute + + Steps: + 1. Download object#2 with attributes [chapter2=w@r] and compare hashes + 2. Download object#2 with attributes [Ch@pter1=peace] and compare hashes + 3. Delete object#2 + 4. Download object#1 with attributes [Writer=Leo Tolstoy] and compare hashes + """ + storage_object_1 = storage_objects_with_attributes[0] + storage_object_2 = storage_objects_with_attributes[1] + + with allure.step( + f'Download object#2 via wget with attributes [chapter2={storage_object_2.attributes["chapter2"]}] / [Ch@pter1={storage_object_2.attributes["Ch@pter1"]}] and compare hashes' + ): + selected_attributes_object2 = [ + {"chapter2": storage_object_2.attributes["chapter2"]}, + {"Ch@pter1": storage_object_2.attributes["Ch@pter1"]}, + ] + for attributes in selected_attributes_object2: + get_object_by_attr_and_verify_hashes( + oid=storage_object_2.oid, + file_name=storage_object_2.file_path, + cid=storage_object_2.cid, + attrs=attributes, + endpoint=f"http://{self.neofs_env.http_gw.address}", + ) + with allure.step("Delete object#2 and verify is the container deleted"): + delete_object( + wallet=self.wallet.path, + cid=storage_object_2.cid, + oid=storage_object_2.oid, + shell=self.shell, + endpoint=self.neofs_env.sn_rpc, + ) + error_pattern = "404 Not Found" + try_to_get_object_and_expect_error( + cid=storage_object_2.cid, + oid=storage_object_2.oid, + error_pattern=error_pattern, + endpoint=f"http://{self.neofs_env.http_gw.address}", + ) + storage_objects_with_attributes.remove(storage_object_2) + + with allure.step( + f'Download object#1 with attributes [Writer={storage_object_1.attributes["Writer"]}] and compare hashes' + ): + key_value_pair = {"Writer": storage_object_1.attributes["Writer"]} + get_object_by_attr_and_verify_hashes( + oid=storage_object_1.oid, + file_name=storage_object_1.file_path, + cid=storage_object_1.cid, + attrs=key_value_pair, + endpoint=f"http://{self.neofs_env.http_gw.address}", + ) + + @allure.title("[Negative] Try to put object and get right after container is deleted") + def test_negative_put_and_get_object3( + self, storage_objects_with_attributes: list[StorageObjectInfo] + ): + """ + Test to attempt to put object and try to download it right after the container has been deleted + + Steps: + 1. [Negative] Allocate and attempt to put object#3 via http with attributes: [Writer=Leo Tolstoy, Writer=peace, peace=peace] + Expected: "Error duplication of attributes detected" + 2. Delete container + 3. [Negative] Try to download object with attributes [peace=peace] + Expected: "HTTP request sent, awaiting response... 404 Not Found" + """ + storage_object_1 = storage_objects_with_attributes[0] + + with allure.step( + "[Negative] Allocate and attemt to put object#3 via http with attributes: [Writer=Leo Tolstoy, Writer=peace, peace=peace]" + ): + file_path_3 = generate_file(storage_object_1.size) + attrs_obj3 = {"Writer": "Leo Tolstoy", "peace": "peace"} + headers = attr_into_str_header_curl(attrs_obj3) + headers.append(" ".join(attr_into_str_header_curl({"Writer": "peace"}))) + error_pattern = f"key duplication error: X-Attribute-Writer" + upload_via_http_gate_curl( + cid=storage_object_1.cid, + filepath=file_path_3, + endpoint=f"http://{self.neofs_env.http_gw.address}", + headers=headers, + error_pattern=error_pattern, + ) + with allure.step("Delete container and verify container deletion"): + delete_container( + wallet=self.wallet.path, + cid=storage_object_1.cid, + shell=self.shell, + endpoint=self.neofs_env.sn_rpc, + ) + self.tick_epochs_and_wait(1) + wait_for_container_deletion( + self.wallet.path, + storage_object_1.cid, + shell=self.shell, + endpoint=self.neofs_env.sn_rpc, + ) + assert storage_object_1.cid not in list_containers( + self.wallet.path, shell=self.shell, endpoint=self.neofs_env.sn_rpc + ) + with allure.step( + "[Negative] Try to download (wget) object via wget with attributes [peace=peace]" + ): + request = f"/get/{storage_object_1.cid}/peace/peace" + error_pattern = "404 Not Found" + try_to_get_object_via_passed_request_and_expect_error( + cid=storage_object_1.cid, + oid="", + error_pattern=error_pattern, + attrs=attrs_obj3, + http_request_path=request, + endpoint=f"http://{self.neofs_env.http_gw.address}", + ) diff --git a/dynamic_env_pytest_tests/tests/services/http_gate/test_http_object.py b/dynamic_env_pytest_tests/tests/services/http_gate/test_http_object.py new file mode 100644 index 000000000..2b544bc0e --- /dev/null +++ b/dynamic_env_pytest_tests/tests/services/http_gate/test_http_object.py @@ -0,0 +1,138 @@ +import logging +import os + +import allure +import pytest +from container import create_container +from file_helper import generate_file +from http_gate import ( + get_object_by_attr_and_verify_hashes, + try_to_get_object_via_passed_request_and_expect_error, +) +from http_gw.http_utils import get_object_and_verify_hashes +from neofs_env.neofs_env_test_base import NeofsEnvTestBase +from python_keywords.neofs_verbs import put_object_to_random_node +from wellknown_acl import PUBLIC_ACL + +logger = logging.getLogger("NeoLogger") + + +@pytest.mark.sanity +@pytest.mark.http_gate +class Test_http_object(NeofsEnvTestBase): + PLACEMENT_RULE = "REP 2 IN X CBF 1 SELECT 4 FROM * AS X" + + @pytest.fixture(scope="class", autouse=True) + @allure.title("[Class/Autouse]: Prepare wallet and deposit") + def prepare_wallet(self, default_wallet): + Test_http_object.wallet = default_wallet + + @allure.title("Test Put over gRPC, Get over HTTP") + @pytest.mark.parametrize( + "object_size", + [pytest.lazy_fixture("simple_object_size"), pytest.lazy_fixture("complex_object_size")], + ids=["simple object", "complex object"], + ) + def test_object_put_get_attributes(self, object_size: int): + """ + Test that object can be put using gRPC interface and get using HTTP. + + Steps: + 1. Create object; + 2. Put objects using gRPC (neofs-cli) with attributes [--attributes chapter1=peace,chapter2=war]; + 3. Download object using HTTP gate (https://github.com/nspcc-dev/neofs-http-gw#downloading); + 4. Compare hashes between original and downloaded object; + 5. [Negative] Try to the get object with specified attributes and `get` request: [get/$CID/chapter1/peace]; + 6. Download the object with specified attributes and `get_by_attribute` request: [get_by_attribute/$CID/chapter1/peace]; + 7. Compare hashes between original and downloaded object; + 8. [Negative] Try to the get object via `get_by_attribute` request: [get_by_attribute/$CID/$OID]; + + + Expected result: + Hashes must be the same. + """ + with allure.step("Create public container"): + cid = create_container( + self.wallet.path, + shell=self.shell, + endpoint=self.neofs_env.sn_rpc, + rule=self.PLACEMENT_RULE, + basic_acl=PUBLIC_ACL, + ) + + # Generate file + file_path = generate_file(object_size) + + # List of Key=Value attributes + obj_key1 = "chapter1" + obj_value1 = "peace" + obj_key2 = "chapter2" + obj_value2 = "war" + + # Prepare for grpc PUT request + key_value1 = obj_key1 + "=" + obj_value1 + key_value2 = obj_key2 + "=" + obj_value2 + + with allure.step("Put objects using gRPC [--attributes chapter1=peace,chapter2=war]"): + oid = put_object_to_random_node( + wallet=self.wallet.path, + path=file_path, + cid=cid, + shell=self.shell, + neofs_env=self.neofs_env, + attributes=f"{key_value1},{key_value2}", + ) + with allure.step("Get object and verify hashes [ get/$CID/$OID ]"): + get_object_and_verify_hashes( + oid=oid, + file_name=file_path, + wallet=self.wallet.path, + cid=cid, + shell=self.shell, + nodes=self.neofs_env.storage_nodes, + endpoint=f"http://{self.neofs_env.http_gw.address}", + ) + with allure.step("[Negative] try to get object: [get/$CID/chapter1/peace]"): + attrs = {obj_key1: obj_value1, obj_key2: obj_value2} + request = f"/get/{cid}/{obj_key1}/{obj_value1}" + expected_err_msg = "Failed to get object via HTTP gate:" + try_to_get_object_via_passed_request_and_expect_error( + cid=cid, + oid=oid, + error_pattern=expected_err_msg, + http_request_path=request, + attrs=attrs, + endpoint=f"http://{self.neofs_env.http_gw.address}", + ) + + with allure.step( + "Download the object with attribute [get_by_attribute/$CID/chapter1/peace]" + ): + get_object_by_attr_and_verify_hashes( + oid=oid, + file_name=file_path, + cid=cid, + attrs=attrs, + endpoint=f"http://{self.neofs_env.http_gw.address}", + ) + with allure.step("[Negative] try to get object: get_by_attribute/$CID/$OID"): + request = f"/get_by_attribute/{cid}/{oid}" + try_to_get_object_via_passed_request_and_expect_error( + cid=cid, + oid=oid, + error_pattern=expected_err_msg, + http_request_path=request, + endpoint=f"http://{self.neofs_env.http_gw.address}", + ) + + with allure.step( + "[Negative] Try to get object with invalid attribute [get_by_attribute/$CID/chapter1/war]" + ): + with pytest.raises(Exception, match=".*object not found.*"): + get_object_by_attr_and_verify_hashes( + oid=oid, + file_name=file_path, + cid=cid, + attrs={obj_key1: obj_value2}, + endpoint=f"http://{self.neofs_env.http_gw.address}", + ) diff --git a/dynamic_env_pytest_tests/tests/services/http_gate/test_http_streaming.py b/dynamic_env_pytest_tests/tests/services/http_gate/test_http_streaming.py new file mode 100644 index 000000000..82c8fef75 --- /dev/null +++ b/dynamic_env_pytest_tests/tests/services/http_gate/test_http_streaming.py @@ -0,0 +1,70 @@ +import logging + +import allure +import pytest +from container import create_container +from file_helper import generate_file +from http_gate import upload_via_http_gate_curl +from http_gw.http_utils import get_object_and_verify_hashes +from neofs_env.neofs_env_test_base import NeofsEnvTestBase +from wellknown_acl import PUBLIC_ACL + +logger = logging.getLogger("NeoLogger") + + +@pytest.mark.sanity +@pytest.mark.http_gate +class Test_http_streaming(NeofsEnvTestBase): + PLACEMENT_RULE = "REP 2 IN X CBF 1 SELECT 4 FROM * AS X" + + @pytest.fixture(scope="class", autouse=True) + @allure.title("[Class/Autouse]: Prepare wallet and deposit") + def prepare_wallet(self, default_wallet): + Test_http_streaming.wallet = default_wallet + + @allure.title("Test Put via pipe (steaming), Get over HTTP and verify hashes") + @pytest.mark.parametrize( + "object_size", + [pytest.lazy_fixture("complex_object_size")], + ids=["complex object"], + ) + def test_object_can_be_put_get_by_streaming(self, object_size: int): + """ + Test that object can be put using gRPC interface and get using HTTP. + + Steps: + 1. Create big object; + 2. Put object using curl with pipe (streaming); + 3. Download object using HTTP gate (https://github.com/nspcc-dev/neofs-http-gw#downloading); + 4. Compare hashes between original and downloaded object; + + Expected result: + Hashes must be the same. + """ + with allure.step("Create public container and verify container creation"): + cid = create_container( + self.wallet.path, + shell=self.shell, + endpoint=self.neofs_env.sn_rpc, + rule=self.PLACEMENT_RULE, + basic_acl=PUBLIC_ACL, + ) + with allure.step("Allocate big object"): + # Generate file + file_path = generate_file(object_size) + + with allure.step( + "Put objects using curl utility and Get object and verify hashes [ get/$CID/$OID ]" + ): + oid = upload_via_http_gate_curl( + cid=cid, filepath=file_path, endpoint=f"http://{self.neofs_env.http_gw.address}" + ) + get_object_and_verify_hashes( + oid=oid, + file_name=file_path, + wallet=self.wallet.path, + cid=cid, + shell=self.shell, + nodes=self.neofs_env.storage_nodes, + endpoint=f"http://{self.neofs_env.http_gw.address}", + ) diff --git a/dynamic_env_pytest_tests/tests/services/http_gate/test_http_system_header.py b/dynamic_env_pytest_tests/tests/services/http_gate/test_http_system_header.py new file mode 100644 index 000000000..5491f51b4 --- /dev/null +++ b/dynamic_env_pytest_tests/tests/services/http_gate/test_http_system_header.py @@ -0,0 +1,417 @@ +import calendar +import datetime +import logging +from typing import Optional + +import allure +import neofs_env.neofs_epoch as neofs_epoch +import pytest +from container import create_container +from file_helper import generate_file +from grpc_responses import OBJECT_NOT_FOUND +from http_gate import ( + attr_into_str_header_curl, + try_to_get_object_and_expect_error, + upload_via_http_gate_curl, +) +from http_gw.http_utils import get_object_and_verify_hashes +from neofs_env.neofs_env_test_base import NeofsEnvTestBase +from python_keywords.neofs_verbs import get_netmap_netinfo, get_object_from_random_node, head_object +from wellknown_acl import PUBLIC_ACL + +logger = logging.getLogger("NeoLogger") +EXPIRATION_TIMESTAMP_HEADER = "__NEOFS__EXPIRATION_TIMESTAMP" +EXPIRATION_EPOCH_HEADER = "__NEOFS__EXPIRATION_EPOCH" +EXPIRATION_DURATION_HEADER = "__NEOFS__EXPIRATION_DURATION" +EXPIRATION_EXPIRATION_RFC = "__NEOFS__EXPIRATION_RFC3339" +NEOFS_EXPIRATION_EPOCH = "Neofs-Expiration-Epoch" +NEOFS_EXPIRATION_DURATION = "Neofs-Expiration-Duration" +NEOFS_EXPIRATION_TIMESTAMP = "Neofs-Expiration-Timestamp" +NEOFS_EXIPRATION_RFC3339 = "Neofs-Expiration-RFC3339" + + +@pytest.mark.http_gate +class Test_http_system_header(NeofsEnvTestBase): + PLACEMENT_RULE = "REP 2 IN X CBF 1 SELECT 2 FROM * AS X" + + @pytest.fixture(scope="class", autouse=True) + @allure.title("[Class/Autouse]: Prepare wallet and deposit") + def prepare_wallet(self, default_wallet): + Test_http_system_header.wallet = default_wallet + + @pytest.fixture(scope="class") + @allure.title("Create container") + def user_container(self): + return create_container( + wallet=self.wallet.path, + shell=self.shell, + endpoint=self.neofs_env.sn_rpc, + rule=self.PLACEMENT_RULE, + basic_acl=PUBLIC_ACL, + ) + + @pytest.fixture(scope="class") + @allure.title("epoch_duration in seconds") + def epoch_duration(self) -> int: + net_info = get_netmap_netinfo( + wallet=self.wallet.path, + endpoint=self.neofs_env.sn_rpc, + shell=self.shell, + ) + epoch_duration_in_blocks = net_info["epoch_duration"] + time_per_block = net_info["time_per_block"] + return int(epoch_duration_in_blocks * time_per_block) + + @allure.title("Return N-epoch count in minutes") + def epoch_count_into_mins(self, epoch_duration: int, epoch: int) -> str: + mins = epoch_duration * epoch / 60 + return f"{mins}m" + + @allure.title("Return future timestamp after N epochs are passed") + def epoch_count_into_timestamp( + self, epoch_duration: int, epoch: int, rfc3339: Optional[bool] = False + ) -> str: + current_datetime = datetime.datetime.utcnow() + epoch_count_in_seconds = epoch_duration * epoch + future_datetime = current_datetime + datetime.timedelta(seconds=epoch_count_in_seconds) + if rfc3339: + return future_datetime.isoformat("T") + "Z" + else: + return str(calendar.timegm(future_datetime.timetuple())) + + @allure.title("Check is (header_output) Key=Value exists and equal in passed (header_to_find)") + def check_key_value_presented_header(self, header_output: dict, header_to_find: dict) -> bool: + header_att = header_output["header"]["attributes"] + for key_to_check, val_to_check in header_to_find.items(): + if key_to_check not in header_att or val_to_check != header_att[key_to_check]: + logger.info(f"Unable to find {key_to_check}: '{val_to_check}' in {header_att}") + return False + return True + + @allure.title( + f"Validate that only {EXPIRATION_EPOCH_HEADER} exists in header and other headers are abesent" + ) + def validation_for_http_header_attr(self, head_info: dict, expected_epoch: int) -> None: + # check that __NEOFS__EXPIRATION_EPOCH attribute has corresponding epoch + assert self.check_key_value_presented_header( + head_info, {EXPIRATION_EPOCH_HEADER: str(expected_epoch)} + ), f'Expected to find {EXPIRATION_EPOCH_HEADER}: {expected_epoch} in: {head_info["header"]["attributes"]}' + # check that {EXPIRATION_EPOCH_HEADER} absents in header output + assert not ( + self.check_key_value_presented_header(head_info, {EXPIRATION_DURATION_HEADER: ""}) + ), f"Only {EXPIRATION_EPOCH_HEADER} can be displayed in header attributes" + # check that {EXPIRATION_TIMESTAMP_HEADER} absents in header output + assert not ( + self.check_key_value_presented_header(head_info, {EXPIRATION_TIMESTAMP_HEADER: ""}) + ), f"Only {EXPIRATION_TIMESTAMP_HEADER} can be displayed in header attributes" + # check that {EXPIRATION_EXPIRATION_RFC} absents in header output + assert not ( + self.check_key_value_presented_header(head_info, {EXPIRATION_EXPIRATION_RFC: ""}) + ), f"Only {EXPIRATION_EXPIRATION_RFC} can be displayed in header attributes" + + @allure.title("Put / get / verify object and return head command result to invoker") + def oid_header_info_for_object(self, file_path: str, attributes: dict, user_container: str): + oid = upload_via_http_gate_curl( + cid=user_container, + filepath=file_path, + endpoint=f"http://{self.neofs_env.http_gw.address}", + headers=attr_into_str_header_curl(attributes), + ) + get_object_and_verify_hashes( + oid=oid, + file_name=file_path, + wallet=self.wallet.path, + cid=user_container, + shell=self.shell, + nodes=self.neofs_env.storage_nodes, + endpoint=f"http://{self.neofs_env.http_gw.address}", + ) + head = head_object( + wallet=self.wallet.path, + cid=user_container, + oid=oid, + shell=self.shell, + endpoint=self.neofs_env.sn_rpc, + ) + return oid, head + + @allure.title("[negative] attempt to put object with expired epoch") + def test_unable_put_expired_epoch(self, user_container: str, simple_object_size: int): + headers = attr_into_str_header_curl( + {"Neofs-Expiration-Epoch": str(neofs_epoch.get_epoch(self.neofs_env) - 1)} + ) + file_path = generate_file(simple_object_size) + with allure.step( + "Put object using HTTP with attribute Expiration-Epoch where epoch is expired" + ): + upload_via_http_gate_curl( + cid=user_container, + filepath=file_path, + endpoint=f"http://{self.neofs_env.http_gw.address}", + headers=headers, + error_pattern="object has expired", + ) + + @allure.title("[negative] attempt to put object with negative Neofs-Expiration-Duration") + def test_unable_put_negative_duration(self, user_container: str, simple_object_size: int): + headers = attr_into_str_header_curl({"Neofs-Expiration-Duration": "-1h"}) + file_path = generate_file(simple_object_size) + with allure.step( + "Put object using HTTP with attribute Neofs-Expiration-Duration where duration is negative" + ): + upload_via_http_gate_curl( + cid=user_container, + filepath=file_path, + endpoint=f"http://{self.neofs_env.http_gw.address}", + headers=headers, + error_pattern=f"{EXPIRATION_DURATION_HEADER} must be positive", + ) + + @allure.title( + "[negative] attempt to put object with Neofs-Expiration-Timestamp value in the past" + ) + def test_unable_put_expired_timestamp(self, user_container: str, simple_object_size: int): + headers = attr_into_str_header_curl({"Neofs-Expiration-Timestamp": "1635075727"}) + file_path = generate_file(simple_object_size) + with allure.step( + "Put object using HTTP with attribute Neofs-Expiration-Timestamp where duration is in the past" + ): + upload_via_http_gate_curl( + cid=user_container, + filepath=file_path, + endpoint=f"http://{self.neofs_env.http_gw.address}", + headers=headers, + error_pattern=f"{EXPIRATION_TIMESTAMP_HEADER} must be in the future", + ) + + @allure.title( + "[negative] Put object using HTTP with attribute Neofs-Expiration-RFC3339 where duration is in the past" + ) + def test_unable_put_expired_rfc(self, user_container: str, simple_object_size: int): + headers = attr_into_str_header_curl({"Neofs-Expiration-RFC3339": "2021-11-22T09:55:49Z"}) + file_path = generate_file(simple_object_size) + upload_via_http_gate_curl( + cid=user_container, + filepath=file_path, + endpoint=f"http://{self.neofs_env.http_gw.address}", + headers=headers, + error_pattern=f"{EXPIRATION_EXPIRATION_RFC} must be in the future", + ) + + @pytest.mark.sanity + @allure.title("priority of attributes epoch>duration") + @pytest.mark.parametrize( + "object_size", + [pytest.lazy_fixture("simple_object_size"), pytest.lazy_fixture("complex_object_size")], + ids=["simple object", "complex object"], + ) + def test_http_attr_priority_epoch_duration( + self, user_container: str, object_size: int, epoch_duration: int + ): + self.tick_epochs_and_wait(1) + epoch_count = 1 + expected_epoch = neofs_epoch.get_epoch(self.neofs_env) + epoch_count + logger.info( + f"epoch duration={epoch_duration}, current_epoch= {neofs_epoch.get_epoch(self.neofs_env)} expected_epoch {expected_epoch}" + ) + attributes = {NEOFS_EXPIRATION_EPOCH: expected_epoch, NEOFS_EXPIRATION_DURATION: "1m"} + file_path = generate_file(object_size) + with allure.step( + f"Put objects using HTTP with attributes and head command should display {EXPIRATION_EPOCH_HEADER}: {expected_epoch} attr" + ): + oid, head_info = self.oid_header_info_for_object( + file_path=file_path, attributes=attributes, user_container=user_container + ) + self.validation_for_http_header_attr(head_info=head_info, expected_epoch=expected_epoch) + with allure.step("Check that object becomes unavailable when epoch is expired"): + self.tick_epochs_and_wait(epoch_count + 1) + assert ( + neofs_epoch.get_epoch(self.neofs_env) == expected_epoch + 1 + ), f"Epochs should be equal: {neofs_epoch.get_epoch(self.neofs_env)} != {expected_epoch + 1}" + + with allure.step("Check object deleted because it expires-on epoch"): + neofs_epoch.wait_for_epochs_align(self.neofs_env) + try_to_get_object_and_expect_error( + cid=user_container, + oid=oid, + error_pattern="404 Not Found", + endpoint=f"http://{self.neofs_env.http_gw.address}", + ) + # check that object is not available via grpc + with pytest.raises(Exception, match=OBJECT_NOT_FOUND): + get_object_from_random_node( + self.wallet.path, + user_container, + oid, + self.shell, + neofs_env=self.neofs_env, + ) + + @allure.title( + f"priority of attributes duration>timestamp, duration time has higher priority and should be converted {EXPIRATION_EPOCH_HEADER}" + ) + @pytest.mark.parametrize( + "object_size", + [pytest.lazy_fixture("simple_object_size"), pytest.lazy_fixture("complex_object_size")], + ids=["simple object", "complex object"], + ) + def test_http_attr_priority_dur_timestamp( + self, user_container: str, object_size: int, epoch_duration: int + ): + self.tick_epochs_and_wait(1) + epoch_count = 2 + expected_epoch = neofs_epoch.get_epoch(self.neofs_env) + epoch_count + logger.info( + f"epoch duration={epoch_duration}, current_epoch= {neofs_epoch.get_epoch(self.neofs_env)} expected_epoch {expected_epoch}" + ) + attributes = { + NEOFS_EXPIRATION_DURATION: self.epoch_count_into_mins( + epoch_duration=epoch_duration, epoch=2 + ), + NEOFS_EXPIRATION_TIMESTAMP: self.epoch_count_into_timestamp( + epoch_duration=epoch_duration, epoch=1 + ), + } + file_path = generate_file(object_size) + with allure.step( + f"Put objects using HTTP with attributes and head command should display {EXPIRATION_EPOCH_HEADER}: {expected_epoch} attr" + ): + oid, head_info = self.oid_header_info_for_object( + file_path=file_path, attributes=attributes, user_container=user_container + ) + self.validation_for_http_header_attr(head_info=head_info, expected_epoch=expected_epoch) + with allure.step("Check that object becomes unavailable when epoch is expired"): + self.tick_epochs_and_wait(epoch_count + 1) + assert ( + neofs_epoch.get_epoch(self.neofs_env) == expected_epoch + 1 + ), f"Epochs should be equal: {neofs_epoch.get_epoch(self.neofs_env)} != {expected_epoch + 1}" + + with allure.step("Check object deleted because it expires-on epoch"): + neofs_epoch.wait_for_epochs_align(self.neofs_env) + try_to_get_object_and_expect_error( + cid=user_container, + oid=oid, + error_pattern="404 Not Found", + endpoint=f"http://{self.neofs_env.http_gw.address}", + ) + # check that object is not available via grpc + with pytest.raises(Exception, match=OBJECT_NOT_FOUND): + get_object_from_random_node( + self.wallet.path, + user_container, + oid, + self.shell, + neofs_env=self.neofs_env, + ) + + @allure.title( + f"priority of attributes timestamp>Expiration-RFC, timestamp has higher priority and should be converted {EXPIRATION_EPOCH_HEADER}" + ) + @pytest.mark.parametrize( + "object_size", + [pytest.lazy_fixture("simple_object_size"), pytest.lazy_fixture("complex_object_size")], + ids=["simple object", "complex object"], + ) + def test_http_attr_priority_timestamp_rfc( + self, user_container: str, object_size: int, epoch_duration: int + ): + self.tick_epochs_and_wait(1) + epoch_count = 2 + expected_epoch = neofs_epoch.get_epoch(self.neofs_env) + epoch_count + logger.info( + f"epoch duration={epoch_duration}, current_epoch= {neofs_epoch.get_epoch(self.neofs_env)} expected_epoch {expected_epoch}" + ) + attributes = { + NEOFS_EXPIRATION_TIMESTAMP: self.epoch_count_into_timestamp( + epoch_duration=epoch_duration, epoch=2 + ), + NEOFS_EXIPRATION_RFC3339: self.epoch_count_into_timestamp( + epoch_duration=epoch_duration, epoch=1, rfc3339=True + ), + } + file_path = generate_file(object_size) + with allure.step( + f"Put objects using HTTP with attributes and head command should display {EXPIRATION_EPOCH_HEADER}: {expected_epoch} attr" + ): + oid, head_info = self.oid_header_info_for_object( + file_path=file_path, attributes=attributes, user_container=user_container + ) + self.validation_for_http_header_attr(head_info=head_info, expected_epoch=expected_epoch) + with allure.step("Check that object becomes unavailable when epoch is expired"): + self.tick_epochs_and_wait(epoch_count + 1) + assert ( + neofs_epoch.get_epoch(self.neofs_env) == expected_epoch + 1 + ), f"Epochs should be equal: {neofs_epoch.get_epoch(self.neofs_env)} != {expected_epoch + 1}" + + with allure.step("Check object deleted because it expires-on epoch"): + neofs_epoch.wait_for_epochs_align(self.neofs_env) + try_to_get_object_and_expect_error( + cid=user_container, + oid=oid, + error_pattern="404 Not Found", + endpoint=f"http://{self.neofs_env.http_gw.address}", + ) + # check that object is not available via grpc + with pytest.raises(Exception, match=OBJECT_NOT_FOUND): + get_object_from_random_node( + self.wallet.path, + user_container, + oid, + self.shell, + neofs_env=self.neofs_env, + ) + + @allure.title("Test that object is automatically delete when expiration passed") + @pytest.mark.parametrize( + "object_size", + [pytest.lazy_fixture("simple_object_size"), pytest.lazy_fixture("complex_object_size")], + ids=["simple object", "complex object"], + ) + def test_http_rfc_object_unavailable_after_expir( + self, user_container: str, object_size: int, epoch_duration: int + ): + self.tick_epochs_and_wait(1) + epoch_count = 2 + expected_epoch = neofs_epoch.get_epoch(self.neofs_env) + epoch_count + logger.info( + f"epoch duration={epoch_duration}, current_epoch= {neofs_epoch.get_epoch(self.neofs_env)} expected_epoch {expected_epoch}" + ) + attributes = { + NEOFS_EXIPRATION_RFC3339: self.epoch_count_into_timestamp( + epoch_duration=epoch_duration, epoch=2, rfc3339=True + ) + } + file_path = generate_file(object_size) + with allure.step( + f"Put objects using HTTP with attributes and head command should display {EXPIRATION_EPOCH_HEADER}: {expected_epoch} attr" + ): + oid, head_info = self.oid_header_info_for_object( + file_path=file_path, + attributes=attributes, + user_container=user_container, + ) + self.validation_for_http_header_attr(head_info=head_info, expected_epoch=expected_epoch) + with allure.step("Check that object becomes unavailable when epoch is expired"): + self.tick_epochs_and_wait(epoch_count + 1) + # check that {EXPIRATION_EXPIRATION_RFC} absents in header output + assert ( + neofs_epoch.get_epoch(self.neofs_env) == expected_epoch + 1 + ), f"Epochs should be equal: {neofs_epoch.get_epoch(self.neofs_env)} != {expected_epoch + 1}" + + with allure.step("Check object deleted because it expires-on epoch"): + neofs_epoch.wait_for_epochs_align(self.neofs_env) + try_to_get_object_and_expect_error( + cid=user_container, + oid=oid, + error_pattern="404 Not Found", + endpoint=f"http://{self.neofs_env.http_gw.address}", + ) + # check that object is not available via grpc + with pytest.raises(Exception, match=OBJECT_NOT_FOUND): + get_object_from_random_node( + self.wallet.path, + user_container, + oid, + self.shell, + neofs_env=self.neofs_env, + ) diff --git a/requirements.txt b/requirements.txt index 3caa37e22..576658da4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -29,7 +29,7 @@ mmh3==3.0.0 multidict==6.0.2 mypy==0.950 mypy-extensions==0.4.3 -neofs-testlib==1.1.14 +neofs-testlib==1.1.16 netaddr==0.8.0 packaging==21.3 paramiko==3.4.0 diff --git a/robot/resources/lib/python_keywords/neofs_verbs.py b/robot/resources/lib/python_keywords/neofs_verbs.py index dde68dab7..fee4aa83d 100644 --- a/robot/resources/lib/python_keywords/neofs_verbs.py +++ b/robot/resources/lib/python_keywords/neofs_verbs.py @@ -1,6 +1,7 @@ import json import logging import os +import random import re import uuid from typing import Any, Optional @@ -8,8 +9,9 @@ import allure import json_transformers from cluster import Cluster -from common import ASSETS_DIR, TEST_OBJECTS_DIR, NEOFS_CLI_EXEC, WALLET_CONFIG +from common import ASSETS_DIR, NEOFS_CLI_EXEC, TEST_OBJECTS_DIR, WALLET_CONFIG from neofs_testlib.cli import NeofsCli +from neofs_testlib.env.env import NeoFSEnv from neofs_testlib.shell import Shell logger = logging.getLogger("NeoLogger") @@ -21,7 +23,8 @@ def get_object_from_random_node( cid: str, oid: str, shell: Shell, - cluster: Cluster, + cluster: Optional[Cluster] = None, + neofs_env: Optional[NeoFSEnv] = None, bearer: Optional[str] = None, write_object: Optional[str] = None, xhdr: Optional[dict] = None, @@ -47,7 +50,10 @@ def get_object_from_random_node( Returns: (str): path to downloaded file """ - endpoint = cluster.get_random_storage_rpc_endpoint() + if cluster: + endpoint = cluster.get_random_storage_rpc_endpoint() + if neofs_env: + endpoint = random.choice(neofs_env.storage_nodes).endpoint return get_object( wallet, cid, @@ -169,7 +175,8 @@ def put_object_to_random_node( path: str, cid: str, shell: Shell, - cluster: Cluster, + cluster: Optional[Cluster] = None, + neofs_env: Optional[NeoFSEnv] = None, bearer: Optional[str] = None, attributes: Optional[dict] = None, xhdr: Optional[dict] = None, @@ -188,9 +195,9 @@ def put_object_to_random_node( cid: ID of Container where we get the Object from shell: executor for cli command cluster: cluster under test + neofs_env: neofs env under test bearer: path to Bearer Token file, appends to `--bearer` key attributes: User attributes in form of Key1=Value1,Key2=Value2 - cluster: cluster under test wallet_config: path to the wallet config no_progress: do not show progress bar lifetime: Lock lifetime - relative to the current epoch. @@ -201,7 +208,10 @@ def put_object_to_random_node( ID of uploaded Object """ - endpoint = cluster.get_random_storage_rpc_endpoint() + if cluster: + endpoint = cluster.get_random_storage_rpc_endpoint() + if neofs_env: + endpoint = random.choice(neofs_env.storage_nodes).endpoint return put_object( wallet, path, diff --git a/venv/no-dev-env-pytest/environment.sh b/venv/no-dev-env-pytest/environment.sh index e71795f61..38b40c702 100644 --- a/venv/no-dev-env-pytest/environment.sh +++ b/venv/no-dev-env-pytest/environment.sh @@ -1,5 +1,6 @@ # DevEnv variables export NEOFS_MORPH_DISABLE_CACHE=true +export WALLET_PASS=password popd > /dev/null export PYTHONPATH=${PYTHONPATH}:${VIRTUAL_ENV}/../robot/resources/lib/:${VIRTUAL_ENV}/../robot/resources/lib/python_keywords:${VIRTUAL_ENV}/../robot/resources/lib/robot:${VIRTUAL_ENV}/../robot/variables:${VIRTUAL_ENV}/../pytest_tests/helpers:${VIRTUAL_ENV}/../pytest_tests/steps:${VIRTUAL_ENV}/../pytest_tests/resources:${VIRTUAL_ENV}/../dynamic_env_pytest_tests/lib