From 39b21a387212cabdecf97d61c70d94bd4f41b59b Mon Sep 17 00:00:00 2001 From: Evgeniy Zayats Date: Fri, 10 May 2024 20:59:01 -0400 Subject: [PATCH 1/3] .git: add neofs-testcases submodule Signed-off-by: Evgeniy Zayats --- .gitmodules | 3 +++ neofs-testcases | 1 + 2 files changed, 4 insertions(+) create mode 100644 .gitmodules create mode 160000 neofs-testcases diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000..150cde6c6 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "neofs-testcases"] + path = neofs-testcases + url = https://github.com/nspcc-dev/neofs-testcases.git diff --git a/neofs-testcases b/neofs-testcases new file mode 160000 index 000000000..fa039ce97 --- /dev/null +++ b/neofs-testcases @@ -0,0 +1 @@ +Subproject commit fa039ce97aa4c9213de7c80892f881bc960aa210 From 10fb532ef64d968d89666a9f9105f2efbbf0c2e0 Mon Sep 17 00:00:00 2001 From: Evgeniy Zayats Date: Sat, 11 May 2024 21:12:31 -0400 Subject: [PATCH 2/3] tests: start to use new dev env Signed-off-by: Evgeniy Zayats --- bearer_rules.json | 81 ++++++++++++++++++ conftest.py | 192 +++++++++++++++++++++++++++++++++++++++++- container_policy.json | 5 ++ pytest.ini | 6 ++ requirements.txt | 26 ++++-- s3tests.conf.SAMPLE | 69 ++++----------- 6 files changed, 319 insertions(+), 60 deletions(-) create mode 100644 bearer_rules.json create mode 100644 container_policy.json diff --git a/bearer_rules.json b/bearer_rules.json new file mode 100644 index 000000000..8e9197a8c --- /dev/null +++ b/bearer_rules.json @@ -0,0 +1,81 @@ +{ + "records": [ + { + "operation": "PUT", + "action": "ALLOW", + "filters": [], + "targets": [ + { + "role": "OTHERS", + "keys": [] + } + ] + }, + { + "operation": "GET", + "action": "ALLOW", + "filters": [], + "targets": [ + { + "role": "OTHERS", + "keys": [] + } + ] + }, + { + "operation": "DELETE", + "action": "ALLOW", + "filters": [], + "targets": [ + { + "role": "OTHERS", + "keys": [] + } + ] + }, + { + "operation": "SEARCH", + "action": "ALLOW", + "filters": [], + "targets": [ + { + "role": "OTHERS", + "keys": [] + } + ] + }, + { + "operation": "GETRANGE", + "action": "ALLOW", + "filters": [], + "targets": [ + { + "role": "OTHERS", + "keys": [] + } + ] + }, + { + "operation": "GETRANGEHASH", + "action": "ALLOW", + "filters": [], + "targets": [ + { + "role": "OTHERS", + "keys": [] + } + ] + }, + { + "operation": "HEAD", + "action": "ALLOW", + "filters": [], + "targets": [ + { + "role": "OTHERS", + "keys": [] + } + ] + } + ] + } diff --git a/conftest.py b/conftest.py index 7fd8c978d..cf580f9c8 100644 --- a/conftest.py +++ b/conftest.py @@ -1,9 +1,52 @@ import configparser +import json +import logging +import os +import re +import shutil +import sys +import time +from pathlib import Path +import allure +import jinja2 +import pexpect import pytest +from helpers.common import ASSETS_DIR +from helpers.wallet_helpers import create_wallet +from neofs_testlib.env.env import NeoFSEnv, NodeWallet +from neofs_testlib.reporter import AllureHandler, get_reporter +from neofs_testlib.utils.wallet import ( + get_last_address_from_wallet, + get_last_public_key_from_wallet, +) from s3tests_boto3.functional import setup +get_reporter().register_handler(AllureHandler()) +logger = logging.getLogger("NeoLogger") + + +def pytest_addoption(parser): + parser.addoption( + "--persist-env", action="store_true", default=False, help="persist deployed env" + ) + parser.addoption("--load-env", action="store", help="load persisted env from file") + + +def _run_with_passwd(cmd: str, password: str) -> str: + child = pexpect.spawn(cmd) + child.delaybeforesend = 1 + child.expect(".*") + child.sendline(f"{password}\r") + if sys.platform == "darwin": + child.expect(pexpect.EOF) + cmd = child.before + else: + child.wait() + cmd = child.read() + return cmd.decode() + def read_config(config_file): config = configparser.ConfigParser() @@ -11,6 +54,153 @@ def read_config(config_file): return config +@allure.step("Init S3 Credentials") +def init_s3_credentials( + wallet: NodeWallet, + neofs_env: NeoFSEnv, +) -> tuple: + gate_public_key = get_last_public_key_from_wallet( + neofs_env.s3_gw.wallet.path, neofs_env.s3_gw.wallet.password + ) + + cmd = ( + f"{neofs_env.neofs_s3_authmate_path} --debug --with-log --timeout 1m " + f"issue-secret --wallet {wallet.path} --gate-public-key={gate_public_key} " + f"--peer {neofs_env.storage_nodes[0].endpoint} " + f"--bearer-rules {os.getcwd()}/bearer_rules.json --container-placement-policy 'REP 1' " + f"--container-policy {os.getcwd()}/container_policy.json" + ) + + logger.info(f"Executing command: {cmd}") + + try: + output = _run_with_passwd(cmd, wallet.password) + + logger.info(f"output: {output}") + + # output contains some debug info and then several JSON structures, so we find each + # JSON structure by curly brackets (naive approach, but works while JSON is not nested) + # and then we take JSON containing secret_access_key + json_blocks = re.findall(r"\{.*?\}", output, re.DOTALL) + for json_block in json_blocks: + try: + parsed_json_block = json.loads(json_block) + if "secret_access_key" in parsed_json_block: + return ( + parsed_json_block["access_key_id"], + parsed_json_block["secret_access_key"], + parsed_json_block["wallet_public_key"], + get_last_address_from_wallet(wallet.path, wallet.password), + ) + except json.JSONDecodeError: + raise AssertionError(f"Could not parse info from output\n{output}") + raise AssertionError(f"Could not find AWS credentials in output:\n{output}") + except Exception as exc: + raise RuntimeError( + f"Failed to init s3 credentials because of error\n{exc}" + ) from exc + + +def create_dir(dir_path: str) -> None: + with allure.step("Create directory"): + remove_dir(dir_path) + os.mkdir(dir_path) + + +def remove_dir(dir_path: str) -> None: + with allure.step("Remove directory"): + shutil.rmtree(dir_path, ignore_errors=True) + + +@pytest.fixture(scope="session") +@allure.title("Prepare tmp directory") +def temp_directory() -> str: + with allure.step("Prepare tmp directory"): + full_path = os.path.join(os.getcwd(), ASSETS_DIR) + create_dir(full_path) + + yield full_path + + with allure.step("Remove tmp directory"): + remove_dir(full_path) + + +@pytest.fixture(scope="session", autouse=True) +def neofs_setup(request, temp_directory): + if request.config.getoption("--load-env"): + neofs_env = NeoFSEnv.load(request.config.getoption("--load-env")) + else: + neofs_env = NeoFSEnv.simple() + + neofs_env.neofs_adm().morph.set_config( + rpc_endpoint=f"http://{neofs_env.morph_rpc}", + alphabet_wallets=neofs_env.alphabet_wallets_dir, + post_data="ContainerFee=0 ContainerAliasFee=0 MaxObjectSize=524288", + ) + time.sleep(30) + + main_wallet = create_wallet() + + ( + main_access_key_id, + main_secret_access_key, + main_wallet_public_key, + main_wallet_address, + ) = init_s3_credentials(main_wallet, neofs_env) + + alt_wallet = create_wallet() + + (alt_access_key_id, alt_secret_access_key, alt_wallet_public_key, _) = ( + init_s3_credentials(alt_wallet, neofs_env) + ) + + jinja_env = jinja2.Environment() + config_template = Path("s3tests.conf.SAMPLE").read_text() + jinja_template = jinja_env.from_string(config_template) + rendered_config = jinja_template.render( + S3_HOST=neofs_env.s3_gw.address.split(":")[0], + S3_PORT=neofs_env.s3_gw.address.split(":")[1], + S3_TLS=True, + S3_MAIN_DISPLAY_NAME=main_wallet_address, + S3_MAIN_USER_ID=main_wallet_public_key, + S3_MAIN_ACCESS_KEY=main_access_key_id, + S3_MAIN_SECRET_KEY=main_secret_access_key, + S3_ALT_USER_ID=alt_wallet_public_key, + S3_ALT_ACCESS_KEY=alt_access_key_id, + S3_ALT_SECRET_KEY=alt_secret_access_key, + S3_TENANT_USER_ID=alt_wallet_public_key, + S3_TENANT_ACCESS_KEY=alt_access_key_id, + S3_TENANT_SECRET_KEY=alt_secret_access_key, + S3_IAM_USER_ID=alt_wallet_public_key, + S3_IAM_ACCESS_KEY=alt_access_key_id, + S3_IAM_SECRET_KEY=alt_secret_access_key, + ) + with open("s3tests.conf", mode="w") as fp: + fp.write(rendered_config) + + os.environ["S3TEST_CONF"] = "s3tests.conf" + + yield neofs_env + + if request.config.getoption("--persist-env"): + neofs_env.persist() + else: + if not request.config.getoption("--load-env"): + neofs_env.kill() + + logs_path = os.path.join(os.getcwd(), ASSETS_DIR, "logs") + os.makedirs(logs_path, exist_ok=True) + + shutil.copyfile(neofs_env.s3_gw.stderr, f"{logs_path}/s3_gw_log.txt") + for idx, ir in enumerate(neofs_env.inner_ring_nodes): + shutil.copyfile(ir.stderr, f"{logs_path}/ir_{idx}_log.txt") + for idx, sn in enumerate(neofs_env.storage_nodes): + shutil.copyfile(sn.stderr, f"{logs_path}/sn_{idx}_log.txt") + + logs_zip_file_path = shutil.make_archive("neofs_logs", "zip", logs_path) + allure.attach.file(logs_zip_file_path, name="neofs logs", extension="zip") + + @pytest.fixture(scope="session", autouse=True) -def nose_setup(): +def nose_setup(neofs_setup): setup() diff --git a/container_policy.json b/container_policy.json new file mode 100644 index 000000000..dac24fa45 --- /dev/null +++ b/container_policy.json @@ -0,0 +1,5 @@ +{ + "rep-1": "REP 1", + "rep-3": "REP 3", + "complex": "REP 1 IN X CBF 1 SELECT 1 FROM * AS X" +} diff --git a/pytest.ini b/pytest.ini index 0e3bcba21..afe1d6057 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,4 +1,10 @@ [pytest] +log_cli = 1 +log_cli_level = debug +log_cli_format = [%(threadName)s] %(asctime)s [%(levelname)4s] %(message)s +log_format = [%(threadName)s] %(asctime)s [%(levelname)4s] %(message)s [%(threadName)s] +log_cli_date_format = %Y-%m-%d %H:%M:%S +log_date_format = %H:%M:%S markers = abac_test appendobject diff --git a/requirements.txt b/requirements.txt index d1f107d26..28fafe315 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,18 +1,32 @@ PyYAML boto >=2.6.0 -boto3 >=1.0.0 -botocore munch >=2.0.0 # 0.14 switches to libev, that means bootstrap needs to change too gevent >=1.0 isodate >=0.4.4 -requests >=2.23.0 pytz >=2011k httplib2 lxml tox -pytest ==7.3.1 -allure-pytest ==2.9.45 pytest-nose-attrib ==0.1.1 pytest-timeout ==2.1.0 -allure-combine ==1.0.8 +allure-combine==1.0.11 +allure-pytest==2.13.4 +base58==2.1.1 +black==24.3.0 +botocore==1.34.72 +boto3==1.34.72 +docker==7.0.0 +isort==5.13.2 +jinja2==3.1.4 +neo-mamba==2.6.0 +paramiko==3.4.0 +pexpect==4.9.0 +pytest==7.3.1 +PyYAML==6.0.1 +python-dotenv==1.0.1 +python-dateutil==2.9.0 +pytest-lazy-fixture==0.6.3 +requests==2.31.0 +tenacity==8.2.3 +urllib3==2.2.1 diff --git a/s3tests.conf.SAMPLE b/s3tests.conf.SAMPLE index 9593fc10a..62491480a 100644 --- a/s3tests.conf.SAMPLE +++ b/s3tests.conf.SAMPLE @@ -2,13 +2,13 @@ ## this section is just used for host, port and bucket_prefix # host set for rgw in vstart.sh -host = localhost +host = {{ S3_HOST }} # port set for rgw in vstart.sh -port = 8000 +port = {{ S3_PORT }} ## say "False" to disable TLS -is_secure = False +is_secure = {{ S3_TLS }} ## say "False" to disable SSL Verify ssl_verify = False @@ -21,10 +21,10 @@ bucket prefix = yournamehere-{random}- [s3 main] # main display_name set in vstart.sh -display_name = M. Tester +display_name = {{ S3_MAIN_DISPLAY_NAME }} # main user_idname set in vstart.sh -user_id = testid +user_id = {{ S3_MAIN_USER_ID }} # main email set in vstart.sh email = tester@ceph.com @@ -33,20 +33,14 @@ email = tester@ceph.com api_name = default ## main AWS access key -access_key = 0555b35654ad1656d804 +access_key = {{ S3_MAIN_ACCESS_KEY }} ## main AWS secret key -secret_key = h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q== +secret_key = {{ S3_MAIN_SECRET_KEY }} ## replace with key id obtained when secret is created, or delete if KMS not tested #kms_keyid = 01234567-89ab-cdef-0123-456789abcdef -## Storage classes -#storage_classes = "LUKEWARM, FROZEN" - -## Lifecycle debug interval (default: 10) -#lc_debug_interval = 20 - [s3 alt] # alt display_name set in vstart.sh display_name = john.doe @@ -54,57 +48,26 @@ display_name = john.doe email = john.doe@example.com # alt user_id set in vstart.sh -user_id = 56789abcdef0123456789abcdef0123456789abcdef0123456789abcdef01234 +user_id = {{ S3_ALT_USER_ID }} # alt AWS access key set in vstart.sh -access_key = NOPQRSTUVWXYZABCDEFG +access_key = {{ S3_ALT_ACCESS_KEY }} # alt AWS secret key set in vstart.sh -secret_key = nopqrstuvwxyzabcdefghijklmnabcdefghijklm - -#[s3 cloud] -## to run the testcases with "cloud_transition" attribute. -## Note: the waiting time may have to tweaked depending on -## the I/O latency to the cloud endpoint. - -## host set for cloud endpoint -# host = localhost - -## port set for cloud endpoint -# port = 8001 - -## say "False" to disable TLS -# is_secure = False - -## cloud endpoint credentials -# access_key = 0555b35654ad1656d804 -# secret_key = h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q== - -## storage class configured as cloud tier on local rgw server -# cloud_storage_class = CLOUDTIER - -## Below are optional - - -## Above configured cloud storage class config options -# retain_head_object = false -# target_storage_class = Target_SC -# target_path = cloud-bucket - -## another regular storage class to test multiple transition rules, -# storage_class = S1 +secret_key = {{ S3_ALT_SECRET_KEY }} [s3 tenant] # tenant display_name set in vstart.sh display_name = testx$tenanteduser # tenant user_id set in vstart.sh -user_id = 9876543210abcdef0123456789abcdef0123456789abcdef0123456789abcdef +user_id = {{ S3_TENANT_USER_ID }} # tenant AWS secret key set in vstart.sh -access_key = HIJKLMNOPQRSTUVWXYZA +access_key = {{ S3_TENANT_ACCESS_KEY }} # tenant AWS secret key set in vstart.sh -secret_key = opqrstuvwxyzabcdefghijklmnopqrstuvwxyzab +secret_key = {{ S3_TENANT_SECRET_KEY }} # tenant email set in vstart.sh email = tenanteduser@example.com @@ -116,13 +79,13 @@ email = tenanteduser@example.com email = s3@example.com #user_id from vstart.sh -user_id = 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef +user_id = {{ S3_IAM_USER_ID }} #access_key from vstart.sh -access_key = ABCDEFGHIJKLMNOPQRST +access_key = {{ S3_IAM_ACCESS_KEY }} #secret_key vstart.sh -secret_key = abcdefghijklmnopqrstuvwxyzabcdefghijklmn +secret_key = {{ S3_IAM_SECRET_KEY }} #display_name from vstart.sh display_name = youruseridhere From 260c39f9bffb5f791c63d7c6abbeec0d862981f8 Mon Sep 17 00:00:00 2001 From: Evgeniy Zayats Date: Sat, 11 May 2024 21:12:48 -0400 Subject: [PATCH 3/3] tests: update tests due to latest s3 gw updates Signed-off-by: Evgeniy Zayats --- s3tests_boto3/functional/test_s3.py | 223 ++++------------------ s3tests_boto3/functional/test_s3_neofs.py | 8 +- 2 files changed, 39 insertions(+), 192 deletions(-) diff --git a/s3tests_boto3/functional/test_s3.py b/s3tests_boto3/functional/test_s3.py index 756fe823f..5b786a8f9 100644 --- a/s3tests_boto3/functional/test_s3.py +++ b/s3tests_boto3/functional/test_s3.py @@ -4,6 +4,7 @@ import hashlib import hmac import json +import logging import os import random import re @@ -69,6 +70,8 @@ generate_random, ) +logger = logging.getLogger("NeoLogger") + @allure.step("Check Bucket is Empty") def _bucket_is_empty(bucket): @@ -2020,7 +2023,7 @@ def do_request(n): thr.start() t.append(thr) _do_wait_completion(t) - + response = client.list_objects(Bucket=bucket.name) assert "Contents" not in response @@ -2090,7 +2093,6 @@ def test_multi_object_delete_key_limit(): assert status == 400 -@pytest.mark.skip(reason="https://github.com/nspcc-dev/neofs-s3-gw/issues/891") def test_multi_objectv2_delete_key_limit(): key_names = [f"key-{i}" for i in range(1001)] bucket_name = _create_objects(keys=key_names) @@ -2120,13 +2122,12 @@ def test_object_head_zero_bytes(): assert response["ContentLength"] == 0 -@pytest.mark.skip(reason="https://github.com/nspcc-dev/neofs-s3-gw/issues/845") def test_object_write_check_etag(): bucket_name = get_new_bucket() client = get_client() response = client.put_object(Bucket=bucket_name, Key="foo", Body="bar") assert response["ResponseMetadata"]["HTTPStatusCode"] == 200 - assert response["ETag"] == '"37b51d194a7513e45b56f6524f2d51f2"' + assert response["ETag"] != '' def test_object_write_cache_control(): @@ -2299,7 +2300,6 @@ def _get_post_url(bucket_name): return "{endpoint}/{bucket_name}".format(endpoint=endpoint, bucket_name=bucket_name) -@pytest.mark.skip(reason="https://github.com/nspcc-dev/neofs-s3-gw/issues/847") def test_post_object_anonymous_request(): bucket_name = get_new_bucket_name() client = get_client() @@ -2315,10 +2315,7 @@ def test_post_object_anonymous_request(): client.create_bucket(ACL="public-read-write", Bucket=bucket_name) r = requests.post(url, files=payload, verify=get_config_ssl_verify()) - assert r.status_code == 204 - response = client.get_object(Bucket=bucket_name, Key="foo.txt") - body = _get_body(response) - assert body == "bar" + assert r.status_code == 501 @pytest.mark.skip(reason="https://github.com/nspcc-dev/s3-tests/issues/46") @@ -2466,7 +2463,6 @@ def test_post_object_authenticated_request_bad_access_key(): assert r.status_code == 403 -@pytest.mark.skip(reason="https://github.com/nspcc-dev/neofs-s3-gw/issues/847") def test_post_object_set_success_code(): bucket_name = get_new_bucket_name() client = get_client() @@ -2484,12 +2480,9 @@ def test_post_object_set_success_code(): ) r = requests.post(url, files=payload, verify=get_config_ssl_verify()) - assert r.status_code == 201 - message = ET.fromstring(r.content).find("Key") - assert message.text == "foo.txt" + assert r.status_code == 501 -@pytest.mark.skip(reason="https://github.com/nspcc-dev/neofs-s3-gw/issues/847") def test_post_object_set_invalid_success_code(): bucket_name = get_new_bucket_name() client = get_client() @@ -2507,9 +2500,7 @@ def test_post_object_set_invalid_success_code(): ) r = requests.post(url, files=payload, verify=get_config_ssl_verify()) - assert r.status_code == 204 - content = r.content.decode() - assert content == "" + assert r.status_code == 501 @pytest.mark.skip(reason="https://github.com/nspcc-dev/s3-tests/issues/46") @@ -4157,7 +4148,6 @@ def test_bucket_head_extended(): assert int(response["ResponseMetadata"]["HTTPHeaders"]["x-rgw-bytes-used"]) == 9 -@pytest.mark.skip(reason="https://github.com/nspcc-dev/neofs-s3-gw/issues/852") def test_object_raw_get_bucket_acl(): bucket_name = _setup_bucket_object_acl("private", "public-read") @@ -4166,7 +4156,6 @@ def test_object_raw_get_bucket_acl(): assert response["ResponseMetadata"]["HTTPStatusCode"] == 200 -@pytest.mark.skip(reason="https://github.com/nspcc-dev/neofs-s3-gw/issues/852") def test_object_raw_get_object_acl(): bucket_name = _setup_bucket_object_acl("public-read", "private") @@ -4306,12 +4295,10 @@ def _test_object_raw_get_x_amz_expires_not_expired(client): assert res["status_code"] == 200 -@pytest.mark.skip(reason="https://github.com/nspcc-dev/neofs-s3-gw/issues/852") def test_object_raw_get_x_amz_expires_not_expired(): _test_object_raw_get_x_amz_expires_not_expired(client=get_client()) -@pytest.mark.skip(reason="https://github.com/nspcc-dev/neofs-s3-gw/issues/852") def test_object_raw_get_x_amz_expires_not_expired_tenant(): _test_object_raw_get_x_amz_expires_not_expired(client=get_tenant_client()) @@ -4329,7 +4316,6 @@ def test_object_raw_get_x_amz_expires_out_range_zero(): assert res["status_code"] == 403 -@pytest.mark.skip(reason="https://github.com/nspcc-dev/neofs-s3-gw/issues/853") def test_object_raw_get_x_amz_expires_out_max_range(): bucket_name = _setup_bucket_object_acl("public-read", "public-read") client = get_client() @@ -4340,7 +4326,7 @@ def test_object_raw_get_x_amz_expires_out_max_range(): ) res = requests.get(url, verify=get_config_ssl_verify()).__dict__ - assert res["status_code"] == 403 + assert res["status_code"] == 400 def test_object_raw_get_x_amz_expires_out_positive_range(): @@ -4376,7 +4362,6 @@ def test_object_anon_put(): assert error_code == "AccessDenied" -@pytest.mark.skip(reason="https://github.com/nspcc-dev/neofs-s3-gw/issues/852") def test_object_anon_put_write_access(): bucket_name = _setup_bucket_acl("public-read-write") client = get_client() @@ -4737,7 +4722,6 @@ def check_grants(got, want): assert g == {"Grantee": {}} -@pytest.mark.skip(reason="https://github.com/nspcc-dev/neofs-s3-gw/issues/854") def test_bucket_acl_default(): bucket_name = get_new_bucket() client = get_client() @@ -4851,7 +4835,6 @@ def test_bucket_acl_canned(): ) -@pytest.mark.skip(reason="https://github.com/nspcc-dev/neofs-s3-gw/issues/857") def test_bucket_acl_canned_publicreadwrite(): bucket_name = get_new_bucket_name() client = get_client() @@ -4892,7 +4875,6 @@ def test_bucket_acl_canned_publicreadwrite(): ) -@pytest.mark.skip(reason="https://github.com/nspcc-dev/neofs-s3-gw/issues/858") def test_bucket_acl_canned_authenticatedread(): bucket_name = get_new_bucket_name() client = get_client() @@ -4926,7 +4908,6 @@ def test_bucket_acl_canned_authenticatedread(): ) -@pytest.mark.skip(reason="https://github.com/nspcc-dev/neofs-s3-gw/issues/859") def test_object_acl_default(): bucket_name = get_new_bucket() client = get_client() @@ -4953,7 +4934,6 @@ def test_object_acl_default(): ) -@pytest.mark.skip(reason="https://github.com/nspcc-dev/neofs-s3-gw/issues/895") def test_object_acl_canned_during_create(): bucket_name = get_new_bucket() client = get_client() @@ -4988,7 +4968,6 @@ def test_object_acl_canned_during_create(): ) -@pytest.mark.skip(reason="https://github.com/nspcc-dev/neofs-s3-gw/issues/895") def test_object_acl_canned(): bucket_name = get_new_bucket() client = get_client() @@ -5043,7 +5022,6 @@ def test_object_acl_canned(): ) -@pytest.mark.skip(reason="https://github.com/nspcc-dev/neofs-s3-gw/issues/895") def test_object_acl_canned_publicreadwrite(): bucket_name = get_new_bucket() client = get_client() @@ -5088,7 +5066,6 @@ def test_object_acl_canned_publicreadwrite(): ) -@pytest.mark.skip(reason="https://github.com/nspcc-dev/neofs-s3-gw/issues/858") def test_object_acl_canned_authenticatedread(): bucket_name = get_new_bucket() client = get_client() @@ -5125,7 +5102,6 @@ def test_object_acl_canned_authenticatedread(): ) -@pytest.mark.skip(reason="https://github.com/nspcc-dev/neofs-s3-gw/issues/895") def test_object_acl_canned_bucketownerread(): bucket_name = get_new_bucket_name() main_client = get_client() @@ -5135,43 +5111,11 @@ def test_object_acl_canned_bucketownerread(): alt_client.put_object(Bucket=bucket_name, Key="foo", Body="bar") - bucket_acl_response = main_client.get_bucket_acl(Bucket=bucket_name) - bucket_owner_id = bucket_acl_response["Grants"][2]["Grantee"]["ID"] - bucket_owner_display_name = bucket_acl_response["Grants"][2]["Grantee"][ - "DisplayName" - ] + # not supported right now + with pytest.raises(ClientError): + alt_client.put_object(ACL="bucket-owner-read", Bucket=bucket_name, Key="foo") - alt_client.put_object(ACL="bucket-owner-read", Bucket=bucket_name, Key="foo") - response = alt_client.get_object_acl(Bucket=bucket_name, Key="foo") - - alt_display_name = get_alt_display_name() - alt_user_id = get_alt_user_id() - grants = response["Grants"] - check_grants( - grants, - [ - dict( - Permission="FULL_CONTROL", - ID=alt_user_id, - DisplayName=alt_display_name, - URI=None, - EmailAddress=None, - Type="CanonicalUser", - ), - dict( - Permission="READ", - ID=bucket_owner_id, - DisplayName=bucket_owner_display_name, - URI=None, - EmailAddress=None, - Type="CanonicalUser", - ), - ], - ) - - -@pytest.mark.skip(reason="https://github.com/nspcc-dev/neofs-s3-gw/issues/895") def test_object_acl_canned_bucketownerfullcontrol(): bucket_name = get_new_bucket_name() main_client = get_client() @@ -5181,42 +5125,11 @@ def test_object_acl_canned_bucketownerfullcontrol(): alt_client.put_object(Bucket=bucket_name, Key="foo", Body="bar") - bucket_acl_response = main_client.get_bucket_acl(Bucket=bucket_name) - bucket_owner_id = bucket_acl_response["Grants"][2]["Grantee"]["ID"] - bucket_owner_display_name = bucket_acl_response["Grants"][2]["Grantee"][ - "DisplayName" - ] - - alt_client.put_object( - ACL="bucket-owner-full-control", Bucket=bucket_name, Key="foo" - ) - response = alt_client.get_object_acl(Bucket=bucket_name, Key="foo") - - alt_display_name = get_alt_display_name() - alt_user_id = get_alt_user_id() - - grants = response["Grants"] - check_grants( - grants, - [ - dict( - Permission="FULL_CONTROL", - ID=alt_user_id, - DisplayName=alt_display_name, - URI=None, - EmailAddress=None, - Type="CanonicalUser", - ), - dict( - Permission="FULL_CONTROL", - ID=bucket_owner_id, - DisplayName=bucket_owner_display_name, - URI=None, - EmailAddress=None, - Type="CanonicalUser", - ), - ], - ) + # not supported right now + with pytest.raises(ClientError): + alt_client.put_object( + ACL="bucket-owner-full-control", Bucket=bucket_name, Key="foo" + ) @pytest.mark.fails_on_aws @@ -5925,7 +5838,6 @@ def test_bucket_acl_grant_email(): ) -@pytest.mark.skip(reason="https://github.com/nspcc-dev/neofs-s3-gw/issues/908") def test_bucket_acl_grant_email_not_exist(): # behavior not documented by amazon bucket_name = get_new_bucket() @@ -5950,8 +5862,8 @@ def test_bucket_acl_grant_email_not_exist(): AccessControlPolicy=grant, ) status, error_code = _get_status_and_error_code(e.response) - assert status == 400 - assert error_code == "UnresolvableGrantByEmailAddress" + assert status == 501 + assert error_code == "BadRequest" @pytest.mark.skip(reason="https://github.com/nspcc-dev/neofs-s3-gw/issues/861") @@ -5983,7 +5895,6 @@ def test_bucket_acl_revoke_all(): # TODO rgw log_bucket.set_as_logging_target() gives 403 Forbidden # http://tracker.newdream.net/issues/984 @pytest.mark.fails_on_rgw -@pytest.mark.skip(reason="https://github.com/nspcc-dev/neofs-s3-gw/issues/907") def test_logging_toggle(): bucket_name = get_new_bucket() client = get_client() @@ -6008,11 +5919,8 @@ def test_logging_toggle(): } } - client.put_bucket_logging(Bucket=bucket_name, BucketLoggingStatus=status) - client.get_bucket_logging(Bucket=bucket_name) - status = {"LoggingEnabled": {}} - client.put_bucket_logging(Bucket=bucket_name, BucketLoggingStatus=status) - # NOTE: this does not actually test whether or not logging works + with pytest.raises(ClientError): + client.put_bucket_logging(Bucket=bucket_name, BucketLoggingStatus=status) @allure.step("Setup Access") @@ -7552,7 +7460,6 @@ def test_multipart_upload_resend_part(): _check_upload_multipart_resend(bucket_name, key, objlen, [0, 1, 2, 3, 4, 5]) -@pytest.mark.skip(reason="https://github.com/nspcc-dev/neofs-s3-gw/issues/843") def test_multipart_upload_multiple_sizes(): bucket_name = get_new_bucket() key = "mymultipart" @@ -8029,7 +7936,10 @@ def _simple_http_req_100_cont(host, port, is_secure, method, resource): s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) if is_secure: - s = ssl.wrap_socket(s) + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2) + context.verify_mode = ssl.CERT_NONE + context.load_default_certs() + s = context.wrap_socket(s) s.settimeout(5) s.connect((host, port)) s.send(req) @@ -8111,7 +8021,6 @@ def _cors_request_and_check( assert r.headers.get("access-control-allow-methods", None) == expect_allow_methods -@pytest.mark.skip(reason="https://github.com/nspcc-dev/neofs-s3-gw/issues/841") def test_cors_origin_response(): bucket_name = _setup_bucket_acl(bucket_acl="public-read") client = get_client() @@ -8347,7 +8256,6 @@ def test_cors_origin_response(): ) -@pytest.mark.skip(reason="https://github.com/nspcc-dev/neofs-s3-gw/issues/841") def test_cors_origin_wildcard(): bucket_name = _setup_bucket_acl(bucket_acl="public-read") client = get_client() @@ -8377,7 +8285,6 @@ def test_cors_origin_wildcard(): ) -@pytest.mark.skip(reason="https://github.com/nspcc-dev/neofs-s3-gw/issues/842") def test_cors_header_option(): bucket_name = _setup_bucket_acl(bucket_acl="public-read") client = get_client() @@ -8463,7 +8370,6 @@ def test_cors_presigned_get_object_tenant(): @pytest.mark.tagging -@pytest.mark.skip(reason="https://github.com/nspcc-dev/neofs-s3-gw/issues/898") def test_set_bucket_tagging(): bucket_name = get_new_bucket() client = get_client() @@ -8621,17 +8527,14 @@ def _test_atomic_read(file_size): _verify_atomic_key_data(bucket_name, "testobj", file_size, "B") -@pytest.mark.skip(reason="https://github.com/nspcc-dev/neofs-s3-gw/issues/885") def test_atomic_read_1mb(): _test_atomic_read(1024 * 1024) -@pytest.mark.skip(reason="https://github.com/nspcc-dev/neofs-s3-gw/issues/885") def test_atomic_read_4mb(): _test_atomic_read(1024 * 1024 * 4) -@pytest.mark.skip(reason="https://github.com/nspcc-dev/neofs-s3-gw/issues/885") def test_atomic_read_8mb(): _test_atomic_read(1024 * 1024 * 8) @@ -8670,17 +8573,14 @@ def _test_atomic_write(file_size): _verify_atomic_key_data(bucket_name, objname, file_size, "B") -@pytest.mark.skip(reason="https://github.com/nspcc-dev/neofs-s3-gw/issues/885") def test_atomic_write_1mb(): _test_atomic_write(1024 * 1024) -@pytest.mark.skip(reason="https://github.com/nspcc-dev/neofs-s3-gw/issues/885") def test_atomic_write_4mb(): _test_atomic_write(1024 * 1024 * 4) -@pytest.mark.skip(reason="https://github.com/nspcc-dev/neofs-s3-gw/issues/885") def test_atomic_write_8mb(): _test_atomic_write(1024 * 1024 * 8) @@ -8711,17 +8611,14 @@ def rewind_put_fp_a(): _verify_atomic_key_data(bucket_name, objname, file_size, "B") -@pytest.mark.skip(reason="https://github.com/nspcc-dev/neofs-s3-gw/issues/885") def test_atomic_dual_write_1mb(): _test_atomic_dual_write(1024 * 1024) -@pytest.mark.skip(reason="https://github.com/nspcc-dev/neofs-s3-gw/issues/885") def test_atomic_dual_write_4mb(): _test_atomic_dual_write(1024 * 1024 * 4) -@pytest.mark.skip(reason="https://github.com/nspcc-dev/neofs-s3-gw/issues/885") def test_atomic_dual_write_8mb(): _test_atomic_dual_write(1024 * 1024 * 8) @@ -8871,7 +8768,6 @@ def trigger(self): self.result = self.action() -@pytest.mark.skip(reason="https://github.com/nspcc-dev/neofs-s3-gw/issues/901") def test_multipart_resend(): bucket_name = get_new_bucket() client = get_client() @@ -9628,7 +9524,6 @@ def test_versioning_multi_object_delete_with_marker_create(): assert key == delete_markers[0]["Key"] -@pytest.mark.skip(reason="https://github.com/nspcc-dev/neofs-s3-gw/issues/897") def test_versioned_object_acl(): bucket_name = get_new_bucket() client = get_client() @@ -12130,7 +12025,6 @@ def test_encryption_sse_c_multipart_invalid_chunks_2(): @pytest.mark.encryption -@pytest.mark.skip(reason="https://github.com/nspcc-dev/neofs-s3-gw/issues/843") def test_encryption_sse_c_multipart_bad_download(): bucket_name = get_new_bucket() client = get_client() @@ -13469,7 +13363,6 @@ def test_versioning_bucket_atomic_upload_return_version_id(): assert not "VersionId" in response -@pytest.mark.skip(reason="https://github.com/nspcc-dev/neofs-s3-gw/issues/843") def test_versioning_bucket_multipart_upload_return_version_id(): content_type = "text/bla" objlen = 30 * 1024 * 1024 @@ -13982,7 +13875,6 @@ def remove_header(**kwargs): @pytest.mark.encryption -@pytest.mark.skip(reason="https://github.com/nspcc-dev/neofs-s3-gw/issues/850") def test_put_obj_enc_conflict_c_s3(): bucket_name = get_new_bucket() client = get_v2_client() @@ -14003,11 +13895,10 @@ def test_put_obj_enc_conflict_c_s3(): e = assert_raises(ClientError, client.put_object, Bucket=bucket_name, Key=key1_str) status, error_code = _get_status_and_error_code(e.response) assert status == 400 - assert error_code == "InvalidArgument" + assert error_code == "AuthorizationQueryParametersError" @pytest.mark.encryption -@pytest.mark.skip(reason="https://github.com/nspcc-dev/neofs-s3-gw/issues/850") def test_put_obj_enc_conflict_c_kms(): kms_keyid = get_main_kms_keyid() if kms_keyid is None: @@ -14032,11 +13923,10 @@ def test_put_obj_enc_conflict_c_kms(): e = assert_raises(ClientError, client.put_object, Bucket=bucket_name, Key=key1_str) status, error_code = _get_status_and_error_code(e.response) assert status == 400 - assert error_code == "InvalidArgument" + assert error_code == "AuthorizationQueryParametersError" @pytest.mark.encryption -@pytest.mark.skip(reason="https://github.com/nspcc-dev/neofs-s3-gw/issues/850") def test_put_obj_enc_conflict_s3_kms(): kms_keyid = get_main_kms_keyid() if kms_keyid is None: @@ -14058,11 +13948,10 @@ def test_put_obj_enc_conflict_s3_kms(): e = assert_raises(ClientError, client.put_object, Bucket=bucket_name, Key=key1_str) status, error_code = _get_status_and_error_code(e.response) assert status == 400 - assert error_code == "InvalidArgument" + assert error_code == "AuthorizationQueryParametersError" @pytest.mark.encryption -@pytest.mark.skip(reason="https://github.com/nspcc-dev/neofs-s3-gw/issues/850") def test_put_obj_enc_conflict_bad_enc_kms(): kms_keyid = get_main_kms_keyid() if kms_keyid is None: @@ -14083,7 +13972,7 @@ def test_put_obj_enc_conflict_bad_enc_kms(): e = assert_raises(ClientError, client.put_object, Bucket=bucket_name, Key=key1_str) status, error_code = _get_status_and_error_code(e.response) assert status == 400 - assert error_code == "InvalidArgument" + assert error_code == "AuthorizationQueryParametersError" @pytest.mark.encryption @@ -14148,7 +14037,6 @@ def test_bucket_policy_put_obj_s3_noenc(): @pytest.mark.encryption @pytest.mark.bucket_policy @pytest.mark.sse_s3 -@pytest.mark.skip(reason="https://github.com/nspcc-dev/neofs-s3-gw/issues/850") def test_bucket_policy_put_obj_s3_kms(): kms_keyid = get_main_kms_keyid() if kms_keyid is None: @@ -14175,22 +14063,8 @@ def test_bucket_policy_put_obj_s3_kms(): # boto3.set_stream_logger(name='botocore') - client.put_bucket_policy(Bucket=bucket_name, Policy=policy_document) - key1_str = "testobj" - - # response = client.get_bucket_policy(Bucket=bucket_name) - # print response - - sse_client_headers = { - "x-amz-server-side-encryption": "aws:kms", - "x-amz-server-side-encryption-aws-kms-key-id": kms_keyid, - } - - lf = lambda **kwargs: kwargs["params"]["headers"].update(sse_client_headers) - client.meta.events.register("before-call.s3.PutObject", lf) - check_access_denied( - client.put_object, Bucket=bucket_name, Key=key1_str, Body=key1_str - ) + with pytest.raises(ClientError): + client.put_bucket_policy(Bucket=bucket_name, Policy=policy_document) @pytest.mark.encryption @@ -14255,7 +14129,6 @@ def test_bucket_policy_put_obj_kms_noenc(): @pytest.mark.encryption @pytest.mark.bucket_policy -@pytest.mark.skip(reason="https://github.com/nspcc-dev/neofs-s3-gw/issues/850") def test_bucket_policy_put_obj_kms_s3(): bucket_name = get_new_bucket() client = get_v2_client() @@ -14279,21 +14152,8 @@ def test_bucket_policy_put_obj_kms_s3(): # boto3.set_stream_logger(name='botocore') - client.put_bucket_policy(Bucket=bucket_name, Policy=policy_document) - key1_str = "testobj" - - # response = client.get_bucket_policy(Bucket=bucket_name) - # print response - - sse_client_headers = { - "x-amz-server-side-encryption": "AES256", - } - - lf = lambda **kwargs: kwargs["params"]["headers"].update(sse_client_headers) - client.meta.events.register("before-call.s3.PutObject", lf) - check_access_denied( - client.put_object, Bucket=bucket_name, Key=key1_str, Body=key1_str - ) + with pytest.raises(ClientError): + client.put_bucket_policy(Bucket=bucket_name, Policy=policy_document) @pytest.mark.tagging @@ -14444,7 +14304,6 @@ def test_object_lock_put_obj_lock_invalid_bucket(): assert error_code == "InvalidBucketState" -@pytest.mark.skip(reason="https://github.com/nspcc-dev/neofs-s3-gw/issues/869") def test_object_lock_put_obj_lock_with_days_and_years(): bucket_name = get_new_bucket_name() client = get_client() @@ -14464,7 +14323,6 @@ def test_object_lock_put_obj_lock_with_days_and_years(): assert error_code == "MalformedXML" -@pytest.mark.skip(reason="https://github.com/nspcc-dev/neofs-s3-gw/issues/869") def test_object_lock_put_obj_lock_invalid_days(): bucket_name = get_new_bucket_name() client = get_client() @@ -14481,10 +14339,9 @@ def test_object_lock_put_obj_lock_invalid_days(): ) status, error_code = _get_status_and_error_code(e.response) assert status == 400 - assert error_code == "InvalidRetentionPeriod" + assert error_code == "MalformedXML" -@pytest.mark.skip(reason="https://github.com/nspcc-dev/neofs-s3-gw/issues/869") def test_object_lock_put_obj_lock_invalid_years(): bucket_name = get_new_bucket_name() client = get_client() @@ -14501,10 +14358,9 @@ def test_object_lock_put_obj_lock_invalid_years(): ) status, error_code = _get_status_and_error_code(e.response) assert status == 400 - assert error_code == "InvalidRetentionPeriod" + assert error_code == "MalformedXML" -@pytest.mark.skip(reason="https://github.com/nspcc-dev/neofs-s3-gw/issues/869") def test_object_lock_put_obj_lock_invalid_mode(): bucket_name = get_new_bucket_name() client = get_client() @@ -14538,7 +14394,6 @@ def test_object_lock_put_obj_lock_invalid_mode(): assert error_code == "MalformedXML" -@pytest.mark.skip(reason="https://github.com/nspcc-dev/neofs-s3-gw/issues/869") def test_object_lock_put_obj_lock_invalid_status(): bucket_name = get_new_bucket_name() client = get_client() @@ -15505,17 +15360,13 @@ def test_get_nonpublicpolicy_deny_bucket_policy_status(): assert resp["PolicyStatus"]["IsPublic"] == True -@pytest.mark.skip(reason="https://github.com/nspcc-dev/neofs-s3-gw/issues/864") def test_get_default_public_block(): # client = get_svc_client(svc='s3control', client_config=Config(s3={'addressing_style': 'path'})) bucket_name = get_new_bucket() client = get_client() - resp = client.get_public_access_block(Bucket=bucket_name) - assert resp["PublicAccessBlockConfiguration"]["BlockPublicAcls"] == False - assert resp["PublicAccessBlockConfiguration"]["BlockPublicPolicy"] == False - assert resp["PublicAccessBlockConfiguration"]["IgnorePublicAcls"] == False - assert resp["PublicAccessBlockConfiguration"]["RestrictPublicBuckets"] == False + with pytest.raises(ClientError): + client.get_public_access_block(Bucket=bucket_name) @pytest.mark.skip(reason="Not Implemented") diff --git a/s3tests_boto3/functional/test_s3_neofs.py b/s3tests_boto3/functional/test_s3_neofs.py index ebbefc512..c4133e56d 100644 --- a/s3tests_boto3/functional/test_s3_neofs.py +++ b/s3tests_boto3/functional/test_s3_neofs.py @@ -155,7 +155,6 @@ def test_set_cors(): assert status == 404 -@pytest.mark.skip(reason="https://github.com/nspcc-dev/neofs-s3-gw/issues/841") def test_cors_origin_response(): bucket_name = _setup_bucket_acl(bucket_acl="public-read") client = get_client() @@ -444,7 +443,6 @@ def test_cors_origin_response_with_credentials(): ) -@pytest.mark.skip(reason="https://github.com/nspcc-dev/neofs-s3-gw/issues/841") def test_cors_origin_wildcard(): bucket_name = _setup_bucket_acl(bucket_acl="public-read") client = get_client() @@ -487,7 +485,6 @@ def test_cors_origin_wildcard(): ) -@pytest.mark.skip(reason="https://github.com/nspcc-dev/neofs-s3-gw/issues/842") def test_cors_header_option(): bucket_name = _setup_bucket_acl(bucket_acl="public-read") client = get_client() @@ -737,7 +734,6 @@ def test_multipart_copy_invalid_range(): client.delete_object(Bucket=src_bucket_name, Key=src_key) -@pytest.mark.skip(reason="https://github.com/nspcc-dev/neofs-s3-gw/issues/843") def test_multipart_upload(): bucket_name = get_new_bucket() key = "mymultipart" @@ -1005,8 +1001,8 @@ def test_bucket_tagging_workflow(): response = client.delete_bucket_tagging(Bucket=bucket_name) assert response["ResponseMetadata"]["HTTPStatusCode"] == 204 - response = client.get_bucket_tagging(Bucket=bucket_name) - assert len(response["TagSet"]) == 0 + with pytest.raises(ClientError): + client.get_bucket_tagging(Bucket=bucket_name) def test_object_tagging_workflow():