Enable ruff rules on tests folder (#3969)

Author: cbeauchesne

pyproject.toml

@@ -207,7 +207,131 @@ ignore = [
 ]
 "utils/waf_rules.py" = ["N801"]  # generated file
 # TODO : remove those ignores
-"tests/*" = ["ALL"]
+"tests/*" = [
+    # lines with [*] can be autofixed with ruff check --fix --unsafe-fixes
+    # though, each change must be reviewed
+    "ANN201",    # 2043 occurences  [ ] missing-return-type-undocumented-public-function
+    "N801",      # 492 occurences   [ ] invalid-class-name
+    "D212",      # 202 occurences   [*] multi-line-summary-first-line
+    "ARG002",    # 177 occurences   [ ] unused-method-argument
+    "E501",      # 159 occurences   [ ] line-too-long
+    "SIM117",    # 127 occurences   [ ] multiple-with-statements
+    "F401",      # 90 occurences    [*] unused-import
+    "TID252",    # 77 occurences    [*] relative-imports
+    "N806",      # 76 occurences    [ ] non-lowercase-variable-in-function
+    "FBT003",    # 67 occurences    [ ] boolean-positional-value-in-call
+    "D403",      # 61 occurences    [*] first-line-capitalized
+    "D200",      # 52 occurences    [*] fits-on-one-line
+    "F405",      # 43 occurences    [ ] undefined-local-with-import-star-usage
+    "TRY002",    # 41 occurences    [ ] raise-vanilla-class
+    "PT018",     # 39 occurences    [ ] pytest-composite-assertion
+    "N802",      # 39 occurences    [ ] invalid-function-name
+    "UP006",     # 35 occurences    [*] non-pep585-annotation
+    "FBT002",    # 30 occurences    [ ] boolean-default-value-positional-argument
+    "SIM300",    # 30 occurences    [*] yoda-conditions
+    "F541",      # 30 occurences    [*] f-string-missing-placeholders
+    "F841",      # 28 occurences    [*] unused-variable
+    "UP015",     # 27 occurences    [*] redundant-open-modes
+    "RET502",    # 25 occurences    [*] implicit-return-value
+    "D404",      # 25 occurences    [ ] docstring-starts-with-this
+    "ANN401",    # 24 occurences    [ ] any-type
+    "D209",      # 24 occurences    [*] new-line-after-last-paragraph
+    "B007",      # 23 occurences    [ ] unused-loop-control-variable
+    "DTZ005",    # 23 occurences    [ ] call-datetime-now-without-tzinfo
+    "INP001",    # 22 occurences    [ ] implicit-namespace-package
+    "ARG001",    # 21 occurences    [ ] unused-function-argument
+    "UP031",     # 17 occurences    [ ] printf-string-formatting
+    "UP032",     # 17 occurences    [*] f-string
+    "ANN205",    # 16 occurences    [ ] missing-return-type-static-method
+    "T201",      # 15 occurences    [*] print
+    "RET505",    # 15 occurences    [*] superfluous-else-return
+    "UP007",     # 13 occurences    [*] non-pep604-annotation
+    "FBT001",    # 12 occurences    [ ] boolean-type-hint-positional-argument
+    "SLF001",    # 12 occurences    [ ] private-member-access
+    "SIM118",    # 12 occurences    [*] in-dict-keys
+    "F811",      # 12 occurences    [*] redefined-while-unused
+    "UP035",     # 12 occurences    [ ] deprecated-import
+    "S105",      # 10 occurences    [ ] hardcoded-password-string
+    "B015",      # 10 occurences    [ ] useless-comparison
+    "RET503",    # 9 occurences     [*] implicit-return
+    "SIM115",    # 9 occurences     [ ] open-file-with-context-handler
+    "PGH004",    # 9 occurences     [ ] blanket-noqa
+    "RUF001",    # 9 occurences     [ ] ambiguous-unicode-character-string
+    "RUF015",    # 9 occurences     [*] unnecessary-iterable-allocation-for-first-element
+    "RUF100",    # 9 occurences     [*] unused-noqa
+    "SIM108",    # 8 occurences     [*] if-else-block-instead-of-if-exp
+    "PTH120",    # 8 occurences     [ ] os-path-dirname
+    "PGH003",    # 8 occurences     [ ] blanket-type-ignore
+    "TRY301",    # 8 occurences     [ ] raise-within-try
+    "ANN206",    # 7 occurences     [ ] missing-return-type-class-method
+    "B011",      # 7 occurences     [*] assert-false
+    "PT015",     # 7 occurences     [ ] pytest-assert-always-false
+    "N815",      # 7 occurences     [ ] mixed-case-variable-in-class-scope
+    "A001",      # 6 occurences     [ ] builtin-variable-shadowing
+    "PT006",     # 6 occurences     [*] pytest-parametrize-names-wrong-type
+    "RET504",    # 6 occurences     [*] unnecessary-assign
+    "SIM114",    # 6 occurences     [*] if-with-same-arms
+    "N803",      # 6 occurences     [ ] invalid-argument-name
+    "E712",      # 6 occurences     [*] true-false-comparison
+    "E741",      # 6 occurences     [ ] ambiguous-variable-name
+    "S113",      # 5 occurences     [ ] request-without-timeout
+    "PT011",     # 5 occurences     [ ] pytest-raises-too-broad
+    "E731",      # 5 occurences     [*] lambda-assignment
+    "RUF005",    # 5 occurences     [ ] collection-literal-concatenation
+    "B006",      # 4 occurences     [*] mutable-argument-default
+    "PIE794",    # 4 occurences     [*] duplicate-class-field-definition
+    "PTH103",    # 4 occurences     [ ] os-makedirs
+    "E711",      # 4 occurences     [*] none-comparison
+    "E721",      # 4 occurences     [ ] type-comparison
+    "PLR5501",   # 4 occurences     [*] collapsible-else-if
+    "PLW2901",   # 4 occurences     [ ] redefined-loop-name
+    "RUF003",    # 4 occurences     [ ] ambiguous-unicode-character-comment
+    "RUF010",    # 4 occurences     [*] explicit-f-string-type-conversion
+    "B904",      # 3 occurences     [ ] raise-without-from-inside-except
+    "A002",      # 3 occurences     [ ] builtin-argument-shadowing
+    "RET507",    # 3 occurences     [*] superfluous-else-continue
+    "PTH112",    # 3 occurences     [ ] os-path-isdir
+    "W291",      # 3 occurences     [*] trailing-whitespace
+    "UP003",     # 3 occurences     [*] type-of-primitive
+    "UP034",     # 3 occurences     [*] extraneous-parentheses
+    "RUF017",    # 3 occurences     [*] quadratic-list-summation
+    "ANN002",    # 2 occurences     [ ] missing-type-args
+    "ASYNC230",  # 2 occurences     [ ] blocking-open-call-in-async-function
+    "S605",      # 2 occurences     [ ] start-process-with-a-shell
+    "C408",      # 2 occurences     [*] unnecessary-collection-call
+    "C416",      # 2 occurences     [*] unnecessary-comprehension
+    "FIX003",    # 2 occurences     [ ] line-contains-xxx
+    "PIE810",    # 2 occurences     [*] multiple-starts-ends-with
+    "T203",      # 2 occurences     [*] p-print
+    "ARG005",    # 2 occurences     [ ] unused-lambda-argument
+    "PTH100",    # 2 occurences     [ ] os-path-abspath
+    "PTH109",    # 2 occurences     [ ] os-getcwd
+    "E401",      # 2 occurences     [*] multiple-imports-on-one-line
+    "D210",      # 2 occurences     [ ] surrounding-whitespace
+    "D300",      # 2 occurences     [*] triple-single-quotes
+    "FURB129",   # 2 occurences     [*] readlines-in-for
+    "RUF002",    # 2 occurences     [ ] ambiguous-unicode-character-docstring
+    "ASYNC251",  # 1 occurences     [ ] blocking-sleep-in-async-function
+    "S108",      # 1 occurences     [ ] hardcoded-temp-file
+    "S110",      # 1 occurences     [ ] try-except-pass
+    "B033",      # 1 occurences     [*] duplicate-value
+    "B905",      # 1 occurences     [*] zip-without-explicit-strict
+    "C400",      # 1 occurences     [*] unnecessary-generator-list
+    "C401",      # 1 occurences     [*] unnecessary-generator-set
+    "C413",      # 1 occurences     [*] unnecessary-call-around-sorted
+    "SIM105",    # 1 occurences     [ ] suppressible-exception
+    "TD005",     # 1 occurences     [ ] missing-todo-description
+    "PTH113",    # 1 occurences     [ ] os-path-isfile
+    "F403",      # 1 occurences     [ ] undefined-local-with-import-star
+    "PLC0206",   # 1 occurences     [ ] dict-index-missing-items
+    "PLR0133",   # 1 occurences     [ ] comparison-of-constant
+    "RUF006",    # 1 occurences     [ ] asyncio-dangling-task
+
+    # keep those rules
+    "PLR1730",   # if-stmt-min-max: not clear that it makes the code easier to read
+    "RET506",    # superfluous-else-raise: requires a slightly higher cognitive effort to understand the code
+    "RET508",    # superfluous-else-break: : requires a slightly higher cognitive effort to understand the code
+]
 "utils/build/*"  = ["ALL"]
 "lib-injection/*" = ["ALL"]
 "utils/{k8s_lib_injection/*,_context/_scenarios/k8s_lib_injection.py}" = [

tests/appsec/waf/test_addresses.py

@@ -450,7 +450,7 @@ class Test_GrpcServerMethod:
 
     def validate_span(self, span, appsec_data):
         tag = "rpc.grpc.full_method"
-        if not tag in span["meta"]:
+        if tag not in span["meta"]:
             logger.info(f"Can't find '{tag}' in span's meta")
             return False
 

tests/debugger/test_debugger_pii.py

@@ -224,8 +224,6 @@ def test_pii_redaction_line_full(self):
         self._assert(REDACTED_KEYS, REDACTED_TYPES, line_probe=True)
 
     ############ old versions ############
-    def filter(keys_to_filter):
-        return [item for item in REDACTED_KEYS if item not in keys_to_filter]
 
     def setup_pii_redaction_java_1_33(self):
         self._setup()

tests/debugger/test_debugger_symdb.py

@@ -42,10 +42,7 @@ def check_scope(scope):
             name = scope.get("name", "")
             if re.search(pattern, name):
                 scope_type = scope.get("scope_type", "")
-                if scope_type in ["CLASS", "class", "MODULE"]:
-                    return True
-
-                return False
+                return scope_type in ["CLASS", "class", "MODULE"]
 
             for nested_scope in scope.get("scopes", []):
                 if check_scope(nested_scope):

tests/fuzzer/core.py

@@ -51,7 +51,7 @@ def __call__(self, payload):
             return
 
         if self.logger is None:
-            self.logger = logging.Logger(__name__)
+            self.logger = logging.getLogger(__name__)
             self.logger.addHandler(RotatingFileHandler(self.filename))
 
         self.logger.info(json.dumps(payload))
@@ -209,7 +209,7 @@ async def _run(self):
         try:
             await self.wait_for_first_response()
         except Exception as e:
-            self.logger.error(str(e))
+            self.logger.exception("First response failed")
             self.loop.stop()
             return
 

tests/integrations/utils.py

@@ -171,7 +171,7 @@ def delete_aws_resource(
     resource_identifier: str,
     resource_type: str,
     error_name: str,
-    get_callable: Callable = None,
+    get_callable: Callable | None = None,
 ):
     """
     Generalized function to delete AWS resources.

tests/parametric/conftest.py

@@ -77,7 +77,7 @@ def apm_test_server(request, library_env, test_id):
     context.scenario.parametrized_tests_metadata[request.node.nodeid] = new_env
 
     new_env.update(apm_test_server_image.env)
-    yield dataclasses.replace(
+    return dataclasses.replace(
         apm_test_server_image, container_name=f"{apm_test_server_image.container_name}-{test_id}", env=new_env
     )
 
@@ -507,7 +507,7 @@ def docker() -> Optional[str]:
     return shutil.which("docker")
 
 
-@pytest.fixture()
+@pytest.fixture
 def docker_network(test_id: str) -> Generator[str, None, None]:
     network = scenarios.parametric.create_docker_network(test_id)
 

tests/parametric/test_128_bit_traceids.py

@@ -608,7 +608,7 @@ def check_128_bit_trace_id(header_trace_id, span_trace_id, dd_p_tid):
 
 def validate_dd_p_tid(dd_p_tid):
     """Validate that dd_p_tid is well-formed."""
-    assert not dd_p_tid is None
+    assert dd_p_tid is not None
     assert len(dd_p_tid) == 16
     assert dd_p_tid != ZERO16
     assert dd_p_tid[8:16] == ZERO8

tests/parametric/test_dynamic_configuration.py

@@ -649,7 +649,7 @@ def test_trace_sampling_rules_override_rate(self, library_env, test_agent, test_
         assert span["meta"]["_dd.p.dm"] == "-3"
 
         # Unset RC to ensure local settings
-        set_and_wait_rc(test_agent, config_overrides={"tracing_sampling_rules": None, "tracing_sampling_rules": None})
+        set_and_wait_rc(test_agent, config_overrides={"tracing_sampling_rules": None})
         trace = get_sampled_trace(test_library, test_agent, service="other_service", name="op_name")
         assert_sampling_rate(trace, DEFAULT_SAMPLE_RATE)
 

tests/parametric/test_library_tracestats.py

@@ -4,7 +4,7 @@
 from typing import Optional
 from typing import List
 
-import numpy
+import numpy as np
 import msgpack
 import pytest
 
@@ -385,11 +385,11 @@ def test_relative_error_TS008(self, library_env, test_agent, test_library):
         assert web_stats["Hits"] == 10
 
         # Validate the sketches
-        np_duration = numpy.array(durations)
+        np_duration = np.array(durations)
         assert web_stats["Duration"] == sum(durations), "Stats duration should match the span duration exactly"
         for quantile in (0.5, 0.75, 0.95, 0.99, 1):
             assert web_stats["OkSummary"].get_quantile_value(quantile) == pytest.approx(
-                numpy.quantile(np_duration, quantile),
+                np.quantile(np_duration, quantile),
                 rel=0.01,
             ), "Quantile mismatch for quantile %r" % quantile
 

tests/parametric/test_sampling_span_tags.py

@@ -14,7 +14,7 @@
 UNSET = -420
 
 
-class AnyRatio(object):
+class AnyRatio:
     def __eq__(self, other):
         return 0 <= other <= 1
 

tests/parametric/test_trace_sampling.py

@@ -91,7 +91,6 @@ def test_trace_kept_in_spite_trace_sampling_rule(self, test_agent, test_library)
             with test_library.dd_start_span(name="web.request", service="webserver") as s1:
                 s1.set_metric("sampling.priority", 2)
                 s1.set_meta("resource.name", "drop-me")
-                pass
         span = find_only_span(test_agent.wait_for_num_traces(1))
 
         assert span["metrics"].get(SAMPLING_PRIORITY_KEY) == 2

tests/perfs/process.py

@@ -4,7 +4,7 @@
 from os import environ
 
 
-LOG_FOLDER = environ["LOG_FOLDER"] if "LOG_FOLDER" in environ else "logs"
+LOG_FOLDER = environ.get("LOG_FOLDER", "logs")
 LIBS = ("golang", "dotnet", "java", "nodejs", "php", "ruby")
 
 

tests/remote_config/test_remote_configuration.py

@@ -159,7 +159,7 @@ def rc_check_request(data, expected, caching):
                         raise ValidationError(f"{file} should not be in cached_target_files", extra_info=content)
     except Exception as e:
         e.args += (expected.get("test_description", "No description"),)
-        raise e
+        raise
 
 
 @rfc("https://docs.google.com/document/d/1u_G7TOr8wJX0dOM_zUDKuRJgxoJU_hVTd5SeaMucQUs/edit#heading=h.octuyiil30ph")

tests/stats/test_stats.py

@@ -10,7 +10,7 @@
         - Must have `is_trace_root` on trace root
         - Must set peer tags
         - Must have span_kind
-        
+
 Config:
 - apm_config.peer_tags_aggregation (we should see peer service tags and aggregation by them, note only works on client or producer kind)
 - apm_config.compute_stats_by_span_kind (span_kind will be set and we will calc stats on these spans even when not "top level")

tests/test_config_consistency.py

@@ -496,7 +496,6 @@ def test_log_injection_enabled(self):
         required_fields = ["trace_id", "span_id", "service", "version", "env"]
         for field in required_fields:
             assert field in dd, f"Missing field: {field}"
-        return
 
 
 @rfc("https://docs.google.com/document/d/1kI-gTAKghfcwI7YzKhqRv2ExUstcHqADIWA4-TZ387o/edit#heading=h.8v16cioi7qxp")

tests/test_standard_tags.py

@@ -270,8 +270,8 @@ def _setup_without_attack(self):
         for header, value in (self.FORWARD_HEADERS | self.FORWARD_HEADERS_VENDOR).items():
             self.requests_without_attack[header] = weblog.get("/waf/", headers={header: value})
 
-    def _test_client_ip(self, forward_headers):
-        for header, _ in forward_headers.items():
+    def _test_client_ip(self, forward_headers: dict[str, str]):
+        for header in forward_headers:
             request = self.requests_without_attack[header]
             meta = self._get_root_span_meta(request)
             assert "http.client_ip" in meta, f"Missing http.client_ip for {header}"

tests/test_the_test/test_scrubber.py

@@ -36,7 +36,7 @@
 @scenarios.test_the_test
 def test_log_scrubber():
     cmd = ["./run.sh", "MOCK_THE_TEST", FILENAME]
-    subprocess.run(cmd, env=scrubbed_names | os.environ, text=True, capture_output=True)
+    subprocess.run(cmd, env=scrubbed_names | os.environ, text=True, capture_output=True, check=False)
 
     redacted_count = 0
 
@@ -51,7 +51,7 @@ def test_log_scrubber():
             for secret in scrubbed_names.values():
                 assert secret not in data, f"{secret} found in {file_path}"
 
-    # extra portection to make sure we redacted all secrets
+    # extra protection to make sure we redacted all secrets
     assert redacted_count != 0, "No secrets were redacted"