chore(iast): migrate test packages to gitlab (DataDog#11827)

Tests packages are failing in main on CircleCi due to timeout. We’re not
sure the root cause because the CI has been broken for over 7 days,
until DataDog#11807 reverted the changes. Making it nearly impossible to
pinpoint where the issue started in CircleCI. So, W've decided it's time
to migrate this to GitLab

CAVEAT: test_packages will only run on the main branch. Therefore, these
tests haven't been executed in this PR, but the [previous
commit](DataDog@0cec964)
included a configuration to run them here to validate functionality
https://gitlab.ddbuild.io/DataDog/apm-reliability/dd-trace-py/-/jobs/748795172


## Checklist
- [x] PR author has checked that all the criteria below are met
- The PR description includes an overview of the change
- The PR description articulates the motivation for the change
- The change includes tests OR the PR description describes a testing
strategy
- The PR description notes risks associated with the change, if any
- Newly-added code is easy to change
- The change follows the [library release note
guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html)
- The change includes or references documentation updates if necessary
- Backport labels are set (if
[applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting))

## Reviewer Checklist
- [x] Reviewer has checked that all the criteria below are met 
- Title is accurate
- All changes are related to the pull request's stated goal
- Avoids breaking
[API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces)
changes
- Testing strategy adequately addresses listed risks
- Newly-added code is easy to change
- Release note makes sense to a user of the library
- If necessary, author has acknowledged and discussed the performance
implications of this PR as reported in the benchmarks PR comment
- Backport labels are set in a manner that is consistent with the
[release branch maintenance
policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)

Author: avara1986

.circleci/config.templ.yml

@@ -404,19 +404,6 @@ jobs:
           paths:
             - "."
 
-  appsec_iast_packages:
-    <<: *machine_executor
-    parallelism: 5
-    steps:
-      - when:
-          condition:
-            matches: { pattern: "main", value: << pipeline.git.branch >> }
-          steps:
-            - run_test:
-                pattern: 'appsec_iast_packages'
-                snapshot: true
-      - run: echo "This test is skipped outside of main branch"
-
   appsec_integrations:
     <<: *machine_executor
     parallelism: 13

hatch.toml

@@ -299,6 +299,32 @@ test = [
 [[envs.appsec_iast_native.matrix]]
 python = ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]
 
+## ASM appsec_iast_packages
+
+[envs.appsec_iast_packages]
+template = "appsec_iast_packages"
+dependencies = [
+    "pytest",
+    "pytest-cov",
+    "requests",
+    "hypothesis",
+    "requests",
+    "astunparse",
+    "flask",
+    "virtualenv-clone"
+]
+
+[envs.appsec_iast_packages.scripts]
+test = [
+    "uname -a",
+    "pip freeze",
+    "DD_CIVISIBILITY_ITR_ENABLED=0 DD_IAST_REQUEST_SAMPLING=100 _DD_APPSEC_DEDUPLICATION_ENABLED=false python -m pytest tests/appsec/iast_packages",
+]
+
+[[envs.appsec_iast_packages.matrix]]
+python = ["3.9", "3.10", "3.11", "3.12"]
+
+
 ## ASM FastAPI
 
 [envs.appsec_threats_fastapi]

scripts/gen_gitlab_config.py

@@ -22,6 +22,7 @@ class JobSpec:
     timeout: t.Optional[int] = None
     skip: bool = False
     paths: t.Optional[t.Set[str]] = None  # ignored
+    only: t.Optional[t.Set[str]] = None  # ignored
 
     def __str__(self) -> str:
         lines = []
@@ -60,6 +61,11 @@ def __str__(self) -> str:
         for key, value in env.items():
             lines.append(f"    {key}: {value}")
 
+        if self.only:
+            lines.append("  only:")
+            for value in self.only:
+                lines.append(f"    - {value}")
+
         if self.parallelism is not None:
             lines.append(f"  parallel: {self.parallelism}")
 

tests/appsec/suitespec.yml

@@ -74,7 +74,7 @@ suites:
     retry: 2
     runner: hatch
   appsec_iast_packages:
-    parallelism: 5
+    parallelism: 4
     paths:
       - '@bootstrap'
       - '@core'
@@ -84,8 +84,11 @@ suites:
       - '@remoteconfig'
       - tests/appsec/iast/*
       - tests/appsec/iast_packages/*
-    runner: riot
-    snapshot: true
+    retry: 2
+    runner: hatch
+    timeout: 50m
+    only:
+      - 'main'
   appsec_integrations:
     parallelism: 7
     paths: