fix: [OSM-855] fixed python 3.12 support (#229)

Author: gemaxim

.circleci/config.yml

@@ -30,17 +30,43 @@ jobs:
     steps:
       - checkout
       - setup_remote_docker
-      - run:
-          name: Run tests
-          command: |
-            BUILDKIT_PROGRESS=plain \
-            DOCKER_BUILDKIT=1 \
-            docker build \
-                --build-arg NODE_VERSION=<< parameters.node_version >> \
-                --build-arg PYTHON_VERSION=<< parameters.python_version >> \
-                -t snyk-python-plugin:integration-tests-<< parameters.python_version >> \
-                -f test/Dockerfile .
-            docker run --rm snyk-python-plugin:integration-tests-<< parameters.python_version >>
+      - when:
+          condition: 
+            equal: [ "3.12", <<parameters.python_version>>]
+          steps:
+          - run:
+              name: Run tests
+              command: |
+                BUILDKIT_PROGRESS=plain \
+                DOCKER_BUILDKIT=1 \
+                docker build \
+                    --build-arg NODE_VERSION=<< parameters.node_version >> \
+                    --build-arg PYTHON_VERSION=<< parameters.python_version >> \
+                    --build-arg PY_TEST_CMD=test:pysrc3_12 \
+                    -t snyk-python-plugin:integration-tests-<< parameters.python_version >> \
+                    -f test/Dockerfile .
+                docker run --rm snyk-python-plugin:integration-tests-<< parameters.python_version >>
+      - when:
+          condition: 
+            or:
+              - equal: [ "3.8", <<parameters.python_version>>]
+              - equal: [ "3.9", <<parameters.python_version>>]
+              - equal: [ "3.10", <<parameters.python_version>>]
+              - equal: [ "3.11", <<parameters.python_version>>]
+          steps:
+          - run:
+              name: Run tests
+              command: |
+                BUILDKIT_PROGRESS=plain \
+                DOCKER_BUILDKIT=1 \
+                docker build \
+                    --build-arg NODE_VERSION=<< parameters.node_version >> \
+                    --build-arg PYTHON_VERSION=<< parameters.python_version >> \
+                    --build-arg PY_TEST_CMD=test:pysrc3 \
+                    -t snyk-python-plugin:integration-tests-<< parameters.python_version >> \
+                    -f test/Dockerfile .
+                docker run --rm snyk-python-plugin:integration-tests-<< parameters.python_version >>
+
 
   build:
     <<: *defaults
@@ -108,6 +134,7 @@ workflows:
                 '3.9',
                 '3.10',
                 '3.11',
+                '3.12',
               ]
           filters:
             branches:

README.md

@@ -31,10 +31,10 @@ Snyk helps you find, fix and monitor for known vulnerabilities in your dependenc
 
 ## Supported Pip & Python versions (requirements.txt)
 
-| Pip / Python   |2.7|3.6|3.7|3.8|3.9|
-|----------------|---|---|---|---|---|
-| 10.0.0         | ✅ | ✅ | ✅ | ✅ | ✅ |
-| 18.1.0         | ✅ | ✅ | ✅ | ✅ | ✅ |
+| Pip / Python   |2.7|3.6|3.7|3.8|3.9|3.10|3.11|3.12|
+|----------------|---|---|---|---|---|---|---|---|
+| 10.0.0         | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅
+| 18.1.0         | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
 
 ## Supported Poetry versions (`pyproject.toml` and `poetry.lock`)
 All known versions are expected to be supported (current latest version is 1.1.6)

dev-requirements.txt

@@ -1,5 +1,13 @@
-pipenv==2022.4.8; python_version >= '3.6'
+pipenv==2022.4.8; python_version == '3.8'
+pipenv==2022.4.8; python_version == '3.9'
+pipenv==2022.4.8; python_version == '3.10'
+pipenv==2022.4.8; python_version == '3.11'
+pipenv==2023.11.15; python_version == '3.12'
 pipenv==2018.11.26; python_version == '2.7'
-virtualenv==20.15.1; python_version >= '3.6'
+virtualenv==20.15.1; python_version == '3.8'
+virtualenv==20.15.1; python_version == '3.9'
+virtualenv==20.15.1; python_version == '3.10'
+virtualenv==20.15.1; python_version == '3.11'
+virtualenv==20.25.0; python_version == '3.12'
 virtualenv==16.2.0; python_version == '2.7'
 mock

lib/dependencies/inspect-implementation.ts

@@ -8,6 +8,9 @@ import { buildDepGraph, PartialDepTree } from './build-dep-graph';
 import { FILENAMES } from '../types';
 import { EmptyManifestError, RequiredPackagesMissingError } from '../errors';
 
+const PYTHON_3_12_REGEX = new RegExp('^Python 3.12.*');
+const UPDATED_SETUPTOOLS_VERSION = '68.0.0';
+
 const returnedTargetFile = (originalTargetFile) => {
   const basename = path.basename(originalTargetFile);
 
@@ -163,6 +166,8 @@ function createAssets() {
     path.join(__dirname, '../../pysrc/pkg_resources_py3/_vendor/pyparsing.py'),
     path.join(__dirname, '../../pysrc/pkg_resources_py3/__init__.py'),
     path.join(__dirname, '../../pysrc/pkg_resources_py3/extern/__init__.py'),
+
+    path.join(__dirname, '../../pysrc/pkg_resources_py3_12/__init__.py'),
   ];
 }
 
@@ -203,6 +208,30 @@ function dumpAllFilesInTempDir(tempDirName: string) {
   });
 }
 
+async function updateSetuptools(
+  setuptoolsVersion: string,
+  dir: string,
+  pythonEnv
+) {
+  // For python 3.12, setuptools needs to be updated
+  // due to removal of some deprecated packages
+  // see: https://github.com/pypa/pip/pull/11997
+
+  const pythonVersion = await subProcess.execute(`python`, ['--version'], {
+    cwd: dir,
+    env: pythonEnv,
+  });
+
+  if (!PYTHON_3_12_REGEX.test(pythonVersion)) {
+    return;
+  }
+
+  await subProcess.execute(`pip install setuptools==${setuptoolsVersion}`, [], {
+    cwd: dir,
+    env: pythonEnv,
+  });
+}
+
 export async function inspectInstalledDeps(
   command: string,
   baseargs: string[],
@@ -221,6 +250,9 @@ export async function inspectInstalledDeps(
   dumpAllFilesInTempDir(tempDirObj.name);
   try {
     const pythonEnv = getPythonEnv(targetFile);
+
+    await updateSetuptools(UPDATED_SETUPTOOLS_VERSION, root, pythonEnv);
+
     // See ../../pysrc/README.md
     const output = await subProcess.execute(
       command,

package.json

@@ -14,9 +14,11 @@
     "format:check": "prettier --check '{lib,test}/**/*.{js,ts}'",
     "format": "prettier --write '{lib,test}/**/*.{js,ts}'",
     "prepare": "npm run build",
-    "test": "npm run test:pysrc && npm run test:jest",
+    "test": "npm run test:jest",
     "test:jest": "jest",
-    "test:pysrc": "python -m unittest discover pysrc",
+    "test:pysrc2": "cd pysrc; python -m unittest test_pip_resolve_py2 test_pipfile",
+    "test:pysrc3": "cd pysrc; python -m unittest test_pip_resolve_py3 test_pipfile",
+    "test:pysrc3_12": "cd pysrc; python -m unittest test_pip_resolve_py3_12 test_pipfile",
     "lint": "npm run build-tests && npm run format:check && eslint --cache '{lib,test}/**/*.{js,ts}'"
   },
   "author": "snyk.io",

pysrc/pkg_resources.py

@@ -1,6 +1,9 @@
 import sys
 
 if sys.version_info >= (3, 0):
-    from pkg_resources_py3 import *
+    if sys.version_info.minor >= 12:
+        from pkg_resources_py3_12 import *
+    else:
+        from pkg_resources_py3 import *
 else:
     from pkg_resources_py2 import *

pysrc/pkg_resources_py3_12/LICENSE

@@ -0,0 +1,19 @@
+Copyright Jason R. Coombs
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to
+deal in the Software without restriction, including without limitation the
+rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+sell copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+IN THE SOFTWARE.