diff --git a/.github/workflows/action.yml b/.github/workflows/action.yml index d193157dd..7e3df3a12 100644 --- a/.github/workflows/action.yml +++ b/.github/workflows/action.yml @@ -17,7 +17,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1 + uses: step-security/harden-runner@cb605e52c26070c328afc4562f0b4ada7618a84e # v2.10.4 - 2025.01.20 with: egress-policy: audit diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index cc387a5df..e99a4b720 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -23,15 +23,15 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1 + uses: step-security/harden-runner@cb605e52c26070c328afc4562f0b4ada7618a84e # v2.10.4 - 2025.01.20 with: egress-policy: audit - name: Checkout CredData - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - 2024.10.23 with: repository: Samsung/CredData - ref: main + ref: 4079250d81c0481ad4b5511448b7efd9d2f15884 - name: Markup hashing run: | @@ -43,14 +43,14 @@ jobs: - name: Cache data id: cache-data - uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 + uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 - 2024.12.05 with: path: data key: cred-data-${{ hashFiles('checksums.md5') }} - name: Set up Python 3.10 if: steps.cache-data.outputs.cache-hit != 'true' - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4.7.1 + uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 - 2025.01.28 with: python-version: "3.10" @@ -79,12 +79,12 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1 + uses: step-security/harden-runner@cb605e52c26070c328afc4562f0b4ada7618a84e # v2.10.4 - 2025.01.20 with: egress-policy: audit - name: Checkout CredData - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - 2024.10.23 with: repository: Samsung/CredData ref: main @@ -99,7 +99,7 @@ jobs: - name: Cache data id: cache-data - uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 + uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 - 2024.12.05 with: path: data key: cred-data-${{ hashFiles('checksums.md5') }} @@ -113,7 +113,7 @@ jobs: run: ls -al . && ls -al data - name: Set up Python 3.10 - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4.7.1 + uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 - 2025.01.28 with: python-version: "3.10" @@ -124,7 +124,7 @@ jobs: run: python -m pip install --requirement requirements.txt - name: Checkout CredSweeper - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - 2024.10.23 with: ref: ${{ github.event.pull_request.head.sha }} path: temp/CredSweeper @@ -144,21 +144,21 @@ jobs: - name: Upload CredSweeper log if: always() - uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 - 2025.01.09 with: name: credsweeper path: credsweeper.${{ github.event.pull_request.head.sha }}.log - name: Upload CredSweeper report if: always() - uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 - 2025.01.09 with: name: report path: report.${{ github.event.pull_request.head.sha }}.json - name: Upload benchmark output if: always() - uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 - 2025.01.09 with: name: benchmark path: benchmark.${{ github.event.pull_request.head.sha }}.log @@ -170,7 +170,7 @@ jobs: # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # performance_benchmark: - # put the benchmark in single job to keep constant environment during test python 3.8 is not applicable + # put the benchmark in single job to keep constant environment during test needs: [ download_data ] runs-on: ubuntu-latest @@ -182,12 +182,12 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1 + uses: step-security/harden-runner@cb605e52c26070c328afc4562f0b4ada7618a84e # v2.10.4 - 2025.01.20 with: egress-policy: audit - name: Checkout CredData - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - 2024.10.23 with: repository: Samsung/CredData ref: main @@ -202,7 +202,7 @@ jobs: - name: Cache data id: cache-data - uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 + uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 - 2024.12.05 with: path: data key: cred-data-${{ hashFiles('checksums.md5') }} @@ -216,7 +216,7 @@ jobs: run: rm -rf data/0* data/2* data/7* data/8* data/a* data/b* data/d* data/e* data/f* - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4.7.1 + uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 - 2025.01.28 with: python-version: ${{ matrix.python-version }} @@ -252,7 +252,7 @@ jobs: python -m pip uninstall -y credsweeper - name: Checkout base CredSweeper - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - 2024.10.23 with: ref: ${{ github.event.pull_request.base.sha }} path: temp/CredSweeper.base @@ -278,7 +278,7 @@ jobs: echo "BASE_TIME=${BASE_TIME}" >> $GITHUB_ENV - name: Checkout current CredSweeper - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - 2024.10.23 with: ref: ${{ github.event.pull_request.head.sha }} path: temp/CredSweeper.head @@ -370,12 +370,12 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1 + uses: step-security/harden-runner@cb605e52c26070c328afc4562f0b4ada7618a84e # v2.10.4 - 2025.01.20 with: egress-policy: audit - name: Checkout CredData - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - 2024.10.23 with: repository: Samsung/CredData ref: main @@ -390,7 +390,7 @@ jobs: - name: Cache data id: cache-data - uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 + uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 - 2024.12.05 with: path: data key: cred-data-${{ hashFiles('checksums.md5') }} @@ -408,7 +408,7 @@ jobs: mv meta ${{ github.workspace }}/CredData/ - name: Set up Python 3.10 - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4.7.1 + uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 - 2025.01.28 with: python-version: "3.10" @@ -416,7 +416,7 @@ jobs: run: python -m pip install --upgrade pip - name: Checkout current CredSweeper - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - 2024.10.23 with: ref: ${{ github.event.pull_request.head.sha }} path: CredSweeper.head @@ -461,19 +461,19 @@ jobs: if: ${{ 'push' == github.event_name }} or ${{ 'Samsung/CredSweeper' == github.event.pull_request.head.repo.full_name }} steps: - name: Harden Runner - uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1 + uses: step-security/harden-runner@cb605e52c26070c328afc4562f0b4ada7618a84e # v2.10.4 - 2025.01.20 with: egress-policy: audit - name: Checkout CredSweeper PR if: ${{ 'pull_request' == github.event_name }} - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - 2024.10.23 with: ref: ${{ github.event.pull_request.head.sha }} - name: Checkout CredSweeper HEAD if: ${{ 'push' == github.event_name }} - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - 2024.10.23 with: ref: ${{ github.event.head }} @@ -488,5 +488,3 @@ jobs: else echo "secrets.SLACK_URL is not available" fi - - # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 1290ea879..7401008e2 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -13,225 +13,212 @@ permissions: jobs: -# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # - checkers: runs-on: ubuntu-latest steps: - # # # MUST be full history to check git workflow - - - name: Harden Runner - uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1 - with: - egress-policy: audit - - - name: Checkout - id: code_checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 0 - ref: ${{ github.event.pull_request.head.sha }} - - # # # integrity for train diagram match - - - name: Check ml_config.json and ml_model.onnx integrity - if: ${{ always() && steps.code_checkout.conclusion == 'success' }} - run: | - md5sum --binary credsweeper/ml_model/ml_config.json | grep 3a4bfcd6f3ea74461b158d4ec073cc06 - md5sum --binary credsweeper/ml_model/ml_model.onnx | grep 9725b166e07e60f94929fea986f84ae2 - - # # # line ending - - - name: Check for text file ending - if: ${{ always() && steps.code_checkout.conclusion == 'success' }} - run: | - n=0 - for f in $(find . -type f -not -wholename '*/.*' -a -not -wholename '*/tests/samples/*' -a -not -wholename '*/corpus/*' -a -not -wholename '*.json'); do - n=$(( 1 + ${n} )) - filetype=$(file ${f}) - if echo "${filetype}" | grep -q '.*text.*'; then - echo "CHECK:'${filetype}'" - lastbyte=$(hexdump -v -e '/1 "%02X\n"' ${f} | tail -1) - echo "Last byte is '${lastbyte}'" - if [ "0A" != "${lastbyte}" ]; then - echo "File ${f} has inappropriate line ending" - tail -1 ${f} | hexdump -C - else - n=$(( ${n} - 1 )) - fi - else - echo "SKIP:'${filetype}'" - n=$(( ${n} - 1 )) - fi - done - exit ${n} - - # # # Python setup - - - name: Set up Python - if: ${{ always() && steps.code_checkout.conclusion == 'success' }} - id: setup_python - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4.7.1 - with: - python-version: "3.12" - - - name: Install CredSweeper and auxiliary packages - id: setup_credsweeper - if: ${{ always() && steps.setup_python.conclusion == 'success' }} - run: | - python --version #dbg - python -m pip install --upgrade pip - pip install --requirement requirements.txt - pip list #dbg - - # # # pylint - - - name: Analysing the code with pylint and minimum Python version 3.8 - if: ${{ always() && steps.setup_credsweeper.conclusion == 'success' }} - run: pylint --py-version=3.8 --errors-only credsweeper - - - name: Analysing the code with pylint and minimum Python version 3.9 - if: ${{ always() && steps.setup_credsweeper.conclusion == 'success' }} - run: pylint --py-version=3.9 --errors-only credsweeper - - - name: Analysing the code with pylint and minimum Python version 3.10 - if: ${{ always() && steps.setup_credsweeper.conclusion == 'success' }} - run: pylint --py-version=3.10 --errors-only credsweeper - - - name: Analysing the code with pylint and minimum Python version 3.11 - if: ${{ always() && steps.setup_credsweeper.conclusion == 'success' }} - run: pylint --py-version=3.11 --errors-only credsweeper - - - name: Analysing the code with pylint and minimum Python version 3.12 - if: ${{ always() && steps.setup_credsweeper.conclusion == 'success' }} - run: pylint --py-version=3.12 --errors-only credsweeper - - # # # mypy - - - name: Analysing the code with mypy and minimum Python version 3.8 - if: ${{ always() && steps.setup_credsweeper.conclusion == 'success' }} - run: | - mypy --config-file .mypy.ini --python-version=3.8 credsweeper - - - name: Analysing the code with mypy and minimum Python version 3.9 - if: ${{ always() && steps.setup_credsweeper.conclusion == 'success' }} - run: | - mypy --config-file .mypy.ini --python-version=3.9 credsweeper - - - name: Analysing the code with mypy and minimum Python version 3.10 - if: ${{ always() && steps.setup_credsweeper.conclusion == 'success' }} - run: | - mypy --config-file .mypy.ini --python-version=3.10 credsweeper - - - name: Analysing the code with mypy and minimum Python version 3.11 - if: ${{ always() && steps.setup_credsweeper.conclusion == 'success' }} - run: | - mypy --config-file .mypy.ini --python-version=3.11 credsweeper - - - name: Analysing the code with mypy and minimum Python version 3.12 - if: ${{ always() && steps.setup_credsweeper.conclusion == 'success' }} - run: | - mypy --config-file .mypy.ini --python-version=3.12 credsweeper - - # # # documentation - - - name: Analysing the code with pylint for NEW missed docstrings of classes or functions - if: ${{ always() && steps.setup_credsweeper.conclusion == 'success' }} - run: | - pylint --disable=E,R,W,C0114,C0103,C0303,C0412,C0413,C0415,C0200,C0201,C0325 --verbose credsweeper - - # # # Documentation check - - - name: Test for creation sphinx documentations - if: ${{ always() && steps.setup_credsweeper.conclusion == 'success' }} - run: | - cd docs - pip install --requirement requirements.txt - make html - cd source - python -m sphinx -T -E -b html -d _build/doctrees -D language=en . ./_html - - # # # yapf - - - name: Check project style - if: ${{ always() && steps.setup_credsweeper.conclusion == 'success' }} - run: | - for f in credsweeper tests docs experiment; do - yapf --style .style.yapf --recursive --in-place --parallel $f - done - if [ 0 -ne $(git ls-files -m | wc -l) ]; then - git diff - echo "<- difference how to apply the style" - exit 1 - fi - - # # # flake8 - - - name: Analysing the code with flake8 - id: test_flake8 - if: ${{ always() && steps.setup_credsweeper.conclusion == 'success' }} - run: | - ERRCNT=$(flake8 credsweeper --count --exit-zero --output-file=flake8.txt) - if ! [ 0 -eq ${ERRCNT} ] ; then - echo "flake8 found '${ERRCNT}' failures:" - cat flake8.txt + # # # MUST be full history to check git workflow + + - name: Harden Runner + uses: step-security/harden-runner@cb605e52c26070c328afc4562f0b4ada7618a84e # v2.10.4 - 2025.01.20 + with: + egress-policy: audit + + - name: Checkout + id: code_checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - 2024.10.23 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + + # # # integrity for train diagram match + + - name: Check ml_config.json and ml_model.onnx integrity + if: ${{ always() && steps.code_checkout.conclusion == 'success' }} + run: | + md5sum --binary credsweeper/ml_model/ml_config.json | grep 3a4bfcd6f3ea74461b158d4ec073cc06 + md5sum --binary credsweeper/ml_model/ml_model.onnx | grep 9725b166e07e60f94929fea986f84ae2 + + # # # line ending + + - name: Check for text file ending + if: ${{ always() && steps.code_checkout.conclusion == 'success' }} + run: | + n=0 + for f in $(find . -type f -not -wholename '*/.*' -a -not -wholename '*/tests/samples/*' -a -not -wholename '*/corpus/*' -a -not -wholename '*.json'); do + n=$(( 1 + ${n} )) + filetype=$(file ${f}) + if echo "${filetype}" | grep -q '.*text.*'; then + echo "CHECK:'${filetype}'" + lastbyte=$(hexdump -v -e '/1 "%02X\n"' ${f} | tail -1) + echo "Last byte is '${lastbyte}'" + if [ "0A" != "${lastbyte}" ]; then + echo "File ${f} has inappropriate line ending" + tail -1 ${f} | hexdump -C + else + n=$(( ${n} - 1 )) + fi + else + echo "SKIP:'${filetype}'" + n=$(( ${n} - 1 )) + fi + done + exit ${n} + + # # # Python setup + + - name: Set up Python + if: ${{ always() && steps.code_checkout.conclusion == 'success' }} + id: setup_python + uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 - 2025.01.28 + with: + python-version: "3.12" + + - name: Install CredSweeper and auxiliary packages + id: setup_credsweeper + if: ${{ always() && steps.setup_python.conclusion == 'success' }} + run: | + python --version #dbg + python -m pip install --upgrade pip + pip install --requirement requirements.txt + pip list #dbg + + # # # pylint + + - name: Analysing the code with pylint and minimum Python version 3.9 + if: ${{ always() && steps.setup_credsweeper.conclusion == 'success' }} + run: pylint --py-version=3.9 --errors-only credsweeper + + - name: Analysing the code with pylint and minimum Python version 3.10 + if: ${{ always() && steps.setup_credsweeper.conclusion == 'success' }} + run: pylint --py-version=3.10 --errors-only credsweeper + + - name: Analysing the code with pylint and minimum Python version 3.11 + if: ${{ always() && steps.setup_credsweeper.conclusion == 'success' }} + run: pylint --py-version=3.11 --errors-only credsweeper + + - name: Analysing the code with pylint and minimum Python version 3.12 + if: ${{ always() && steps.setup_credsweeper.conclusion == 'success' }} + run: pylint --py-version=3.12 --errors-only credsweeper + + # # # mypy + + - name: Analysing the code with mypy and minimum Python version 3.9 + if: ${{ always() && steps.setup_credsweeper.conclusion == 'success' }} + run: | + mypy --config-file .mypy.ini --python-version=3.9 credsweeper + + - name: Analysing the code with mypy and minimum Python version 3.10 + if: ${{ always() && steps.setup_credsweeper.conclusion == 'success' }} + run: | + mypy --config-file .mypy.ini --python-version=3.10 credsweeper + + - name: Analysing the code with mypy and minimum Python version 3.11 + if: ${{ always() && steps.setup_credsweeper.conclusion == 'success' }} + run: | + mypy --config-file .mypy.ini --python-version=3.11 credsweeper + + - name: Analysing the code with mypy and minimum Python version 3.12 + if: ${{ always() && steps.setup_credsweeper.conclusion == 'success' }} + run: | + mypy --config-file .mypy.ini --python-version=3.12 credsweeper + + # # # documentation + + - name: Analysing the code with pylint for NEW missed docstrings of classes or functions + if: ${{ always() && steps.setup_credsweeper.conclusion == 'success' }} + run: | + pylint --disable=E,R,W,C0114,C0103,C0303,C0412,C0413,C0415,C0200,C0201,C0325 --verbose credsweeper + + # # # Documentation check + + - name: Test for creation sphinx documentations + if: ${{ always() && steps.setup_credsweeper.conclusion == 'success' }} + run: | + cd docs + pip install --requirement requirements.txt + make html + cd source + python -m sphinx -T -E -b html -d _build/doctrees -D language=en . ./_html + + # # # yapf + + - name: Check project style + if: ${{ always() && steps.setup_credsweeper.conclusion == 'success' }} + run: | + for f in credsweeper tests docs experiment; do + yapf --style .style.yapf --recursive --in-place --parallel $f + done + if [ 0 -ne $(git ls-files -m | wc -l) ]; then + git diff + echo "<- difference how to apply the style" exit 1 - fi - - - name: FLAKE 8 reports - if: ${{ failure() && steps.test_flake8.conclusion == 'failure' }} - uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 - with: - name: flake8_report - path: flake8.txt - - # # # Banner crc32 - - - name: Setup crc32 tool - id: setup_crc32 - if: ${{ always() && steps.setup_credsweeper.conclusion == 'success' }} - run: sudo apt-get update && sudo apt-get install libarchive-zip-perl && crc32 /etc/fstab - - - name: Banner and version check - if: ${{ always() && steps.setup_crc32.conclusion == 'success' }} - continue-on-error: true - run: | - crc32_int=0 - for f in $(find credsweeper -iregex '.*\.\(py\|json\|yaml\|txt\|onnx\)$'); do - file_crc32_hex=$(crc32 $f) - file_crc32_int=$((16#${file_crc32_hex})) - crc32_int=$(( ${crc32_int} ^ ${file_crc32_int} )) - done - version_with_crc="$(python -m credsweeper --version | head -1) crc32:$(printf '%x' ${crc32_int})" - echo "version_with_crc = '${version_with_crc}'" - banner=$(python -m credsweeper --banner | head -1) - echo "banner = '${banner}'" - if ! [ -n "${version_with_crc}" ] && [ -n "${banner}" ] && [ "${version_with_crc}" == "${banner}" ]; then - echo "'${version_with_crc}' != '${banner}'" + fi + + # # # flake8 + + - name: Analysing the code with flake8 + id: test_flake8 + if: ${{ always() && steps.setup_credsweeper.conclusion == 'success' }} + run: | + ERRCNT=$(flake8 credsweeper --count --exit-zero --output-file=flake8.txt) + if ! [ 0 -eq ${ERRCNT} ] ; then + echo "flake8 found '${ERRCNT}' failures:" + cat flake8.txt + exit 1 + fi + + - name: FLAKE 8 reports + if: ${{ failure() && steps.test_flake8.conclusion == 'failure' }} + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 - 2025.01.09 + with: + name: flake8_report + path: flake8.txt + + # # # Banner crc32 + + - name: Setup crc32 tool + id: setup_crc32 + if: ${{ always() && steps.setup_credsweeper.conclusion == 'success' }} + run: sudo apt-get update && sudo apt-get install libarchive-zip-perl && crc32 /etc/fstab + + - name: Banner and version check + if: ${{ always() && steps.setup_crc32.conclusion == 'success' }} + continue-on-error: true + run: | + crc32_int=0 + for f in $(find credsweeper -iregex '.*\.\(py\|json\|yaml\|txt\|onnx\)$'); do + file_crc32_hex=$(crc32 $f) + file_crc32_int=$((16#${file_crc32_hex})) + crc32_int=$(( ${crc32_int} ^ ${file_crc32_int} )) + done + version_with_crc="$(python -m credsweeper --version | head -1) crc32:$(printf '%x' ${crc32_int})" + echo "version_with_crc = '${version_with_crc}'" + banner=$(python -m credsweeper --banner | head -1) + echo "banner = '${banner}'" + if ! [ -n "${version_with_crc}" ] && [ -n "${banner}" ] && [ "${version_with_crc}" == "${banner}" ]; then + echo "'${version_with_crc}' != '${banner}'" + exit 1 + fi + + # # # SECURITY.md check + + - name: SECURITY.md check + if: ${{ always() && steps.setup_credsweeper.conclusion == 'success' }} + run: | + # get actual version (major.minor) from credsweeper package + V=$(python -c "from packaging.version import Version as V; import credsweeper; v=V(credsweeper.__version__); print(f'{v.major}.{v.minor}');") + # check whether current version exists in the file + if ! grep $V SECURITY.md; then + echo $V + cat --number SECURITY.md exit 1 - fi - - # # # SECURITY.md check - - - name: SECURITY.md check - if: ${{ always() && steps.setup_credsweeper.conclusion == 'success' }} - run: | - # get actual version (major.minor) from credsweeper package - V=$(python -c "from packaging.version import Version as V; import credsweeper; v=V(credsweeper.__version__); print(f'{v.major}.{v.minor}');") - # check whether current version exists in the file - if ! grep $V SECURITY.md; then - echo $V - cat --number SECURITY.md - exit 1 - fi - - # # # from https://github.com/step-security-bot/CredSweeper/commit/dbc01f2709c56f69e2d8fd717156385f42b7bbf5 + fi - - name: Dependency Review - if: ${{ 'push' != github.event_name }} - uses: actions/dependency-review-action@3b139cfc5fae8b618d3eae3675e383bb1769c019 # v4.5.0 + # # # from https://github.com/step-security-bot/CredSweeper/commit/dbc01f2709c56f69e2d8fd717156385f42b7bbf5 -# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # + - name: Dependency Review + if: ${{ 'push' != github.event_name }} + uses: actions/dependency-review-action@3b139cfc5fae8b618d3eae3675e383bb1769c019 # v4.5.0 - 2024.11.20 diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml index 0461bbf4b..95b1a0972 100644 --- a/.github/workflows/fuzz.yml +++ b/.github/workflows/fuzz.yml @@ -15,98 +15,94 @@ permissions: jobs: -# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # - fuzz: runs-on: ubuntu-latest steps: - - name: Harden Runner - uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1 - with: - egress-policy: audit - - - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - ref: ${{ github.event.pull_request.head.sha }} - - - name: Backup corpus - run: cp -r fuzz/corpus corpus.bak - - - name: Set up Python - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4.7.1 - with: - python-version: "3.11" - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - python -m pip install --requirement requirements.txt - python -m pip install --requirement fuzz/requirements.txt - - - name: Run fuzzing test with COVERAGE - id: run_fuzz - run: | - fuzz/coveraging.sh - - - name: Store coverage report - if: always() - uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 - with: - name: htmlcov - path: htmlcov - - - name: Check coverage of dynamic testing - if: always() - run: | - COVERAGE=$(tail -1 report.txt | awk '{print $6}' | tr --delete '%') - # additionally check correctness of the value - should be an integer - FUZZ_COVERAGE_LIMIT=75 - if ! [ ${FUZZ_COVERAGE_LIMIT} -le ${COVERAGE} ]; then - echo "Fuzzing coverage '${COVERAGE}' does not satisfy the limit ${FUZZ_COVERAGE_LIMIT}%" + - name: Harden Runner + uses: step-security/harden-runner@cb605e52c26070c328afc4562f0b4ada7618a84e # v2.10.4 - 2025.01.20 + with: + egress-policy: audit + + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - 2024.10.23 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: Backup corpus + run: cp -r fuzz/corpus corpus.bak + + - name: Set up Python + uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 - 2025.01.28 + with: + python-version: "3.11" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install --requirement requirements.txt + python -m pip install --requirement fuzz/requirements.txt + + - name: Run fuzzing test with COVERAGE + id: run_fuzz + run: | + fuzz/coveraging.sh + + - name: Store coverage report + if: always() + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 - 2025.01.09 + with: + name: htmlcov + path: htmlcov + + - name: Check coverage of dynamic testing + if: always() + run: | + COVERAGE=$(tail -1 report.txt | awk '{print $6}' | tr --delete '%') + # additionally check correctness of the value - should be an integer + FUZZ_COVERAGE_LIMIT=75 + if ! [ ${FUZZ_COVERAGE_LIMIT} -le ${COVERAGE} ]; then + echo "Fuzzing coverage '${COVERAGE}' does not satisfy the limit ${FUZZ_COVERAGE_LIMIT}%" + exit 1 + fi + + - name: Detect new corpus to upload as artifact + if: always() + run: | + ls fuzz/corpus | sort >corpus.txt + ls corpus.bak | sort >corpus.bak.txt + mkdir -vp new_corpus + for f in $(comm -3 corpus.txt corpus.bak.txt); do cp -vf fuzz/corpus/${f} new_corpus/; done + echo "NEW_CORPUS=$(ls new_corpus | wc -l)" >> $GITHUB_ENV + + - name: New corpus upload + if: ${{ env.NEW_CORPUS > 0 }} + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 - 2025.01.09 + with: + name: new_corpus + path: new_corpus + + - name: Detect crash files + if: always() + id: crash_detect + run: | + mkdir -vp crash_corpus + CRASH_CORPUS=0 + for f in $(find . -maxdepth 1 -regextype 'posix-extended' -regex '.*-[0-9a-f]{40}'); do + mv -vf ${f} crash_corpus/ + CRASH_CORPUS=$(( 1 + ${CRASH_CORPUS} )) + done + echo "CRASH_CORPUS=${CRASH_CORPUS}" >> $GITHUB_ENV + if [ 0 -ne ${CRASH_CORPUS} ]; then + echo "${CRASH_CORPUS} crashes were found" exit 1 - fi - - - name: Detect new corpus to upload as artifact - if: always() - run: | - ls fuzz/corpus | sort >corpus.txt - ls corpus.bak | sort >corpus.bak.txt - mkdir -vp new_corpus - for f in $(comm -3 corpus.txt corpus.bak.txt); do cp -vf fuzz/corpus/${f} new_corpus/; done - echo "NEW_CORPUS=$(ls new_corpus | wc -l)" >> $GITHUB_ENV - - - name: New corpus upload - if: ${{ env.NEW_CORPUS > 0 }} - uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 - with: - name: new_corpus - path: new_corpus - - - name: Detect crash files - if: always() - id: crash_detect - run: | - mkdir -vp crash_corpus - CRASH_CORPUS=0 - for f in $(find . -maxdepth 1 -regextype 'posix-extended' -regex '.*-[0-9a-f]{40}'); do - mv -vf ${f} crash_corpus/ - CRASH_CORPUS=$(( 1 + ${CRASH_CORPUS} )) - done - echo "CRASH_CORPUS=${CRASH_CORPUS}" >> $GITHUB_ENV - if [ 0 -ne ${CRASH_CORPUS} ]; then - echo "${CRASH_CORPUS} crashes were found" - exit 1 - fi - - - name: Crash corpus upload - if: ${{ env.CRASH_CORPUS > 0 }} - uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 - with: - name: crash_corpus - path: crash_corpus - -# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # + fi + + - name: Crash corpus upload + if: ${{ env.CRASH_CORPUS > 0 }} + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 - 2025.01.09 + with: + name: crash_corpus + path: crash_corpus diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index 924aa35bf..9dac01de0 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -18,35 +18,35 @@ jobs: deploy: runs-on: ubuntu-latest steps: - - name: Harden Runner - uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1 - with: - egress-policy: audit - - - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - ref: ${{ github.event.pull_request.head.sha }} - - - name: Set up Python - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4.7.1 - with: - python-version: "3.10" - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install --requirement requirements.txt - pip freeze - python -m build - twine check dist/* - # dbg - find dist -name "*.whl" -exec unzip -d dbg {} + - find dbg -name METADATA -type f -exec cat --number {} + - - - name: Publish - if: ${{ 'release' == github.event_name }} - uses: pypa/gh-action-pypi-publish@67339c736fd9354cd4f8cb0b744f2b82a74b5c70 # v1.12.3 - with: - user: __token__ - password: ${{ secrets.PYPI_PASSWORD }} + - name: Harden Runner + uses: step-security/harden-runner@cb605e52c26070c328afc4562f0b4ada7618a84e # v2.10.4 - 2025.01.20 + with: + egress-policy: audit + + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - 2024.10.23 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: Set up Python + uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 - 2025.01.28 + with: + python-version: "3.10" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install --requirement requirements.txt + pip freeze + python -m build + twine check dist/* + # dbg + find dist -name "*.whl" -exec unzip -d dbg {} + + find dbg -name METADATA -type f -exec cat --number {} + + + - name: Publish + if: ${{ 'release' == github.event_name }} + uses: pypa/gh-action-pypi-publish@67339c736fd9354cd4f8cb0b744f2b82a74b5c70 # v1.12.3 + with: + user: __token__ + password: ${{ secrets.PYPI_PASSWORD }} diff --git a/.github/workflows/rottenness.yml b/.github/workflows/rottenness.yml index 6a6466231..dbbc6e6cb 100644 --- a/.github/workflows/rottenness.yml +++ b/.github/workflows/rottenness.yml @@ -15,7 +15,7 @@ permissions: jobs: -# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # + # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # rottenness: @@ -23,48 +23,46 @@ jobs: steps: - - name: Harden Runner - uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1 - with: - egress-policy: audit + - name: Harden Runner + uses: step-security/harden-runner@cb605e52c26070c328afc4562f0b4ada7618a84e # v2.10.4 - 2025.01.20 + with: + egress-policy: audit - - name: Check last release date - continue-on-error: true - run: | - latest_release="$(curl --silent https://api.github.com/repos/Samsung/CredSweeper/releases/latest)" - published_date=$(echo "${latest_release}" | jq --raw-output '.published_at') - release_age=$(( $(date +%s) - $(date --date="${published_date}" +%s) )) - if [ 0 -ge ${release_age} ]; then - echo "Probably, release: ${published_date} and current timezone were different" - release_age=0 - fi - tag_name=$(echo "${latest_release}" | jq --raw-output '.tag_name') - if [ $(( 60 * 60 * 24 * 28 )) -gt ${release_age} ]; then - echo "Release is fresh" - echo "TAG_NAME=" >> $GITHUB_ENV - else - echo "Release is rotten" - echo "TAG_NAME=${tag_name}" >> $GITHUB_ENV - fi + - name: Check last release date + continue-on-error: true + run: | + latest_release="$(curl --silent https://api.github.com/repos/Samsung/CredSweeper/releases/latest)" + published_date=$(echo "${latest_release}" | jq --raw-output '.published_at') + release_age=$(( $(date +%s) - $(date --date="${published_date}" +%s) )) + if [ 0 -ge ${release_age} ]; then + echo "Probably, release: ${published_date} and current timezone were different" + release_age=0 + fi + tag_name=$(echo "${latest_release}" | jq --raw-output '.tag_name') + if [ $(( 60 * 60 * 24 * 28 )) -gt ${release_age} ]; then + echo "Release is fresh" + echo "TAG_NAME=" >> $GITHUB_ENV + else + echo "Release is rotten" + echo "TAG_NAME=${tag_name}" >> $GITHUB_ENV + fi - - name: Checkout current code of default branch - if: env.TAG_NAME - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - repository: Samsung/CredSweeper - ref: main - path: main + - name: Checkout current code of default branch + if: env.TAG_NAME + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - 2024.10.23 + with: + repository: Samsung/CredSweeper + ref: main + path: main - - name: Checkout last release code - if: env.TAG_NAME - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - repository: Samsung/CredSweeper - ref: ${{ env.TAG_NAME }} - path: ${{ env.TAG_NAME }} + - name: Checkout last release code + if: env.TAG_NAME + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - 2024.10.23 + with: + repository: Samsung/CredSweeper + ref: ${{ env.TAG_NAME }} + path: ${{ env.TAG_NAME }} - - name: Compare source code of versions - if: env.TAG_NAME - run: diff --recursive ${{ env.TAG_NAME }}/credsweeper ${{ github.event.repository.default_branch }}/credsweeper - -# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # + - name: Compare source code of versions + if: env.TAG_NAME + run: diff --recursive ${{ env.TAG_NAME }}/credsweeper ${{ github.event.repository.default_branch }}/credsweeper diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1be0f3f33..972c38bc4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,7 +16,7 @@ permissions: jobs: -# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # + # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # release_test: @@ -25,63 +25,63 @@ jobs: fail-fast: false matrix: os: [ ubuntu-latest, windows-latest, macos-latest ] - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: [ "3.9", "3.10", "3.11", "3.12" ] steps: - - name: Harden Runner - uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1 - with: - egress-policy: audit + - name: Harden Runner + uses: step-security/harden-runner@cb605e52c26070c328afc4562f0b4ada7618a84e # v2.10.4 - 2025.01.20 + with: + egress-policy: audit - - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - ref: ${{ github.event.pull_request.head.sha }} + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - 2024.10.23 + with: + ref: ${{ github.event.pull_request.head.sha }} - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4.7.1 - with: - python-version: ${{ matrix.python-version }} - cache: 'pip' + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 - 2025.01.28 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' - - name: Upgrade PIP - run: | - # windows requires update pip via python module - python -m pip install --upgrade pip + - name: Upgrade PIP + run: | + # windows requires update pip via python module + python -m pip install --upgrade pip - - name: Install application - run: | - python -m pip install . - python -m pip freeze + - name: Install application + run: | + python -m pip install . + python -m pip freeze - - name: Remove sources dir to check installation - if: runner.os != 'Windows' - run: rm -rf credsweeper + - name: Remove sources dir to check installation + if: runner.os != 'Windows' + run: rm -rf credsweeper - - name: Remove sources dir to check installation WINDOWS PowerShell - if: runner.os == 'Windows' - run: Remove-Item -Path credsweeper -Force -Recurse + - name: Remove sources dir to check installation WINDOWS PowerShell + if: runner.os == 'Windows' + run: Remove-Item -Path credsweeper -Force -Recurse - - name: CLI tool check - run: | - credsweeper --help + - name: CLI tool check + run: | + credsweeper --help - - name: Install test framework dependencies - run: | - pip install pytest pytest-random-order deepdiff + - name: Install test framework dependencies + run: | + pip install pytest pytest-random-order deepdiff - - name: Suppress warning ``...Unsupported Windows version (2022server)...`` - if: ${{ matrix.python-version == '3.12' && matrix.os == 'windows-latest' }} - run: | - echo "PYTHONWARNINGS=ignore::UserWarning:onnxruntime.capi.onnxruntime_validation:26" >> $env:GITHUB_ENV + - name: Suppress warning ``...Unsupported Windows version (2022server)...`` + if: ${{ matrix.python-version == '3.12' && matrix.os == 'windows-latest' }} + run: | + echo "PYTHONWARNINGS=ignore::UserWarning:onnxruntime.capi.onnxruntime_validation:26" >> $env:GITHUB_ENV - - name: UnitTest with pytest - run: | - # put the command into one line to use in various OS to avoid processing differences in new line char sequence - pytest --random-order --random-order-bucket=global tests + - name: UnitTest with pytest + run: | + # put the command into one line to use in various OS to avoid processing differences in new line char sequence + pytest --random-order --random-order-bucket=global tests -# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # + # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # development_test: @@ -89,87 +89,85 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: [ "3.9", "3.10", "3.11", "3.12" ] steps: - - name: Harden Runner - uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1 - with: - egress-policy: audit - - - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - ref: ${{ github.event.pull_request.head.sha }} - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4.7.1 - with: - python-version: ${{ matrix.python-version }} - - - name: Install requirements - run: | - python -m pip install --upgrade pip - python -m pip install --requirement requirements.txt - python -m pip freeze - - - name: UnitTest with pytest and coverage - run: | - mkdir -vp xmlcov - python -m \ - pytest \ - --random-order \ - --random-order-bucket=global \ - --ignore=docs \ - --ignore=experiment \ - --ignore=fuzz \ - --ignore=tests/test_app.py \ - --cov=credsweeper \ - --cov-report html:coverage_html/ \ - --cov-report xml:xmlcov/coverage.xml \ - tests \ - ; - - - name: ApplicationTest with pytest - run: | - python -m \ - pytest \ - --random-order \ - --random-order-bucket=global \ - tests/test_app.py \ - ; - - - name: Check unit-test coverage - run: | - if [ ! -f xmlcov/coverage.xml ]; then echo "xmlcov/coverage.xml does not exist"; exit 1; fi - COVERED=$(grep '' xmlcov/coverage.xml | sed 's/.* lines-covered="\([0-9]\+\)" .*/\1/') - echo "COVERED=${COVERED}" - VALID=$(grep '' xmlcov/coverage.xml | sed 's/.* lines-valid="\([0-9]\+\)" .*/\1/') - echo "VALID=${VALID}" - if [ -z "${COVERED}" ] || [ -z "${VALID}" ] || [ ${VALID} -eq 0 ]; then echo "'${VALID}' or '${COVERED}' fail"; exit 1; fi - COVERAGE=$(python -c "print (round(100 * ${COVERED} / ${VALID}, 2))") - DESCRIPTION="Coverage of lines: ${COVERED} : ${VALID} = ${COVERAGE}%" - echo "${DESCRIPTION}" - if [ $(( 1000 * ${COVERED} / ${VALID} )) -lt 800 ]; then - echo "Coverage should be not less than 80% !" - exit 1 - else - echo "Satisfied coverage" - fi - - - name: HTML coverage reports - if: always() - uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 - with: - name: coverage_html-${{ matrix.python-version }} - path: coverage_html - - - name: Upload coverage reports to Codecov - if: ${{ matrix.python-version == '3.10' }} - uses: codecov/codecov-action@ab904c41d6ece82784817410c45d8b8c02684457 # v3.1.6 - with: - token: ${{ secrets.CODECOV_TOKEN }} - files: xmlcov/coverage.xml - -# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # + - name: Harden Runner + uses: step-security/harden-runner@cb605e52c26070c328afc4562f0b4ada7618a84e # v2.10.4 - 2025.01.20 + with: + egress-policy: audit + + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - 2024.10.23 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 - 2025.01.28 + with: + python-version: ${{ matrix.python-version }} + + - name: Install requirements + run: | + python -m pip install --upgrade pip + python -m pip install --requirement requirements.txt + python -m pip freeze + + - name: UnitTest with pytest and coverage + run: | + mkdir -vp xmlcov + python -m \ + pytest \ + --random-order \ + --random-order-bucket=global \ + --ignore=docs \ + --ignore=experiment \ + --ignore=fuzz \ + --ignore=tests/test_app.py \ + --cov=credsweeper \ + --cov-report html:coverage_html/ \ + --cov-report xml:xmlcov/coverage.xml \ + tests \ + ; + + - name: ApplicationTest with pytest + run: | + python -m \ + pytest \ + --random-order \ + --random-order-bucket=global \ + tests/test_app.py \ + ; + + - name: Check unit-test coverage + run: | + if [ ! -f xmlcov/coverage.xml ]; then echo "xmlcov/coverage.xml does not exist"; exit 1; fi + COVERED=$(grep '' xmlcov/coverage.xml | sed 's/.* lines-covered="\([0-9]\+\)" .*/\1/') + echo "COVERED=${COVERED}" + VALID=$(grep '' xmlcov/coverage.xml | sed 's/.* lines-valid="\([0-9]\+\)" .*/\1/') + echo "VALID=${VALID}" + if [ -z "${COVERED}" ] || [ -z "${VALID}" ] || [ ${VALID} -eq 0 ]; then echo "'${VALID}' or '${COVERED}' fail"; exit 1; fi + COVERAGE=$(python -c "print (round(100 * ${COVERED} / ${VALID}, 2))") + DESCRIPTION="Coverage of lines: ${COVERED} : ${VALID} = ${COVERAGE}%" + echo "${DESCRIPTION}" + if [ $(( 1000 * ${COVERED} / ${VALID} )) -lt 800 ]; then + echo "Coverage should be not less than 80% !" + exit 1 + else + echo "Satisfied coverage" + fi + + - name: HTML coverage reports + if: always() + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 - 2025.01.09 + with: + name: coverage_html-${{ matrix.python-version }} + path: coverage_html + + - name: Upload coverage reports to Codecov + if: ${{ matrix.python-version == '3.10' }} + uses: codecov/codecov-action@13ce06bfc6bbe3ecf90edbbf1bc32fe5978ca1d3 # v5.3.1 - 2025.01.24 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: xmlcov/coverage.xml diff --git a/docs/source/install.rst b/docs/source/install.rst index e34e8cb9f..d5ca94af9 100644 --- a/docs/source/install.rst +++ b/docs/source/install.rst @@ -3,7 +3,7 @@ Installation Currently `CredSweeper` requires the following prerequisites: -* Python version 3.8, 3.9, 3.10, 3.11 +* Python version 3.9, 3.10, 3.11, 3.12 .. note:: We recommend to use credsweeper in a separate virtual enviroment. Some heave dependencies as Tensorflow diff --git a/pyproject.toml b/pyproject.toml index 0cbe362c7..45101e780 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,7 +27,7 @@ dependencies = [ "PyYAML", "whatthepatch", ] -requires-python = ">=3.8" +requires-python = ">=3.9" readme = "README.md" license = {text = "MIT"} classifiers = [ @@ -38,7 +38,6 @@ classifiers = [ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Topic :: Security", "Topic :: Software Development :: Quality Assurance", diff --git a/requirements.txt b/requirements.txt index 709277515..fa34f4aeb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -29,8 +29,7 @@ onnxruntime==1.20.1; python_version >= '3.10' openpyxl==3.1.5 # pandas - ML requirement and excel data reading -pandas==2.0.3; python_version < '3.9' -pandas==2.2.3; python_version >= '3.9' +pandas==2.2.3 password-strength==0.0.3.post2 pdfminer.six==20240706 @@ -40,8 +39,7 @@ python-dateutil==2.9.0.post0 python-docx==1.1.2 python-pptx==1.0.2 PyYAML==6.0.2 -whatthepatch==1.0.6; python_version < '3.9' -whatthepatch==1.0.7; python_version >= '3.9' +whatthepatch==1.0.7 # Auxiliary # Tests and maintenance packages