diff --git a/.github/workflows/check-links-in-documentation.yml b/.github/workflows/internal-check-links-in-documentation.yml similarity index 93% rename from .github/workflows/check-links-in-documentation.yml rename to .github/workflows/internal-check-links-in-documentation.yml index 62d831b68..b48cf2a12 100644 --- a/.github/workflows/check-links-in-documentation.yml +++ b/.github/workflows/internal-check-links-in-documentation.yml @@ -9,6 +9,7 @@ on: - 'README.md' - 'COMMANDS.md' - 'GETTING_STARTED.md' + - 'INTEGRATION.md' - '.github/workflows/check-links-in-documentation.yml' # also run when this file was changed schedule: - cron: "15 6 1 * *" # On the first day of each month at 6:15 o'clock @@ -36,6 +37,6 @@ jobs: - name: Check links in top level documentation Markdown files if: ${{ ! env.skip_link_check}} - run: npx --yes markdown-link-check@3.13.6 --verbose --alive=200,202,206 --retry README.md COMMANDS.md GETTING_STARTED.md + run: npx --yes markdown-link-check@3.13.6 --verbose --alive=200,202,206 --retry README.md COMMANDS.md GETTING_STARTED.md INTEGRATION.md # Temporarily, everything is done using command line options rather than with the config file, which doesn't seem to work. # Maybe related to https://github.com/tcort/markdown-link-check/issues/379 ? \ No newline at end of file diff --git a/.github/workflows/check-renovate-config.yml b/.github/workflows/internal-check-renovate-config.yml similarity index 100% rename from .github/workflows/check-renovate-config.yml rename to .github/workflows/internal-check-renovate-config.yml diff --git a/.github/workflows/cypher-reference-documentation.yml b/.github/workflows/internal-cypher-reference-documentation.yml similarity index 100% rename from .github/workflows/cypher-reference-documentation.yml rename to .github/workflows/internal-cypher-reference-documentation.yml diff --git a/.github/workflows/environment-variables-reference-documentation.yml b/.github/workflows/internal-environment-variables-reference-documentation.yml similarity index 100% rename from .github/workflows/environment-variables-reference-documentation.yml rename to .github/workflows/internal-environment-variables-reference-documentation.yml diff --git a/.github/workflows/internal-java-code-analysis.yml b/.github/workflows/internal-java-code-analysis.yml new file mode 100644 index 000000000..9930a32a1 --- /dev/null +++ b/.github/workflows/internal-java-code-analysis.yml @@ -0,0 +1,159 @@ +name: Java Code Structure Graph Analysis + +on: + push: + branches: + - main + # Ignore changes in documentation, general configuration and reports for push events + paths-ignore: + - 'results/**' + - '**/*.md' + - '**/*.txt' + - '**/*.css' + - '**/*.html' + - '**/*.js' + - '.gitignore' + - '.gitattributes' + - 'renovate.json' + - 'changelogTemplate.mustache' + - '**.code-workspace' + - '.github/workflows/typescript-code-analysis.yml' + - '.github/workflows/*documentation.yml' + pull_request: + branches: + - main + # Ignore changes in documentation, general configuration and reports for pull request events + paths-ignore: + - 'results/**' + - '**/*.md' + - '**/*.txt' + - '**/*.css' + - '**/*.html' + - '**/*.js' + - '.gitignore' + - '.gitattributes' + - 'renovate.json' + - 'changelogTemplate.mustache' + - '**.code-workspace' + - '.github/workflows/typescript-code-analysis.yml' + - '.github/workflows/*documentation.yml' + +# Requires the secret NEO4J_INITIAL_PASSWORD to be configured +jobs: + prepare-code-to-analyze: + runs-on: ubuntu-latest + outputs: + analysis-name: ${{ steps.set-analysis-name.outputs.analysis-name }} + sources-upload-name: ${{ steps.set-sources-upload-name.outputs.sources-upload-name }} + artifacts-upload-name: ${{ steps.set-artifacts-upload-name.outputs.artifacts-upload-name }} + + env: + PROJECT_NAME: AxonFramework + # Version variable names matches renovate.json configuration entry + AXON_FRAMEWORK_VERSION: 4.10.3 + + steps: + - name: Checkout GIT Repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + + - name: Set Set output variable 'analysis-name' + id: set-analysis-name + run: echo "analysis-name=${{ env.PROJECT_NAME }}-${{ env.AXON_FRAMEWORK_VERSION }}" >> "$GITHUB_OUTPUT" + + - name: Setup temp directory if missing + run: mkdir -p ./temp + + - name: Download ${{ steps.set-analysis-name.outputs.analysis-name }} + working-directory: temp + run: | + mkdir -p ${{ steps.set-analysis-name.outputs.analysis-name }} + cd ${{ steps.set-analysis-name.outputs.analysis-name }} + echo "Working directory: $( pwd -P )" + ./../../scripts/downloader/downloadAxonFramework.sh ${{ env.AXON_FRAMEWORK_VERSION }} + + - name: Debug folder structure in temp directory + if: runner.debug == '1' + working-directory: temp + run: | + ls -R | grep ":$" | sed -e 's/:$//' -e 's/[^-][^\/]*\//--/g' -e 's/^/ /' -e 's/-/|/' + + - name: (Prepare Code to Analyze) Generate ARTIFACT_UPLOAD_ID + run: echo "ARTIFACT_UPLOAD_ID=$(LC_ALL=C tr -dc 'A-Za-z0-9' < /dev/urandom | head -c 10)" >> $GITHUB_ENV + + - name: (Prepare Code to Analyze) Set sources-upload-name + id: set-sources-upload-name + run: echo "sources-upload-name=${{ steps.set-analysis-name.outputs.analysis-name }}-analysis-sources_input-${{ env.ARTIFACT_UPLOAD_ID }}" >> "$GITHUB_OUTPUT" + + - name: (Prepare Code to Analyze) Set output variable 'artifacts-upload-name' + id: set-artifacts-upload-name + run: echo "artifacts-upload-name=${{ steps.set-analysis-name.outputs.analysis-name }}-analysis-artifacts-input-${{ env.ARTIFACT_UPLOAD_ID }}" >> "$GITHUB_OUTPUT" + + - name: (Prepare Code to Analyze) Upload sources to analyze + if: success() + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4 + with: + name: ${{ steps.set-sources-upload-name.outputs.sources-upload-name }} + path: ./temp/${{ steps.set-analysis-name.outputs.analysis-name }}/source + include-hidden-files: true + if-no-files-found: error + retention-days: 1 + + - name: (Prepare Code to Analyze) Upload artifacts to analyze + if: success() + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4 + with: + name: ${{ steps.set-artifacts-upload-name.outputs.artifacts-upload-name }} + path: ./temp/${{ steps.set-analysis-name.outputs.analysis-name }}/artifacts + if-no-files-found: error + retention-days: 1 + + + + analyze-code-graph: + needs: [prepare-code-to-analyze] + uses: ./.github/workflows/public-analyze-code-graph.yml + with: + analysis-name: ${{ needs.prepare-code-to-analyze.outputs.analysis-name }} + artifacts-upload-name: ${{ needs.prepare-code-to-analyze.outputs.artifacts-upload-name }} + sources-upload-name: ${{ needs.prepare-code-to-analyze.outputs.sources-upload-name }} + + + + commit-analysis-results: + if: github.event_name == 'push' + needs: [prepare-code-to-analyze, analyze-code-graph] + runs-on: ubuntu-latest + + env: + CI_COMMIT_MESSAGE: Automated code structure analysis analysis-results (CI) + CI_COMMIT_AUTHOR: ${{ github.event.repository.name }} Continuous Integration + + steps: + - name: Checkout GIT Repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + token: ${{ secrets.WORKFLOW_GIT_ACCESS_TOKEN }} + + - name: (Code Analysis Setup) Download source code and artifacts for analysis + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4 + with: + name: ${{ needs.analyze-code-graph.outputs.uploaded-analysis-results }} + path: ./results/${{ needs.prepare-code-to-analyze.outputs.analysis-name }} + + - name: Commit "results" directory containing the reports + # Only run when a pull request gets merged or a commit is pushed to the main branch + # git add parameters need to match paths-ignore parameters above + # Git pull before add/commit/push to reduce race conditions on parallel builds + run: | + git config --global user.name '${{ env.CI_COMMIT_AUTHOR }}' + git config --global user.email "7671054+JohT@users.noreply.github.com" + git config --local http.postBuffer 524288000 + git fetch origin + git status + git add results + git status + git commit -m "${{ env.CI_COMMIT_MESSAGE }}" + git status + git rebase --strategy-option=theirs origin/main --verbose + git status + git push --verbose \ No newline at end of file diff --git a/.github/workflows/report-reference-documentation.yml b/.github/workflows/internal-report-reference-documentation.yml similarity index 100% rename from .github/workflows/report-reference-documentation.yml rename to .github/workflows/internal-report-reference-documentation.yml diff --git a/.github/workflows/scripts-reference-documentation.yml b/.github/workflows/internal-scripts-reference-documentation.yml similarity index 100% rename from .github/workflows/scripts-reference-documentation.yml rename to .github/workflows/internal-scripts-reference-documentation.yml diff --git a/.github/workflows/internal-typescript-code-analysis.yml b/.github/workflows/internal-typescript-code-analysis.yml new file mode 100644 index 000000000..2e175916f --- /dev/null +++ b/.github/workflows/internal-typescript-code-analysis.yml @@ -0,0 +1,160 @@ +name: Typescript Code Structure Graph Analysis + +on: + push: + branches: + - main + # Ignore changes in documentation, general configuration and reports for push events + paths-ignore: + - 'results/**' + - '**/*.md' + - '**/*.txt' + - '**/*.css' + - '**/*.html' + - '**/*.js' + - '.gitignore' + - '.gitattributes' + - 'renovate.json' + - 'changelogTemplate.mustache' + - '**.code-workspace' + - '.github/workflows/java-code-analysis.yml' + - '.github/workflows/*documentation.yml' + pull_request: + branches: + - main + # Ignore changes in documentation, general configuration and reports for pull request events + paths-ignore: + - 'results/**' + - '**/*.md' + - '**/*.txt' + - '**/*.css' + - '**/*.html' + - '**/*.js' + - '.gitignore' + - '.gitattributes' + - 'renovate.json' + - 'changelogTemplate.mustache' + - '**.code-workspace' + - '.github/workflows/java-code-analysis.yml' + - '.github/workflows/*documentation.yml' + +jobs: + + prepare-code-to-analyze: + runs-on: ubuntu-latest + outputs: + analysis-name: ${{ steps.set-analysis-name.outputs.analysis-name }} + sources-upload-name: ${{ steps.set-sources-upload-name.outputs.sources-upload-name }} + + env: + PROJECT_NAME: react-router + # Version variable name matches renovate.json configuration entry + REACT_ROUTER_VERSION: 6.28.1 + + steps: + - name: (Prepare Code to Analyze) Checkout GIT repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + + + - name: Set Set output variable 'analysis-name' + id: set-analysis-name + run: echo "analysis-name=${{ env.PROJECT_NAME }}-${{ env.REACT_ROUTER_VERSION }}" >> "$GITHUB_OUTPUT" + + - name: Setup temp directory if missing + run: mkdir -p ./temp + + - name: Setup Cache for "temp/downloads" folder + uses: actions/cache@v4 + with: + path: ./temp/downloads + key: + ${{ runner.os }}-${{ hashFiles('**/*.sh') }} + + - name: Download ${{ steps.set-analysis-name.outputs.analysis-name }} + working-directory: temp + run: | + mkdir -p ${{ steps.set-analysis-name.outputs.analysis-name }} + cd ${{ steps.set-analysis-name.outputs.analysis-name }} + echo "Working directory: $( pwd -P )" + ./../../scripts/downloader/downloadReactRouter.sh ${{ env.REACT_ROUTER_VERSION }} + + - name: (Prepare Code to Analyze) Setup pnpm for react-router + uses: pnpm/action-setup@fe02b34f77f8bc703788d5817da081398fad5dd2 # v4.0.0 + with: + package_json_file: temp/${{ steps.set-analysis-name.outputs.analysis-name }}/source/${{ steps.set-analysis-name.outputs.analysis-name }}/package.json + + - name: (Prepare Code to Analyze) Install dependencies with pnpm + working-directory: temp/${{ steps.set-analysis-name.outputs.analysis-name }}/source/${{ steps.set-analysis-name.outputs.analysis-name }} + run: pnpm install --frozen-lockfile --strict-peer-dependencies + + - name: Debug folder structure in temp directory + if: runner.debug == '1' + working-directory: temp + run: | + ls -R | grep ":$" | sed -e 's/:$//' -e 's/[^-][^\/]*\//--/g' -e 's/^/ /' -e 's/-/|/' + + - name: (Prepare Code to Analyze) Generate ARTIFACT_UPLOAD_ID + run: echo "ARTIFACT_UPLOAD_ID=$(LC_ALL=C tr -dc 'A-Za-z0-9' < /dev/urandom | head -c 10)" >> $GITHUB_ENV + + - name: (Prepare Code to Analyze) Set sources-upload-name + id: set-sources-upload-name + run: echo "sources-upload-name=${{ steps.set-analysis-name.outputs.analysis-name }}-analysis-sources_input-${{ env.ARTIFACT_UPLOAD_ID }}" >> "$GITHUB_OUTPUT" + + - name: (Prepare Code to Analyze) Upload code to analyze + if: success() + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4 + with: + name: ${{ steps.set-sources-upload-name.outputs.sources-upload-name }} + path: ./temp/${{ steps.set-analysis-name.outputs.analysis-name }}/source + if-no-files-found: error + retention-days: 1 + + + + analyze-code-graph: + needs: [prepare-code-to-analyze] + uses: ./.github/workflows/public-analyze-code-graph.yml + with: + analysis-name: ${{ needs.prepare-code-to-analyze.outputs.analysis-name }} + sources-upload-name: ${{ needs.prepare-code-to-analyze.outputs.sources-upload-name }} + + + + commit-analysis-results: + if: github.event_name == 'push' + needs: [prepare-code-to-analyze, analyze-code-graph] + runs-on: ubuntu-latest + + env: + CI_COMMIT_MESSAGE: Automated code structure analysis analysis-results (CI) + CI_COMMIT_AUTHOR: ${{ github.event.repository.name }} Continuous Integration + + steps: + - name: Checkout GIT Repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + token: ${{ secrets.WORKFLOW_GIT_ACCESS_TOKEN }} + + - name: (Code Analysis Setup) Download source code and artifacts for analysis + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4 + with: + name: ${{ needs.analyze-code-graph.outputs.uploaded-analysis-results }} + path: results/${{ needs.prepare-code-to-analyze.outputs.analysis-name }} + + - name: Commit "results" directory containing the reports + # Only run when a pull request gets merged or a commit is pushed to the main branch + # git add parameters need to match paths-ignore parameters above + # Git pull before add/commit/push to reduce race conditions on parallel builds + run: | + git config --global user.name '${{ env.CI_COMMIT_AUTHOR }}' + git config --global user.email "7671054+JohT@users.noreply.github.com" + git config --local http.postBuffer 524288000 + git fetch origin + git status + git add results + git status + git commit -m "${{ env.CI_COMMIT_MESSAGE }}" + git status + git rebase --strategy-option=theirs origin/main --verbose + git status + git push --verbose \ No newline at end of file diff --git a/.github/workflows/java-code-analysis.yml b/.github/workflows/java-code-analysis.yml deleted file mode 100644 index 5c0ccee41..000000000 --- a/.github/workflows/java-code-analysis.yml +++ /dev/null @@ -1,183 +0,0 @@ -name: Java Code Structure Graph Analysis - -on: - push: - branches: - - main - # Ignore changes in documentation, general configuration and reports for push events - paths-ignore: - - 'results/**' - - '**/*.md' - - '**/*.txt' - - '**/*.css' - - '**/*.html' - - '**/*.js' - - '.gitignore' - - '.gitattributes' - - 'renovate.json' - - 'changelogTemplate.mustache' - - '**.code-workspace' - - '.github/workflows/typescript-code-analysis.yml' - - '.github/workflows/*documentation.yml' - pull_request: - branches: - - main - # Ignore changes in documentation, general configuration and reports for pull request events - paths-ignore: - - 'results/**' - - '**/*.md' - - '**/*.txt' - - '**/*.css' - - '**/*.html' - - '**/*.js' - - '.gitignore' - - '.gitattributes' - - 'renovate.json' - - 'changelogTemplate.mustache' - - '**.code-workspace' - - '.github/workflows/typescript-code-analysis.yml' - - '.github/workflows/*documentation.yml' - -# Requires the secret NEO4J_INITIAL_PASSWORD to be configured -jobs: - reports: - runs-on: ubuntu-latest - strategy: - matrix: - include: - - os: ubuntu-latest - java: 17 - python: 3.11 - miniforge: 24.9.0-0 - - env: - CI_COMMIT_MESSAGE: Automated code structure analysis reports (CI) - CI_COMMIT_AUTHOR: ${{ github.event.repository.name }} Continuous Integration - PROJECT_NAME: AxonFramework - # Version variable name matches renovate.json configuration entry - AXON_FRAMEWORK_VERSION: 4.10.3 - - steps: - - name: Checkout GIT Repository - uses: actions/checkout@v4 - with: - token: ${{ secrets.WORKFLOW_GIT_ACCESS_TOKEN }} - - - name: Setup Java JDK ${{ matrix.java }} - uses: actions/setup-java@v4 - with: - distribution: 'adopt' - java-version: ${{ matrix.java }} - - - name: Setup Cache for Conda package manager Miniforge - uses: actions/cache@v4 - env: - # Increase this value to reset cache if etc/example-environment.yml has not changed - # Reference: https://github.com/conda-incubator/setup-miniconda#caching - CACHE_NUMBER: 0 - with: - path: ~/conda_pkgs_dir - key: - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-environments-${{hashFiles('**/environment.yml', '.github/workflows/*.yml') }} - - # "Setup Python" can be skipped if jupyter notebook reports aren't needed - - name: Setup Python ${{ matrix.python }} with Conda package manager Miniforge - uses: conda-incubator/setup-miniconda@v3 - with: - python-version: ${{ matrix.python }} - miniforge-version: ${{ matrix.miniforge }} - activate-environment: codegraph - environment-file: ./jupyter/environment.yml - auto-activate-base: false - use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly! - - - name: Conda environment info - shell: bash -el {0} - run: conda info - - - name: Setup temp directory if missing - run: mkdir -p ./temp - - - name: Setup Cache for "temp/downloads" folder - uses: actions/cache@v4 - with: - path: ./temp/downloads - key: - ${{ runner.os }}-${{ hashFiles('**/*.sh') }} - - - name: Download ${{ env.PROJECT_NAME }}-${{ env.AXON_FRAMEWORK_VERSION }} - working-directory: temp - run: | - mkdir -p ${{ env.PROJECT_NAME }}-${{ env.AXON_FRAMEWORK_VERSION }} - cd ${{ env.PROJECT_NAME }}-${{ env.AXON_FRAMEWORK_VERSION }} - echo "Working directory: $( pwd -P )" - ./../../scripts/downloader/downloadAxonFramework.sh ${{ env.AXON_FRAMEWORK_VERSION }} - - - name: Analyze ${{ env.PROJECT_NAME }}-${{ env.AXON_FRAMEWORK_VERSION }} - working-directory: temp/${{ env.PROJECT_NAME }}-${{ env.AXON_FRAMEWORK_VERSION }} - # Shell type can be skipped if jupyter notebook reports (and therefore conda) aren't needed - shell: bash -el {0} - env: - NEO4J_INITIAL_PASSWORD: ${{ secrets.NEO4J_INITIAL_PASSWORD }} - ENABLE_JUPYTER_NOTEBOOK_PDF_GENERATION: "true" - IMPORT_GIT_LOG_DATA_IF_SOURCE_IS_PRESENT: "" # Options: "none", "aggregated", "full". default = "plugin" or "" - run: | - ./../../scripts/analysis/analyze.sh --profile Neo4jv5-low-memory - - - name: Move reports from the temp to the results directory preserving their surrounding directory - working-directory: temp - run: ./../scripts/copyReportsIntoResults.sh - - # Upload logs and unfinished reports in case of an error for troubleshooting - - name: Archive failed run with logs and unfinished results - if: failure() - uses: actions/upload-artifact@v4 - with: - name: java-code-analysis-logs-java-${{ matrix.java }}-python-${{ matrix.python }}-miniforge-${{ matrix.miniforge }} - path: | - ./temp/**/runtime/* - ./temp/**/reports/* - retention-days: 5 - - # Upload successful results in case they are needed for troubleshooting - - name: Archive successful results - if: success() - uses: actions/upload-artifact@v4 - with: - name: java-code-analysis-results-java-${{ matrix.java }}-python-${{ matrix.python }}-miniforge-${{ matrix.miniforge }} - path: ./temp/**/reports/* - if-no-files-found: error - retention-days: 5 - - # Upload Database Export - # Only possible after an export with "./../../scripts/analysis/analyze.sh --report DatabaseCsvExport" - # Won't be done here because of performance and security concerns - #- name: Archive exported database - # uses: actions/upload-artifact@v3 - # with: - # name: java-code-analysis-database-export-${{ matrix.java }}-python-${{ matrix.python }}-miniforge-${{ matrix.miniforge }} - # path: ./temp/**/import - # if-no-files-found: error - # retention-days: 5 - - # Commit and push the native image agent results - - name: Display environment variable "github.event_name" - run: echo "github.event_name=${{ github.event_name }}" - - name: Commit "results" directory containing the reports - # Only run when a pull request gets merged or a commit is pushed to the main branch - # git add parameters need to match paths-ignore parameters above - # Git pull before add/commit/push to reduce race conditions on parallel builds - if: github.event_name == 'push' - run: | - git config --global user.name '${{ env.CI_COMMIT_AUTHOR }}' - git config --global user.email "7671054+JohT@users.noreply.github.com" - git config --local http.postBuffer 524288000 - git fetch origin - git status - git add results - git status - git commit -m "${{ env.CI_COMMIT_MESSAGE }}" - git status - git rebase --strategy-option=theirs origin/main --verbose - git status - git push --verbose diff --git a/.github/workflows/public-analyze-code-graph.yml b/.github/workflows/public-analyze-code-graph.yml new file mode 100644 index 000000000..bddded158 --- /dev/null +++ b/.github/workflows/public-analyze-code-graph.yml @@ -0,0 +1,194 @@ +# This is the public version of the code graph analysis workflow that can be used by other projects. +name: Code Graph Analysis + +on: + workflow_call: + inputs: + analysis-name: + description: > + The name of the project to analyze. + Example: MyProject-1.0.0 + required: true + type: string + artifacts-upload-name: + description: > + The name of the artifacts uploaded with 'actions/upload-artifact' + containing the content of the 'artifacts' directory for the analysis. + Use it to analyze Java JARs, WARs, EARs, etc. + required: false + type: string + default: '' + sources-upload-name: + description: > + The name of the sources uploaded with 'actions/upload-artifact' + containing the content of the 'source' directory for the analysis. + Also supports sub-folders for multiple source code bases. + required: false + type: string + default: '' + ref: + description: > + The branch, tag or SHA of the code-graph-analysis-pipeline to checkout. + Default: "main" + required: false + type: string + default: '' + analysis-arguments: + description: > + The arguments to pass to the analysis script. + Default: '--profile Neo4jv5-low-memory' + required: false + type: string + default: '--profile Neo4jv5-low-memory' + typescript-scan-heap-memory: + description: > + The heap memory size in MB to use for the TypeScript code scans (default=4096). + This value is only used for the TypeScript code scans and is ignored for other scans. + required: false + type: string + default: '4096' + outputs: + uploaded-analysis-results: + description: > + The name of the artifact uploaded with 'actions/upload-artifact' + containing all analysis results. + value: ${{ jobs.analyze-code-graph.outputs.uploaded-analysis-results-artifact-name }} + +jobs: + analyze-code-graph: + runs-on: ubuntu-22.04 + outputs: + uploaded-analysis-results-artifact-name: ${{ steps.set-analysis-results-artifact-name.outputs.uploaded-analysis-results-artifact-name }} + strategy: + matrix: + include: + - os: ubuntu-22.04 + java: 17 + python: 3.11 + miniforge: 24.9.0-0 + steps: + - name: Assure that either artifacts-upload-name or sources-upload-name is set + if: inputs.artifacts-upload-name == '' && inputs.sources-upload-name == '' + run: echo "Please specify either the input parameter 'artifacts-upload-name' or 'sources-upload-name'."; exit 1 + + - name: Checkout code-graph-analysis-pipeline + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + repository: JohT/code-graph-analysis-pipeline + ref: ${{ inputs.ref }} + persist-credentials: false + + - name: (Java Setup) Java Development Kit (JDK) ${{ matrix.java }} + uses: actions/setup-java@7a6d8a8234af8eb26422e24e3006232cccaa061b # v4 + with: + distribution: "temurin" + java-version: ${{ matrix.java }} + + # "Setup Python" can be skipped if jupyter notebook analysis-results aren't needed + - name: (Python Setup) Setup Cache for Conda package manager Miniforge + uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4 + env: + # Increase this value to reset cache if etc/example-environment.yml has not changed + # Reference: https://github.com/conda-incubator/setup-miniconda#caching + CACHE_NUMBER: 0 + with: + path: ~/conda_pkgs_dir + key: + ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-environments-${{hashFiles('**/environment.yml', '.github/workflows/*.yml') }} + + - name: (Python Setup) Use version ${{ matrix.python }} with Conda package manager Miniforge + uses: conda-incubator/setup-miniconda@d2e6a045a86077fb6cad6f5adf368e9076ddaa8d # v3 + with: + python-version: ${{ matrix.python }} + miniforge-version: ${{ matrix.miniforge }} + activate-environment: codegraph + environment-file: ./jupyter/environment.yml + auto-activate-base: false + use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly! + - name: (Python Setup) Conda environment info + shell: bash -el {0} + run: conda info + + - name: (Code Analysis Setup) Setup Cache Analysis Downloads + uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4 + with: + path: ./temp/downloads + key: + ${{ runner.os }}-${{ hashFiles('**/*.sh') }} + + - name: (Code Analysis Setup) Generate Neo4j Initial Password + id: generate-neo4j-initial-password + run: | + generated_password=$( LC_ALL=C tr -dc '[:graph:]' > "$GITHUB_OUTPUT" + + - name: (Code Analysis Setup) Initialize Analysis + env: + NEO4J_INITIAL_PASSWORD: ${{ steps.generate-neo4j-initial-password.outputs.neo4j-initial-password }} + run: ./init.sh ${{ inputs.analysis-name }} + + - name: (Code Analysis Setup) Download sources for analysis + if: inputs.sources-upload-name != '' + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4 + with: + name: ${{ inputs.sources-upload-name }} + path: temp/${{ inputs.analysis-name }}/source/${{ inputs.analysis-name }} + + - name: (Code Analysis Setup) Download artifacts for analysis + if: inputs.artifacts-upload-name != '' + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4 + with: + name: ${{ inputs.artifacts-upload-name }} + path: temp/${{ inputs.analysis-name }}/artifacts + + - name: (Code Analysis) Analyze ${{ inputs.analysis-name }} + working-directory: temp/${{ inputs.analysis-name }} + # Shell type can be skipped if jupyter notebook analysis-results (and therefore conda) aren't needed + shell: bash -el {0} + env: + NEO4J_INITIAL_PASSWORD: ${{ steps.generate-neo4j-initial-password.outputs.neo4j-initial-password }} + ENABLE_JUPYTER_NOTEBOOK_PDF_GENERATION: "true" + IMPORT_GIT_LOG_DATA_IF_SOURCE_IS_PRESENT: "" # Options: "none", "aggregated", "full". default = "plugin" or "" + run: | + TYPESCRIPT_SCAN_HEAP_MEMORY=${{ inputs.typescript-scan-heap-memory }} ./../../scripts/analysis/analyze.sh ${{ inputs.analysis-arguments }} + + - name: Assemble ENVIRONMENT_INFO + run: echo "ENVIRONMENT_INFO=-java-${{ matrix.java }}-python-${{ matrix.python }}-miniforge-${{ matrix.miniforge }}" >> $GITHUB_ENV + + - name: Set artifact name for uploaded analysis results + id: set-analysis-results-artifact-name + run: echo "uploaded-analysis-results-artifact-name=code-analysis-results-${{ env.ENVIRONMENT_INFO }}" >> $GITHUB_OUTPUT + + # Upload successful analysis-results in case they are needed for troubleshooting + - name: (Code Analysis Results) Archive successful analysis-results + if: success() + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4 + with: + name: ${{ steps.set-analysis-results-artifact-name.outputs.uploaded-analysis-results-artifact-name }} + path: ./temp/${{ inputs.analysis-name }}/reports/* + if-no-files-found: error + retention-days: 5 + + + # Upload logs and unfinished analysis-results in case of an error for troubleshooting + - name: (Code Analysis Results) Archive failed run with logs and unfinished analysis-results + if: failure() + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4 + with: + name: code-analysis-logs-${{ env.ENVIRONMENT_INFO }} + path: | + ./temp/**/runtime/* + ./temp/**/reports/* + retention-days: 5 + + # Upload Database Export + # Only possible after an export with "./../../scripts/analysis/analyze.sh --report DatabaseCsvExport" + # Won't be done here because of performance and security concerns + #- name: Archive exported database + # uses: actions/upload-artifact@v3 + # with: + # name: typescript-code-analysis-database-export-${{ matrix.java }}-python-${{ matrix.python }}-miniforge-${{ matrix.miniforge }} + # path: ./temp/**/import + # if-no-files-found: error + # retention-days: 5 \ No newline at end of file diff --git a/.github/workflows/typescript-code-analysis.yml b/.github/workflows/typescript-code-analysis.yml deleted file mode 100644 index 699053316..000000000 --- a/.github/workflows/typescript-code-analysis.yml +++ /dev/null @@ -1,197 +0,0 @@ -name: Typescript Code Structure Graph Analysis - -on: - push: - branches: - - main - # Ignore changes in documentation, general configuration and reports for push events - paths-ignore: - - 'results/**' - - '**/*.md' - - '**/*.txt' - - '**/*.css' - - '**/*.html' - - '**/*.js' - - '.gitignore' - - '.gitattributes' - - 'renovate.json' - - 'changelogTemplate.mustache' - - '**.code-workspace' - - '.github/workflows/java-code-analysis.yml' - - '.github/workflows/*documentation.yml' - pull_request: - branches: - - main - # Ignore changes in documentation, general configuration and reports for pull request events - paths-ignore: - - 'results/**' - - '**/*.md' - - '**/*.txt' - - '**/*.css' - - '**/*.html' - - '**/*.js' - - '.gitignore' - - '.gitattributes' - - 'renovate.json' - - 'changelogTemplate.mustache' - - '**.code-workspace' - - '.github/workflows/java-code-analysis.yml' - - '.github/workflows/*documentation.yml' - -# Requires the secret NEO4J_INITIAL_PASSWORD to be configured -jobs: - reports: - runs-on: ubuntu-latest - strategy: - matrix: - include: - - os: ubuntu-latest - java: 17 - python: 3.11 - miniforge: 24.9.0-0 - - env: - CI_COMMIT_MESSAGE: Automated code structure analysis reports (CI) - CI_COMMIT_AUTHOR: ${{ github.event.repository.name }} Continuous Integration - PROJECT_NAME: react-router - # Version variable name matches renovate.json configuration entry - REACT_ROUTER_VERSION: 6.28.1 - - steps: - - name: Checkout GIT Repository - uses: actions/checkout@v4 - with: - token: ${{ secrets.WORKFLOW_GIT_ACCESS_TOKEN }} - - - name: Setup Java JDK ${{ matrix.java }} - uses: actions/setup-java@v4 - with: - distribution: 'adopt' - java-version: ${{ matrix.java }} - - - name: Setup Cache for Conda package manager Miniforge - uses: actions/cache@v4 - env: - # Increase this value to reset cache if etc/example-environment.yml has not changed - # Reference: https://github.com/conda-incubator/setup-miniconda#caching - CACHE_NUMBER: 0 - with: - path: ~/conda_pkgs_dir - key: - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-environments-${{hashFiles('**/environment.yml', '.github/workflows/*.yml') }} - - # "Setup Python" can be skipped if jupyter notebook reports aren't needed - - name: Setup Python ${{ matrix.python }} with Conda package manager Miniforge - uses: conda-incubator/setup-miniconda@v3 - with: - python-version: ${{ matrix.python }} - miniforge-version: ${{ matrix.miniforge }} - activate-environment: codegraph - environment-file: ./jupyter/environment.yml - auto-activate-base: false - use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly! - - - name: Conda environment info - shell: bash -el {0} - run: conda info - - - name: Conda environment list updates - shell: bash -el {0} - run: conda update --all --dry-run | grep -Fe '-->' - - - name: Setup temp directory if missing - run: mkdir -p ./temp - - - name: Setup Cache for "temp/downloads" folder - uses: actions/cache@v4 - with: - path: ./temp/downloads - key: - ${{ runner.os }}-${{ hashFiles('**/*.sh') }} - - - name: Download ${{ env.PROJECT_NAME }}-${{ env.REACT_ROUTER_VERSION }} - working-directory: temp - run: | - mkdir -p ${{ env.PROJECT_NAME }}-${{ env.REACT_ROUTER_VERSION }} - cd ${{ env.PROJECT_NAME }}-${{ env.REACT_ROUTER_VERSION }} - echo "Working directory: $( pwd -P )" - ./../../scripts/downloader/downloadReactRouter.sh ${{ env.REACT_ROUTER_VERSION }} - - - name: Setup pnpm for react-router - uses: pnpm/action-setup@v4.0.0 - with: - package_json_file: temp/${{env.PROJECT_NAME}}-${{env.REACT_ROUTER_VERSION}}/source/${{env.PROJECT_NAME}}-${{env.REACT_ROUTER_VERSION}}/package.json - - - name: Install dependencies with pnpm - working-directory: temp/${{ env.PROJECT_NAME }}-${{ env.REACT_ROUTER_VERSION }}/source/${{ env.PROJECT_NAME }}-${{ env.REACT_ROUTER_VERSION }} - run: | - pnpm install --frozen-lockfile --strict-peer-dependencies - - - name: Analyze ${{ env.PROJECT_NAME }}-${{ env.REACT_ROUTER_VERSION }} - working-directory: temp/${{ env.PROJECT_NAME }}-${{ env.REACT_ROUTER_VERSION }} - # Shell type can be skipped if jupyter notebook reports (and therefore conda) aren't needed - shell: bash -el {0} - env: - NEO4J_INITIAL_PASSWORD: ${{ secrets.NEO4J_INITIAL_PASSWORD }} - ENABLE_JUPYTER_NOTEBOOK_PDF_GENERATION: "true" - IMPORT_GIT_LOG_DATA_IF_SOURCE_IS_PRESENT: "" # Options: "none", "aggregated", "full". default = "plugin" or "" - run: | - ./../../scripts/analysis/analyze.sh --profile Neo4jv5-low-memory - - - name: Move reports from the temp to the results directory preserving their surrounding directory - working-directory: temp - run: ./../scripts/copyReportsIntoResults.sh - - # Upload logs and unfinished reports in case of an error for troubleshooting - - name: Archive failed run with logs and unfinished results - if: failure() - uses: actions/upload-artifact@v4 - with: - name: typescript-code-analysis-logs-java-${{ matrix.java }}-python-${{ matrix.python }}-miniforge-${{ matrix.miniforge }} - path: | - ./temp/**/runtime/* - ./temp/**/reports/* - retention-days: 5 - - # Upload successful results in case they are needed for troubleshooting - - name: Archive successful results - if: success() - uses: actions/upload-artifact@v4 - with: - name: typescript-code-analysis-results-java-${{ matrix.java }}-python-${{ matrix.python }}-miniforge-${{ matrix.miniforge }} - path: ./temp/**/reports/* - if-no-files-found: error - retention-days: 5 - - # Upload Database Export - # Only possible after an export with "./../../scripts/analysis/analyze.sh --report DatabaseCsvExport" - # Won't be done here because of performance and security concerns - #- name: Archive exported database - # uses: actions/upload-artifact@v3 - # with: - # name: typescript-code-analysis-database-export-${{ matrix.java }}-python-${{ matrix.python }}-miniforge-${{ matrix.miniforge }} - # path: ./temp/**/import - # if-no-files-found: error - # retention-days: 5 - - # Commit and push the native image agent results - - name: Display environment variable "github.event_name" - run: echo "github.event_name=${{ github.event_name }}" - - name: Commit "results" directory containing the reports - # Only run when a pull request gets merged or a commit is pushed to the main branch - # git add parameters need to match paths-ignore parameters above - # Git pull before add/commit/push to reduce race conditions on parallel builds - if: github.event_name == 'push' - run: | - git config --global user.name '${{ env.CI_COMMIT_AUTHOR }}' - git config --global user.email "7671054+JohT@users.noreply.github.com" - git config --local http.postBuffer 524288000 - git fetch origin - git status - git add results - git status - git commit -m "${{ env.CI_COMMIT_MESSAGE }}" - git status - git rebase --strategy-option=theirs origin/main --verbose - git status - git push --verbose diff --git a/COMMANDS.md b/COMMANDS.md index 4554f325a..2f7414c09 100644 --- a/COMMANDS.md +++ b/COMMANDS.md @@ -193,7 +193,7 @@ Change into the [scripts](./scripts/) directory e.g. with `cd scripts` and then The following command shows how to use [markdown-link-check](https://github.com/tcort/markdown-link-check) to for example check the links in the [README.md](./README.md) file: ```script -npx --yes markdown-link-check --quiet --progress --config=markdown-lint-check-config.json README.md COMMANDS.md GETTING_STARTED.md +npx --yes markdown-link-check --quiet --progress --config=markdown-lint-check-config.json README.md COMMANDS.md GETTING_STARTED.md INTEGRATION.md ``` ## Manual Setup diff --git a/INTEGRATION.md b/INTEGRATION.md new file mode 100644 index 000000000..f3d10b70d --- /dev/null +++ b/INTEGRATION.md @@ -0,0 +1,39 @@ +# Code Graph Analysis Pipeline - Integration guide + +This document describes the steps to get started as quickly as possible. +:point_right: For more details on what else you can do see [README](./README.md). +:point_right: For more details on how to analyze your code locally see [GETTING_STARTED](./GETTING_STARTED.md). + +## :rocket: How to use it + +This repository provides a reusable GitHub Actions Workflow to analyze code. The workflow is defined in [public-analyze-code-graph.yml](./.github/workflows/public-analyze-code-graph.yml). + +The main idea is to have three workflow jobs: + +1. **Collect source code and build artifacts**: Gather the source code and any build artifacts. +2. **Run the analysis**: Use the reusable workflow to analyze the collected code and artifacts. +3. **Download the reports**: Retrieve the analysis reports generated by the workflow. + +The workflow requires the names of the uploaded artifacts (source code and build artifacts) and provides the names of the artifact containing the analysis results for download. + +You can find examples in: + +- [internal-java-code-analysis.yml](./.github/workflows/internal-java-code-analysis.yml) +- [internal-typescript-code-analysis.yml](./.github/workflows/internal-typescript-code-analysis.yml) + +:warning: Note: Workflows with names starting with `internal-` are private and should not be used outside this repository. They may change at any time without notice. + +## :gear: Parameters + +The workflow parameters are as follows: + +- **analysis-name**: The name of the project to analyze. Example: MyProject-1.0.0. This parameter is required and should be a string. +- **artifacts-upload-name**: The name of the artifacts uploaded with [actions/upload-artifact](https://github.com/actions/upload-artifact/tree/65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08) containing the content of the 'artifacts' directory for the analysis. This is used to analyze Java JARs, WARs, EARs, etc. This parameter is optional and defaults to an empty string. +- **sources-upload-name**: The name of the sources uploaded with [actions/upload-artifact](https://github.com/actions/upload-artifact/tree/65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08) containing the content of the 'source' directory for the analysis. It also supports sub-folders for multiple source code bases. This parameter is optional and defaults to an empty string. +- **ref**: The branch, tag, or SHA of the code-graph-analysis-pipeline to checkout. This parameter is optional and defaults to "main". +- **analysis-arguments**: The arguments to pass to the analysis script. This parameter is optional and defaults to '--profile Neo4jv5-low-memory'. +- **typescript-scan-heap-memory**: The heap memory size in MB to use for the TypeScript code scans. This value is only used for the TypeScript code scans and is ignored for other scans. This parameter is optional and defaults to '4096'. + +The workflow also provides an output parameter: + +- **uploaded-analysis-results**: The name of the artifact uploaded with 'actions/upload-artifact' containing all analysis \ No newline at end of file diff --git a/README.md b/README.md index 96d8d8ede..f70cfccca 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ Contained within this repository is a comprehensive and automated code graph ana - Fully automated [pipeline for Java](./.github/workflows/java-code-analysis.yml) from tool installation to report generation - Fully automated [pipeline for Typescript](./.github/workflows/typescript-code-analysis.yml) from tool installation to report generation - Fully automated [local run](./GETTING_STARTED.md) +- Easy integratable in your [continuous integration pipeline](./INTEGRATION.md) - More than 130 CSV reports for dependencies, metrics, cycles, annotations, algorithms and many more - Jupyter notebook reports for dependencies, metrics, visibility and many more - Graph structure visualization @@ -105,6 +106,11 @@ This could be as simple as running the following command in your Typescript proj See [GETTING_STARTED.md](./GETTING_STARTED.md) on how to get started on your local machine. +## :rocket: Integration + +See [INTEGRATION.md](./INTEGRATION.md) on how to integrate code analysis in your continuous integration pipeline. +Currently (2025), only GitHub Actions are supported. + ## :building_construction: Pipeline and Tools The [Code Structure Analysis Pipeline](./.github/workflows/java-code-analysis.yml) utilizes [GitHub Actions](https://docs.github.com/de/actions) to automate the whole analysis process: