diff --git a/.github/actions/install_node_dependencies/action.yml b/.github/actions/install_node_dependencies/action.yml new file mode 100644 index 0000000000..bd69ddf4dd --- /dev/null +++ b/.github/actions/install_node_dependencies/action.yml @@ -0,0 +1,26 @@ +name: install_node_dependencies +description: Install Node dependencies + +runs: + using: composite + steps: + - uses: actions/setup-node@v4.0.0 + with: + node-version: 16.13.2 + # Sets up the Node environment with the specified version. + + - id: npm-cache-dir + run: echo "dir=$(npm config get cache)" >> $GITHUB_OUTPUT + shell: bash + # Retrieves the cache directory path for npm and stores it in the $GITHUB_OUTPUT environment variable. + + - uses: actions/cache@v3.3.2 + with: + path: "${{ steps.npm-cache-dir.outputs.dir }}" + key: "${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}" + restore-keys: "${{ runner.os }}-node-" + # Caches the Node dependencies using the specified cache key and restore keys. + + - run: npm ci + shell: bash + # Installs the Node dependencies using the npm ci command. diff --git a/.github/actions/setup_cypress_requirements/action.yml b/.github/actions/setup_cypress_requirements/action.yml new file mode 100644 index 0000000000..aa0c50684c --- /dev/null +++ b/.github/actions/setup_cypress_requirements/action.yml @@ -0,0 +1,11 @@ +name: setup_cypress_requirements +description: Setup Cypress requirements +runs: + using: composite + steps: + - name: Install cypress system dependencies + run: |- + sudo sed -i 's/archive.ubuntu.com/us-east-1.ec2.archive.ubuntu.com/g' /etc/apt/sources.list + sudo apt-get update + sudo apt-get install libgtk2.0-0 libgtk-3-0 libgbm-dev libnotify-dev libgconf-2-4 libnss3 libxss1 libasound2 libxtst6 xauth xvfb + shell: bash \ No newline at end of file diff --git a/.github/actions/setup_python_env/action.yml b/.github/actions/setup_python_env/action.yml new file mode 100644 index 0000000000..5d228b8c07 --- /dev/null +++ b/.github/actions/setup_python_env/action.yml @@ -0,0 +1,17 @@ +name: setup_python_env +description: Setup Python environment +runs: + using: composite + steps: + - name: Install Python dependencies + run: |- + pip install git+https://github.com/kedro-org/kedro@main + pip install wheel + pip install toposort + pip install -r package/test_requirements.txt -r demo-project/src/docker_requirements.txt -U + shell: bash + - name: Echo package versions + run: |- + python -V + pip freeze + shell: bash \ No newline at end of file diff --git a/.github/workflows/all-checks.yml b/.github/workflows/all-checks.yml new file mode 100644 index 0000000000..aebbd5c678 --- /dev/null +++ b/.github/workflows/all-checks.yml @@ -0,0 +1,44 @@ +name: Run all checks on Kedro-Viz +on: + workflow_call: + workflow_dispatch: +jobs: + e2e_tests: + strategy: + matrix: + os: [ windows-latest, ubuntu-latest ] + python-version: [ "3.9", "3.10", "3.11" ] + uses: ./.github/workflows/e2e-tests.yml + with: + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} + + unit_tests: + strategy: + matrix: + os: [ windows-latest, ubuntu-latest ] + python-version: [ "3.9", "3.10", "3.11" ] + uses: ./.github/workflows/unit-tests.yml + with: + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} + + lint: + strategy: + matrix: + os: [ ubuntu-latest ] + python-version: [ "3.9", "3.10", "3.11" ] + uses: ./.github/workflows/lint.yml + with: + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} + + javascript_lint_and_tests: + strategy: + matrix: + os: [ ubuntu-latest ] + python-version: [ "3.9" ] + uses: ./.github/workflows/javascript-lint-and-tests.yml + with: + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/build-backend.yml b/.github/workflows/build-backend.yml new file mode 100644 index 0000000000..e7dc4f5d6e --- /dev/null +++ b/.github/workflows/build-backend.yml @@ -0,0 +1,41 @@ +name: Build backend +on: + push: + paths: + - 'package/**' + - '.github/**' + pull_request: + paths: + - 'package/**' + - '.github/**' + workflow_dispatch: +jobs: + e2e_tests: + strategy: + matrix: + os: [ windows-latest, ubuntu-latest ] + python-version: [ "3.9", "3.10", "3.11" ] + uses: ./.github/workflows/e2e-tests.yml + with: + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} + + unit_tests: + strategy: + matrix: + os: [ windows-latest, ubuntu-latest ] + python-version: [ "3.9", "3.10", "3.11" ] + uses: ./.github/workflows/unit-tests.yml + with: + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} + + lint: + strategy: + matrix: + os: [ ubuntu-latest ] + python-version: [ "3.9", "3.10", "3.11" ] + uses: ./.github/workflows/lint.yml + with: + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/build-frontend.yml b/.github/workflows/build-frontend.yml new file mode 100644 index 0000000000..6224bab0b9 --- /dev/null +++ b/.github/workflows/build-frontend.yml @@ -0,0 +1,19 @@ +name: Build frontend +on: + push: + paths-ignore: + - 'package/**' + pull_request: + paths-ignore: + - 'package/**' + workflow_dispatch: +jobs: + javascript_lint_and_tests: + strategy: + matrix: + os: [ ubuntu-latest ] + python-version: [ "3.9" ] + uses: ./.github/workflows/javascript-lint-and-tests.yml + with: + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} \ No newline at end of file diff --git a/.github/workflows/check-release.yml b/.github/workflows/check-release.yml index 8b73b1cc8d..b0c4742e74 100644 --- a/.github/workflows/check-release.yml +++ b/.github/workflows/check-release.yml @@ -36,9 +36,17 @@ jobs: package_name: ${{ steps.version_check.outputs.package_name }} package_version: ${{ steps.version_check.outputs.package_version }} - prepare-release: - needs: [check-version] + test-kedro-viz: + needs: check-version if: ${{ needs.check-version.outputs.new_release == 'true' }} + uses: ./.github/workflows/all-checks.yml + + prepare-release: + needs: [check-version, test-kedro-viz] + if: | + !contains(needs.*.result, 'failure') && + !contains(needs.*.result, 'cancelled') && + needs.check-version.outputs.new_release == 'true' runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml new file mode 100644 index 0000000000..1e003e1bf3 --- /dev/null +++ b/.github/workflows/daily.yml @@ -0,0 +1,8 @@ +name: Daily all checks +on: + schedule: + - cron: 0 1 * * * + workflow_dispatch: +jobs: + all-checks: + uses: ./.github/workflows/all-checks.yml diff --git a/.github/workflows/deploy-demo.yml b/.github/workflows/deploy-demo.yml new file mode 100644 index 0000000000..fce412d8bb --- /dev/null +++ b/.github/workflows/deploy-demo.yml @@ -0,0 +1,38 @@ +name: Deploy demo +on: + workflow_dispatch: + push: + branches: + - demo +jobs: + deploy_demo: + runs-on: ubuntu-latest + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + steps: + - name: Checkout code + uses: actions/checkout@v4.1.0 + + - name: Setup environment + run: |- + cd demo-project + echo "AWS_ECR_URL=public.ecr.aws/h1b3x4b6/jitu5-kedro-viz" >> $GITHUB_ENV + echo "KEDRO_VIZ_VERSION=$(cat .version)" >> $GITHUB_ENV + + - name: Install AWS CLI + run: pip3 install awscli + + - name: Build demo container image + run: |- + cd demo-project + echo "kedro_viz==$KEDRO_VIZ_VERSION" >> src/docker_requirements.txt + docker build -t $AWS_ECR_URL:$KEDRO_VIZ_VERSION . + aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws + docker push $AWS_ECR_URL:$KEDRO_VIZ_VERSION + + - name: Create a new lightsail deployment + run: |- + cd demo-project + # run https://docs.aws.amazon.com/cli/latest/reference/lightsail/create-container-service-deployment.html#create-container-service-deployment + aws lightsail create-container-service-deployment --region eu-west-2 --cli-input-json file://./lightsail.json diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml new file mode 100644 index 0000000000..37d5f92bcc --- /dev/null +++ b/.github/workflows/e2e-tests.yml @@ -0,0 +1,49 @@ +name: Run e2e tests on Kedro-Viz + +on: + workflow_call: + inputs: + os: + type: string + python-version: + type: string +jobs: + e2e_tests: + runs-on: ${{ inputs.os }} + if: inputs.os == 'ubuntu-latest' || ((github.ref == 'refs/heads/main' || github.ref == 'refs/heads/demo') && inputs.os == 'windows-latest') + steps: + - name: Checkout code + uses: actions/checkout@v4.1.0 + + - name: Set up Python ${{inputs.python-version}} + uses: actions/setup-python@v5 + with: + python-version: ${{inputs.python-version}} + + - name: Cache python packages for Linux + if: inputs.os == 'ubuntu-latest' + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: ${{inputs.os}}-python-${{inputs.python-version}} + + - name: Cache python packages for Windows + if: inputs.os == 'windows-latest' + uses: actions/cache@v4 + with: + path: ~\AppData\Local\pip\Cache + key: ${{inputs.os}}-python-${{inputs.python-version}} + + - name: Setup Python environment + uses: "./.github/actions/setup_python_env" + + - name: Install Node dependencies + uses: "./.github/actions/install_node_dependencies" + + - name: Build React application + run: |- + node -v + make build + + - name: Run Python tests + run: make e2e-tests \ No newline at end of file diff --git a/.github/workflows/javascript-lint-and-tests.yml b/.github/workflows/javascript-lint-and-tests.yml new file mode 100644 index 0000000000..da88f96755 --- /dev/null +++ b/.github/workflows/javascript-lint-and-tests.yml @@ -0,0 +1,54 @@ +name: Run javascript linters and tests on Kedro-Viz + +on: + workflow_call: + inputs: + os: + type: string + python-version: + type: string +jobs: + javascript_lint_and_tests: + runs-on: ${{ inputs.os }} + steps: + - name: Checkout code + uses: actions/checkout@v4.1.0 + + - name: Set up Python ${{inputs.python-version}} + uses: actions/setup-python@v5 + with: + python-version: ${{inputs.python-version}} + + - name: Cache python packages for Linux + if: inputs.os == 'ubuntu-latest' + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: ${{inputs.os}}-python-${{inputs.python-version}} + + - name: Setup Python environment + uses: "./.github/actions/setup_python_env" + + - name: Install Node dependencies + uses: "./.github/actions/install_node_dependencies" + + - name: Setup Cypress requirements + uses: "./.github/actions/setup_cypress_requirements" + + - name: Test lib transpilation + run: npm run lib + + - name: Test JS library imports + run: |- + npm run lib-test:setup + cd tools/test-lib/react-app + npm run test:ci + + - name: Run Eslint + run: npm run lint + + - name: Run JavaScript tests + run: npm run test:ci + + - name: Run Javascript end to end tests + run: npm run cy:ci diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000000..20082a05f7 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,42 @@ +name: Run linters on Kedro-Viz + +on: + workflow_call: + inputs: + os: + type: string + python-version: + type: string +jobs: + lint: + runs-on: ${{ inputs.os }} + steps: + - name: Checkout code + uses: actions/checkout@v4.1.0 + + - name: Set up Python ${{inputs.python-version}} + uses: actions/setup-python@v5 + with: + python-version: ${{inputs.python-version}} + + - name: Cache python packages for Linux + if: inputs.os == 'ubuntu-latest' + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: ${{inputs.os}}-python-${{inputs.python-version}} + + - name: Setup Python environment + uses: "./.github/actions/setup_python_env" + + - name: Run secret scan + run: make secret-scan + + - name: Run security scan + run: make security-scan + + - name: Verify GraphQL schema is up to date + run: make schema-check + + - name: Run Python formatters and linters + run: make format-check lint-check diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000000..35183d1d30 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,75 @@ +name: Release +on: + release: + types: [published] + workflow_dispatch: +env: + # AWS_REGION: xxxxst-2 + # CACHE_VERSION: xxxx + # SNYK_TOKEN: xxxxa727 + TWINE_NON_INTERACTIVE: true + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} + TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} + TWINE_REPOSITORY_URL: ${{ secrets.TWINE_REPOSITORY_URL }} + TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }} +jobs: + release_to_npm: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4.1.0 + + - uses: actions/setup-node@v4 + with: + node-version: 16.13.2 + registry-url: 'https://registry.npmjs.org' + + - name: Install Node dependencies + run: npm ci + + - name: Build React application + run: |- + node -v + make build + + - name: Publish to npm + run: npm publish --access=public + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + + release_to_pypi: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4.1.0 + + - name: Set up Python 3.9 + uses: actions/setup-python@v5 + with: + python-version: 3.9 + + - name: Cache python packages for Linux + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: ubuntu-latest-python-3.9 + + - name: Setup Python environment + uses: "./.github/actions/setup_python_env" + + - name: Install Node dependencies + uses: "./.github/actions/install_node_dependencies" + + - name: Build React application + run: |- + node -v + make build + + - name: Make Python package + run: make package + + - name: Install twine + run: python -m pip install -U twine + + - name: Publish to PyPI + run: python -m twine upload package/dist/* diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml new file mode 100644 index 0000000000..6c33fcd2d8 --- /dev/null +++ b/.github/workflows/unit-tests.yml @@ -0,0 +1,49 @@ +name: Run unit-tests on Kedro-Viz + +on: + workflow_call: + inputs: + os: + type: string + python-version: + type: string +jobs: + unit_tests: + runs-on: ${{ inputs.os }} + if: inputs.os == 'ubuntu-latest' || ((github.ref == 'refs/heads/main' || github.ref == 'refs/heads/demo') && inputs.os == 'windows-latest') + steps: + - name: Checkout code + uses: actions/checkout@v4.1.0 + + - name: Set up Python ${{inputs.python-version}} + uses: actions/setup-python@v5 + with: + python-version: ${{inputs.python-version}} + + - name: Cache python packages for Linux + if: inputs.os == 'ubuntu-latest' + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: ${{inputs.os}}-python-${{inputs.python-version}} + + - name: Cache python packages for Windows + if: inputs.os == 'windows-latest' + uses: actions/cache@v4 + with: + path: ~\AppData\Local\pip\Cache + key: ${{inputs.os}}-python-${{inputs.python-version}} + + - name: Setup Python environment + uses: "./.github/actions/setup_python_env" + + - name: Install Node dependencies + uses: "./.github/actions/install_node_dependencies" + + - name: Build React application + run: |- + node -v + make build + + - name: Run Python tests + run: make pytest \ No newline at end of file