Merge branch 'main' into maikypedia/javascript-cors

Author: maikypedia

.bazelrc

@@ -1,9 +1,28 @@
 common --enable_platform_specific_config
+common --enable_bzlmod
+# because we use --override_module with `%workspace%`, the lock file is not stable
+common --lockfile_mode=off
+
+# when building from this repository in isolation, the internal repository will not be found at ..
+# where `MODULE.bazel` looks for it. The following will get us past the module loading phase, so
+# that we can build things that do not rely on that
+common --override_module=semmle_code=%workspace%/misc/bazel/semmle_code_stub
 
 build --repo_env=CC=clang --repo_env=CXX=clang++
 
-build:linux --cxxopt=-std=c++20
-build:macos --cxxopt=-std=c++20 --cpu=darwin_x86_64
-build:windows --cxxopt=/std:c++20 --cxxopt=/Zc:preprocessor
+# we use transitions that break builds of `...`, so for `test` to work with that we need the following
+test --build_tests_only
+
+# this requires developer mode, but is required to have pack installer functioning
+startup --windows_enable_symlinks
+common --enable_runfiles
+
+# with the above, we can avoid building python zips which is the default on windows as that's expensive
+build --nobuild_python_zip
+
+common --registry=file:///%workspace%/misc/bazel/registry
+common --registry=https://bcr.bazel.build
+
+common --@rules_dotnet//dotnet/settings:strict_deps=false
 
 try-import %workspace%/local.bazelrc

.bazelrc.internal

@@ -0,0 +1,10 @@
+# this file should contain bazel settings required to build things from `semmle-code`
+
+common --registry=file:///%workspace%/ql/misc/bazel/registry
+common --registry=https://bcr.bazel.build
+
+# See bazelbuild/rules_dotnet#413: strict_deps in C# also appliy to 3rd-party deps, and when we pull
+# in (for example) the xunit package, there's no code in this at all, it just depends transitively on
+# its implementation packages without providing any code itself.
+# We either can depend on internal implementation details, or turn of strict deps.
+common --@rules_dotnet//dotnet/settings:strict_deps=false

.bazelversion

@@ -1 +1 @@
-6.3.1
+7.2.1

.clang-format

@@ -0,0 +1 @@
+DisableFormat: true

.gitattributes

@@ -50,24 +50,40 @@
 *.dll -text
 *.pdb -text
 
-java/ql/test/stubs/**/*.java linguist-generated=true
-java/ql/test/experimental/stubs/**/*.java linguist-generated=true
+/java/ql/test/stubs/**/*.java linguist-generated=true
+/java/ql/test/experimental/stubs/**/*.java linguist-generated=true
+/java/kotlin-extractor/deps/*.jar filter=lfs diff=lfs merge=lfs -text
 
 # Force git not to modify line endings for go or html files under the go/ql directory
-go/ql/**/*.go -text
-go/ql/**/*.html -text
+/go/ql/**/*.go -text
+/go/ql/**/*.html -text
 # Force git not to modify line endings for go dbschemes
-go/*.dbscheme -text
+/go/*.dbscheme -text
 # Preserve unusual line ending from codeql-go merge
-go/extractor/opencsv/CSVReader.java -text
+/go/extractor/opencsv/CSVReader.java -text
 
 # For some languages, upgrade script testing references really old dbscheme
 # files from legacy upgrades that have CRLF line endings. Since upgrade
 # resolution relies on object hashes, we must suppress line ending conversion
 # for those testing dbscheme files.
-*/ql/lib/upgrades/initial/*.dbscheme -text
+/*/ql/lib/upgrades/initial/*.dbscheme -text
 
-# Generated test files - these are synced from the standard JavaScript libraries using
-# `javascript/ql/experimental/adaptivethreatmodeling/test/update_endpoint_test_files.py`.
-javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/**/*.js linguist-generated=true -merge
-javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/**/*.ts linguist-generated=true -merge
+# Auto-generated modeling for Python
+/python/ql/lib/semmle/python/frameworks/data/internal/subclass-capture/*.yml linguist-generated=true
+
+# auto-generated bazel lock file
+/ruby/extractor/cargo-bazel-lock.json linguist-generated=true
+/ruby/extractor/cargo-bazel-lock.json -merge
+
+# auto-generated files for the C# build
+/csharp/paket.lock linguist-generated=true
+# needs eol=crlf, as `paket` touches this file and saves it as crlf
+/csharp/.paket/Paket.Restore.targets linguist-generated=true eol=crlf
+/csharp/paket.main.bzl linguist-generated=true
+/csharp/paket.main_extension.bzl linguist-generated=true
+
+# ripunzip tool
+/misc/ripunzip/ripunzip-* filter=lfs diff=lfs merge=lfs -text
+
+# swift prebuilt resources
+/swift/third_party/resource-dir/*.zip filter=lfs diff=lfs merge=lfs -text

.github/dependabot.yml

@@ -19,11 +19,24 @@ updates:
         update-types: ['version-update:semver-patch', 'version-update:semver-minor']
 
   - package-ecosystem: "gomod"
-    directory: "go"
+    directory: "go/extractor"
     schedule:
       interval: "daily"
     allow:
       - dependency-name: "golang.org/x/mod"
       - dependency-name: "golang.org/x/tools"
+    groups:
+      extractor-dependencies:
+        patterns:
+          - "golang.org/x/*"
+    reviewers:
+      - "github/codeql-go"
+
+  - package-ecosystem: "gomod"
+    directory: "go/ql/test"
+    schedule:
+      interval: "monthly"
+    ignore:
+      - dependency-name: "*"
     reviewers:
       - "github/codeql-go"

.github/labeler.yml

@@ -15,12 +15,12 @@ Java:
   - change-notes/**/*java.*
 
 JS:
-  - any: [ 'javascript/**/*', '!javascript/ql/experimental/adaptivethreatmodeling/**/*' ]
+  - any: [ 'javascript/**/*' ]
   - change-notes/**/*javascript*
 
 Kotlin:
   - java/kotlin-extractor/**/*
-  - java/ql/test/kotlin/**/*
+  - java/ql/test-kotlin*/**/*
 
 Python:
   - python/**/*
@@ -46,6 +46,3 @@ documentation:
 # Since these are all shared files that need to be synced, just pick _one_ copy of each.
 "DataFlow Library":
   - "shared/dataflow/**/*"
-
-"ATM":
-  - javascript/ql/experimental/adaptivethreatmodeling/**/*

.github/workflows/build-ripunzip.yml

@@ -0,0 +1,74 @@
+name: Build runzip
+
+on:
+  workflow_dispatch:
+    inputs:
+      ripunzip-version:
+        description: "what reference to checktout from google/runzip"
+        required: false
+        default: v1.2.1
+      openssl-version:
+        description: "what reference to checkout from openssl/openssl for Linux"
+        required: false
+        default: openssl-3.3.0
+
+jobs:
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-20.04, macos-12, windows-2019]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          repository: google/ripunzip
+          ref: ${{ inputs.ripunzip-version }}
+      # we need to avoid ripunzip dynamically linking into libssl
+      # see https://github.com/sfackler/rust-openssl/issues/183
+      - if: runner.os == 'Linux'
+        name: checkout openssl
+        uses: actions/checkout@v4
+        with:
+          repository: openssl/openssl
+          path: openssl
+          ref: ${{ inputs.openssl-version }}
+      - if: runner.os == 'Linux'
+        name: build and install openssl with fPIC
+        shell: bash
+        working-directory: openssl
+        run: |
+          ./config -fPIC --prefix=$HOME/.local --openssldir=$HOME/.local/ssl
+          make -j $(nproc)
+          make install_sw -j $(nproc)
+      - if: runner.os == 'Linux'
+        name: build (linux)
+        shell: bash
+        run: |
+          env OPENSSL_LIB_DIR=$HOME/.local/lib64 OPENSSL_INCLUDE_DIR=$HOME/.local/include OPENSSL_STATIC=yes cargo build --release
+          mv target/release/ripunzip ripunzip-linux
+      - if: runner.os == 'Windows'
+        name: build (windows)
+        shell: bash
+        run: |
+          cargo build --release
+          mv target/release/ripunzip ripunzip-windows
+      - name: build (macOS)
+        if: runner.os == 'macOS'
+        shell: bash
+        run: |
+          rustup target install x86_64-apple-darwin
+          rustup target install aarch64-apple-darwin
+          cargo build --target x86_64-apple-darwin --release
+          cargo build --target aarch64-apple-darwin --release
+          lipo -create -output ripunzip-macos \
+            -arch x86_64 target/x86_64-apple-darwin/release/ripunzip \
+            -arch arm64 target/aarch64-apple-darwin/release/ripunzip
+      - uses: actions/upload-artifact@v4
+        with:
+          name: ripunzip-${{ runner.os }}
+          path: ripunzip-*
+      - name: Check built binary
+        shell: bash
+        run: |
+          ./ripunzip-* --version

.github/workflows/buildifier.yml

@@ -0,0 +1,28 @@
+name: Check bazel formatting
+
+on:
+  pull_request:
+    paths:
+      - "**.bazel"
+      - "**.bzl"
+    branches:
+      - main
+      - "rc/*"
+
+permissions:
+  contents: read
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Check bazel formatting
+        uses: pre-commit/action@646c83fcd040023954eafda54b4db0192ce70507
+        with:
+          extra_args: >
+            buildifier --all-files 2>&1 ||
+            (
+              echo -e "In order to format all bazel files, please run:\n  bazel run //misc/bazel:buildifier"; exit 1
+            )

.github/workflows/check-change-note.yml

@@ -1,5 +1,8 @@
 name: Check change note
 
+permissions:
+  pull-requests: read
+
 on:
   pull_request_target:
     types: [labeled, unlabeled, opened, synchronize, reopened, ready_for_review]
@@ -9,26 +12,42 @@ on:
       - "*/ql/lib/**/*.ql"
       - "*/ql/lib/**/*.qll"
       - "*/ql/lib/**/*.yml"
+      - "shared/**/*.ql"
+      - "shared/**/*.qll"
       - "!**/experimental/**"
       - "!ql/**"
       - ".github/workflows/check-change-note.yml"
 
 jobs:
   check-change-note:
+    env: 
+      REPO: ${{ github.repository }}
+      PULL_REQUEST_NUMBER: ${{ github.event.number }}
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     runs-on: ubuntu-latest
     steps:
+
       - name: Fail if no change note found. To fix, either add one, or add the `no-change-note-required` label.
         if: |
           github.event.pull_request.draft == false &&
           !contains(github.event.pull_request.labels.*.name, 'no-change-note-required')
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
-          gh api 'repos/${{github.repository}}/pulls/${{github.event.number}}/files' --paginate --jq 'any(.[].filename ; test("/change-notes/.*[.]md$"))' |
-          grep true -c
+          change_note_files=$(gh api "repos/$REPO/pulls/$PULL_REQUEST_NUMBER/files" --paginate --jq '.[].filename | select(test("/change-notes/.*[.]md$"))')
+          
+          if [ -z "$change_note_files" ]; then
+            echo "No change note found. Either add one, or add the 'no-change-note-required' label."
+            exit 1
+          fi
+
+          echo "Change notes found:"
+          echo "$change_note_files"
+
       - name: Fail if the change note filename doesn't match the expected format. The file name must be of the form 'YYYY-MM-DD.md', 'YYYY-MM-DD-{title}.md', where '{title}' is arbitrary text, or released/x.y.z.md for released change-notes
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
-          gh api 'repos/${{github.repository}}/pulls/${{github.event.number}}/files' --paginate --jq '[.[].filename | select(test("/change-notes/.*[.]md$"))] | all(test("/change-notes/[0-9]{4}-[0-9]{2}-[0-9]{2}.*[.]md$") or test("/change-notes/released/[0-9]*[.][0-9]*[.][0-9]*[.]md$"))' |
-          grep true -c
+          bad_change_note_file_names=$(gh api "repos/$REPO/pulls/$PULL_REQUEST_NUMBER/files" --paginate --jq '[.[].filename | select(test("/change-notes/.*[.]md$"))][] | select((test("/change-notes/[0-9]{4}-[0-9]{2}-[0-9]{2}.*[.]md$") or test("/change-notes/released/[0-9]*[.][0-9]*[.][0-9]*[.]md$")) | not)')
+
+          if [ -n "$bad_change_note_file_names" ]; then
+            echo "The following change note file names are invalid:"
+            echo "$bad_change_note_file_names"
+            exit 1
+          fi