Merge pull request #353 from JohT/feature/auto-update-conda-environment

Auto update Conda Environment

Author: JohT

.github/workflows/internal-java-code-analysis.yml

@@ -56,6 +56,10 @@ jobs:
     - name: Checkout GIT Repository
       uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
 
+    - name: Run script tests
+      id: script-tests
+      run: ./scripts/runTests.sh
+
     - name: Set Set output variable 'analysis-name'
       id: set-analysis-name
       run: echo "analysis-name=${{ env.PROJECT_NAME }}-${{ env.AXON_FRAMEWORK_VERSION }}" >> "$GITHUB_OUTPUT"

.github/workflows/public-analyze-code-graph.yml

@@ -97,6 +97,7 @@ jobs:
 
       # "Setup Python" can be skipped if jupyter notebook analysis-results aren't needed
       - name: (Python Setup) Use version ${{ matrix.python }} with Conda package manager Miniforge
+        id: prepare-conda-environment
         uses: conda-incubator/setup-miniconda@505e6394dae86d6a5c7fbb6e3fb8938e3e863830 # v3
         with:
           python-version: ${{ matrix.python }}
@@ -160,6 +161,7 @@ jobs:
           NEO4J_INITIAL_PASSWORD: ${{ steps.generate-neo4j-initial-password.outputs.neo4j-initial-password }}
           ENABLE_JUPYTER_NOTEBOOK_PDF_GENERATION: "true"
           IMPORT_GIT_LOG_DATA_IF_SOURCE_IS_PRESENT: "" # Options: "none", "aggregated", "full". default = "plugin" or ""
+          PREPARE_CONDA_ENVIRONMENT: "false" # Had already been done in step with id "prepare-conda-environment".
         run: |
           TYPESCRIPT_SCAN_HEAP_MEMORY=${{ inputs.typescript-scan-heap-memory }} ./../../scripts/analysis/analyze.sh ${{ inputs.analysis-arguments }}
     

scripts/activateCondaEnvironment.sh

@@ -17,25 +17,35 @@ set -o errexit -o pipefail
 # CDPATH reduces the scope of the cd command to potentially prevent unintended directory changes.
 # This way non-standard tools like readlink aren't needed.
 SCRIPTS_DIR=${SCRIPTS_DIR:-$( CDPATH=. cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P )} # Repository directory containing the shell scripts
-echo "activateCondaEnvironment: SCRIPTS_DIR=$SCRIPTS_DIR"
+echo "activateCondaEnvironment: SCRIPTS_DIR=${SCRIPTS_DIR}"
 
 # Get the "jupyter" directory by taking the path of this script and going two directory up and then to "jupyter".
 JUPYTER_NOTEBOOK_DIRECTORY=${JUPYTER_NOTEBOOK_DIRECTORY:-"${SCRIPTS_DIR}/../jupyter"} # Repository directory containing the Jupyter Notebooks
-echo "activateCondaEnvironment: JUPYTER_NOTEBOOK_DIRECTORY=$JUPYTER_NOTEBOOK_DIRECTORY"
+echo "activateCondaEnvironment: JUPYTER_NOTEBOOK_DIRECTORY=${JUPYTER_NOTEBOOK_DIRECTORY}"
+
+# Get the file name of the environment description file for the conda package and environment manager 
+# that contains all dependencies and their versions.
+CONDA_ENVIRONMENT_FILE=${CONDA_ENVIRONMENT_FILE:-"${JUPYTER_NOTEBOOK_DIRECTORY}/environment.yml"} # Conda (package manager for Python) environment file path
+if [ ! -f "${CONDA_ENVIRONMENT_FILE}" ] ; then
+    echo "activateCondaEnvironment: Couldn't find environment file ${CONDA_ENVIRONMENT_FILE}."
+    exit 2
+fi
 
 # Define conda environment to use for code structure analysis. Default "codegraph"
 CODEGRAPH_CONDA_ENVIRONMENT=${CODEGRAPH_CONDA_ENVIRONMENT:-"codegraph"} # Name of the conda environment to use for code graph analysis
 echo "activateCondaEnvironment: CONDA_PREFIX=${CONDA_PREFIX}"
 echo "activateCondaEnvironment: Current conda environment=${CONDA_DEFAULT_ENV}"
 echo "activateCondaEnvironment: Target conda environment=${CODEGRAPH_CONDA_ENVIRONMENT}"
 
-if [ "${CONDA_DEFAULT_ENV}" = "${CODEGRAPH_CONDA_ENVIRONMENT}" ] ; then
+PREPARE_CONDA_ENVIRONMENT=${PREPARE_CONDA_ENVIRONMENT:-"true"} # Wether to prepare then Conda environment if needed (default, "true") or use an already prepared Conda environment ("false")
+
+if [ "${CONDA_DEFAULT_ENV}" = "${CODEGRAPH_CONDA_ENVIRONMENT}" ] && [ "${PREPARE_CONDA_ENVIRONMENT}" = "false" ]; then
     echo "activateCondaEnvironment: Skipping activation. Target conda environment ${CODEGRAPH_CONDA_ENVIRONMENT} is already activated."
     # "return" needs to be used here instead of "exit".
     # This script is included in another script by using "source". 
     # "exit" would end the main script, "return" just ends this sub script.
     return 0
-fi
+fi 
 
 # Include operation system function to for example detect Windows.
 source "${SCRIPTS_DIR}/operatingSystemFunctions.sh"
@@ -60,18 +70,24 @@ echo "activateCondaEnvironment: scriptExtension=${scriptExtension}"
 eval "$(${pathToConda}conda${scriptExtension} shell.bash hook)"
 echo "activateCondaEnvironment: Current conda environment after shell hook=${CONDA_DEFAULT_ENV}"
 
-# Create (if missing) and activate Conda environment for code structure graph analysis
-if { "${pathToConda}conda" env list | grep "$CODEGRAPH_CONDA_ENVIRONMENT "; } >/dev/null 2>&1; then
-    echo "activateCondaEnvironment: Conda environment $CODEGRAPH_CONDA_ENVIRONMENT already created"
-else
-    if [ ! -f "${JUPYTER_NOTEBOOK_DIRECTORY}/environment.yml" ] ; then
-        echo "activateCondaEnvironment: Couldn't find environment file ${jupyter_notebook_file_path}/environment.yml."
-        exit 2
+# If missing, create Conda environment for code graph analysis
+# Note: The curly braces are grouping the outputs of both (piped) operations together to suppress them later (dev/null).
+if "${pathToConda}conda" env list | grep "${CODEGRAPH_CONDA_ENVIRONMENT} " >/dev/null 2>&1; then
+    echo "activateCondaEnvironment: Conda environment ${CODEGRAPH_CONDA_ENVIRONMENT} has already been created."
+    
+    # Check if the declaration in the environment file matches the actual environment to find out if it needs to be updated.
+    if "${pathToConda}conda" compare --name "${CODEGRAPH_CONDA_ENVIRONMENT}" "${CONDA_ENVIRONMENT_FILE}" >/dev/null 2>&1; then
+        echo "activateCondaEnvironment: Conda environment ${CODEGRAPH_CONDA_ENVIRONMENT} is up-to-date."
+    else
+        echo "activateCondaEnvironment: Conda environment ${CODEGRAPH_CONDA_ENVIRONMENT} needs to be updated."
+        "${pathToConda}conda" env update --file "${CONDA_ENVIRONMENT_FILE}" --name ${CODEGRAPH_CONDA_ENVIRONMENT} --prune
     fi
-    echo "activateCondaEnvironment: Creating Conda environment ${CODEGRAPH_CONDA_ENVIRONMENT}"
-    "${pathToConda}conda" env create --file "${jupyter_notebook_file_path}/environment.yml" --name "${CODEGRAPH_CONDA_ENVIRONMENT}"
+else
+    echo "activateCondaEnvironment: Creating Conda environment ${CODEGRAPH_CONDA_ENVIRONMENT}..."
+    "${pathToConda}conda" env create --file "${CONDA_ENVIRONMENT_FILE}" --name "${CODEGRAPH_CONDA_ENVIRONMENT}"
 fi
 
+# Activate code graph Conda environment
 echo "activateCondaEnvironment: Activating Conda environment ${CODEGRAPH_CONDA_ENVIRONMENT}"
 "${pathToConda}conda" activate ${CODEGRAPH_CONDA_ENVIRONMENT}
 

scripts/detectChangedFiles.sh

@@ -21,11 +21,17 @@ ARTIFACTS_CHANGE_DETECTION_HASH_FILE=${ARTIFACTS_CHANGE_DETECTION_HASH_FILE:-"ar
 CHANGE_DETECTION_HASH_FILE=${CHANGE_DETECTION_HASH_FILE:-"${ARTIFACTS_CHANGE_DETECTION_HASH_FILE}"} # Name of the file that contains the hash code of the file list for change detection
 CHANGE_DETECTION_HASH_FILE_PATH=${CHANGE_DETECTION_HASH_FILE_PATH:-"./${ARTIFACTS_DIRECTORY}/${CHANGE_DETECTION_HASH_FILE}"} # Default path of the file that contains the hash code of the file list for change detection. Can be overridden by a command line option.
 
+COLOR_INFO='\033[0;30m' # dark grey
+COLOR_ERROR='\033[0;31m' # red
+COLOR_DEFAULT='\033[0m'    
+
 # Function to display script usage
 usage() {
-  echo "Usage: $0 [--readonly]"
-  echo "          [--paths <comma separated list of file and directory names> (default=artifacts)]"
-  echo "          [--hashfile <path to the file that contains the hash for change detection> (default=env var CHANGE_DETECTION_HASH_FILE_PATH)]"
+  echo -e "${COLOR_ERROR}" >&2
+  echo "Usage: $0 [--readonly]" >&2
+  echo "          [--paths <comma separated list of file and directory names> (default=artifacts)]" >&2
+  echo "          [--hashfile <path to the file that contains the hash for change detection> (default=env var CHANGE_DETECTION_HASH_FILE_PATH)]" >&2
+  echo -e "${COLOR_DEFAULT}" >&2
   exit 1
 }
 
@@ -52,25 +58,52 @@ while [[ $# -gt 0 ]]; do
       shift
       ;;
     *)
-      echo "detectChangedFiles: Error: Unknown option: ${key}"
+      echo -e "${COLOR_ERROR}detectChangedFiles: Error: Unknown option: ${key}${COLOR_DEFAULT}" >&2
       usage
       ;;
   esac
   shift || true # ignore error when there are no more arguments
 done
 
+exit_failed() {
+  case "$0" in
+    */sh) return 1 ;; # Script is sourced
+    *) exit 1 ;;      # Script is executed directly
+  esac
+}
+
+exit_successful() {
+  case "$0" in
+    */sh) return 0 ;; # Script is sourced
+    *) exit 0 ;;      # Script is executed directly
+  esac
+}
+
 if ${readonlyMode}; then
-  echo "detectChangedFiles: Readonly mode activated. Change detection file won't be created." >&2
+  echo -e "${COLOR_INFO}detectChangedFiles: Readonly mode activated. Change detection file won't be created.${COLOR_DEFAULT}" >&2
 else
-  echo "detectChangedFiles: ${hashFilePath} will be used as change detection file." >&2
+  echo -e "${COLOR_INFO}detectChangedFiles: ${hashFilePath} will be used as change detection file.${COLOR_DEFAULT}" >&2
 fi
 
 # Check if the paths parameter exist
-if [ -z "${paths}" ] ; then
-    echo 0 # 0=No change detected. The path list is empty. There is nothing to compare. Therefore assume that there are no changes.
-    exit 0
+if [ -z "${paths}" ]; then
+  echo 0 # 0=No change detected. The path list is empty. There is nothing to compare. Therefore assume that there are no changes.
+  exit_successful
 fi
 
+# Check all paths if they are valid files or valid directories
+for path in ${paths//,/ }; do
+    pathWithoutProtocolPrefix=${path/#*::/}
+    if [ -f "${pathWithoutProtocolPrefix}" ] ; then
+      continue # Valid file
+    elif [ -d "${pathWithoutProtocolPrefix}" ] ; then
+      continue # Valid directory
+    fi
+    # Neither a valid directory and file
+    echo -e "${COLOR_ERROR}detectChangedFiles: Error: Invalid path: ${pathWithoutProtocolPrefix}${COLOR_DEFAULT}" >&2
+    exit_failed
+done
+
 # Function to get file size
 get_file_size() {
     if [ -f "$1" ]; then
@@ -80,23 +113,46 @@ get_file_size() {
     fi
 }
 
+isMacOS() {
+    [ "$(uname -s)" = "Darwin" ]
+}
+
 # Function to process a single path
 file_names_and_sizes() {
     if [ -d "$1" ]; then
+        # TODO Remove after debugging
+        echo "detectChangedFiles: Checking directory $1" >&2
+        
         # If it's a directory, list all files inside 
         # except for "node_modules", "target", "temp" and the change detection file itself
-        find -L "$1" \
-          -type d -name "node_modules" -prune -o \
-          -type d -name "target" -prune -o \
-          -type d -name "temp" -prune -o \
-          -type d -name ".reports" -prune -o \
-          -not -path "${hashFilePath}" \
-          -type f \
-          -exec stat -f "%N %z" {} + \
-        | sort 
+        if isMacOS; then
+          find -L "$1" \
+            -type d -name "node_modules" -prune -o \
+            -type d -name "target" -prune -o \
+            -type d -name "temp" -prune -o \
+            -type d -name ".reports" -prune -o \
+            -not -path "${hashFilePath}" \
+            -type f \
+            -exec stat -f "%N %z" {} + \
+          | sort 
+        else
+          find -L "$1" \
+            -type d -name "node_modules" -prune -o \
+            -type d -name "target" -prune -o \
+            -type d -name "temp" -prune -o \
+            -type d -name ".reports" -prune -o \
+            -not -path "${hashFilePath}" \
+            -type f \
+            -exec stat --printf="%n %s\n" {} + \
+          | sort 
+        fi
     elif [ -f "$1" ]; then
-        # If it's a file, just echo the file path
-        stat -f "%N %z" < "$1"
+    # The path is a file. Print its path and size.
+      if isMacOS; then
+        stat -f "%N %z" "$1"
+      else
+        stat --printf="%n %s\n" "$1"
+      fi
     fi
 }
 
@@ -110,10 +166,11 @@ get_md5_checksum_of_all_file_names_and_sizes() {
   local processed_paths=0
 
   for path in ${paths//,/ }; do
-      local files_and_their_size; files_and_their_size=$(file_names_and_sizes "${path}")
+      pathWithoutProtocolPrefix=${path/#*::/}
+      local files_and_their_size; files_and_their_size=$(file_names_and_sizes "${pathWithoutProtocolPrefix}")
       all_files_and_sizes="${all_files_and_sizes}${files_and_their_size}"
       processed_paths=$((processed_paths + 1))
-      echo -ne "detectChangedFiles: Calculate checksum progress: ($processed_paths/$total_paths)\r" >&2
+      echo -ne "${COLOR_INFO}detectChangedFiles: Calculate checksum progress: ($processed_paths/$total_paths)\r${COLOR_DEFAULT}" >&2
   done
   echo "" >&2
   echo "${all_files_and_sizes}" | openssl md5 | awk '{print $2}'
@@ -133,12 +190,12 @@ if [ ! -f "${hashFilePath}" ] ; then
         mkdir -p "${hash_file_directory}"
         # Create the file containing the hash of the files list to a new file for the next call
         echo "${CURRENT_FILES_HASH}" > "${hashFilePath}"
-        echo "detectChangedFiles: Change detection file created" >&2
+        echo -e "${COLOR_INFO}detectChangedFiles: Change detection file created.${COLOR_DEFAULT}" >&2
     else
-        echo "detectChangedFiles: Skipping file creation with content (=hash) ${CURRENT_FILES_HASH}" >&2
+        echo -e "${COLOR_INFO}detectChangedFiles: Skipping file creation with content (=hash) ${CURRENT_FILES_HASH}${COLOR_DEFAULT}" >&2
     fi
     echo 1 # 1=Change detected and change detection file created
-    exit 0
+    exit_successful
 fi
 
 # Assume that there is no change if the saved hash is equal to the current one.
@@ -149,9 +206,9 @@ else
     if ! ${readonlyMode}; then
         # Write the updated hash into the file containing the hash of the files list for the next call
         echo "${CURRENT_FILES_HASH}" > "${hashFilePath}"
-        echo "detectChangedFiles: Change detection file updated" >&2
+        echo -e "${COLOR_INFO}detectChangedFiles: Change detection file updated.${COLOR_DEFAULT}" >&2
     else
-        echo "detectChangedFiles: Skipping file update with content (=hash) ${CURRENT_FILES_HASH}" >&2
+        echo -e "${COLOR_INFO}detectChangedFiles: Skipping file update with content (=hash) ${CURRENT_FILES_HASH}.${COLOR_DEFAULT}" >&2
     fi
     echo 2 # 2=Change detected and change detection file updated
 fi

scripts/download.sh

@@ -45,11 +45,6 @@ if [[ -z ${downloadUrl} ]]; then
   exit 1
 fi
 
-if ! curl --head --fail ${downloadUrl} >/dev/null 2>&1; then
-  echo "download: Error: Invalid URL: ${downloadUrl}"
-  exit 1
-fi
-
 if [[ -z ${filename} ]]; then
   filename=$(basename -- "${downloadUrl}")
 fi
@@ -65,6 +60,14 @@ fi
 if [ ! -f "${SHARED_DOWNLOADS_DIRECTORY}/${filename}" ] ; then
     echo "download: Downloading ${filename} from ${downloadUrl} into ${SHARED_DOWNLOADS_DIRECTORY}"
 
+    # Check if the URL is valid
+    # The check is deferred and not done in the input validation block at the beginning.
+    # This is because the check needs a network connection which shouldn't be required when the file had already been downloaded.
+    if ! curl --head --fail ${downloadUrl} >/dev/null 2>&1; then
+      echo "download: Error: Invalid URL: ${downloadUrl}"
+      exit 1
+    fi
+
     # Download the file
     if ! curl -L --fail-with-body -o "${SHARED_DOWNLOADS_DIRECTORY}/${filename}" "${downloadUrl}"; then
         echo "download: Error: Failed to download ${filename}"

scripts/runTests.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+
+# Runs all test scripts (no Python and Chromium required).
+# It only considers scripts in the "scripts" directory.
+
+# Fail on any error ("-e" = exit on first error, "-o pipefail" exist on errors within piped commands)
+set -o errexit -o pipefail
+
+# Overrideable Constants (defaults also defined in sub scripts)
+LOG_GROUP_START=${LOG_GROUP_START:-"::group::"} # Prefix to start a log group. Defaults to GitHub Actions log group start command.
+LOG_GROUP_END=${LOG_GROUP_END:-"::endgroup::"} # Prefix to end a log group. Defaults to GitHub Actions log group end command.
+
+## Get this "scripts" directory if not already set
+# Even if $BASH_SOURCE is made for Bourne-like shells it is also supported by others and therefore here the preferred solution. 
+# CDPATH reduces the scope of the cd command to potentially prevent unintended directory changes.
+# This way non-standard tools like readlink aren't needed.
+SCRIPTS_DIR=${SCRIPTS_DIR:-$( CDPATH=. cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P )} # Repository directory containing the shell scripts
+echo "runTests: SCRIPTS_DIR=${SCRIPTS_DIR}" >&2
+
+# Run all report scripts
+for test_script_file in "${SCRIPTS_DIR}"/test*.sh; do
+    test_script_filename=$(basename -- "${test_script_file}");
+    test_script_filename="${test_script_filename%.*}" # Remove file extension
+
+    echo "${LOG_GROUP_START}Run ${test_script_filename}";
+    echo "runTests: $(date +'%Y-%m-%dT%H:%M:%S%z') Starting ${test_script_filename}...";
+
+    source "${test_script_file}"
+
+    echo "runTests: $(date +'%Y-%m-%dT%H:%M:%S%z') Finished ${test_script_filename}";
+    echo "${LOG_GROUP_END}";
+done

scripts/scanTypescript.sh

@@ -115,14 +115,10 @@ is_valid_scan_result() {
 }
 
 is_change_detected() {
-    local COLOR_DARK_GREY='\033[0;30m'
-    local COLOR_DEFAULT='\033[0m'    
     local source_directory_name; source_directory_name=$(basename "${source_directory}");
 
-    echo -e "${COLOR_DARK_GREY}"
     changeDetectionHashFilePath="./${SOURCE_DIRECTORY}/typescriptScanChangeDetection-${source_directory_name}.sha"
     changeDetectionReturnCode=$( source "${SCRIPTS_DIR}/detectChangedFiles.sh" --readonly --hashfile "${changeDetectionHashFilePath}" --paths "${source_directory}")
-    echo -e "${COLOR_DEFAULT}"
 
     if [ "${changeDetectionReturnCode}" == "0" ] && [ "${TYPESCRIPT_SCAN_CHANGE_DETECTION}" = true ]; then
         true