diff --git a/.evg.yml b/.evg.yml index 05bb3065..86697958 100644 --- a/.evg.yml +++ b/.evg.yml @@ -11,10 +11,52 @@ variables: role_arn: ${assume_role_arn} duration_seconds: 3600 - &evg_bucket_config - aws_key: ${AWS_ACCESS_KEY_ID} - aws_secret: ${AWS_SECRET_ACCESS_KEY} - aws_session_token: ${AWS_SESSION_TOKEN} - bucket: evg-bucket-mongo-jdbc-driver + aws_key: ${AWS_ACCESS_KEY_ID} + aws_secret: ${AWS_SECRET_ACCESS_KEY} + aws_session_token: ${AWS_SESSION_TOKEN} + bucket: evg-bucket-mongo-jdbc-driver + - &check_and_allow_for_eap_build + command: shell.exec + type: test + params: + shell: bash + working_dir: mongo-jdbc-driver + script: | + ${PREPARE_SHELL} + if [ "${BUILD_TYPE}" != "eap" ]; then + echo "BUILD_TYPE (${BUILD_TYPE}) is not 'eap'. Stopping task execution." + curl -d '{"status":"success", "should_continue": false}' -H "Content-Type: application/json" -X POST localhost:2285/task_status + else + echo "BUILD_TYPE is 'eap'. Allowing task execution to continue." + fi + - &check_and_allow_for_standard_and_snapshot_build + command: shell.exec + type: test + params: + shell: bash + working_dir: mongo-jdbc-driver + script: | + ${PREPARE_SHELL} + if [ "${BUILD_TYPE}" == "eap" ]; then + echo "BUILD_TYPE (${BUILD_TYPE}) is 'eap'. Stopping task execution." + curl -d '{"status":"success", "should_continue": false}' -H "Content-Type: application/json" -X POST localhost:2285/task_status + else + echo "BUILD_TYPE is ${BUILD_TYPE}. Allowing task execution to continue." + fi + - &check_and_allow_for_eap_and_snapshot_build + command: shell.exec + type: test + params: + shell: bash + working_dir: mongo-jdbc-driver + script: | + ${PREPARE_SHELL} + if [ "${BUILD_TYPE}" == "standard" ]; then + echo "BUILD_TYPE (${BUILD_TYPE}) is 'standard'. Stopping task execution." + curl -d '{"status":"success", "should_continue": false}' -H "Content-Type: application/json" -X POST localhost:2285/task_status + else + echo "BUILD_TYPE is ${BUILD_TYPE}. Allowing task execution to continue." + fi pre: - func: "fetch source" @@ -54,7 +96,8 @@ buildvariants: tasks: - name: semgrep - name: sbom - - name: ssdlc-artifacts-snapshot + - name: ssdlc-artifacts-snapshot-standard + - name: ssdlc-artifacts-snapshot-eap - name: ubuntu2204-64-jdk-8 display_name: Ubuntu 22.04 jdk-8 @@ -65,10 +108,8 @@ buildvariants: - name: "test-unit" - name: "test-mongo-sql-translate" - name: "test-adf-integration" - # SQL-2289: integration tests. Use the actual mongosqltranslate library for testing instead of mock. - # - name: "test-dc-integration" - # SQL-2544: Merge jdbc EAP branch into master - # - name: "test-x509-integration" + - name: "test-dc-integration" + - name: "test-x509-integration" - name: amazon2-arm64-jdk-11 display_name: Amazon Linux 2 ARM64 jdk-11 @@ -88,6 +129,8 @@ buildvariants: - name: "build" - name: "test-smoke" - name: "publish-maven" + - name: "make-docs" + - name: "publish-s3" - name: "ssdlc-artifacts-release" - name: "download-center-update" @@ -112,48 +155,12 @@ tasks: - name: "test-dc-integration" commands: + - func: "check and allow for eap and snapshot build" - func: "run dc integration test" - name: "test-x509-integration" commands: - # SQL-2544: Merge jdbc EAP branch into master - - # # fetch mongosqltranslate libraries for each platform before building - # # linux arm - # - func: "fetch libmongosqltranslate" - # vars: - # arch: "arm" - # platform: "linux" - # lib_prefix: "lib" - # ext: "so" - # # linux x86 - # - func: "fetch libmongosqltranslate" - # vars: - # arch: "x86_64" - # platform: "linux" - # lib_prefix: "lib" - # ext: "so" - # # macos arm - # - func: "fetch libmongosqltranslate" - # vars: - # arch: "arm" - # platform: "macos" - # lib_prefix: "lib" - # ext: "dylib" - # # macos x86 - # - func: "fetch libmongosqltranslate" - # vars: - # arch: "x86_64" - # platform: "macos" - # lib_prefix: "lib" - # ext: "dylib" - # # windows - # - func: "fetch libmongosqltranslate" - # vars: - # arch: "x86_64" - # platform: "win" - # lib_prefix: "" - # ext: "dll" + - func: "check and allow for eap and snapshot build" - func: "run dc integration test" - func: "run X509 integration test" @@ -174,9 +181,66 @@ tasks: - name: sbom variant: code-quality-and-correctness commands: + - func: "check and allow for standard and snapshot build" - func: "publish maven" - func: "trace artifacts" + # make and upload docs + - name: make-docs + allowed_requesters: ["ad_hoc", "github_tag", "patch"] + commands: + - func: "build jdbc docs" + - func: "upload jdbc docs" + + # Publishes the EAP release artifacts to S3 + - name: publish-s3 + allowed_requesters: ["ad_hoc", "github_tag", "patch"] + depends_on: + - name: "build" + variant: "release" + - name: "test-smoke" + variant: "release" + commands: + - *check_and_allow_for_eap_build + - *assume_role_cmd + - command: s3.get + params: + working_dir: mongo-jdbc-driver + <<: *evg_bucket_config + local_file: mongodb-jdbc-eap-${MDBJDBC_VER}-all.jar + remote_file: ${S3_ARTIFACTS_DIR}/mongodb-jdbc-eap-${MDBJDBC_VER}-all.jar + - command: s3.get + params: + working_dir: mongo-jdbc-driver + <<: *evg_bucket_config + local_file: mongodb-jdbc-eap-${MDBJDBC_VER}.jar + remote_file: ${S3_ARTIFACTS_DIR}/mongodb-jdbc-eap-${MDBJDBC_VER}.jar + - command: s3.put + params: + aws_key: ${release_aws_key} + aws_secret: ${release_aws_secret} + local_files_include_filter: + - mongodb-jdbc-eap-*.jar + remote_file: eap/mongo-jdbc-driver-eap/ + bucket: translators-connectors-releases + permissions: public-read + content_type: application/java-archive + - command: s3.get + params: + work_dir: mongo-jdbc-driver + <<: *evg_bucket_config + local_file: docs/MongoDB_JDBC_Guide.pdf + remote_file: ${S3_ARTIFACTS_DIR}/docs/MongoDB_JDBC_Guide.pdf + - command: s3.put + params: + aws_key: ${release_aws_key} + aws_secret: ${release_aws_secret} + local_file: docs/MongoDB_JDBC_Guide.pdf + remote_file: eap/mongo-jdbc-driver-eap/docs/MongoDB_JDBC_Guide.pdf + bucket: translators-connectors-releases + permissions: public-read + content_type: application/pdf + - name: spotless commands: - func: "check spotless" @@ -187,6 +251,7 @@ tasks: - name: "publish-maven" variant: "release" commands: + - func: "check and allow for standard and snapshot build" - func: "update download center feed" - name: semgrep @@ -211,12 +276,13 @@ tasks: variant: code-quality-and-correctness exec_timeout_secs: 300 # 5m commands: + - func: "check and allow for standard and snapshot build" - func: "publish augmented SBOM" - func: "publish static code analysis" - func: "generate compliance report" - func: "publish compliance report" - - name: ssdlc-artifacts-snapshot + - name: ssdlc-artifacts-snapshot-standard run_on: ubuntu2204-small allow_for_git_tag: false depends_on: @@ -226,12 +292,35 @@ tasks: variant: code-quality-and-correctness exec_timeout_secs: 300 # 5m commands: + - func: "check and allow for standard and snapshot build" - func: "publish augmented SBOM" - func: "publish static code analysis" - func: "generate compliance report" - func: "publish compliance report" + - name: ssdlc-artifacts-snapshot-eap + run_on: ubuntu2204-small + allow_for_git_tag: false + depends_on: + - name: sbom + variant: code-quality-and-correctness + - name: semgrep + variant: code-quality-and-correctness + exec_timeout_secs: 300 # 5m + commands: + - func: "check and allow for eap build" + - func: "generate compliance report" + functions: + "check and allow for standard and snapshot build": + - *check_and_allow_for_standard_and_snapshot_build + + "check and allow for eap and snapshot build": + - *check_and_allow_for_eap_and_snapshot_build + + "check and allow for eap build": + - *check_and_allow_for_eap_build + "augment sbom": - command: ec2.assume_role display_name: Assume IAM role with permissions to pull Kondukto API token @@ -243,7 +332,7 @@ functions: silent: true shell: bash working_dir: mongo-jdbc-driver - include_expansions_in_env: [AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN] + include_expansions_in_env: [ AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN ] script: | # use AWS CLI to get the Kondukto API token from AWS Secrets Manager kondukto_token=$(aws secretsmanager get-secret-value --secret-id "kondukto-token" --region "us-east-1" --query 'SecretString' --output text) @@ -259,11 +348,85 @@ functions: working_dir: mongo-jdbc-driver script: | ${PREPARE_SHELL} + + ls -lrt $SSDLC_DIR + + echo "SBOM_LITE_NAME = $SBOM_LITE_NAME" + + echo "AUGMENTED_SBOM_NAME = $AUGMENTED_SBOM_NAME" + + echo "-- Augmenting SBOM Lite --" + docker run -i --platform="linux/amd64" --rm -v "$PWD":/pwd \ + --env-file ${workdir}/kondukto_credentials.env \ + artifactory.corp.mongodb.com/release-tools-container-registry-public-local/silkbomb:2.0 \ + augment --repo mongodb/mongo-jdbc-driver --branch ${branch_name} --sbom-in /pwd/artifacts/ssdlc/$SBOM_LITE_NAME --sbom-out /pwd/artifacts/ssdlc/$AUGMENTED_SBOM_NAME + echo "-------------------------------" + + ls -lrt $SSDLC_DIR + - command: ec2.assume_role + params: + role_arn: ${assume_role_arn} + duration_seconds: 3600 + - command: s3.put + params: + <<: *evg_bucket_config + local_file: mongo-jdbc-driver/artifacts/ssdlc/${AUGMENTED_SBOM_NAME} + remote_file: artifacts/${version_id}/ssdlc/${AUGMENTED_SBOM_NAME} + content_type: application/json + permissions: public-read + + "build jdbc docs": + - command: subprocess.exec + params: + binary: bash + working_dir: mongo-jdbc-driver + args: + - "./evergreen/make_docs.sh" + + "upload jdbc docs": + - *assume_role_cmd + - command: s3.put + params: + <<: *evg_bucket_config + local_file: mongo-jdbc-driver/docs/MongoDB_JDBC_Guide.pdf + remote_file: ${S3_ARTIFACTS_DIR}/docs/MongoDB_JDBC_Guide.pdf + permissions: public-read + content_type: application/pdf + + "push SBOM Lite to Silk": + - command: shell.exec + type: test + params: + shell: bash + working_dir: mongo-jdbc-driver + script: | + ${PREPARE_SHELL} ls -lrt $SSDLC_DIR echo "SBOM_LITE_NAME = $SBOM_LITE_NAME" + echo "-- Uploading initial SBOM Lite to Silk --" + docker run -i --platform="linux/amd64" --rm -v "$PWD":/pwd \ + --env-file silkbomb.env \ + artifactory.corp.mongodb.com/release-tools-container-registry-public-local/silkbomb:1.0 \ + upload --silk-asset-group ${SILK_ASSET_GROUP} --sbom-in /pwd/artifacts/ssdlc/$SBOM_LITE_NAME + echo "-------------------------------" + + "pull augmented SBOM from Silk": + - *assume_role_cmd + - command: shell.exec + type: test + params: + shell: bash + working_dir: mongo-jdbc-driver + script: | + ${PREPARE_SHELL} + cat << EOF > silkbomb.env + SILK_CLIENT_ID=${SILK_CLIENT_ID} + SILK_CLIENT_SECRET=${SILK_CLIENT_SECRET} + EOF + echo "AUGMENTED_SBOM_NAME = $AUGMENTED_SBOM_NAME" echo "-- Augmenting SBOM Lite --" @@ -386,15 +549,22 @@ functions: type: test params: shell: bash + add_expansions_to_env: true working_dir: mongo-jdbc-driver script: | ${PREPARE_SHELL} if [[ "${triggered_by_git_tag}" != "" ]]; then EXTRA_PROP="-PisTagTriggered" + else + if [[ "${BUILD_TYPE}" == "eap" ]]; then + # Sets EAP build flag when not triggered by a git tag. Useful for testing the eap build + # without needing to rely on Tag-triggered build + EXTRA_PROP="-PisEapBuild=true" + fi fi ./gradlew clean generateLicenseReport --rerun-tasks && \ echo -e "$(cat resources/third_party_header.txt)\n$(cat build/reports/dependency-license/THIRD-PARTY-LICENSE.txt)" > build/reports/dependency-license/THIRD-PARTY-LICENSE.txt && \ - ./gradlew -Dorg.gradle.java.home=${JAVA_HOME} $EXTRA_PROP -x test -x integrationTest spotlessApply build shadowjar --rerun-tasks + ./gradlew -Dorg.gradle.java.home=${JAVA_HOME} $EXTRA_PROP clean -x test -x integrationTest spotlessApply build shadowjar --rerun-tasks "check spotless": command: shell.exec @@ -437,6 +607,10 @@ functions: # Test loading library from same directory as where the JDBC driver is located ./gradlew runMongoSQLTranslateLibTest -PtestMethod=testLibraryLoadingFromDriverPath + # Test loading library from an invalid path specified in ENV variable + MONGOSQL_TRANSLATE_PATH="$PWD/src/test/resources/MongoSqlLibraryTest/thisIsNotCorrect" \ + ./gradlew runMongoSQLTranslateLibTest -PtestMethod=testLibraryLoadingWithInvalidEnvironmentVariableFallback + "generate github token": command: github.generate_token params: @@ -453,59 +627,59 @@ functions: shell: bash working_dir: mongo-jdbc-driver script: | - ${PREPARE_SHELL} - ./resources/run_adf.sh start && - ./gradlew -Dorg.gradle.java.home=${JAVA_HOME} runDataLoader && - ./gradlew -Dorg.gradle.java.home=${JAVA_HOME} clean integrationTest \ - -x test --tests ADFIntegrationTest > gradle_output.log 2>&1 & - GRADLE_PID=$! - - echo "Gradle process started with PID $GRADLE_PID" - - # On Amazon Linux 2 hosts, the gradlew integrationTest command was hanging indefinitely. - # This monitoring approach will detect build completion or failure even when the Gradle - # process doesn't terminate properly and allows the task to complete. - SECONDS=0 - TIMEOUT=1800 # 30 minute timeout - - while true; do - if grep -q "BUILD SUCCESSFUL" gradle_output.log; then - echo "Build successful!" - EXITCODE=0 - break - fi + ${PREPARE_SHELL} + ./resources/run_adf.sh start && + ./gradlew -Dorg.gradle.java.home=${JAVA_HOME} runDataLoader && + ./gradlew -Dorg.gradle.java.home=${JAVA_HOME} clean integrationTest \ + -x test --tests ADFIntegrationTest > gradle_output.log 2>&1 & + GRADLE_PID=$! - if grep -q "BUILD FAILED" gradle_output.log; then - echo "Build failed!" - EXITCODE=1 - break - fi + echo "Gradle process started with PID $GRADLE_PID" - if (( SECONDS > TIMEOUT )); then - echo "$TIMEOUT second timeout reached. Exiting with failure." - EXITCODE=1 - break - fi + # On Amazon Linux 2 hosts, the gradlew integrationTest command was hanging indefinitely. + # This monitoring approach will detect build completion or failure even when the Gradle + # process doesn't terminate properly and allows the task to complete. + SECONDS=0 + TIMEOUT=1800 # 30 minute timeout - # Check if Gradle process is still running - if ! kill -0 $GRADLE_PID 2>/dev/null; then - echo "Gradle process has finished." - wait $GRADLE_PID - EXITCODE=$? - break - fi + while true; do + if grep -q "BUILD SUCCESSFUL" gradle_output.log; then + echo "Build successful!" + EXITCODE=0 + break + fi + + if grep -q "BUILD FAILED" gradle_output.log; then + echo "Build failed!" + EXITCODE=1 + break + fi + + if (( SECONDS > TIMEOUT )); then + echo "$TIMEOUT second timeout reached. Exiting with failure." + EXITCODE=1 + break + fi - sleep 5 - done + # Check if Gradle process is still running + if ! kill -0 $GRADLE_PID 2>/dev/null; then + echo "Gradle process has finished." + wait $GRADLE_PID + EXITCODE=$? + break + fi - cat gradle_output.log + sleep 5 + done - kill $GRADLE_PID 2>/dev/null || true + cat gradle_output.log - ./resources/run_adf.sh stop + kill $GRADLE_PID 2>/dev/null || true - echo "Integration test exit code: $EXITCODE" - exit $EXITCODE + ./resources/run_adf.sh stop + + echo "Integration test exit code: $EXITCODE" + exit $EXITCODE "run dc integration test": command: shell.exec @@ -537,7 +711,7 @@ functions: ./resources/start_local_mdb.sh $mdb_version_com $mdb_version_ent $arch # Run the tests. - ./gradlew integrationTest \ + ./gradlew integrationTest -PisEapBuild=true \ -x test --tests DCIntegrationTest > gradle_output.log 2>&1 & GRADLE_PID=$! @@ -578,9 +752,7 @@ functions: done cat gradle_output.log - kill $GRADLE_PID 2>/dev/null || true - pkill mongod echo "Integration test exit code: $EXITCODE" @@ -622,7 +794,7 @@ functions: -Djavax.net.ssl.trustStoreType=JKS -Djavax.net.ssl.trustStorePassword=$ADF_TEST_LOCAL_PWD" # Run the tests. - ./gradlew integrationTest \ + ./gradlew integrationTest -PisEapBuild=true \ -x test --tests AuthX509IntegrationTest > gradle_output.log 2>&1 & GRADLE_PID=$! @@ -679,30 +851,32 @@ functions: work_dir: mongo-jdbc-driver key_id: ${papertrail_id} secret_key: ${papertrail_key} - product: mongo-jdbc-driver + product: mongo-jdbc-driver-eap version: ${MDBJDBC_VER} filenames: - "build/libs/*.jar" "run smoke test": - command: shell.exec - type: test - params: - env: - GITHUB_TOKEN: "${github_token}" - working_dir: mongo-jdbc-driver - script: | - ${PREPARE_SHELL} - - #Smoke test are loading the "*-all.jar" from mongo-jdbc-driver/build/libs/ - ls -lrt build/libs/ - - ./resources/run_adf.sh start && - ./gradlew runDataLoader && - ./gradlew :smoketest:test -Psmoketest - EXITCODE=$? - ./resources/run_adf.sh stop - exit $EXITCODE + - command: shell.exec + type: test + params: + env: + GITHUB_TOKEN: "${github_token}" + working_dir: mongo-jdbc-driver + script: | + ${PREPARE_SHELL} + + #Smoke test are loading the "*-all.jar" from mongo-jdbc-driver/build/libs/ + ls -lrt build/libs/ + echo "Build Type:" + echo ${BUILD_TYPE} + + ./resources/run_adf.sh start && + ./gradlew runDataLoader && + ./gradlew :smoketest:test -Psmoketest + EXITCODE=$? + ./resources/run_adf.sh stop + exit $EXITCODE "export variables": - command: shell.exec @@ -718,9 +892,19 @@ functions: # Tag triggered runs are releases and the version is set in the tag. # Other runs are snapshot builds (periodic builds or patches) if [[ "${triggered_by_git_tag}" != "" ]]; then + echo "Tag triggered build: ${triggered_by_git_tag}" + + if [[ "${triggered_by_git_tag}" == *"-libv"* ]]; then + echo "Detected EAP tag format." + export BUILD_TYPE="eap" + else + echo "Setting standard build." + export BUILD_TYPE="standard" + fi export MDBJDBC_VER=$(echo ${triggered_by_git_tag} | sed s/v//) else export MDBJDBC_VER=snapshot + export BUILD_TYPE=snapshot fi # Set JAVA_HOME based on platform @@ -764,6 +948,7 @@ functions: echo "Compliance Report Name = $COMPLIANCE_REPORT_NAME" echo "Static Code Analysis Name = $STATIC_CODE_ANALYSIS_NAME" echo "SSDLC Directory = $SSDLC_DIR" + echo "BUILD_TYPE = $BUILD_TYPE" # Write calculated values to expansions file mkdir -p $ARTIFACTS_DIR @@ -780,6 +965,91 @@ functions: COMPLIANCE_REPORT_NAME: "$COMPLIANCE_REPORT_NAME" STATIC_CODE_ANALYSIS_NAME: "$STATIC_CODE_ANALYSIS_NAME" SSDLC_DIR: "$SSDLC_DIR" + BUILD_TYPE: "$BUILD_TYPE" + EOT + - command: expansions.update + params: + file: mongo-jdbc-driver/artifacts/initial_expansions.yml + - command: shell.exec + params: + shell: bash + silent: true + working_dir: mongo-jdbc-driver + script: | + ARTIFACTS_DIR=artifacts + S3_ARTIFACTS_DIR='mongo-jdbc-driver/artifacts/${version_id}/${build_variant}' + + # Get the version from trigger. + # Tag triggered runs are releases and the version is set in the tag. + # Other runs are snapshot builds (periodic builds or patches) + if [[ "${triggered_by_git_tag}" != "" ]]; then + # tag should be formatted as 'v..-libv.. + export MDBJDBC_VER=$(echo ${triggered_by_git_tag} | awk -F'-libv' '{print $1}' | sed s/v// ) + export LIBMONGOSQLTRANSLATE_VER=$(echo ${triggered_by_git_tag} | awk -F'-libv' '{print $2}') + else + export MDBJDBC_VER=snapshot + export LIBMONGOSQLTRANSLATE_VER=snapshot + fi + + # Set JAVA_HOME based on platform + if [[ "${_platform}" == "ubuntu2204-64-jdk-8" ]]; then + export JAVA_HOME=/opt/java/jdk8 + elif [[ "${_platform}" == "ubuntu2204-64-jdk-11" ]]; then + export JAVA_HOME=/opt/java/jdk11 + elif [[ "${_platform}" == "amazon2-arm64-jdk-11" ]]; then + export JAVA_HOME=/usr/lib/jvm/java-11 + else + # According to DEVPROD, the Java toolchain should always be under `/opt/java/` + if [ -d "/opt/java/jdk11" ]; then + export JAVA_HOME=/opt/java/jdk11 + else + echo >&2 "Can't find Java. `/opt/java/jdk11` does not exist" + exit 1 + fi + fi + + # set the state needed irrespective of _platform + SBOM_WITHOUT_TEAM_NAME=$ARTIFACTS_DIR/ssdlc/sbom_without_team_name.json + SBOM_TOOL_DIR="sbom_generations" + SBOM_LITE_NAME="mongo-jdbc-driver.cdx.json" + SBOM_LITE="$ARTIFACTS_DIR/ssdlc/$SBOM_LITE_NAME" + AUGMENTED_SBOM_NAME="mongo-jdbc-driver.augmented.sbom.json" + COMPLIANCE_REPORT_NAME="mongodb-jdbc-compliance-report.md" + STATIC_CODE_ANALYSIS_NAME="mongo-jdbc-driver.sast.sarif" + SSDLC_DIR="$ARTIFACTS_DIR/ssdlc" + mkdir -p "$SSDLC_DIR" + + echo "=== Generated Values ===" + echo "JDBC version = $MDBJDBC_VER" + echo "Mongosqltranslate version = $LIBMONGOSQLTRANSLATE_VER" + echo "Java home = $JAVA_HOME" + echo "S3 artifacts dir = $S3_ARTIFACTS_DIR" + echo "=== SSDLC Values ===" + echo "SBOM_WITHOUT_TEAM_NAME = $SBOM_WITHOUT_TEAM_NAME" + echo "SBOM_TOOL_DIR = $SBOM_TOOL_DIR" + echo "SBOM Lite Name = $SBOM_LITE_NAME" + echo "SBOM_LITE = $SBOM_LITE" + echo "Augmented SBOM Name = $AUGMENTED_SBOM_NAME" + echo "Compliance Report Name = $COMPLIANCE_REPORT_NAME" + echo "Static Code Analysis Name = $STATIC_CODE_ANALYSIS_NAME" + echo "SSDLC Directory = $SSDLC_DIR" + + # Write calculated values to expansions file + mkdir -p $ARTIFACTS_DIR + cat < $ARTIFACTS_DIR/initial_expansions.yml + MDBJDBC_VER: "$MDBJDBC_VER" + LIBMONGOSQLTRANSLATE_VER: "$LIBMONGOSQLTRANSLATE_VER" + JAVA_HOME: "$JAVA_HOME" + ARTIFACTS_DIR: "$ARTIFACTS_DIR" + S3_ARTIFACTS_DIR: "$S3_ARTIFACTS_DIR" + SBOM_WITHOUT_TEAM_NAME: "$SBOM_WITHOUT_TEAM_NAME" + SBOM_TOOL_DIR: "$SBOM_TOOL_DIR" + SBOM_LITE_NAME: "$SBOM_LITE_NAME" + SBOM_LITE: "$SBOM_LITE" + AUGMENTED_SBOM_NAME: "$AUGMENTED_SBOM_NAME" + COMPLIANCE_REPORT_NAME: "$COMPLIANCE_REPORT_NAME" + STATIC_CODE_ANALYSIS_NAME: "$STATIC_CODE_ANALYSIS_NAME" + SSDLC_DIR: "$SSDLC_DIR" EOT - command: expansions.update params: @@ -805,12 +1075,14 @@ functions: cat < $ARTIFACTS_DIR/expansions.yml S3_ARTIFACTS_DIR: "$S3_ARTIFACTS_DIR" MDBJDBC_VER: "$MDBJDBC_VER" + LIBMONGOSQLTRANSLATE_VER: "$LIBMONGOSQLTRANSLATE_VER" JAVA_HOME: "$JAVA_HOME" SBOM_LITE_NAME: "$SBOM_LITE_NAME" AUGMENTED_SBOM_NAME: "$AUGMENTED_SBOM_NAME" STATIC_CODE_ANALYSIS_NAME: "$STATIC_CODE_ANALYSIS_NAME" COMPLIANCE_REPORT_NAME: "$COMPLIANCE_REPORT_NAME" SSDLC_DIR: "$SSDLC_DIR" + BUILD_TYPE: "$BUILD_TYPE" PREPARE_SHELL: | export ADF_TEST_LOCAL_USER=${adf_test_local_user} export ADF_TEST_LOCAL_PWD=${adf_test_local_pwd} @@ -821,8 +1093,8 @@ functions: export ADF_TEST_HOST=${adf_test_host} export ADF_TEST_AUTH_DB=${adf_test_auth_db} export SRV_TEST_HOST=${srv_test_host} - export SRV_TEST_USER=${srv_test_user} - export SRV_TEST_PWD=${srv_test_pwd} + export SRV_TEST_USER=${srv_test_user_dc} + export SRV_TEST_PWD=${srv_test_pwd_dc} export SRV_TEST_AUTH_DB=${srv_test_auth_db} export LOCAL_MDB_PORT_COM=${local_mdb_port_com} export LOCAL_MDB_PORT_ENT=${local_mdb_port_ent} @@ -831,6 +1103,7 @@ functions: export JAVA_HOME=${JAVA_HOME} export PROJECT_DIRECTORY=${PROJECT_DIRECTORY} export MDBJDBC_VER=${MDBJDBC_VER} + export LIBMONGOSQLTRANSLATE_VER=${LIBMONGOSQLTRANSLATE_VER} # ssdlc relevant variables export SBOM_WITHOUT_TEAM_NAME=$SBOM_WITHOUT_TEAM_NAME @@ -841,19 +1114,38 @@ functions: export SBOM_LITE="$SBOM_LITE" export COMPLIANCE_REPORT_NAME="$COMPLIANCE_REPORT_NAME" export STATIC_CODE_ANALYSIS_NAME="$STATIC_CODE_ANALYSIS_NAME" + export BUILD_TYPE="$BUILD_TYPE" EOT - - command: expansions.update params: file: mongo-jdbc-driver/artifacts/expansions.yml "fetch jdbc shadow jar": + - command: shell.exec + params: + shell: bash + script: | + ${PREPARE_SHELL} + + mkdir -p mongo-jdbc-driver/build/libs + + # Determine which JAR file to download based on BUILD_TYPE + if [ "${BUILD_TYPE}" == "eap" ]; then + echo "Fetching EAP JAR file" + echo "JAR_FILENAME: mongodb-jdbc-eap-${MDBJDBC_VER}-all.jar" > jdbc_jar_expansion.yml + else + echo "Fetching standard JAR file" + echo "JAR_FILENAME: mongodb-jdbc-${MDBJDBC_VER}-all.jar" > jdbc_jar_expansion.yml + fi + - command: expansions.update + params: + file: jdbc_jar_expansion.yml - *assume_role_cmd - command: s3.get params: <<: *evg_bucket_config - local_file: mongo-jdbc-driver/build/libs/mongodb-jdbc-${MDBJDBC_VER}-all.jar - remote_file: ${S3_ARTIFACTS_DIR}/mongodb-jdbc-${MDBJDBC_VER}-all.jar + local_file: mongo-jdbc-driver/build/libs/${JAR_FILENAME} + remote_file: ${S3_ARTIFACTS_DIR}/${JAR_FILENAME} "fetch source": - command: shell.exec @@ -968,9 +1260,12 @@ functions: params: silent: true <<: *evg_bucket_config - local_file: mongo-jdbc-driver/build/reports/tests/integrationTest/index.html - remote_file: ${S3_ARTIFACTS_DIR}/integrationTest/index.html + preserve_path: true + local_files_include_filter: + - "mongo-jdbc-driver/build/reports/tests/integrationTest/*" + # local_file: mongo-jdbc-driver/build/reports/tests/integrationTest/ content_type: text/html + remote_file: ${S3_ARTIFACTS_DIR}/integrationTest/ permissions: public-read display_name: "Integration Test Results" @@ -1081,20 +1376,32 @@ functions: type: test params: shell: bash + add_expansions_to_env: true working_dir: mongo-jdbc-driver script: | ${PREPARE_SHELL} if [ -d "build/libs" ]; then # A sanity check to see if expected files are indeed here ls -lrt build/libs - # Rename files according to our naming scheme - for f in ./build/libs/mongodb-jdbc-*.jar; do - if [[ "$f" == *"-all"* ]]; then - mv "$f" build/libs/mongodb-jdbc-${MDBJDBC_VER}-all.jar - else - mv "$f" build/libs/mongodb-jdbc-${MDBJDBC_VER}.jar - fi - done + + if [[ "${BUILD_TYPE}" == "eap" ]]; then + for f in ./build/libs/mongodb-jdbc-*.jar; do + if [[ "$f" == *"-all"* ]]; then + mv "$f" build/libs/mongodb-jdbc-eap-${MDBJDBC_VER}-all.jar + else + mv "$f" build/libs/mongodb-jdbc-eap-${MDBJDBC_VER}.jar + fi + done + else + # Rename files according to our naming scheme + for f in ./build/libs/mongodb-jdbc-*.jar; do + if [[ "$f" == *"-all"* ]]; then + mv "$f" build/libs/mongodb-jdbc-${MDBJDBC_VER}-all.jar + else + mv "$f" build/libs/mongodb-jdbc-${MDBJDBC_VER}.jar + fi + done + fi fi - command: s3.put params: diff --git a/.gitignore b/.gitignore index 9cb2ab00..5c409c17 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,9 @@ local_adf # Ignore generated tests resources/generated_test + +# Ignore mongosqltranslate libraries +src/main/resources/**/libmongosqltranslate.* +src/main/resources/**/mongosqltranslate.* +.library_cache/ + diff --git a/README.md b/README.md index ec173869..24123c00 100644 --- a/README.md +++ b/README.md @@ -114,6 +114,27 @@ You can find the generated jar in build/libs/ ``` ./gradlew spotlessApply ``` +### Downloading mongosqltranslate Library +The driver requires the `mongosqltranslate` library for direct cluster SQL translation. For initial gradle builds the +library files will be automatically downloaded to the cache directory `${project.rootDir}/.library_cache/`. +If a specific version of the library already exists in the cache, it will be used. +#### Specifying library version: +By default, the `snapshot` version will be downloaded. To use a specific version, use the libmongosqltranslateVersion +property: +```bash +./gradlew clean build -PlibmongosqltranslateVersion=1.0.0-beta-1 +``` +#### Force Re-download: +Unless the updateLibs property is set to true, cached versions will be used to avoid repeated downloads. +```bash +# Override cached versions +./gradlew clean build -PupdateLibs=true +``` +#### Run download task only +```bash +./gradlew downloadLibMongosqlTranslate +``` + ## Integration Testing Integration testing requires a local MongoDB and Atlas Data Federation instance to be running #### Environment Variables diff --git a/RELEASE.md b/RELEASE.md index efd8d5e5..665cf6b8 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -1,25 +1,43 @@ # Versioning + +## Versioning + +Versions will follow the [semantic versioning](https://semver.org/) system. + +The following guidelines will be used to determine when each version component will be updated: +- **major**: backwards-breaking changes +- **minor**: functionality added in a backwards compatible manner +- **patch**: backwards compatible bug fixes. +- **pre-release**: The pre-release version, used for preview builds and when updating a preview version of `libmongosqltranslate` +- **libv**: to specify which version of `libmongosqltranslate` gets bundled with the driver during a release + - Our build system will fetch *exactly* the version of `libmongosqltranslate` specified after `libv` and bundle it with the driver. If you are unsure + what version to use, check the [releases](https://jira.mongodb.org/projects/SQL?selectedItem=com.atlassian.jira.jira-projects-plugin:release-page&status=released&contains=libv) page and use the latest **released** tag. + +### The following applies to non-eap releases + Version number policy could be referred from here: https://docs.oracle.com/middleware/1212/core/MAVEN/maven_version.htm#MAVEN8855 Among all the version standards Maven supports, we will use MajorVersion, IncrementalVersion and Qualifier inside this project. -The current version number is specified inside `gradle.properties` file. - -# Release Process +## Release Process -## Snapshot Versions +### Snapshot Versions Every successful untagged build in evergreen will release the artifacts to the Sonatype SNAPSHOT repo in https://oss.sonatype.org/#view-repositories;snapshots~browsestorage -## Release Versions +### Release Versions Follow these instructions for creating a Release Version. ### Pre-Release Tasks #### Determine the correct version to be released + Go to the [SQL releases page](https://jira.mongodb.org/projects/SQL?selectedItem=com.atlassian.jira.jira-projects-plugin%3Arelease-page&status=unreleased), and check the content of the tickets that are included in the current release. The fix version by default is a patch version. If there is a backwards incompatible API change in the tickets that are set to be released, we should instead update the major version; if there are new features added in the tickets set to be released, we should instead update the minor version. To do so, update the version on the [SQL releases page](https://jira.mongodb.org/projects/SQL?selectedItem=com.atlassian.jira.jira-projects-plugin%3Arelease-page&status=unreleased) under "Actions". This will update the fix version on all of the tickets as well. +To determine which version of `libmongosqltranslate` to bundle, go to the [releases](https://jira.mongodb.org/projects/SQL?selectedItem=com.atlassian.jira.jira-projects-plugin:release-page&status=released&contains=libv) page +and choose the latest **released** version. Add this fix version to the release ticket in JIRA. + #### Start Release Ticket Move the JIRA ticket for the release to the "In Progress" state. Ensure that its fixVersion matches the version being released, and update it if it changed in the previous step. @@ -40,11 +58,22 @@ Close the release on JIRA, adding the current date (you may need to ask the SQL Ensure you have the `master` branch checked out, and that you have pulled the latest commit from `mongodb/mongo-jdbc-driver`. #### Create the tag and push -Create an annotated tag and push it: -``` -git tag -a -m X.Y.Z vX.Y.Z + +The tag is formatted as follows: + +`..[-]-libv..[-]` + +To specify which version of [`libmongosqltranslate`](https://github.com/10gen/mongosql-rs) should be built with the release, the version +of `libmongosqltranslate` must be annotated in the git tag with `libv`. This version was obtained previously when updating the release ticket. + +Create an annotated tag and push it (using the correct versions): + +```sh +# git tag -a -m X.Y.Z[--libvXX.YY.ZZ[-]] +git tag -am 3.0.0-alpha-2-libv1.0.0-alpha-3 git push --tags ``` + This should trigger an Evergreen version that can be viewed on the [mongo-jdbc-driver waterfall](https://evergreen.mongodb.com/waterfall/mongo-jdbc-driver). If it does not, you may have to ask the project manager to give you the right permissions to do so. Make sure to run the 'release' task, if it is not run automatically. @@ -60,12 +89,12 @@ priority. #### Wait for evergreen Wait for the evergreen version to finish, and ensure that the release task completes successfully. -#### Verify release artifacts -Check that the version just released is available in the [Sonatype Nexus Repo Manager](https://oss.sonatype.org/#nexus-search;quick~mongodb-jdbc). +#### Verify release artifacts (ignore for EAP) +Check that the version just released is available in the [Sonatype Nexus Repo Manager](https://oss.sonatype.org/#nexus-search;quick~mongodb-jdbc). The release artifacts should appear on [Maven Central](https://search.maven.org/search?q=g:org.mongodb%20AND%20a:mongodb-jdbc) after a while. -#### Notify the Web team about the new release -Create a ticket through the [service desk](https://jira.mongodb.org/plugins/servlet/desk/portal/61/create/926) and request the link for the `Download from Maven` button on the Download Center page `https://www.mongodb.com/try/download/jdbc-driver` to be updated. +#### Notify the Web team about the new release (ignore for EAP) +Create a ticket through the [service desk](https://jira.mongodb.org/plugins/servlet/desk/portal/61/create/926) and request the link for the `Download from Maven` button on the Download Center page `https://www.mongodb.com/try/download/jdbc-driver` to be updated. You can find the new link in the [json feed](https://translators-connectors-releases.s3.amazonaws.com/mongo-jdbc-driver/mongo-jdbc-downloads.json) under `versions[0].download_link`. Include this link in your ticket. #### Close Release Ticket diff --git a/build.gradle b/build.gradle index cc60ebdc..1a5e7a04 100644 --- a/build.gradle +++ b/build.gradle @@ -12,6 +12,8 @@ import com.github.jk1.license.render.TextReportRenderer if (project.hasProperty('isTagTriggered')) { version = getAbbreviatedGitVersion() +} else if (project.hasProperty('isEapBuild')){ + version = getLatestEapTag() } else { version = getGitVersion() } @@ -20,6 +22,7 @@ ext { println("Driver version = " + version) releaseVersion = getReleaseVersion() println("Artifacts version = " + releaseVersion) + javaDataLoader = "com.mongodb.jdbc.integration.testharness.DataLoader" javaTestGenerator = "com.mongodb.jdbc.integration.testharness.TestGenerator" aspectjVersion = '1.9.7' @@ -168,8 +171,9 @@ task integrationTest(type: Test) { description = 'Runs integration tests.' group = 'verification' + dependsOn tasks.named('jar') testClassesDirs = sourceSets.integrationTest.output.classesDirs - classpath = sourceSets.integrationTest.runtimeClasspath + classpath = sourceSets.integrationTest.runtimeClasspath + files(tasks.jar.archiveFile.get()) - sourceSets.main.output shouldRunAfter test } @@ -270,6 +274,16 @@ def getGitVersion() { out.toString().substring(1).trim() } +def getLatestEapTag() { + def stdout = new ByteArrayOutputStream() + exec { + commandLine 'git', 'tag', '--list', '*libv*', '--sort=-creatordate' + standardOutput = stdout + } + def tags = stdout.toString().trim().readLines() + return tags ? tags[0].toString().substring(1).trim() : null +} + def getAbbreviatedGitVersion() { def out = new ByteArrayOutputStream() exec { @@ -279,6 +293,110 @@ def getAbbreviatedGitVersion() { out.toString().substring(1).trim() } +// Determines the version of libmongosqltranslate to use based on the following priority: +// Command line property 'libVersion' ie -PlibmongosqltranslateVersion=1.2.3 for manual testing +// If build is triggered by a tag, check that LIBMONGOSQLTRANSLATE_VER environment variable is set and use that value +// Otherwise default to "snapshot" version +def getLibMongosqlTranslateVersion() { + if (project.hasProperty('libmongosqltranslateVersion')) { + logger.lifecycle("Using manually specified libVersion: ${project.property('libmongosqltranslateVersion')}") + return project.property('libmongosqltranslateVersion') + } + if (project.hasProperty('isTagTriggered')) { + if (System.getenv('LIBMONGOSQLTRANSLATE_VER')) { + logger.lifecycle("Using version from environment: ${System.getenv('LIBMONGOSQLTRANSLATE_VER')}") + return System.getenv('LIBMONGOSQLTRANSLATE_VER') + } else { + throw new GradleException("Build is tag-triggered but LIBMONGOSQLTRANSLATE_VER " + + "environment variable is not set. This is required for tag-triggered builds.") + } + } + logger.lifecycle("Using snapshot version") + return "snapshot" +} + +def libraryCache = new File("${project.rootDir}/.library_cache") + +task downloadLibMongosqlTranslate { + def libraryPlatforms = [ + [platform: 'linux', arch: 'arm', libPrefix: 'lib', ext: 'so'], + [platform: 'linux', arch: 'x86_64', libPrefix: 'lib', ext: 'so'], + [platform: 'macos', arch: 'arm', libPrefix: 'lib', ext: 'dylib'], + [platform: 'macos', arch: 'x86_64', libPrefix: 'lib', ext: 'dylib'], + [platform: 'win', arch: 'x86_64', libPrefix: '', ext: 'dll'] + ] + description = 'Downloads mongosqltranslate libraries for all platforms' + group = 'Build Setup' + + // Read the force-update flag from the command line `-PupdateLibs=true` + def updateLibs = project.hasProperty('updateLibs') ? project.property('updateLibs').toBoolean() : false + doLast { + def libVersion = getLibMongosqlTranslateVersion() + logger.lifecycle("Using libmongosqltranslate version: ${libVersion}") + + libraryCache.mkdirs() + + libraryPlatforms.each { platform -> + def libraryFileName = "${platform.libPrefix}mongosqltranslate.${platform.ext}" + def s3FileName = + "${platform.libPrefix}mongosqltranslate-v${libVersion}-${platform.platform}-${platform.arch}.${platform.ext}" + def s3Url = "https://translators-connectors-releases.s3.amazonaws.com/mongosqltranslate/${s3FileName}" + + def cacheFile = new File(libraryCache, s3FileName) + def resourceDir = new File("${project.rootDir}/src/main/resources/${platform.arch}/${platform.platform}") + resourceDir.mkdirs() + + def destinationFile = new File(resourceDir, libraryFileName) + + // Skip the download if the force-update flag is not set and the library already exists in library cache + if (!updateLibs && cacheFile.exists() && cacheFile.length() > 0) { + logger.lifecycle("Using cached version of ${s3FileName} for ${platform.platform}-${platform.arch}") + destinationFile.bytes = cacheFile.bytes + return + } + + try { + logger.lifecycle("Downloading ${s3Url}...") + + def connection = new URL(s3Url).openConnection() + connection.connectTimeout = 30000 + connection.readTimeout = 30000 + + cacheFile.withOutputStream { outputStream -> + connection.getInputStream().withCloseable { inputStream -> + outputStream << inputStream + } + } + + // Verify we downloaded actual content + if (cacheFile.length() == 0) { + throw new IOException("Downloaded file is empty") + } + + destinationFile.bytes = cacheFile.bytes + logger.lifecycle("Successfully downloaded ${s3FileName} for ${platform.platform}-${platform.arch}") + + } catch (Exception e) { + logger.warn("Could not download ${s3FileName}: ${e.message}") + + if (cacheFile.exists() && cacheFile.length() > 0) { + logger.lifecycle("Using cached version from ${cacheFile.path}") + destinationFile.bytes = cacheFile.bytes + } else { + logger.error("ERROR: Could not download ${s3FileName} and no valid cached version available.") + logger.error("S3 URL attempted: ${s3Url}") + throw new GradleException("Failed to download " + s3FileName + + " and no valid cached version exists. Build cannot continue.") + } + } + } + } +} + +tasks.named('compileJava').configure { + dependsOn tasks.named('downloadLibMongosqlTranslate') +} + tasks.register('runMongoSQLTranslateLibTest', Test) { description = 'Runs MongoSQLTranslateLibTest' group = 'verification' diff --git a/docs/overview.md b/docs/overview.md new file mode 100644 index 00000000..fc973367 --- /dev/null +++ b/docs/overview.md @@ -0,0 +1,678 @@ +# Introduction + +## Overview + +The MongoDB Atlas SQL JDBC driver supports connections from a SQL compliant +client, enabling you to query your data in MongoDB with SQL queries. + +### Features + +Key Features: Native object and array support. + +Compatibility: The MongoDB Atlas SQL JDBC driver is compatible with MongoDB versions greater +than or equal to 6.0.6. + +## System Requirements + +### Hardware Requirements + +The MongoDB Atlas SQL JDBC driver has no hard system requirements itself. Any system +capable of running a modern SQL tool (such as Tableau) is capable of running +the MongoDB Atlas SQL JDBC driver. + +### Software Requirements + +#### Supported Operating Systems + +The MongoDB JDBC driver is compatible with: +- Windows x86_64 +- macOS x86_64 and macOS aarch64 architectures +- linux x86_64 and linux arm64 architectures + +#### Dependencies + +None + +## Installation + +1. [Download the JAR](TODO). +2. Install and Configure the JDBC Driver. +Copy the JAR file to the appropriate directory. For example, to install the JDBC driver for Tableau on MacOS, +copy the file to `~/Library/Tableau/Drivers`. + +## Usage + +### Escape Fields + +It may be required to select a field with a name that conflicts with an operator +keyword. To do so, surround the field name with backticks. + +For example, to select a field called "select" from the table "SQL": + +```sh +SELECT `select` from SQL +``` + +### Databases + +The driver will use the database specified in the following order: +1. Query +2. JDBC Connection String/DSN + +For example, if your JDBC connection string or DSN contains the DATABASE value **Store1**, +the query `SELECT * FROM Sales` will query the Sales collection in the Store1 database. + +You may also specify the database in the query. The following query will target the Sales collection +in the Store2 database. + +```sql +SELECT * FROM Store2.Sales +``` + +### Collection/Table + +The driver treats MongoDB collections and views as tables. See [Databases](#databases) +for more information about specifying a collection to query. + +### Field/Column + +The driver maps documents fields to column names. Note that by default, the driver +does not flatten objects or unwind arrays. Instead, it returns these types, as well +as ObjectID, UUID, and other complex data types in their JSON form. + +Given the following document in the users collection: + +```json +{ + "name": "Jon Snow", + "username": "AzureDiamond", + "favorites": ["irc", "hunter2"], + "address": { + "street": "1234 Password Way", + "city": "Anywhere", + "state": "CA", + "zip": 90510 + } +} +``` + +The query `SELECT * FROM users WHERE username='AzureDiamond'` will return the following: + +| name | username | favorites | address | +|------|------------|------------|-----------| +| "Jon Snow" | "AzureDiamond" | "[\"irc\", \"hunter2\"]" | "{\"street\": \"1234 Password Way\"}..." | + +Note: In the previous example, the output of address was shortened for brevity. In actual results, the full address +will be returned in string JSON form. + +### Work with Objects + +Building on the previous example, if you only wish to get the zip field from the address object, the query +SELECT name, username, favorite, address.zip FROM users WHERE username='AzureDiamond' will return the following columns: +- name, username, favorites, address.zip + +If you require the entire object in a flattened state, use the FLATTEN operator. +The query SELECT * from FLATTEN(users) will return the following columns, with the values mapped appropriately: +- name, username, favorites, address_street, address_city, address_state, address_zip + +### Work with Arrays + +Arrays can be unwound with the UNWIND operator. You specify which array to unwind with the +`WITH PATH` identifier. The following query: + +```sql +SELECT * FROM UNWIND(users WITH PATH => users.favorites) WHERE username = 'AzureDiamond' +``` + +Will result in two rows in the result set, each with an entry from the `favorites` array. + +- "Jon Snow", "AzureDiamond", "irc", ... +- "Jon Snow", "AzureDiamond", "hunter2", ... + +### Convert Data Types + +Convert data types using the CAST() operator, or the :: shorthand. + +```sql +SELECT CAST(saleDate AS string), saleDate +FROM Sales; + +SELECT saleDate::string, saleDate +FROM Sales; +``` + +### String Literals + +Use single quotes for string literals: + +```sql +SELECT * FROM Sales WHERE customer.gender = 'M' LIMIT 2; +``` + +Notice that 'M' is enclosed in single quotes. + +### Query Syntax + +Retrieve data using the SELECT statement: + +```sql +SELECT * FROM Sales LIMIT 2; +SELECT purchaseMethod, customer, items FROM Sales LIMIT 2; +``` + +Note: Combining \* with specific column names (e.g., SELECT *, FieldA FROM Table) is not supported and will produce an error. + +### CASE + +Use the CASE expression for conditional logic: + +```sql +SELECT + CASE + WHEN customer.age <= 20 THEN '20 years old or younger' + WHEN customer.age > 20 AND customer.age <= 30 THEN '21-30 year olds' + WHEN customer.age > 30 AND customer.age <= 40 THEN '31-40 year olds' + WHEN customer.age > 40 AND customer.age <= 50 THEN '41-50 year olds' + WHEN customer.age > 50 AND customer.age <= 60 THEN '51-60 year olds' + WHEN customer.age > 60 AND customer.age <= 70 THEN '61-70 year olds' + WHEN customer.age > 70 THEN '70 years and older' + ELSE 'Other' + END AS ageRange, + customer.age, + customer.gender, + customer.email +FROM Sales; +``` + +This example categorizes customer ages using a CASE expression with dot notation for nested fields. + +### FROM + +Specify the collection or table in the FROM clause: + +```sql +SELECT * FROM Sales LIMIT 2; +``` + +### JOIN + +Perform joins between collections: + +```sql +SELECT + b.ProductSold, + CAST(b._id AS string) AS ID, + (b.Price * b.Quantity) AS totalAmount +FROM + (SELECT * FROM Sales a WHERE customer.gender = 'F') a +INNER JOIN + Transactions b +ON + (CAST(a._id AS string) = CAST(b._id AS string)); +``` + +Best Practices: Filter or limit data as much as possible to improve query execution speed. +Supported Joins: INNER JOIN, (CROSS) JOIN, LEFT OUTER JOIN, and RIGHT OUTER JOIN. + +### UNION ALL + +Combine result sets using UNION ALL: + +```sql +SELECT * FROM Sales +UNION ALL +SELECT * FROM Transactions; +``` + +Note: UNION (which removes duplicates) is not supported. Only UNION ALL is supported. + +### Nested Selects + +Use subqueries with aliases: + +```sql +SELECT * +FROM (SELECT * FROM Sales) AS subSelect; +``` + +Note: MongoSQL requires nested selects to have an alias, although this is not a SQL-92 requirement. + +### WHERE + +Filter records using the WHERE clause: + +```sql +SELECT * FROM Sales WHERE customer.gender = 'M'; +SELECT * FROM Sales WHERE customer.age > 20; +``` + +### LIKE + +Use LIKE for pattern matching: + +```sql +SELECT purchaseMethod FROM Sales WHERE purchaseMethod LIKE 'In%'; +``` + +### ESCAPE + +Specify an escape character in LIKE patterns: + +```sql +SELECT customer FROM Sales WHERE customer.email LIKE '%_%' ESCAPE '_'; +``` + +Note: Escape characters indicate that any wildcard character following the escape character should be treated as a regular character. + +### GROUP BY + +Group records using GROUP BY: + +```sql +SELECT customer.age AS customerAge, COUNT(*) +FROM Sales +GROUP BY customer.age; +``` + +### HAVING + +Filter grouped records with HAVING: + +```sql +SELECT customer.gender AS customerGender, customer.age AS customerAge, COUNT(*) +FROM Sales +GROUP BY customer.gender, customer.age +HAVING COUNT(*) > 1; +``` + +### ORDER BY + +Order results using ORDER BY: + +```sql +SELECT customer.gender AS customerGender, COUNT(*) +FROM Sales +GROUP BY customer.gender +ORDER BY customerGender; +``` + +### LIMIT and OFFSET + +Limit the number of records and specify an offset: + +```sql +SELECT * FROM Sales LIMIT 3; +SELECT couponUsed FROM Sales OFFSET 2; +``` + +### AS + +Alias columns and expressions using AS: + +```sql +SELECT couponUsed AS Coupons FROM Sales OFFSET 2; +SELECT customer.age AS customerAge, COUNT(*) +FROM Sales +GROUP BY customer.age; +``` + +Note: Alias assignments work as expected. When using aggregates with nested fields, the syntax may require attention. + +### Arithmetic Operators + +Perform calculations using arithmetic operators: + +- Addition (+) +- Subtraction (-) +- Multiplication (*) +- Division (/) +- Modulus (MOD function) + +Example: + +```sql +SELECT ProductSold, Price, Quantity, (Price * Quantity) AS TotalCost +FROM Transactions +LIMIT 2; +``` + +Modulus: + +```sql +SELECT MOD(Value1, Value2) FROM TableName; +``` + +### Comparison Operators + +Use comparison operators in conditions: + +- Equals (=) +- Not Equal (!= or <>) +- Greater Than (>) +- Greater Than or Equal (>=) +- Less Than (<) +- Less Than or Equal (<=) + +Examples: + +```sql +SELECT * FROM Sales WHERE customer.age > 20; +SELECT * FROM Sales WHERE customer.gender = 'F'; +``` + +### Logical/Boolean Operators + +Combine conditions using logical operators: + +- AND +- OR +- NOT + +Examples: + +```sql +SELECT * FROM Sales WHERE customer.age > 20 AND customer.gender = 'M'; +SELECT * FROM Sales WHERE customer.age = 20 OR customer.gender = 'M'; +SELECT * FROM Sales WHERE customer.age > 20 AND NOT customer.gender = 'M'; +``` + +### Aggregate Expressions + +Use aggregate functions for calculations: + +#### SUM() + +```sql +SELECT ProductSold, SUM(Price) +FROM Transactions +GROUP BY ProductSold; +``` + +#### AVG() + +```sql +SELECT ProductSold, AVG(Price) +FROM Transactions +GROUP BY ProductSold; +``` + +#### COUNT() + +```sql +SELECT ProductSold, COUNT(Price) +FROM Transactions +GROUP BY ProductSold; +``` + +#### MIN() + +```sql +SELECT ProductSold, MIN(Price) +FROM Transactions +GROUP BY ProductSold; +``` + +#### MAX() + +```sql +SELECT ProductSold, MAX(Price) +FROM Transactions +GROUP BY ProductSold; +``` + +#### COUNT(DISTINCT) + +```sql +SELECT COUNT(DISTINCT purchaseMethod) FROM Sales; +``` + +Note: May not work if the aggregated field is not comparable (e.g., documents, arrays). + +#### SUM(DISTINCT) + +```sql +SELECT ProductSold, SUM(DISTINCT Price) +FROM Transactions +GROUP BY ProductSold; +``` + +### Scalar Functions + +#### String Functions + +##### Concatenation + +Use || to concatenate strings: + +```sql +SELECT purchaseMethod || ' ' || storeLocation AS purchaseDetails +FROM Sales; +``` + +##### SUBSTRING() + +Extract a substring from a string: + +```sql +SELECT ProductSold, SUBSTRING(ProductSold, 0, 2) +FROM Transactions; +``` + +Note: Uses zero-based indexing. + +##### UPPER() and LOWER() + +Convert strings to uppercase or lowercase: + +```sql +SELECT ProductSold, UPPER(SUBSTRING(ProductSold, 0, 2)) +FROM Transactions; + +SELECT ProductSold, LOWER(SUBSTRING(ProductSold, 0, 2)) +FROM Transactions; +``` + +##### TRIM() + +Remove leading and trailing spaces or specified characters: + +```sql +SELECT TRIM(purchaseMethod) +FROM Sales; + +SELECT TRIM('In' FROM purchaseMethod) +FROM Sales; +``` + +##### CHAR_LENGTH() + +Get the length of a string: + +```sql +SELECT CHAR_LENGTH(ProductSold) +FROM Transactions; +``` + +##### POSITION() + +Find the position of a substring: + +```sql +SELECT purchaseMethod, POSITION('i' IN purchaseMethod) +FROM Sales; +``` + +Returns -1 if the substring is not found. + +##### LEFT() and RIGHT() + +###### LEFT(): + +Use SUBSTRING with a starting position of 0. + +```sql +SELECT ProductSold, SUBSTRING(ProductSold, 0, 2) +FROM Transactions; +``` + +###### RIGHT(): + +Use a combination of SUBSTRING and CHAR_LENGTH minus the length from the SUBSTRING +argument. + +```sql +SELECT SUBSTRING(ProductSold, CHAR_LENGTH(ProductSold) - 2, 2) +FROM Transactions; +``` + +Combines SUBSTRING() with CHAR_LENGTH() to get characters from the end of the string. + +#### Date and Time Functions + +##### DATETRUNC() + +Truncate a timestamp to a specified unit: + +```sql +SELECT DATETRUNC(DAY, saleDate) +FROM Sales; +``` + +Supported Units: YEAR, MONTH, DAY, HOUR, MINUTE, SECOND, WEEK, DAY_OF_YEAR, ISO_WEEK, ISO_WEEKDAY. + +##### DATEADD() + +Add an interval to a timestamp: + +```sql +SELECT DATEADD(YEAR, 1, saleDate), saleDate +FROM Sales; +``` + +##### DATEDIFF() + +Calculate the difference between two timestamps: + +```sql +SELECT DATEDIFF(YEAR, CURRENT_TIMESTAMP, saleDate), saleDate +FROM Sales; +``` + +##### EXTRACT() + +Extract a part of a timestamp: + +```sql +SELECT EXTRACT(YEAR FROM saleDate), saleDate +FROM Sales; +``` + +##### CASTING TO/FROM DATE, TIMESTAMP + +```sql +SELECT CAST(EXTRACT(YEAR FROM saleDate) AS integer), saleDate +FROM Sales; + +SELECT CAST('1975-01-23' AS TIMESTAMP) AS Birthdate, saleDate +FROM Sales; +``` + +Note: MongoDB supports only the TIMESTAMP type. + +##### CURRENT_TIMESTAMP + +Get the current timestamp: + +```sql +SELECT CURRENT_TIMESTAMP +FROM Sales; +``` + +##### ISO_WEEKDAY + +Get the ISO day of the week: + +```sql +SELECT EXTRACT(ISO_WEEKDAY FROM saleDate), saleDate +FROM Sales; +``` + +#### Numeric Functions + +##### TO/FROM EPOCH + +To Epoch: + +```sql +SELECT CAST(saleDate AS LONG) +FROM Sales; +``` + +From Epoch: + +```sql +SELECT CAST(epochValue AS TIMESTAMP) +FROM SomeTable; +``` + +#### Unsupported Functions + +- SIMILAR TO +- RANDOM +- Timezone Conversion (MongoDB stores dates in UTC) +- GROUP_CONCAT + +### Additional Notes + +- Polymorphic Schemas: +Be cautious when using aggregate functions on fields with polymorphic schemas or non-comparable types like documents and arrays. +The term "polymorphic" schema is used to refer to a field/column that can have multiple types, e.g. int and string. + +- Escape Characters: +Use the ESCAPE clause to specify custom escape characters in LIKE patterns. + +- Alias Assignments: +Required when using nested selects or derived tables. + +- Date Functions: +MongoSQL supports various date functions, but only the TIMESTAMP data type is available. + +## Additional Features + +### Security Features + +The MongoDB Atlas SQL JDBC driver supports all authentication mechanisms +supported by MongoDB (x509, OAuth, LDAP, etc...). See [authentication mechanisms](https://www.mongodb.com/docs/manual/core/authentication/#authentication-mechanisms) +for a full list of supported authentication mechanisms. + +If [configured](https://www.mongodb.com/docs/manual/tutorial/configure-ssl/), the MongoDB Atlas SQL JDBC driver supports TLS/SSL connections. + +### Logging and Diagnostics + +The driver emits log messages that your preferred BI tool can intercept and log. For tool specific logging +help, see your tool vendor's documentation. + +## Troubleshooting + +### Common Issues + +- Enterprise edition detected, but mongosqltranslate library not found. + +The translation library was unable to be located. The translation library is included in the JAR and copied to a +system location determined by your JVM. Ensure your user has write access to this location. + +### Debugging Tips + +Often times, an error from MongoDB that can't be translated +into an JDBC error will be in the logs. Look for ERROR and WARN entries. + +## Uninstallation + +Delete the JAR file. + +## Appendix + +### Error Codes Reference + +There are many error codes available to help trouble shoot queries and operations. +You can reference them [in the official MongoDB documentation](https://www.mongodb.com/docs/atlas/data-federation/query/sql/errors/). diff --git a/evergreen/make_docs.sh b/evergreen/make_docs.sh new file mode 100644 index 00000000..2631761b --- /dev/null +++ b/evergreen/make_docs.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +DIR="$PWD" + +if [[ $(lsb_release -is 2>/dev/null) == "Ubuntu" ]]; then + sudo apt install texlive texlive-latex-extra texlive-fonts-recommended -y +else + echo "skipping installation of deps for non-Ubuntu OS" +fi + +MDFILE="$DIR/docs/overview.md" + +# Define the output PDF file name +OUTPUT_PDF="MongoDB_JDBC_Guide.pdf" + +# Use pandoc to convert the markdown file to a PDF +pandoc -f gfm -V geometry:a4paper -V geometry:margin=2cm --toc -s -o "$DIR/docs/$OUTPUT_PDF" $MDFILE + +# Inform the user of the output file +echo "PDF generated: $OUTPUT_PDF" diff --git a/resources/third_party_header.txt b/resources/third_party_header.txt new file mode 100644 index 00000000..27961c57 --- /dev/null +++ b/resources/third_party_header.txt @@ -0,0 +1,13 @@ +MongoDB uses third-party libraries or other resources that may +be distributed under licenses different than the MongoDB software. + +In the event that we accidentally failed to list a required notice, +please bring it to our attention through our JIRA system at: + + https://jira.mongodb.org + +The attached notices are provided for information only. + + + -------------------------- + diff --git a/smoketest/src/test/java/com/mongodb/jdbc/smoketest/SmokeTest.java b/smoketest/src/test/java/com/mongodb/jdbc/smoketest/SmokeTest.java index d0c4b675..41003372 100644 --- a/smoketest/src/test/java/com/mongodb/jdbc/smoketest/SmokeTest.java +++ b/smoketest/src/test/java/com/mongodb/jdbc/smoketest/SmokeTest.java @@ -11,6 +11,8 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; +import java.util.HashMap; +import java.util.Map; import java.util.Properties; /** @@ -21,9 +23,10 @@ public class SmokeTest { static final String URL = "jdbc:mongodb://localhost"; static final String DB = "integration_test"; - private Connection conn; + // Connection and simple query to use for sanity check. + private Map connections = new HashMap<>(); - public static Connection getBasicConnection(String url, String db) + public static Connection getADFInstanceConnection(String url, String db) throws SQLException { Properties p = new java.util.Properties(); p.setProperty("user", System.getenv("ADF_TEST_LOCAL_USER")); @@ -34,32 +37,84 @@ public static Connection getBasicConnection(String url, String db) return DriverManager.getConnection(URL, p); } + private Connection getDirectRemoteInstanceConnection() throws SQLException { + String mongoHost = System.getenv("SRV_TEST_HOST"); + String mongoURI = + "mongodb+srv://" + + mongoHost + + "/?readPreference=secondaryPreferred&connectTimeoutMS=300000"; + String fullURI = "jdbc:" + mongoURI; + + String user = System.getenv("SRV_TEST_USER"); + String pwd = System.getenv("SRV_TEST_PWD"); + String authSource = System.getenv("SRV_TEST_AUTH_DB"); + + Properties p = new java.util.Properties(); + p.setProperty("user", user); + p.setProperty("password", pwd); + p.setProperty("authSource", authSource); + p.setProperty("database", "test"); + + return DriverManager.getConnection(fullURI, p); + } + @BeforeEach public void setupConnection() throws SQLException { - conn = getBasicConnection(URL, DB); + String buildType = System.getenv("BUILD_TYPE"); + boolean isEapBuild = "eap".equalsIgnoreCase(buildType); + System.out.println("Read environment variable BUILD_TYPE: '" + buildType + "', Detected EAP build: " + isEapBuild); + + connections.put(getADFInstanceConnection(URL, DB), "SELECT * from class"); + + if (isEapBuild) { + try { + Connection directConnection = getDirectRemoteInstanceConnection(); + connections.put(directConnection, "Select * from accounts limit 5"); + } catch (SQLException e) { + System.err.println("Failed to connect to direct remote instance: " + e.getMessage()); + throw e; + } + } else { + try { + Connection directConnection = getDirectRemoteInstanceConnection(); + directConnection.close(); + throw new AssertionError("Expected direct remote connection to fail for non-EAP build"); + } catch (SQLException e) { + if (!"Connection failed.".equals(e.getMessage())) { + throw new AssertionError("Expected 'Connection failed.' but got: " + e.getMessage()); + } + } + } } @AfterEach protected void cleanupTest() throws SQLException { - conn.close(); + for (Connection conn : connections.keySet()) { + conn.close(); + } } @Test public void databaseMetadataTest() throws SQLException { - DatabaseMetaData dbMetadata = conn.getMetaData(); - System.out.println(dbMetadata.getDriverName()); - System.out.println(dbMetadata.getDriverVersion()); + System.out.println("Running databaseMetadataTest"); + for (Connection conn : connections.keySet()) { + DatabaseMetaData dbMetadata = conn.getMetaData(); + System.out.println(dbMetadata.getDriverName()); + System.out.println(dbMetadata.getDriverVersion()); - ResultSet rs = dbMetadata.getColumns(null, "%", "%", "%"); - rowsReturnedCheck(rs); + ResultSet rs = dbMetadata.getColumns(null, "%", "%", "%"); + rowsReturnedCheck(rs); + } } @Test public void queryTest() throws SQLException { - try (Statement stmt = conn.createStatement()) { - String query = "SELECT * from class"; - ResultSet rs = stmt.executeQuery(query); - rowsReturnedCheck(rs); + System.out.println("Running queryTest"); + for (Map.Entry entry : connections.entrySet()) { + try (Statement stmt = entry.getKey().createStatement()) { + ResultSet rs = stmt.executeQuery(entry.getValue()); + rowsReturnedCheck(rs); + } } } @@ -68,6 +123,7 @@ public static void rowsReturnedCheck(ResultSet rs) throws SQLException { while (rs.next()) { actualCount++; } + System.out.println("Rows returned count: " + actualCount); assertTrue(actualCount >= 1, "No rows returned in result set"); } } diff --git a/src/integration-test/java/com/mongodb/jdbc/integration/DCIntegrationTest.java b/src/integration-test/java/com/mongodb/jdbc/integration/DCIntegrationTest.java index 35f3e72d..3d13d854 100644 --- a/src/integration-test/java/com/mongodb/jdbc/integration/DCIntegrationTest.java +++ b/src/integration-test/java/com/mongodb/jdbc/integration/DCIntegrationTest.java @@ -20,9 +20,9 @@ import static org.junit.jupiter.api.Assertions.*; import com.mongodb.jdbc.MongoConnection; +import com.mongodb.jdbc.MongoDatabaseMetaData; import com.mongodb.jdbc.Pair; -import java.sql.DriverManager; -import java.sql.SQLException; +import java.sql.*; import java.util.Properties; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; @@ -30,9 +30,13 @@ @TestInstance(TestInstance.Lifecycle.PER_CLASS) public class DCIntegrationTest { - /** Tests that the driver can work with SRV-style URIs. */ - @Test - public void testConnectWithSRVURI() throws SQLException { + /** + * Connect to a remote cluster to use for the tests. + * + * @return the connection to the enterprise cluster to use for the tests. + * @throws SQLException If the connection failed. + */ + private Connection remoteTestInstanceConnect() throws SQLException { String mongoHost = System.getenv("SRV_TEST_HOST"); assertNotNull(mongoHost, "SRV_TEST_HOST variable not set in environment"); String mongoURI = @@ -54,8 +58,16 @@ public void testConnectWithSRVURI() throws SQLException { p.setProperty("authSource", authSource); p.setProperty("database", "test"); - MongoConnection conn = (MongoConnection) DriverManager.getConnection(fullURI, p); - conn.close(); + return DriverManager.getConnection(fullURI, p); + } + + /** Tests that the driver can work with SRV-style URIs. */ + @Test + public void testConnectWithSRVURI() throws SQLException { + try (Connection conn = remoteTestInstanceConnect(); ) { + // Let's use the connection to make sure everything is working fine. + conn.getMetaData().getDriverVersion(); + } } /** @@ -84,27 +96,173 @@ private Pair createLocalMongodConnInfo(String typeEnvVar) { return new Pair<>(uri, p); } + /** + * Execute the given SQL query and checks the table and column names from the metadata, also + * verifies that the cursor return is working Ok. + * + * @param query The SQL query to execute. + * @param expectedTableNames The expected table names in the metadata. + * @param expectedColumnLabels The expected column names in the metadata. + * @throws SQLException if an error occurs. + */ + private void executeQueryAndValidateResults( + String query, String[] expectedTableNames, String[] expectedColumnLabels) + throws SQLException { + try (Connection conn = remoteTestInstanceConnect(); + Statement stmt = conn.createStatement(); ) { + ResultSet rs = stmt.executeQuery(query); + ResultSetMetaData rsmd = rs.getMetaData(); + + assert (rsmd.getColumnCount() == expectedColumnLabels.length); + int i = 1; + for (String expectColumnLabel : expectedColumnLabels) { + assertEquals( + rsmd.getColumnName(i), + (expectColumnLabel), + rsmd.getColumnName(1) + " != " + expectColumnLabel); + i++; + } + assert (rs.next()); + // Let's also check that we can access the data and don't blow up. + rs.getString(1); + rs.close(); + } + } + /** Tests that the driver rejects the community edition of the server. */ @Test public void testConnectionToCommunityServerFails() { Pair info = createLocalMongodConnInfo("LOCAL_MDB_PORT_COM"); + try (MongoConnection conn = + (MongoConnection) DriverManager.getConnection(info.left(), info.right()); ) { + assertThrows(java.sql.SQLException.class, () -> {}); - assertThrows( - java.sql.SQLException.class, - () -> { - MongoConnection conn = - (MongoConnection) - DriverManager.getConnection(info.left(), info.right()); - }); + } catch (SQLException e) { + assertTrue( + e.getCause().getMessage().contains("Community edition detected"), + e.getCause().getMessage() + " doesn't contain \"Community edition detected\""); + } } /** Tests that the driver connects to the enterprise edition of the server. */ @Test public void testConnectionToEnterpriseServerSucceeds() throws SQLException { Pair info = createLocalMongodConnInfo("LOCAL_MDB_PORT_ENT"); - MongoConnection conn = - (MongoConnection) DriverManager.getConnection(info.left(), info.right()); - conn.close(); + try (Connection conn = DriverManager.getConnection(info.left(), info.right()); ) { + // Let's use the connection to make sure everything is working fine. + conn.getMetaData().getDriverVersion(); + } + } + + @Test + public void testInvalidQueryShouldFail() throws SQLException { + try (Connection conn = remoteTestInstanceConnect(); + Statement stmt = conn.createStatement(); ) { + // Invalid SQL query should fail + assertThrows( + java.sql.SQLException.class, + () -> { + try { + stmt.executeQuery("This is not valid SQL"); + } catch (SQLException e) { + // Let's make sure that we fail for the reason we expect it to. + assert (e.getMessage().contains("Error 2001")); + throw e; + } + }); + } + } + + @Test + public void testValidSimpleQueryShouldSucceed() throws SQLException { + String[] expectedTableNames = {"acc", "acc", "acc", "acc", "t", "t", "t", "t", "t", "t"}; + String[] expectedColumnLabels = { + "_id", + "account_id", + "limit", + "products", + "_id", + "account_id", + "bucket_end_date", + "bucket_start_date", + "transaction_count", + "transactions" + }; + executeQueryAndValidateResults( + "SELECT * from accounts acc JOIN transactions t on acc.account_id = t.account_id limit 5", + expectedTableNames, + expectedColumnLabels); + } + + @Test + public void testCollectionLessQueryShouldSucceed() throws SQLException { + String[] expectedTableNames = {""}; + String[] expectedColumnLabels = {"_1"}; + executeQueryAndValidateResults("SELECT 1", expectedTableNames, expectedColumnLabels); + } + + @Test + public void testValidSimpleQueryNoSchemaForCollectionShouldSucceed() throws SQLException { + String[] expectedTableNames = {""}; + String[] expectedColumnLabels = {"account_id"}; + executeQueryAndValidateResults( + "SELECT account_id from acc_limit_over_1000 limit 5", + expectedTableNames, + expectedColumnLabels); + } + + @Test + public void testListDatabase() throws SQLException { + try (Connection conn = remoteTestInstanceConnect(); ) { + ResultSet rs = conn.getMetaData().getCatalogs(); + while (rs.next()) { + // Verify that none of the system databases are returned + assert (!MongoDatabaseMetaData.DISALLOWED_DB_NAMES + .matcher(rs.getString(1)) + .matches()); + } + rs.close(); + } + } + + @Test + public void testListTables() throws SQLException { + try (Connection conn = remoteTestInstanceConnect(); ) { + ResultSet rs = conn.getMetaData().getTables(null, null, "%", null); + while (rs.next()) { + // Verify that none of the system collections are returned + assert (!MongoDatabaseMetaData.DISALLOWED_COLLECTION_NAMES + .matcher(rs.getString(3)) + .matches()); + } + rs.close(); + } + } + + @Test + public void testColumnsMetadataForCollectionWithSchema() throws SQLException { + String[] expectedColumnLabels = {"_id", "account_id", "limit", "products"}; + try (Connection conn = remoteTestInstanceConnect(); ) { + ResultSet rs = conn.getMetaData().getColumns(null, null, "accounts", "%"); + for (String expectColumnLabel : expectedColumnLabels) { + assert (rs.next()); + assertEquals( + rs.getString(4), + (expectColumnLabel), + rs.getString(4) + " != " + expectColumnLabel); + } + rs.close(); + } + } + + @Test + public void testColumnsMetadataForCollectionWithNoSchema() throws SQLException { + try (Connection conn = remoteTestInstanceConnect(); ) { + ResultSet rs = conn.getMetaData().getColumns(null, null, "acc_limit_over_1000", "%"); + // Check that the result set is empty and we don't blow up when calling next. + assert (!rs.next()); + rs.close(); + } } @Test diff --git a/src/integration-test/java/com/mongodb/jdbc/integration/testharness/DataLoader.java b/src/integration-test/java/com/mongodb/jdbc/integration/testharness/DataLoader.java index 718f96ee..e89d8296 100644 --- a/src/integration-test/java/com/mongodb/jdbc/integration/testharness/DataLoader.java +++ b/src/integration-test/java/com/mongodb/jdbc/integration/testharness/DataLoader.java @@ -88,6 +88,7 @@ public DataLoader(String dataDirectory) throws IOException { this.collections = new HashSet<>(); this.databases = new HashSet<>(); this.mdbUri = new ConnectionString(LOCAL_MDB_URL); + System.out.println(this.mdbUri); this.adfUri = new ConnectionString(LOCAL_ADF_URL); readDataFiles(dataDirectory); diff --git a/src/main/java/com/mongodb/jdbc/BuildInfo.java b/src/main/java/com/mongodb/jdbc/BuildInfo.java index 7f344bd1..601af8bf 100644 --- a/src/main/java/com/mongodb/jdbc/BuildInfo.java +++ b/src/main/java/com/mongodb/jdbc/BuildInfo.java @@ -16,13 +16,68 @@ package com.mongodb.jdbc; +import java.util.List; import java.util.Set; -import org.bson.BsonValue; +import org.bson.codecs.pojo.annotations.BsonCreator; +import org.bson.codecs.pojo.annotations.BsonProperty; -// Simple POJO for deserializing buildInfo results. public class BuildInfo { - public String version; + private String fullVersion; + private List versionArray; public Set modules; - public BsonValue dataLake; public int ok; + + public DataLake dataLake; + + @BsonCreator + public BuildInfo( + @BsonProperty("version") String version, + @BsonProperty("versionArray") List versionArray, + @BsonProperty("modules") Set modules, + @BsonProperty("ok") int ok, + @BsonProperty("dataLake") DataLake dataLake) + throws IndexOutOfBoundsException { + this.fullVersion = version; + this.versionArray = versionArray; + if (dataLake != null) { + this.fullVersion += "." + dataLake.version + "." + dataLake.mongoSQLVersion; + } + this.dataLake = dataLake; + this.ok = ok; + this.modules = modules; + } + + public String getFullVersion() { + return this.fullVersion; + } + + public int getMajorVersion() throws IndexOutOfBoundsException { + return this.versionArray.get(0); + } + + public int getMinorVersion() throws IndexOutOfBoundsException { + return this.versionArray.get(1); + } + + // Override toString for logging + @Override + public String toString() { + return "BuildInfo{" + + "fullVersion='" + + fullVersion + + '\'' + + ", versionArray=" + + versionArray + + ", majorVersion=" + + this.getMajorVersion() + + ", minorVersion=" + + this.getMinorVersion() + + ", modules=" + + modules + + ", ok=" + + ok + + ", dataLake=" + + dataLake + + '}'; + } } diff --git a/src/test/java/com/mongodb/jdbc/MongosqlLibTest.java b/src/main/java/com/mongodb/jdbc/DataLake.java similarity index 53% rename from src/test/java/com/mongodb/jdbc/MongosqlLibTest.java rename to src/main/java/com/mongodb/jdbc/DataLake.java index 0795cbf6..497fb32b 100644 --- a/src/test/java/com/mongodb/jdbc/MongosqlLibTest.java +++ b/src/main/java/com/mongodb/jdbc/DataLake.java @@ -16,19 +16,19 @@ package com.mongodb.jdbc; -import static org.junit.jupiter.api.Assertions.fail; +public class DataLake { + public String version; + public String mongoSQLVersion; -class MongosqlLibTest { - private native String runCommand(String command); - - public void testRunCommand() { - try { - String result = runCommand("SELECT * FROM test"); - assert result.contains("mongosql translation test result success"); - } catch (UnsatisfiedLinkError e) { - System.err.println("Error: Unable to link with native library"); - e.printStackTrace(); - fail("UnsatisfiedLinkError: Native library could not be loaded"); - } + // Override toString for logging + @Override + public String toString() { + return "DataLake{" + + "version='" + + version + + '\'' + + ", mongoSQLVersion=" + + mongoSQLVersion + + '}'; } } diff --git a/src/main/java/com/mongodb/jdbc/MongoConnection.java b/src/main/java/com/mongodb/jdbc/MongoConnection.java index 8d8fe637..e42628dc 100644 --- a/src/main/java/com/mongodb/jdbc/MongoConnection.java +++ b/src/main/java/com/mongodb/jdbc/MongoConnection.java @@ -38,43 +38,16 @@ import com.mongodb.jdbc.utils.X509Authentication; import java.io.File; import java.io.IOException; -import java.sql.Array; -import java.sql.Blob; -import java.sql.CallableStatement; -import java.sql.Clob; -import java.sql.Connection; -import java.sql.DatabaseMetaData; -import java.sql.NClob; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLClientInfoException; -import java.sql.SQLException; -import java.sql.SQLFeatureNotSupportedException; -import java.sql.SQLWarning; -import java.sql.SQLXML; -import java.sql.Savepoint; -import java.sql.Statement; -import java.sql.Struct; +import java.sql.*; +import java.util.Arrays; import java.util.HashMap; import java.util.Map; import java.util.Properties; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Executor; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; +import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicInteger; -import java.util.logging.ConsoleHandler; -import java.util.logging.FileHandler; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.logging.SimpleFormatter; +import java.util.logging.*; import org.bson.BsonDocument; import org.bson.BsonInt32; -import org.bson.Document; import org.bson.UuidRepresentation; @AutoLoggable @@ -115,6 +88,10 @@ public int getServerMinorVersion() { return serverMinorVersion; } + public String getServerVersion() { + return this.serverVersion; + } + protected enum MongoClusterType { AtlasDataFederation, Community, @@ -268,7 +245,7 @@ private MongoClusterType determineClusterType() { // the type of the cluster. BuildInfo buildInfoRes = mongoClient - .getDatabase("admin") + .getDatabase(currentDB) .withCodecRegistry(MongoDriver.REGISTRY) .runCommand(buildInfoCmd, BuildInfo.class); @@ -277,8 +254,21 @@ private MongoClusterType determineClusterType() { return MongoClusterType.UnknownTarget; } + logger.log(Level.FINE, buildInfoRes.toString()); + + this.serverVersion = buildInfoRes.getFullVersion(); + + try { + this.serverMajorVersion = buildInfoRes.getMajorVersion(); + this.serverMinorVersion = buildInfoRes.getMinorVersion(); + // Only log issues happening while trying to compute the server version as this is not a blocker. + } catch (Exception e) { + logger.log(Level.SEVERE, e.toString()); + } + // If the "dataLake" field is present, it must be an ADF cluster. if (buildInfoRes.dataLake != null) { + // append datalake and mongosql version to server version return MongoClusterType.AtlasDataFederation; } else if (buildInfoRes.modules != null) { // Otherwise, if "modules" is present and contains "enterprise", @@ -322,19 +312,6 @@ String getUser() { return user; } - String getServerVersion() throws SQLException { - checkConnection(); - - BsonDocument command = new BsonDocument(); - command.put("buildInfo", new BsonInt32(1)); - try { - Document result = mongoClient.getDatabase("admin").runCommand(command); - return (String) result.get("version"); - } catch (Exception e) { - throw new SQLException(e); - } - } - protected MongoDatabase getDatabase(String DBName) { return mongoClient.getDatabase(DBName); } @@ -627,30 +604,56 @@ class ConnValidation implements Callable { @Override public Void call() throws SQLException, MongoSQLException, MongoSerializationException { MongoClusterType actualClusterType = determineClusterType(); + String serverInfo = + "Connecting to cluster type " + + actualClusterType.toString() + + " with server version " + + serverVersion; + logger.log(Level.INFO, serverInfo); switch (actualClusterType) { case AtlasDataFederation: + logger.log(Level.FINE, "Connecting to Atlas Data Federation."); break; case Community: // Community edition is disallowed. throw new SQLException( "Community edition detected. The JDBC driver is intended for use with MongoDB Enterprise edition or Atlas Data Federation."); case Enterprise: - // Ensure the library is loaded if Enterprise edition detected. - if (!MongoDriver.isMongoSqlTranslateLibraryLoaded()) { - throw new SQLException( - "Enterprise edition detected, but mongosqltranslate library not found"); - } - String mongosqlTranslateVersion = - mongosqlTranslate.getMongosqlTranslateVersion().version; - if (!mongosqlTranslate.checkDriverVersion().compatible) { + String version = MongoDriver.getVersion(); + if (MongoDriver.isEapBuild()) { + // Ensure the library is loaded if Enterprise edition detected. + if (!MongoDriver.isMongoSqlTranslateLibraryLoaded()) { + throw new SQLException( + "Enterprise edition detected, but mongosqltranslate library not found", + MongoDriver.getMongoSqlTranslateLibraryLoadError()); + } else if (MongoDriver.getMongoSqlTranslateLibraryLoadError() != null) { + logger.log( + Level.INFO, + "Error while loading the library using the environment variable. Library bundled with the driver used instead.\n" + + Arrays.stream( + MongoDriver + .getMongoSqlTranslateLibraryLoadError() + .getStackTrace()) + .map(StackTraceElement::toString)); + } + String mongosqlTranslateVersion = + mongosqlTranslate.getMongosqlTranslateVersion().version; + if (!mongosqlTranslate.checkDriverVersion().compatible) { + throw new SQLException( + "Incompatible driver version. The JDBC driver version, " + + version + + ", is not compatible with mongosqltranslate library version, " + + mongosqlTranslateVersion); + } + appName = appName + "|libmongosqltranslate+" + mongosqlTranslateVersion; + } else { throw new SQLException( - "Incompatible driver version. The JDBC driver version, " - + MongoDriver.getVersion() - + ", is not compatible with mongosqltranslate library version, " - + mongosqlTranslateVersion); + "Direct Cluster connection is only supported in EAP driver builds. " + + "Your driver version ('" + + version + + "') is not an EAP build."); } - appName = appName + "|libmongosqltranslate+" + mongosqlTranslateVersion; break; case UnknownTarget: // Target could not be determined. @@ -660,6 +663,14 @@ public Void call() throws SQLException, MongoSQLException, MongoSerializationExc // Set the cluster type. clusterType = actualClusterType; + boolean resultExists; + try (Statement statement = createStatement()) { + resultExists = statement.execute("SELECT 1"); + } + if (!resultExists) { + // no resultSet returned + throw new SQLException("Connection error"); + } return null; } } diff --git a/src/main/java/com/mongodb/jdbc/MongoDatabaseMetaData.java b/src/main/java/com/mongodb/jdbc/MongoDatabaseMetaData.java index e0b2cae5..33a6647a 100644 --- a/src/main/java/com/mongodb/jdbc/MongoDatabaseMetaData.java +++ b/src/main/java/com/mongodb/jdbc/MongoDatabaseMetaData.java @@ -17,20 +17,13 @@ package com.mongodb.jdbc; import static com.mongodb.jdbc.BsonTypeInfo.*; -import static com.mongodb.jdbc.mongosql.MongoSQLTranslate.SQL_SCHEMAS_COLLECTION; import com.mongodb.client.ListIndexesIterable; import com.mongodb.client.MongoDatabase; import com.mongodb.jdbc.logging.AutoLoggable; import com.mongodb.jdbc.logging.MongoLogger; import com.mongodb.jdbc.mongosql.MongoSQLException; -import java.sql.Connection; -import java.sql.DatabaseMetaData; -import java.sql.ResultSet; -import java.sql.ResultSetMetaData; -import java.sql.RowIdLifetime; -import java.sql.SQLException; -import java.sql.Types; +import java.sql.*; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -189,6 +182,9 @@ public class MongoDatabaseMetaData implements DatabaseMetaData { private static final String FUNC_DEFAULT_CATALOG = "def"; private static final String YES = "YES"; + private int serverMajorVersion; + private int serverMinorVersion; + private static final List GET_TABLES_SORT_SPECS = Arrays.asList( new SortableBsonDocument.SortSpec( @@ -227,7 +223,7 @@ public class MongoDatabaseMetaData implements DatabaseMetaData { private static final List GET_INDEX_INFO_SORT_SPECS = Arrays.asList( new SortableBsonDocument.SortSpec( - NON_UNIQUE, SortableBsonDocument.ValueType.String), + NON_UNIQUE, SortableBsonDocument.ValueType.Boolean), new SortableBsonDocument.SortSpec( INDEX_NAME, SortableBsonDocument.ValueType.String), new SortableBsonDocument.SortSpec( @@ -236,6 +232,11 @@ public class MongoDatabaseMetaData implements DatabaseMetaData { private static final com.mongodb.jdbc.MongoFunctions MongoFunctions = com.mongodb.jdbc.MongoFunctions.getInstance(); + public static final Pattern DISALLOWED_COLLECTION_NAMES = + Pattern.compile("(system\\.(namespace|indexes|profiles|js|views))|__sql_schemas"); + + public static final Pattern DISALLOWED_DB_NAMES = Pattern.compile("admin|config|local|system"); + private final MongoConnection conn; private String serverVersion; private MongoLogger logger; @@ -407,8 +408,8 @@ public ResultSet getTableTypes() throws SQLException { } // MHOUSE-7119: ADF quickstarts return empty strings and the admin database, so we filter them out - static boolean filterEmptiesAndAdmin(String dbName) { - return !dbName.isEmpty() && !dbName.equals("admin"); + static boolean filterEmptiesAndInternalDBs(String dbName) { + return !dbName.isEmpty() && !DISALLOWED_DB_NAMES.matcher(dbName).matches(); } // Helper for getting a stream of all database names. @@ -418,7 +419,7 @@ private Stream getDatabaseNames() { .listDatabaseNames() .into(new ArrayList<>()) .stream() - .filter(dbName -> filterEmptiesAndAdmin(dbName)); + .filter(dbName -> filterEmptiesAndInternalDBs(dbName)); } // Helper for getting a list of collection names from the db @@ -487,10 +488,13 @@ private Stream getTableDataFromDB( List types, BiFunction bsonSerializer) { + // Filter out __sql_schemas, system.namespaces, system.indexes,system.profile,system.js,system.views return this.getTableDataFromDB( dbName, res -> - (tableNamePatternRE == null + // Don't list system collections + (!DISALLOWED_COLLECTION_NAMES.matcher(res.name).matches()) + && (tableNamePatternRE == null || tableNamePatternRE.matcher(res.name).matches()) && (types == null || types.contains(res.type.toLowerCase()))) @@ -570,11 +574,7 @@ public String getDatabaseProductName() throws SQLException { @Override public String getDatabaseProductVersion() throws SQLException { - if (serverVersion != null) { - return serverVersion; - } - serverVersion = conn.getServerVersion(); - return serverVersion; + return conn.getServerVersion(); } @Override @@ -1248,12 +1248,12 @@ public int getResultSetHoldability() throws SQLException { @Override public int getDatabaseMajorVersion() throws SQLException { - return MongoDriver.MAJOR_VERSION; + return conn.getServerMajorVersion(); } @Override public int getDatabaseMinorVersion() throws SQLException { - return MongoDriver.MINOR_VERSION; + return conn.getServerMinorVersion(); } @Override @@ -1551,9 +1551,10 @@ private Stream getColumnsFromDB( // filter only for collections matching the pattern, and exclude the `__sql_schemas` collection .filter( tableName -> - (tableNamePatternRE == null - || tableNamePatternRE.matcher(tableName).matches()) - && !tableName.equals(SQL_SCHEMAS_COLLECTION)) + // Don't list system collections + (!DISALLOWED_COLLECTION_NAMES.matcher(tableName).matches()) + && (tableNamePatternRE == null + || tableNamePatternRE.matcher(tableName).matches())) // map the collection names into triples of (dbName, tableName, tableSchema) .map( @@ -2661,6 +2662,18 @@ private Stream toGetIndexInfoDocs( return keys.keySet() .stream() + .filter( + key -> { + // If the index is not an integer (e.g., a geospatial index), `keys.getInteger(key)` + // will throw a ClassCastException. In this case, we skip the index because the + // sort sequence is not supported by JDBC. + try { + keys.getInteger(key); + } catch (ClassCastException e) { + return false; + } + return true; + }) .map( key -> { BsonValue ascOrDesc = diff --git a/src/main/java/com/mongodb/jdbc/MongoDriver.java b/src/main/java/com/mongodb/jdbc/MongoDriver.java index 25b3fded..5e7821e4 100644 --- a/src/main/java/com/mongodb/jdbc/MongoDriver.java +++ b/src/main/java/com/mongodb/jdbc/MongoDriver.java @@ -16,21 +16,19 @@ package com.mongodb.jdbc; -import static com.mongodb.AuthenticationMechanism.MONGODB_OIDC; -import static com.mongodb.AuthenticationMechanism.MONGODB_X509; +import static com.mongodb.AuthenticationMechanism.*; import static com.mongodb.jdbc.MongoDriver.MongoJDBCProperty.*; import static org.bson.codecs.configuration.CodecRegistries.fromProviders; import com.mongodb.AuthenticationMechanism; import com.mongodb.ConnectionString; import com.mongodb.MongoClientSettings; +import com.mongodb.MongoConfigurationException; import com.mongodb.client.MongoClient; -import java.io.File; -import java.io.UnsupportedEncodingException; +import com.mongodb.jdbc.utils.NativeLoader; +import java.io.*; import java.lang.ref.WeakReference; -import java.net.URL; import java.net.URLEncoder; -import java.nio.file.Path; import java.nio.file.Paths; import java.sql.Connection; import java.sql.Driver; @@ -49,6 +47,8 @@ import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.logging.Level; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import java.util.stream.Stream; import org.bson.codecs.BsonValueCodecProvider; import org.bson.codecs.ValueCodecProvider; @@ -65,6 +65,30 @@ */ public class MongoDriver implements Driver { + // The regular expression to validate and manipulate the mongoDB uri. + protected static final Pattern MONGODB_URI_PATTERN = + Pattern.compile( + "(mongodb(?:\\+srv)?://)(?(?:\\S+:)?\\S+@)?([^\\r\\n\\t\\f\\v ?]+(\\?(?.*))?)"); + // The regular expression to extract the authentication mechanism. + protected static final Pattern AUTH_MECH_TO_AUGMENT_PATTERN = + Pattern.compile( + "authMechanism=(?(" + + PLAIN.getMechanismName() + + "|" + + SCRAM_SHA_1.getMechanismName() + + "|" + + SCRAM_SHA_256.getMechanismName() + + "|" + + GSSAPI.getMechanismName() + + "))"); + //The list of mechanism for which a username and/or password must be present for the first uri parsing pass. + protected static final List MECHANISMS_TO_AUGMENT = + Arrays.asList( + PLAIN.getMechanismName(), + SCRAM_SHA_1.getMechanismName(), + SCRAM_SHA_256.getMechanismName(), + GSSAPI.getMechanismName()); + /** * The list of connection options specific to the JDBC driver which can only be provided through * a Properties Object. @@ -128,6 +152,8 @@ public static String getVersion() { } private static boolean mongoSqlTranslateLibraryLoaded = false; + private static Exception mongoSqlTranslateLibraryLoadingError = null; + private static String mongoSqlTranslateLibraryPath = null; private static final String MONGOSQL_TRANSLATE_NAME = "mongosqltranslate"; public static final String MONGOSQL_TRANSLATE_PATH = "MONGOSQL_TRANSLATE_PATH"; @@ -138,6 +164,32 @@ public static String getVersion() { MongoClientSettings.getDefaultCodecRegistry(), PojoCodecProvider.builder().automatic(true).build()); + static String getAbbreviatedGitVersion() { + Process p = null; + try { + // Unit and integration tests can't rely on the manifest from the jar + // Get the git tag and use it as the version + String command = "git describe --abbrev=0"; + p = Runtime.getRuntime().exec(command); + try (BufferedReader input = + new BufferedReader(new InputStreamReader(p.getInputStream())); ) { + StringBuilder version_sb = new StringBuilder(); + String line; + while ((line = input.readLine()) != null) { + version_sb.append(line); + } + return version_sb.append("-SNAPSHOT").substring(1).trim(); + } + } catch (IOException e) { + throw new RuntimeException( + new SQLException("Internal error retrieving driver version")); + } finally { + if (p != null) { + p.destroy(); + } + } + } + static { MongoDriver unit = new MongoDriver(); try { @@ -145,83 +197,77 @@ public static String getVersion() { } catch (SQLException e) { throw new RuntimeException(e); } - VERSION = unit.getClass().getPackage().getImplementationVersion(); - if (VERSION != null) { - String[] verSp = VERSION.split("[.]"); - if (verSp.length < 2) { - throw new RuntimeException( - new SQLException( - "version was not specified correctly, must contain at least major and minor parts")); - } - MAJOR_VERSION = Integer.parseInt(verSp[0]); - MINOR_VERSION = Integer.parseInt(verSp[1]); + + String version = unit.getClass().getPackage().getImplementationVersion(); + if (version == null) { + VERSION = getAbbreviatedGitVersion(); } else { - // final requires this. - MAJOR_VERSION = 0; - MINOR_VERSION = 0; + VERSION = version; } + String[] verSp = VERSION.split("[.]"); + if (verSp.length < 2) { + throw new RuntimeException( + new SQLException( + "version was not specified correctly, must contain at least major and minor parts")); + } + MAJOR_VERSION = Integer.parseInt(verSp[0]); + MINOR_VERSION = Integer.parseInt(verSp[1]); + String name = unit.getClass().getPackage().getImplementationTitle(); NAME = (name != null) ? name : "mongodb-jdbc"; Runtime.getRuntime().addShutdownHook(new Thread(MongoDriver::closeAllClients)); - initializeMongoSqlTranslateLibrary(); + try { + loadMongoSqlTranslateLibrary(); + } + // Store the error so that we can log it later. + catch (Exception e) { + mongoSqlTranslateLibraryLoadingError = e; + } catch (Error e) { + // Note, linkage issues are reported as linkage error and not as Exception. We need to track both. + mongoSqlTranslateLibraryLoadingError = new Exception(e); + } } /** * Attempts to initialize the MongoSQL Translate library from various paths and sets * mongoSqlTranslateLibraryLoaded to indicate success or failure. */ - private static void initializeMongoSqlTranslateLibrary() { - try { - String[] libraryPaths = resolveLibraryPaths(); - for (String path : libraryPaths) { - if (loadMongoSqlTranslateLibrary(path)) { - mongoSqlTranslateLibraryLoaded = true; - return; - } - mongoSqlTranslateLibraryLoaded = false; + private static void loadMongoSqlTranslateLibrary() throws IOException { + // The `MONGOSQL_TRANSLATE_PATH` environment variable allows specifying an alternative library path. + // This provides a backdoor mechanism to override the default library path of being colocated with the + // driver library and load the MongoSQL Translate library from a different location. + // Intended primarily for development and testing purposes. + String envPath = System.getenv(MONGOSQL_TRANSLATE_PATH); + if (envPath != null && !envPath.isEmpty()) { + String absolutePath = Paths.get(envPath).toAbsolutePath().normalize().toString(); + try { + System.load(absolutePath); + mongoSqlTranslateLibraryPath = absolutePath; + mongoSqlTranslateLibraryLoaded = true; + return; + } catch (Error e) { + // Store the error and then try loading the library from inside the jar next. + mongoSqlTranslateLibraryLoadingError = new Exception(e); } - } catch (Throwable t) { - mongoSqlTranslateLibraryLoaded = false; - } - } - - private static boolean loadMongoSqlTranslateLibrary(String libraryPath) { - try { - System.load(libraryPath); - return true; - } catch (Throwable t) { - return false; } + mongoSqlTranslateLibraryPath = NativeLoader.loadLibraryFromJar(MONGOSQL_TRANSLATE_NAME); + mongoSqlTranslateLibraryLoaded = true; } public static CodecRegistry getCodecRegistry() { return REGISTRY; } - // Resolves the potential paths where the MongoSQL Translate library are expected be located. - private static String[] resolveLibraryPaths() throws Exception { - String libraryPath = getLibraryPath(); - - // The `MONGOSQL_TRANSLATE_PATH` environment variable allows specifying an alternative library path. - // This provides a backdoor mechanism to override the default library path of being colocated with the - // driver library and load the MongoSQL Translate library from a different location. - // Intended primarily for development and testing purposes. - String envPath = System.getenv(MONGOSQL_TRANSLATE_PATH); - - List paths = new ArrayList<>(); - paths.add(libraryPath); - if (envPath != null && !envPath.isEmpty()) { - paths.add(envPath); + public static boolean isEapBuild() { + String version = getVersion(); + // Return false if the version string is null or empty + if (version == null || version.isEmpty()) { + return false; } - return paths.toArray(new String[0]); - } - private static String getLibraryPath() throws Exception { - URL url = MongoDriver.class.getProtectionDomain().getCodeSource().getLocation(); - Path driverPath = Paths.get(url.toURI()); - Path driverDir = driverPath.getParent(); - return driverDir.resolve(System.mapLibraryName(MONGOSQL_TRANSLATE_NAME)).toString(); + // Our EAP builds contain `libv` in the tag + return version.contains("libv"); } @Override @@ -489,6 +535,14 @@ public static boolean isMongoSqlTranslateLibraryLoaded() { return mongoSqlTranslateLibraryLoaded; } + public static String getMongoSqlTranslateLibraryPath() { + return mongoSqlTranslateLibraryPath; + } + + public static Exception getMongoSqlTranslateLibraryLoadError() { + return mongoSqlTranslateLibraryLoadingError; + } + // removePrefix removes a prefix from a String. private static String removePrefix(String prefix, String s) { if (s != null && prefix != null && s.startsWith(prefix)) { @@ -604,6 +658,63 @@ public static MongoConnectionConfig getConnectionSettings(String url, Properties } } + /** + * Parse the original uri provided by the user. If the parsing failed, we try to augment the URI + * with the username and password provided in the properties. The reason behind it is that new + * ConnectionString(xx) validates the uri as is parses it and for some authentication mechanisms + * these info are mandatory, but the user can provide them separately to the driver. + * + * @param url The original uri as provided by the user. + * @param info The extra properties. + * @return the uri unchanged or augmented with uid and pwd from info. + * @throws IllegalArgumentException + * @throws MongoConfigurationException + */ + protected static ConnectionString buildConnectionString(String url, Properties info) + throws IllegalArgumentException, MongoConfigurationException { + String actualURL = removePrefix(JDBC, url); + try { + return new ConnectionString(actualURL); + } catch (IllegalArgumentException ea) { + Matcher uri_matcher = MONGODB_URI_PATTERN.matcher(actualURL); + if (uri_matcher.find()) { + String username = + info.getProperty(USER) != null + ? URLEncoder.encode(info.getProperty(USER)) + : null; + String password = + info.getProperty(PASSWORD) != null + ? URLEncoder.encode(info.getProperty(PASSWORD)) + : null; + String options = uri_matcher.group("options"); + if (uri_matcher.group("uidpwd") == null && username != null && options != null) { + Matcher authMec_matcher = AUTH_MECH_TO_AUGMENT_PATTERN.matcher(options); + if (authMec_matcher.find()) { + String authMech = authMec_matcher.group("authMech"); + if (MECHANISMS_TO_AUGMENT.contains(authMech.toUpperCase())) { + StringBuilder sb = new StringBuilder(); + sb.append(uri_matcher.group(1)); // protocol + sb.append(username); + if (password != null) { + sb.append(":"); + sb.append(password); + } + sb.append("@"); + sb.append(uri_matcher.group(3)); // host and options + + return new ConnectionString(sb.toString()); + } + } + } + // The error is not related to a missing uid/pwd for the mechanisms which need them + throw ea; + } else { + // Credential information were present in the URI, this issue is not related to missing username and/or password + throw ea; + } + } + } + private static interface NullCoalesce { T coalesce(T left, T right); } diff --git a/src/main/java/com/mongodb/jdbc/MongoJsonSchema.java b/src/main/java/com/mongodb/jdbc/MongoJsonSchema.java index eea27569..0f883622 100644 --- a/src/main/java/com/mongodb/jdbc/MongoJsonSchema.java +++ b/src/main/java/com/mongodb/jdbc/MongoJsonSchema.java @@ -18,7 +18,6 @@ import static com.mongodb.jdbc.BsonTypeInfo.*; -import com.mongodb.jdbc.utils.BsonUtils; import java.sql.DatabaseMetaData; import java.sql.SQLException; import java.util.HashMap; @@ -37,9 +36,7 @@ import org.bson.codecs.pojo.annotations.BsonIgnore; public class MongoJsonSchema { - - private static final Codec CODEC = - MongoDriver.getCodecRegistry().get(MongoJsonSchema.class); + private static final Codec CODEC = MongoDriver.REGISTRY.get(JsonSchema.class); public static class ScalarProperties { protected String name; @@ -419,11 +416,6 @@ public int hashCode() { return Objects.hash(bsonType, properties, anyOf, required, items, additionalProperties); } - @Override - public String toString() { - return BsonUtils.toString(CODEC, this); - } - // Any is represented by the empty json schema {}, so all fields // will be null or false public boolean isAny() { diff --git a/src/main/java/com/mongodb/jdbc/MongoResultSet.java b/src/main/java/com/mongodb/jdbc/MongoResultSet.java index 97c579f3..8b2471ca 100644 --- a/src/main/java/com/mongodb/jdbc/MongoResultSet.java +++ b/src/main/java/com/mongodb/jdbc/MongoResultSet.java @@ -652,8 +652,13 @@ public void clearWarnings() throws SQLException { @Override public String getCursorName() throws SQLException { - throw new SQLFeatureNotSupportedException( - Thread.currentThread().getStackTrace()[1].toString()); + if (this.statement.cursorName != null) { + return this.statement.cursorName; + } + if (this.cursor != null) { + return String.valueOf(this.cursor.getServerCursor().getId()); + } + return ""; } @Override diff --git a/src/main/java/com/mongodb/jdbc/MongoStatement.java b/src/main/java/com/mongodb/jdbc/MongoStatement.java index 84881b98..836e6cc5 100644 --- a/src/main/java/com/mongodb/jdbc/MongoStatement.java +++ b/src/main/java/com/mongodb/jdbc/MongoStatement.java @@ -29,10 +29,9 @@ import com.mongodb.jdbc.mongosql.MongoSQLTranslate; import com.mongodb.jdbc.mongosql.TranslateResult; import java.sql.*; -import java.util.*; +import java.util.Collections; +import java.util.List; import java.util.concurrent.TimeUnit; -import java.util.logging.Level; -import org.apache.commons.text.StringEscapeUtils; import org.bson.BsonDocument; import org.bson.BsonInt32; import org.bson.BsonString; @@ -51,8 +50,10 @@ public class MongoStatement implements Statement { protected boolean closeOnCompletion = false; private int fetchSize = 0; private int maxQuerySec = 0; + private String currentDBName; private MongoLogger logger; private int statementId; + String cursorName; public MongoStatement(MongoConnection conn, String databaseName) throws SQLException { Preconditions.checkNotNull(conn); @@ -60,6 +61,7 @@ public MongoStatement(MongoConnection conn, String databaseName) throws SQLExcep this.statementId = conn.getNextStatementId(); logger = new MongoLogger(this.getClass().getCanonicalName(), conn.getLogger(), statementId); this.conn = conn; + currentDBName = databaseName; try { currentDB = conn.getDatabase(databaseName); @@ -186,6 +188,7 @@ public void clearWarnings() throws SQLException { @Override public void setCursorName(String name) throws SQLException { checkClosed(); + this.cursorName = name; } // ----------------------- Multiple Results -------------------------- @@ -214,15 +217,14 @@ private ResultSet executeAtlasDataFederationQuery(String sql) throws SQLExceptio currentDB .withCodecRegistry(MongoDriver.REGISTRY) .runCommand(getSchemaCmd, MongoJsonSchemaResult.class); - MongoJsonSchema resultsetSchema = schemaResult.schema.mongoJsonSchema; + MongoJsonSchema schema = schemaResult.schema.mongoJsonSchema; List> selectOrder = schemaResult.selectOrder; - logger.setResultSetSchema(resultsetSchema); - logger.log(Level.FINE, "ResultSet schema: " + resultsetSchema); + resultSet = new MongoResultSet( this, cursor, - resultsetSchema, + schema, selectOrder, conn.getExtJsonMode(), conn.getUuidRepresentation()); @@ -235,12 +237,10 @@ private ResultSet executeDirectClusterQuery(String sql) MongoSQLTranslate mongoSQLTranslate = conn.getMongosqlTranslate(); String dbName = currentDB.getName(); - // Retrieve the namespaces for the query GetNamespacesResult namespaceResult = mongoSQLTranslate.getNamespaces(currentDB.getName(), sql); - - logger.log(Level.FINE, "Namespaces: " + namespaceResult); List namespaces = namespaceResult.namespaces; + // Check to see if namespaces returned a database. It would only do this // if the query contains a qualified namespace. In this event, we must // switch currentDB to the query's database for proper operation. @@ -249,16 +249,9 @@ private ResultSet executeDirectClusterQuery(String sql) currentDB = conn.getDatabase(dbName); } - // Translate the SQL query BsonDocument catalogDoc = mongoSQLTranslate.buildCatalogDocument(currentDB, dbName, namespaces); - logger.log(Level.FINE, "Query catalog: " + catalogDoc); - logger.setNamespacesSchema(catalogDoc); TranslateResult translateResponse = mongoSQLTranslate.translate(sql, dbName, catalogDoc); - logger.setPipeline(translateResponse.pipeline); - logger.setResultSetSchema(translateResponse.resultSetSchema); - logger.log(Level.FINE, "Translate response: " + translateResponse); - MongoIterable iterable = null; if (translateResponse.targetCollection != null && !translateResponse.targetCollection.isEmpty()) { @@ -296,29 +289,20 @@ private ResultSet executeDirectClusterQuery(String sql) public ResultSet executeQuery(String sql) throws SQLException { checkClosed(); closeExistingResultSet(); - logger.setSqlQuery(sql); - long startTime = System.nanoTime(); - logger.log(Level.INFO, StringEscapeUtils.escapeJava(sql)); - ResultSet result = null; + try { if (conn.getClusterType() == MongoConnection.MongoClusterType.AtlasDataFederation) { - result = executeAtlasDataFederationQuery(sql); + return executeAtlasDataFederationQuery(sql); } else if (conn.getClusterType() == MongoConnection.MongoClusterType.Enterprise) { - result = executeDirectClusterQuery(sql); + return executeDirectClusterQuery(sql); } else { throw new SQLException("Unsupported cluster type: " + conn.clusterType); } } catch (MongoExecutionTimeoutException e) { throw new SQLTimeoutException(e); } catch (MongoSQLException | MongoSerializationException e) { - throw new RuntimeException(e); + throw new SQLException(e); } - long endTime = System.nanoTime(); - logger.log( - Level.FINE, - "Query executed in " + ((endTime - startTime) / 1000000000d) + " seconds"); - - return result; } @Override diff --git a/src/main/java/com/mongodb/jdbc/SortableBsonDocument.java b/src/main/java/com/mongodb/jdbc/SortableBsonDocument.java index 2e486897..bb3dc7e8 100644 --- a/src/main/java/com/mongodb/jdbc/SortableBsonDocument.java +++ b/src/main/java/com/mongodb/jdbc/SortableBsonDocument.java @@ -34,6 +34,7 @@ static class SortSpec { enum ValueType { String, Int, + Boolean, } List sortSpecs; @@ -63,6 +64,12 @@ public int compareTo(SortableBsonDocument o) { .getInt32(sortSpec.key) .compareTo(o.nestedDocValue.getInt32(sortSpec.key)); break; + case Boolean: + r = + this.nestedDocValue + .getBoolean(sortSpec.key) + .compareTo(o.nestedDocValue.getBoolean(sortSpec.key)); + break; } if (r != 0) { diff --git a/src/main/java/com/mongodb/jdbc/mongosql/BaseResult.java b/src/main/java/com/mongodb/jdbc/mongosql/BaseResult.java new file mode 100644 index 00000000..ed677335 --- /dev/null +++ b/src/main/java/com/mongodb/jdbc/mongosql/BaseResult.java @@ -0,0 +1,47 @@ +/* + * Copyright 2024-present MongoDB, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.mongodb.jdbc.mongosql; + +import org.bson.codecs.pojo.annotations.BsonProperty; + +/** Base class for result types, includes error handling. */ +public class BaseResult { + @BsonProperty("error") + protected final String error; + + @BsonProperty("error_is_internal") + protected final Boolean errorIsInternal; + + public BaseResult( + @BsonProperty("error") String error, + @BsonProperty("error_is_internal") Boolean errorIsInternal) { + this.error = (error != null) ? error : ""; + this.errorIsInternal = (errorIsInternal != null) ? errorIsInternal : false; + } + + public boolean hasError() { + return !error.isEmpty(); + } + + public String getError() { + return error; + } + + public Boolean getErrorIsInternal() { + return errorIsInternal; + } +} diff --git a/src/main/java/com/mongodb/jdbc/mongosql/CheckDriverVersionResult.java b/src/main/java/com/mongodb/jdbc/mongosql/CheckDriverVersionResult.java index d324cdf6..893524cf 100644 --- a/src/main/java/com/mongodb/jdbc/mongosql/CheckDriverVersionResult.java +++ b/src/main/java/com/mongodb/jdbc/mongosql/CheckDriverVersionResult.java @@ -16,15 +16,13 @@ package com.mongodb.jdbc.mongosql; -import static com.mongodb.jdbc.utils.BsonUtils.JSON_WRITER_NO_INDENT_SETTINGS; - import com.mongodb.jdbc.MongoDriver; import com.mongodb.jdbc.utils.BsonUtils; import org.bson.codecs.Codec; import org.bson.codecs.pojo.annotations.BsonCreator; import org.bson.codecs.pojo.annotations.BsonProperty; -public class CheckDriverVersionResult { +public class CheckDriverVersionResult extends BaseResult { private static final Codec CODEC = MongoDriver.getCodecRegistry().get(CheckDriverVersionResult.class); @@ -33,12 +31,16 @@ public class CheckDriverVersionResult { public final Boolean compatible; @BsonCreator - public CheckDriverVersionResult(@BsonProperty("compatible") Boolean compatible) { + public CheckDriverVersionResult( + @BsonProperty("compatible") Boolean compatible, + @BsonProperty("error") String error, + @BsonProperty("error_is_internal") Boolean errorIsInternal) { + super(error, errorIsInternal); this.compatible = (compatible != null) ? compatible : false; } @Override public String toString() { - return BsonUtils.toString(CODEC, this, JSON_WRITER_NO_INDENT_SETTINGS); + return BsonUtils.toString(CODEC, this); } } diff --git a/src/main/java/com/mongodb/jdbc/mongosql/GetMongosqlTranslateVersionResult.java b/src/main/java/com/mongodb/jdbc/mongosql/GetMongosqlTranslateVersionResult.java index 678c9c87..6fa01d9a 100644 --- a/src/main/java/com/mongodb/jdbc/mongosql/GetMongosqlTranslateVersionResult.java +++ b/src/main/java/com/mongodb/jdbc/mongosql/GetMongosqlTranslateVersionResult.java @@ -16,15 +16,13 @@ package com.mongodb.jdbc.mongosql; -import static com.mongodb.jdbc.utils.BsonUtils.JSON_WRITER_NO_INDENT_SETTINGS; - import com.mongodb.jdbc.MongoDriver; import com.mongodb.jdbc.utils.BsonUtils; import org.bson.codecs.Codec; import org.bson.codecs.pojo.annotations.BsonCreator; import org.bson.codecs.pojo.annotations.BsonProperty; -public class GetMongosqlTranslateVersionResult { +public class GetMongosqlTranslateVersionResult extends BaseResult { private static final Codec CODEC = MongoDriver.getCodecRegistry().get(GetMongosqlTranslateVersionResult.class); @@ -33,12 +31,16 @@ public class GetMongosqlTranslateVersionResult { public final String version; @BsonCreator - public GetMongosqlTranslateVersionResult(@BsonProperty("version") String version) { + public GetMongosqlTranslateVersionResult( + @BsonProperty("version") String version, + @BsonProperty("error") String error, + @BsonProperty("error_is_internal") Boolean errorIsInternal) { + super(error, errorIsInternal); this.version = version; } @Override public String toString() { - return BsonUtils.toString(CODEC, this, JSON_WRITER_NO_INDENT_SETTINGS); + return BsonUtils.toString(CODEC, this); } } diff --git a/src/main/java/com/mongodb/jdbc/mongosql/GetNamespacesResult.java b/src/main/java/com/mongodb/jdbc/mongosql/GetNamespacesResult.java index c628eb3b..060e9392 100644 --- a/src/main/java/com/mongodb/jdbc/mongosql/GetNamespacesResult.java +++ b/src/main/java/com/mongodb/jdbc/mongosql/GetNamespacesResult.java @@ -16,8 +16,6 @@ package com.mongodb.jdbc.mongosql; -import static com.mongodb.jdbc.utils.BsonUtils.JSON_WRITER_NO_INDENT_SETTINGS; - import com.mongodb.jdbc.MongoDriver; import com.mongodb.jdbc.utils.BsonUtils; import java.util.List; @@ -25,7 +23,7 @@ import org.bson.codecs.pojo.annotations.BsonCreator; import org.bson.codecs.pojo.annotations.BsonProperty; -public class GetNamespacesResult { +public class GetNamespacesResult extends BaseResult { private static final Codec CODEC = MongoDriver.getCodecRegistry().get(GetNamespacesResult.class); @@ -34,14 +32,15 @@ public class GetNamespacesResult { public final List namespaces; @BsonCreator - public GetNamespacesResult(@BsonProperty("namespaces") List namespaces) { + public GetNamespacesResult( + @BsonProperty("namespaces") List namespaces, + @BsonProperty("error") String error, + @BsonProperty("error_is_internal") Boolean errorIsInternal) { + super(error, errorIsInternal); this.namespaces = namespaces; } public static class Namespace { - private static final Codec CODEC = - MongoDriver.getCodecRegistry().get(Namespace.class); - @BsonProperty("database") public final String database; @@ -55,15 +54,10 @@ public Namespace( this.database = database; this.collection = collection; } - - @Override - public String toString() { - return BsonUtils.toString(CODEC, this, JSON_WRITER_NO_INDENT_SETTINGS); - } } @Override public String toString() { - return BsonUtils.toString(CODEC, this, JSON_WRITER_NO_INDENT_SETTINGS); + return BsonUtils.toString(CODEC, this); } } diff --git a/src/main/java/com/mongodb/jdbc/mongosql/MongoSQLTranslate.java b/src/main/java/com/mongodb/jdbc/mongosql/MongoSQLTranslate.java index 08022108..e806fb3b 100644 --- a/src/main/java/com/mongodb/jdbc/mongosql/MongoSQLTranslate.java +++ b/src/main/java/com/mongodb/jdbc/mongosql/MongoSQLTranslate.java @@ -24,9 +24,7 @@ import com.mongodb.client.model.Field; import com.mongodb.client.model.Filters; import com.mongodb.client.model.Projections; -import com.mongodb.jdbc.MongoDriver; -import com.mongodb.jdbc.MongoJsonSchemaResult; -import com.mongodb.jdbc.MongoSerializationException; +import com.mongodb.jdbc.*; import com.mongodb.jdbc.logging.AutoLoggable; import com.mongodb.jdbc.logging.MongoLogger; import com.mongodb.jdbc.utils.BsonUtils; @@ -41,14 +39,16 @@ @AutoLoggable public class MongoSQLTranslate { + public static final String SQL_SCHEMAS_COLLECTION = "__sql_schemas"; private final MongoLogger logger; - /** Native method to send commands via JNI. */ - public native byte[] runCommand(byte[] command, int length); - - public static final String ERROR_KEY = "error"; - public static final String IS_INTERNAL_ERROR_KEY = "error_is_internal"; + /** + * Native method to send commands via JNI. pub extern "C" fn + * Java_com_mongodb_jdbc_mongosql_MongoSQLTranslate_runCommand( env: JNIEnv, _class: JClass, + * command: JByteArray, ) -> jbyteArray + */ + public native byte[] runCommand(byte[] command); public MongoSQLTranslate(MongoLogger logger) { this.logger = logger; @@ -64,28 +64,30 @@ public MongoSQLTranslate(MongoLogger logger) { * deserialization. * @throws MongoSQLException If an error occurs during command execution. */ - public T runCommand(BsonDocument command, Class responseClass) + public T runCommand(BsonDocument command, Class responseClass) throws MongoSerializationException, MongoSQLException { byte[] commandBytes = BsonUtils.serialize(command); - byte[] responseBytes = runCommand(commandBytes, commandBytes.length); + byte[] responseBytes = runCommand(commandBytes); BsonDocument responseDoc = BsonUtils.deserialize(responseBytes); BsonDocumentReader reader = new BsonDocumentReader(responseDoc); - BsonValue error = responseDoc.get(ERROR_KEY); - if (error != null) { + T result = + MongoDriver.getCodecRegistry() + .get(responseClass) + .decode(reader, DecoderContext.builder().build()); + + if (result.hasError()) { String errorMessage = String.format( - responseDoc.getBoolean(IS_INTERNAL_ERROR_KEY).getValue() + result.getErrorIsInternal() ? "Internal error: %s" : "Error executing command: %s", - error.asString().getValue()); + result.getError()); throw new MongoSQLException(errorMessage); } - return MongoDriver.getCodecRegistry() - .get(responseClass) - .decode(reader, DecoderContext.builder().build()); + return result; } /** @@ -183,13 +185,30 @@ public GetNamespacesResult getNamespaces(String dbName, String sql) return runCommand(command, GetNamespacesResult.class); } - // Builds a catalog document containing the schema information for the specified collections. + /** + * Builds a catalog document containing the schema information for the specified collections. + * + * @param collections The list of collections to retrieve the schemas for. + * @param dbName The name of the database where the collections must be. + * @param mongoDatabase The current database for this connection. + * @return the schema catalog for all the specified collections. The catalog document format is + * : { "dbName": { "collection1" : "Schema1", "collection2" : "Schema2", ... }} + */ public BsonDocument buildCatalogDocument( MongoDatabase mongoDatabase, String dbName, List collections) throws MongoSQLException { + // There is no collection tied to the query + // For example "SELECT 1" + if (collections == null || collections.isEmpty()) { + MongoJsonSchema emptyObjectSchema = MongoJsonSchema.createEmptyObjectSchema(); + // Create a catalog with an empty collection name and an empty schema + // {"test": {"": {}}} + return new BsonDocument(dbName, new BsonDocument("", new BsonDocument())); + } + // Create an aggregation pipeline to fetch the schema information for the specified collections. // The pipeline uses $in to query all the specified collections and projects them into the desired format: // "dbName": { "collection1" : "Schema1", "collection2" : "Schema2", ... } @@ -265,8 +284,14 @@ public BsonDocument buildCatalogDocument( foundResult = true; } if (!foundResult) { - throw new MongoSQLException( - "No schema information returned for the requested collections."); + logger.log( + Level.SEVERE, + "No schema information found for any of the requested collections. Will use empty schemas. Hint: Generate schemas for your collections."); + BsonDocument schemas = new BsonDocument(); + for (String collectionName : collectionNames) { + schemas.append(collectionName, new BsonDocument()); + } + catalog = new BsonDocument(dbName, schemas); } // Check that all expected collections are present in the result @@ -283,7 +308,6 @@ public BsonDocument buildCatalogDocument( throw new MongoSQLException( "Could not retrieve schema for collections: " + missingCollections); } - return catalog; } @@ -319,7 +343,10 @@ public MongoJsonSchemaResult getSchema(MongoDatabase mongoDatabase, String colle BsonDocument resultDoc = result.first(); if (resultDoc == null) { - throw new MongoSQLException("No schema found for collection: " + collectionName); + logger.log( + Level.SEVERE, + "No schema information returned for the requested collections. Using an empty schema."); + resultDoc = new BsonDocument(); } BsonDocumentReader reader = new BsonDocumentReader(resultDoc); diff --git a/src/main/java/com/mongodb/jdbc/mongosql/TranslateResult.java b/src/main/java/com/mongodb/jdbc/mongosql/TranslateResult.java index d4fa5ff9..dfc108b0 100644 --- a/src/main/java/com/mongodb/jdbc/mongosql/TranslateResult.java +++ b/src/main/java/com/mongodb/jdbc/mongosql/TranslateResult.java @@ -16,8 +16,6 @@ package com.mongodb.jdbc.mongosql; -import static com.mongodb.jdbc.utils.BsonUtils.JSON_WRITER_NO_INDENT_SETTINGS; - import com.mongodb.jdbc.JsonSchema; import com.mongodb.jdbc.MongoDriver; import com.mongodb.jdbc.MongoJsonSchema; @@ -28,7 +26,7 @@ import org.bson.codecs.pojo.annotations.BsonCreator; import org.bson.codecs.pojo.annotations.BsonProperty; -public class TranslateResult { +public class TranslateResult extends BaseResult { private static final Codec CODEC = MongoDriver.getCodecRegistry().get(TranslateResult.class); @@ -45,7 +43,10 @@ public TranslateResult( @BsonProperty("target_collection") String targetCollection, @BsonProperty("pipeline") List pipeline, @BsonProperty("result_set_schema") JsonSchema resultSetSchema, - @BsonProperty("select_order") List> selectOrder) { + @BsonProperty("select_order") List> selectOrder, + @BsonProperty("error") String error, + @BsonProperty("error_is_internal") Boolean errorIsInternal) { + super(error, errorIsInternal); this.targetDb = targetDb; this.targetCollection = targetCollection; this.pipeline = pipeline; @@ -58,6 +59,6 @@ public TranslateResult( @Override public String toString() { - return BsonUtils.toString(CODEC, this, JSON_WRITER_NO_INDENT_SETTINGS); + return BsonUtils.toString(CODEC, this); } } diff --git a/src/main/java/com/mongodb/jdbc/utils/BsonUtils.java b/src/main/java/com/mongodb/jdbc/utils/BsonUtils.java index 35b2db4d..545fbc66 100644 --- a/src/main/java/com/mongodb/jdbc/utils/BsonUtils.java +++ b/src/main/java/com/mongodb/jdbc/utils/BsonUtils.java @@ -21,7 +21,10 @@ import java.io.IOException; import java.io.StringWriter; import java.nio.ByteBuffer; -import org.bson.*; +import org.bson.BsonBinaryReader; +import org.bson.BsonBinaryWriter; +import org.bson.BsonDocument; +import org.bson.BsonDocumentWriter; import org.bson.codecs.*; import org.bson.io.BasicOutputBuffer; import org.bson.json.JsonMode; diff --git a/src/main/java/com/mongodb/jdbc/utils/NativeLoader.java b/src/main/java/com/mongodb/jdbc/utils/NativeLoader.java new file mode 100644 index 00000000..a3881720 --- /dev/null +++ b/src/main/java/com/mongodb/jdbc/utils/NativeLoader.java @@ -0,0 +1,194 @@ +/* + * Copyright 2024-present MongoDB, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.mongodb.jdbc.utils; + +import com.mongodb.MongoException; +import com.mongodb.jdbc.MongoDriver; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.nio.file.*; +import java.util.HashSet; +import java.util.Set; +import java.util.regex.Pattern; +import org.apache.commons.lang3.SystemUtils; + +/** + * A helper based on the NativeUtils library of Adam Heinrich: + * + *

A simple library class which helps with loading dynamic libraries stored in the JAR archive. + * These libraries usualy contain implementation of some methods in native code (using JNI - Java + * Native Interface). + * + * @see "http://adamheinrich.com/blog/2012/how-to-load-native-jni-library-from-jar" + * @see "https://github.com/adamheinrich/native-utils" + */ +public class NativeLoader { + + private static final String NATIVE_FOLDER_PATH_PREFIX = "mongosql_native"; + + /** Temporary directory which will contain the DLLs. */ + private static File temporaryLibDir; + + // List of libraries loaded using the loader. + // A library can only be loaded once. + private static Set loadedLibs = new HashSet(); + + // This pattern was constructed using OpenJDK platform keys logic. + // See https://github.com/openjdk/jtreg/blob/master/make/Platform.gmk#L103 + private static final Pattern X86_64_ARCH_PATTERN = + Pattern.compile("^(x86_64|amd64|ia32e|em64t|x64|x86-64|8664|intel64)$"); + private static final Pattern ARM_ARCH_PATTERN = Pattern.compile("^(aarch64|arm64)$"); + + private static final String ARM = "arm"; + private static final String X86_64 = "x86_64"; + + private static final String MACOS = "macos"; + private static final String LINUX = "linux"; + private static final String WINDOWS = "win"; + + /** Private constructor - this class will never be instanced. */ + private NativeLoader() {} + + private static String normalizeOS() throws MongoException { + if (SystemUtils.IS_OS_LINUX) { + return LINUX; + } else if (SystemUtils.IS_OS_WINDOWS) { + return WINDOWS; + } else if (SystemUtils.IS_OS_MAC) { + return MACOS; + } + + // Unsupported OS + throw new MongoException("Unsupported OS : " + SystemUtils.OS_NAME); + } + + private static String normalizeArch() throws MongoException { + String arch = SystemUtils.OS_ARCH.toLowerCase(); + if (X86_64_ARCH_PATTERN.matcher(arch).matches()) { + return X86_64; + } else if (ARM_ARCH_PATTERN.matcher(arch).matches()) { + return ARM; + } + + // Unsupported architecture + throw new MongoException("Unsupported architecture : " + arch); + } + + /** + * Loads library from current JAR archive. + * + *

The file from JAR is copied into system temporary directory and then loaded. The temporary + * file is deleted after exiting. Method uses String as filename because the pathname is + * "abstract", not system-dependent. + * + * @param libraryName The name of the library to load. + * @return the path of the loaded library. + * @throws IOException If temporary file creation or read/write operation fails + * @throws IllegalArgumentException If source file (param libPath) does not exist + * @throws IllegalArgumentException If the libPath is not absolute or if the filename is shorter + * than three characters (restriction of {@link File#createTempFile(java.lang.String, + * java.lang.String)}). + * @throws FileNotFoundException If the file could not be found inside the JAR. + */ + public static String loadLibraryFromJar(String libraryName) + throws IOException, IllegalArgumentException, FileNotFoundException { + + String libName = System.mapLibraryName(libraryName); + if (loadedLibs.contains(libName)) { + // Don't reload. + return libName; + } + + // Build the library path using the os and arch information. + String resourcePath = + normalizeArch().toLowerCase() + "/" + normalizeOS().toLowerCase() + "/" + libName; + + URL resource = + MongoDriver.class.getProtectionDomain().getClassLoader().getResource(resourcePath); + + if (resource != null) { + + // Create a temporary directory to copy the library into. + if (temporaryLibDir == null) { + temporaryLibDir = createTempDirectory(); + temporaryLibDir.deleteOnExit(); + } + + // Copy the library in the temporary directory. + File libFile; + libFile = new File(temporaryLibDir, libName); + + try (InputStream is = resource.openStream()) { + Files.copy(is, libFile.toPath()); + } catch (FileAlreadyExistsException e) { + // Do nothing, the library is already there which means that the JVM already loaded it. + } catch (IOException e) { + libFile.delete(); + // Unexpected error. + throw e; + } catch (NullPointerException e) { + libFile.delete(); + throw new FileNotFoundException( + "Resource " + resourcePath + " was not found inside JAR."); + } + + try { + System.load(libFile.getAbsolutePath()); + } finally { + if (isPosixCompliant()) { + // Assume POSIX compliant file system, can be deleted after loading + libFile.delete(); + } else { + // Assume non-POSIX, and don't delete until last file descriptor closed + libFile.deleteOnExit(); + } + } + + return libFile.getAbsolutePath(); + } + throw new FileNotFoundException("Resource " + resourcePath + " was not found inside JAR."); + } + + private static boolean isPosixCompliant() { + try { + return FileSystems.getDefault().supportedFileAttributeViews().contains("posix"); + } catch (FileSystemNotFoundException | ProviderNotFoundException | SecurityException e) { + e.printStackTrace(); + return false; + } + } + + /** + * Creates a temporary directory under the file system path of a temporary directory for use by + * the java runtime. The path will look like {java.io.tmpdir}/{prefix}{nanoTime} + * + * @return The path to the created directory. + * @throws IOException If an error occurs. + */ + private static File createTempDirectory() throws IOException { + String tempDir = System.getProperty("java.io.tmpdir"); + File generatedDir = new File(tempDir, NATIVE_FOLDER_PATH_PREFIX + System.nanoTime()); + + if (!generatedDir.mkdir() && !Files.exists(generatedDir.toPath())) { + throw new IOException("Failed to create temp directory " + generatedDir.getName()); + } + return generatedDir; + } +} diff --git a/src/test/java/com/mongodb/jdbc/MongoDatabaseMetaDataTest.java b/src/test/java/com/mongodb/jdbc/MongoDatabaseMetaDataTest.java index a4f8f4b3..30dfcd11 100644 --- a/src/test/java/com/mongodb/jdbc/MongoDatabaseMetaDataTest.java +++ b/src/test/java/com/mongodb/jdbc/MongoDatabaseMetaDataTest.java @@ -16,7 +16,7 @@ package com.mongodb.jdbc; -import static com.mongodb.jdbc.MongoDatabaseMetaData.filterEmptiesAndAdmin; +import static com.mongodb.jdbc.MongoDatabaseMetaData.filterEmptiesAndInternalDBs; import static org.junit.jupiter.api.Assertions.*; import com.mongodb.ConnectionString; @@ -451,7 +451,7 @@ void testFilterEmptiesAndAdmin() { assertEquals( expected, input.stream() - .filter(dbName -> filterEmptiesAndAdmin(dbName)) + .filter(dbName -> filterEmptiesAndInternalDBs(dbName)) .collect(Collectors.toList())); } } diff --git a/src/test/java/com/mongodb/jdbc/MongoMock.java b/src/test/java/com/mongodb/jdbc/MongoMock.java index d087e082..f76204b6 100644 --- a/src/test/java/com/mongodb/jdbc/MongoMock.java +++ b/src/test/java/com/mongodb/jdbc/MongoMock.java @@ -560,7 +560,7 @@ protected static MongoJsonSchema generateMongoJsonSchemaAllTypes() { + " \"required\": [\"all\"]" + "}"; - return MongoDriver.getCodecRegistry() + return MongoDriver.REGISTRY .get(MongoJsonSchema.class) .decode(new JsonReader(schema), DecoderContext.builder().build()); } diff --git a/src/test/java/com/mongodb/jdbc/MongoResultSetTest.java b/src/test/java/com/mongodb/jdbc/MongoResultSetTest.java index bc5bbff3..ea9e2e16 100644 --- a/src/test/java/com/mongodb/jdbc/MongoResultSetTest.java +++ b/src/test/java/com/mongodb/jdbc/MongoResultSetTest.java @@ -589,6 +589,12 @@ public void testGetObjectToStringMatchesGetString() throws Exception { mongoResultSetAllTypes.getObject(ALL_MAX_KEY_COL_LABEL).toString()); } + @Test + void testGetCursorName() throws Exception { + mongoStatement.setCursorName("test"); + assertEquals("test", mongoResultSet.getCursorName()); + } + @Test void testGetArithmeticValues() throws Exception { // Test Double values are as expected @@ -909,11 +915,6 @@ void closedResultSets() throws Exception { () -> { closedMongoResultSet.clearWarnings(); }); - assertThrows( - SQLException.class, - () -> { - closedMongoResultSet.getCursorName(); - }); assertThrows( SQLException.class, () -> { diff --git a/src/test/java/com/mongodb/jdbc/MongoSQLTranslateLibTest.java b/src/test/java/com/mongodb/jdbc/MongoSQLTranslateLibTest.java index f3bf7fa2..77b71dce 100644 --- a/src/test/java/com/mongodb/jdbc/MongoSQLTranslateLibTest.java +++ b/src/test/java/com/mongodb/jdbc/MongoSQLTranslateLibTest.java @@ -18,20 +18,42 @@ import static org.junit.jupiter.api.Assertions.*; +import com.mongodb.jdbc.mongosql.MongoSQLTranslate; import java.lang.reflect.Field; import java.lang.reflect.Method; +import java.nio.charset.StandardCharsets; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; public class MongoSQLTranslateLibTest { + + /** Helper function to call the runCommand endpoint of the translation library. */ + private static void testRunCommand() { + MongoSQLTranslate mongosqlTranslate = new MongoSQLTranslate(null); + byte[] bytes = + mongosqlTranslate.runCommand("SendingSomething".getBytes(StandardCharsets.UTF_8)); + assert bytes.length > 0; + } + @BeforeEach void setup() throws Exception { // Reset the mongoSqlTranslateLibraryLoaded flag to false before each test case. // This ensures that the flag starts with a known value at the start of the test // as it can be set during the static initialization or test interference. - Field field = MongoDriver.class.getDeclaredField("mongoSqlTranslateLibraryLoaded"); - field.setAccessible(true); - field.set(null, false); + Field mongoSqlTranslateLibraryLoadedField = + MongoDriver.class.getDeclaredField("mongoSqlTranslateLibraryLoaded"); + mongoSqlTranslateLibraryLoadedField.setAccessible(true); + mongoSqlTranslateLibraryLoadedField.set(null, false); + + Field mongoSqlTranslateLibraryPathField = + MongoDriver.class.getDeclaredField("mongoSqlTranslateLibraryPath"); + mongoSqlTranslateLibraryPathField.setAccessible(true); + mongoSqlTranslateLibraryPathField.set(null, null); + + Field mongoSqlTranslateLibraryLoadingError = + MongoDriver.class.getDeclaredField("mongoSqlTranslateLibraryLoadingError"); + mongoSqlTranslateLibraryPathField.setAccessible(true); + mongoSqlTranslateLibraryPathField.set(null, null); } @Test @@ -40,17 +62,23 @@ void testLibraryLoadingFromDriverPath() throws Exception { System.getenv(MongoDriver.MONGOSQL_TRANSLATE_PATH), "MONGOSQL_TRANSLATE_PATH should not be set"); - Method initMethod = - MongoDriver.class.getDeclaredMethod("initializeMongoSqlTranslateLibrary"); + Method initMethod = MongoDriver.class.getDeclaredMethod("loadMongoSqlTranslateLibrary"); initMethod.setAccessible(true); initMethod.invoke(null); assertTrue( MongoDriver.isMongoSqlTranslateLibraryLoaded(), "Library should be loaded successfully from the driver directory"); + String tempDir = System.getProperty("java.io.tmpdir"); + assertTrue( + MongoDriver.getMongoSqlTranslateLibraryPath().contains(tempDir), + "Expected library path to contain '" + + tempDir + + "' but didn't. Actual path is " + + MongoDriver.getMongoSqlTranslateLibraryPath()); - MongosqlLibTest test = new MongosqlLibTest(); - test.testRunCommand(); + // The library was loaded successfully. Now, let's make sure that we can call the runCommand endpoint. + testRunCommand(); } @Test @@ -58,17 +86,59 @@ void testLibraryLoadingWithEnvironmentVariable() throws Exception { String envPath = System.getenv(MongoDriver.MONGOSQL_TRANSLATE_PATH); assertNotNull(envPath, "MONGOSQL_TRANSLATE_PATH should be set"); - // Test initializeMongoSqlTranslateLibrary, with Environment variable set it should find the library - Method initMethod = - MongoDriver.class.getDeclaredMethod("initializeMongoSqlTranslateLibrary"); + // Test loadMongoSqlTranslateLibrary, with Environment variable set it should find the library + Method initMethod = MongoDriver.class.getDeclaredMethod("loadMongoSqlTranslateLibrary"); initMethod.setAccessible(true); initMethod.invoke(null); + assertNull(MongoDriver.getMongoSqlTranslateLibraryLoadError()); + assertTrue( MongoDriver.isMongoSqlTranslateLibraryLoaded(), "Library should be loaded when MONGOSQL_TRANSLATE_PATH is set"); - MongosqlLibTest test = new MongosqlLibTest(); - test.testRunCommand(); + assertTrue( + MongoDriver.getMongoSqlTranslateLibraryPath() + .contains("resources/MongoSqlLibraryTest"), + "Expected library path to contain 'resources/main' but didn't. Actual path is " + + MongoDriver.getMongoSqlTranslateLibraryPath()); + + // The library was loaded successfully. Now, let's make sure that we can call the runCommand endpoint. + testRunCommand(); + } + + @Test + void testLibraryLoadingWithInvalidEnvironmentVariableFallback() throws Exception { + String envPath = System.getenv(MongoDriver.MONGOSQL_TRANSLATE_PATH); + assertNotNull(envPath, "MONGOSQL_TRANSLATE_PATH should be set"); + + // Test loadMongoSqlTranslateLibrary, with invalid Environment variable set should fallback to driver directory + Method initMethod = MongoDriver.class.getDeclaredMethod("loadMongoSqlTranslateLibrary"); + initMethod.setAccessible(true); + initMethod.invoke(null); + + assertNotNull(MongoDriver.getMongoSqlTranslateLibraryLoadError()); + + assertTrue( + MongoDriver.getMongoSqlTranslateLibraryLoadError() + .getMessage() + .contains("java.lang.UnsatisfiedLinkError: Can't load library"), + "Expected error to be a loading error but is " + + MongoDriver.getMongoSqlTranslateLibraryLoadError().getMessage()); + + // The library must be loaded and it should be the one from inside the driver. + assertTrue( + MongoDriver.isMongoSqlTranslateLibraryLoaded(), + "Library should be loaded successfully from the driver directory"); + String tempDir = System.getProperty("java.io.tmpdir"); + assertTrue( + MongoDriver.getMongoSqlTranslateLibraryPath().contains(tempDir), + "Expected library path to contain '" + + tempDir + + "' but didn't. Actual path is " + + MongoDriver.getMongoSqlTranslateLibraryPath()); + + // The library was loaded successfully. Now, let's make sure that we can call the runCommand endpoint. + testRunCommand(); } } diff --git a/src/test/java/com/mongodb/jdbc/TestConnectionString.java b/src/test/java/com/mongodb/jdbc/TestConnectionString.java new file mode 100644 index 00000000..b47afea2 --- /dev/null +++ b/src/test/java/com/mongodb/jdbc/TestConnectionString.java @@ -0,0 +1,309 @@ +/* + * Copyright 2023-present MongoDB, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.mongodb.jdbc; + +import static com.mongodb.jdbc.MongoDriver.*; +import static org.junit.jupiter.api.Assertions.*; + +import com.mongodb.AuthenticationMechanism; +import com.mongodb.ConnectionString; +import java.util.Arrays; +import java.util.Properties; +import java.util.regex.Matcher; +import org.junit.jupiter.api.Test; + +class TestConnectionString { + static final String localhost = "mongodb://localhost"; + static final String localhostWithOnlyDB = "mongodb://localhost/authDB"; + static final String onlyAuthSource = "mongodb://localhost/?authSource=authDB"; + static final String dbAndAuthSource = "mongodb://localhost/pouet?authSource=authDB"; + static final String USER_CONN_KEY = "user"; + static final String PWD_CONN_KEY = "password"; + static final String USER = "AzureDiamond"; + static final String PWD = "hunter2"; + static final String DATABASE = "database"; + static final String DB = "foo"; + static final String AUTHDB = "authDB"; + static final String POUET = "pouet"; + + @Test + void testLocalHost() throws Exception { + Properties p = new Properties(); + p.setProperty(USER_CONN_KEY, USER); + p.setProperty(PWD_CONN_KEY, PWD); + p.setProperty(DATABASE, DB); + + MongoConnectionConfig result = getConnectionSettings(localhost, p); + + assertEquals(USER, result.connectionString.getCredential().getUserName()); + assertEquals(DB, result.connectionString.getCredential().getSource()); + } + + @Test + void testLocalHostWithOnlyDBNoPropsDB() throws Exception { + Properties p = new Properties(); + p.setProperty(USER_CONN_KEY, USER); + p.setProperty(PWD_CONN_KEY, PWD); + + MongoConnectionConfig result = getConnectionSettings(localhostWithOnlyDB, p); + + assertEquals(USER, result.connectionString.getCredential().getUserName()); + assertEquals(AUTHDB, result.connectionString.getCredential().getSource()); + } + + @Test + void testPropsDBOverridesURIDBNoAuthSource() throws Exception { + Properties p = new Properties(); + p.setProperty(USER_CONN_KEY, USER); + p.setProperty(PWD_CONN_KEY, PWD); + p.setProperty(DATABASE, DB); + + MongoConnectionConfig result = getConnectionSettings(localhostWithOnlyDB, p); + + assertEquals(USER, result.connectionString.getCredential().getUserName()); + assertEquals(DB, result.connectionString.getCredential().getSource()); + assertEquals(DB, result.connectionString.getDatabase()); + } + + @Test + void testPropsDBWithURIAuthSource() throws Exception { + Properties p = new Properties(); + p.setProperty(USER_CONN_KEY, USER); + p.setProperty(PWD_CONN_KEY, PWD); + p.setProperty(DATABASE, DB); + + MongoConnectionConfig result = getConnectionSettings(onlyAuthSource, p); + + assertEquals(USER, result.connectionString.getCredential().getUserName()); + assertEquals(AUTHDB, result.connectionString.getCredential().getSource()); + assertEquals(DB, result.connectionString.getDatabase()); + } + + @Test + void testUriDBWithAuthSource() throws Exception { + Properties p = new Properties(); + p.setProperty(USER_CONN_KEY, USER); + p.setProperty(PWD_CONN_KEY, PWD); + + MongoConnectionConfig result = getConnectionSettings(dbAndAuthSource, p); + + assertEquals(USER, result.connectionString.getCredential().getUserName()); + assertEquals(AUTHDB, result.connectionString.getCredential().getSource()); + assertEquals(POUET, result.connectionString.getDatabase()); + } + + @Test + void testPropsOverrideURIDBWithAuthSource() throws Exception { + Properties p = new Properties(); + p.setProperty(USER_CONN_KEY, USER); + p.setProperty(PWD_CONN_KEY, PWD); + p.setProperty(DATABASE, DB); + MongoConnectionConfig result = getConnectionSettings(dbAndAuthSource, p); + + assertEquals(USER, result.connectionString.getCredential().getUserName()); + assertEquals(AUTHDB, result.connectionString.getCredential().getSource()); + assertEquals(DB, result.connectionString.getDatabase()); + } + + // Tests for the work-around required to be able to parse URI when the username and password is mandatory but provided as part of the properties. + @Test + void testBuildConnectionStringWithMissingUidPwdForAllAuthMech() throws Exception { + for (AuthenticationMechanism authMech : AuthenticationMechanism.values()) { + String url = + localhostWithOnlyDB + + "?authSource=$external&authMechanism=" + + authMech.getMechanismName(); + System.out.println(url); + + Properties p = new Properties(); + p.setProperty(USER_CONN_KEY, USER); + if (authMech != AuthenticationMechanism.MONGODB_OIDC + && authMech != AuthenticationMechanism.MONGODB_X509) { + p.setProperty(PWD_CONN_KEY, PWD); + } + + ConnectionString result = buildConnectionString(url, p); + + // For PLAIN,SCRAM-SHA-1, SCRAM-SHA-256 AND GSSAPI the uri must be augmented with the username and password provided in the properties + if (Arrays.asList( + AuthenticationMechanism.PLAIN, + AuthenticationMechanism.SCRAM_SHA_1, + AuthenticationMechanism.SCRAM_SHA_256, + AuthenticationMechanism.GSSAPI) + .contains(authMech)) { + assertNotEquals( + result.getConnectionString(), + url, + "The original URL should have been augmented with the username and password information but it wasn't."); + assertNotNull(result.getCredential()); + assertNotNull(result.getCredential().getAuthenticationMechanism()); + assertEquals( + authMech.getMechanismName(), + result.getCredential().getAuthenticationMechanism().getMechanismName()); + if (null != result.getCredential().getUserName()) { + assertEquals(USER, result.getCredential().getUserName()); + } + if (null != result.getCredential().getPassword()) { + assertEquals(PWD, new String(result.getCredential().getPassword())); + } + } else { + assertEquals( + result.getConnectionString(), + url, + "The original URL should stay unchanged"); + } + } + } + + /** + * Validate that the MONGODB_URI_PATTERN is correct and work as expected. + * + * @param uri The uri to test. + * @param shouldMatch True, if the uri should match the pattern. False otherwise. + * @param hasUidPWd True, if the uri contains a username and/or password. False otherwise. + * @param expectedAuthMech The expected authentication mechanism extracted from the uri. + */ + void testPatternsHelper( + String uri, boolean shouldMatch, boolean hasUidPWd, String expectedAuthMech) { + Matcher uri_matcher = MONGODB_URI_PATTERN.matcher(uri); + boolean match = uri_matcher.find(); + + assertEquals( + match, + shouldMatch, + "The URI " + + uri + + " matching result is not as expected. Expected: " + + shouldMatch + + " , Actual: " + + match); + if (shouldMatch) { + String uidpwd = uri_matcher.group("uidpwd"); + String options = uri_matcher.group("options"); + if (hasUidPWd) { + assertNotNull(uidpwd, "No UID/PWD detected when expected. URI = " + uri); + } else { + assertNull(uidpwd, "UID/PWD detected when none expected. URI = " + uri); + } + + if (options != null) { + Matcher authMec_matcher = AUTH_MECH_TO_AUGMENT_PATTERN.matcher(options); + match = authMec_matcher.find(); + assertEquals( + match, + expectedAuthMech != null, + "The authentication mechanism matching result is not as expected. Expected: " + + (expectedAuthMech != null) + + " , Actual: " + + match); + if (match && expectedAuthMech != null) { + assertTrue( + match, + "No authentication mechanism was found in the URI when " + + expectedAuthMech + + " was expected."); + String authMech = authMec_matcher.group("authMech"); + assertEquals( + authMech, + expectedAuthMech, + "Expected authentication mechanism " + + expectedAuthMech + + " but got " + + authMech); + } + } + } + } + + @Test + void testPatterns() { + // Non matching URIs + testPatternsHelper("mongodb:/localhost", false, false, null); + testPatternsHelper("blabla", false, false, null); + + // No user name or password + testPatternsHelper("mongodb://localhost", true, false, null); + testPatternsHelper("mongodb://localhost?connectTimeoutms=600000", true, false, null); + testPatternsHelper( + "mongodb+srv://localhost?authSource=$external&connectTimeoutms=600000", + true, + false, + null); + + // User name or password + testPatternsHelper("mongodb://toto@localhost", true, true, null); + testPatternsHelper("mongodb+srv://toto:tutu@localhost", true, true, null); + testPatternsHelper( + "mongodb+srv://toto@localhost?connectTimeoutms=600000", true, true, null); + testPatternsHelper( + "mongodb://toto:tutu@localhost?connectTimeoutms=600000", true, true, null); + + // No user name or password, with auth mech + for (String authMech : MECHANISMS_TO_AUGMENT) { + testPatternsHelper( + "mongodb://localhost?authSource=$external&authMechanism=" + authMech, + true, + false, + authMech); + } + testPatternsHelper( + "mongodb://localhost?authSource=$external&authMechanism=MONGODB-OIDC", + true, + false, + null); + testPatternsHelper( + "mongodb://localhost?authSource=$external&authMechanism=SCRAM_SHA_1", + true, + false, + null); + + for (String authMech : MECHANISMS_TO_AUGMENT) { + testPatternsHelper( + "mongodb://localhost?authSource=$external&authMechanism=" + + authMech + + "&connectTimeoutms=600000", + true, + false, + authMech); + } + testPatternsHelper( + "mongodb://localhost?authSource=$external&authMechanism=MONGODB-OIDC&connectTimeoutms=600000", + true, + false, + null); + for (String authMech : MECHANISMS_TO_AUGMENT) { + testPatternsHelper( + "mongodb+srv://localhost?authMechanism=" + + authMech + + "&authSource=$external&connectTimeoutms=600000", + true, + false, + authMech); + } + testPatternsHelper( + "mongodb+srv://localhost?authMechanism=MONGODB-OIDC&authSource=$external&connectTimeoutms=600000", + true, + false, + null); + + for (String authMech : MECHANISMS_TO_AUGMENT) { + testPatternsHelper( + "mongodb://localhost?authMechanism=" + authMech, true, false, authMech); + } + testPatternsHelper("mongodb://localhost?authMechanism=MONGODB-OIDC", true, false, null); + } +} diff --git a/src/test/resources/MongoSqlLibraryTest/libmongosqltranslate.so b/src/test/resources/MongoSqlLibraryTest/libmongosqltranslate.so index 3109e4c5..f5604386 100755 Binary files a/src/test/resources/MongoSqlLibraryTest/libmongosqltranslate.so and b/src/test/resources/MongoSqlLibraryTest/libmongosqltranslate.so differ