Bump aws to v1.5

.github/workflows/MainDistributionPipeline.yml

@@ -14,21 +14,21 @@ concurrency:
 jobs:
   duckdb-stable-build:
     name: Build extension binaries
-    uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@v1.4.0
+    uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@main
     with:
       extension_name: aws
-      duckdb_version: v1.4.0
+      duckdb_version: 75a25b5404700262f965129571a1995fb802242d
       ci_tools_version: main
       exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads;windows_amd64_rtools;windows_amd64_mingw' # Doesn't work anyway: env local file or env access possible
 
   duckdb-stable-deploy:
     name: Deploy extension binaries
     needs: duckdb-stable-build
-    uses: duckdb/extension-ci-tools/.github/workflows/_extension_deploy.yml@v1.4.0
+    uses: duckdb/extension-ci-tools/.github/workflows/_extension_deploy.yml@main
     secrets: inherit
     with:
       extension_name: aws
-      duckdb_version: v1.4.0
+      duckdb_version: 75a25b5404700262f965129571a1995fb802242d
       ci_tools_version: main
       exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads;windows_amd64_rtools;windows_amd64_mingw' # Doesn't work anyway: env local file or env access possible
       deploy_latest: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }}

.github/workflows/MinioTests.yml

@@ -52,7 +52,7 @@ jobs:
       - name: Setup vcpkg
         uses: lukka/run-vcpkg@v11.1
         with:
-          vcpkgGitCommitId: 5e5d0e1cd7785623065e77eff011afdeec1a3574
+          vcpkgGitCommitId: 84bab45d415d22042bd0b9081aea57f362da3f35
 
       - name: Build
         shell: bash
@@ -61,12 +61,7 @@ jobs:
       - name: Start S3/HTTP test server
         shell: bash
         run: |
-          cd duckdb
-          mkdir data/attach_test
-          touch data/attach_test/attach.db
-          sudo ./scripts/install_s3_test_server.sh
-          source ./scripts/run_s3_test_server.sh
-          sleep 30
+          ./scripts/install_and_run_httpfs_s3_test_server.sh
 
       - name: Write AWS credentials file
         shell: bash

extension_config.cmake

@@ -5,3 +5,9 @@ duckdb_extension_load(aws
     SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR}
     LOAD_TESTS
 )
+
+duckdb_extension_load(httpfs
+    GIT_URL https://github.com/duckdb/duckdb-httpfs
+    GIT_TAG c5a73542cfa9d4b2be73670e5ff0ab973f7a9b27
+    INCLUDE_DIR src/include
+)

scripts/create_minio_credential_file.sh

@@ -24,16 +24,10 @@ aws_access_key_id=minio_duckdb_user_invalid
 aws_secret_access_key=thispasswordiscompletelywrong
 aws_session_token=completelybogussessiontoken
 
-[assume-role-arn]
-source_profile = default
-role_arn = arn:aws:iam::840140254803:role/pyiceberg-etl-role
-region = us-east-2
-
-[assume-role-arn-external-id]
-source_profile = default
-role_arn = arn:aws:iam::840140254803:role/pyiceberg-etl-role
-region = us-east-2
-external_id = 128289344
+[minio-testing-empty]
+aws_access_key_id=
+aws_secret_access_key=
+aws_session_token=
 "
 
 # Write the credentials configuration to the file
@@ -48,7 +42,22 @@ region=eu-west-1
 
 [profile minio-testing-invalid]
 region=the-moon-123
+
+[profile minio-testing-empty]
+region=
+
+[profile assume-role-arn]
+source_profile = default
+role_arn = arn:aws:iam::840140254803:role/pyiceberg-etl-role
+region = us-east-2
+
+[profile assume-role-arn-external-id]
+source_profile = default
+role_arn = arn:aws:iam::840140254803:role/pyiceberg-etl-role
+region = us-east-2
+external_id = 128289344
 "
 
 # Write the config to the file
-echo "$config_str" > "$config_file"
+echo "$config_str" >"$config_file"
+

scripts/install_and_run_httpfs_s3_test_server.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+[[ $(basename $PWD) = "duckdb-aws" && -d "./duckdb" ]] || {
+	echo "$0: assert failed -- expect to be in .../duckdb-aws (in $PWD)" >&2
+	exit 1
+}
+
+set -xv
+mkdir -p test/test_data
+touch test/test_data/attach.db
+sudo ./scripts/install_s3_test_server.sh
+source ./scripts/run_s3_test_server.sh
+sleep 30

scripts/install_s3_test_server.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+# Note: needs sudo
+
+unamestr=$(uname)
+if [[ "$unamestr" == 'Linux' ]]; then
+	apt-get install -y docker.io
+fi
+
+docker --version
+echo '127.0.0.1 duckdb-minio.com' >> /etc/hosts
+echo '127.0.0.1 test-bucket.duckdb-minio.com' >> /etc/hosts
+echo '127.0.0.1 test-bucket-2.duckdb-minio.com' >> /etc/hosts
+echo '127.0.0.1 test-bucket-public.duckdb-minio.com' >> /etc/hosts

scripts/minio_s3.yml

@@ -0,0 +1,82 @@
+services:
+  minio:
+    image: minio/minio:RELEASE.2021-11-03T03-36-36Z
+    hostname: duckdb-minio.com
+    ports:
+      - "9000:9000"
+      - "9001:9001"
+    volumes:
+      - /tmp/minio_test_data:/data
+      - /tmp/minio_root_data:/root/.minio
+    environment:
+      - MINIO_ROOT_USER=duckdb_minio_admin
+      - MINIO_ROOT_PASSWORD=duckdb_minio_admin_password
+      - MINIO_REGION_NAME=eu-west-1
+      - MINIO_DOMAIN=duckdb-minio.com
+      - MINIO_ACCESS_KEY=duckdb_minio_admin
+      - MINIO_SECRET_KEY=duckdb_minio_admin_password
+    command: server /data --console-address ":9001"
+
+  minio_setup:
+    image: minio/mc:RELEASE.2021-11-05T10-05-06Z
+    depends_on:
+      - minio
+    links:
+      - minio
+    volumes:
+      - ${PWD}/duckdb/data:/duckdb/data
+      - ${PWD}/test/test_data:/duckdb/test_data
+
+    entrypoint: >
+      /bin/sh -c "
+        until (
+          /usr/bin/mc config host add myminio http://duckdb-minio.com:9000 duckdb_minio_admin duckdb_minio_admin_password
+        ) do
+          echo '...waiting...' && sleep 1;
+        done;
+
+        /usr/bin/mc admin user add myminio minio_duckdb_user minio_duckdb_user_password
+        /usr/bin/mc admin user list myminio
+        /usr/bin/mc admin user info myminio minio_duckdb_user
+        /usr/bin/mc admin policy set myminio readwrite user=minio_duckdb_user
+
+        /usr/bin/mc admin user add myminio minio_duckdb_user_2 minio_duckdb_user_2_password
+        /usr/bin/mc admin user list myminio
+        /usr/bin/mc admin user info myminio minio_duckdb_user_2
+        /usr/bin/mc admin policy set myminio readwrite user=minio_duckdb_user_2
+
+        /usr/bin/mc rb --force myminio/test-bucket
+        /usr/bin/mc mb myminio/test-bucket
+        /usr/bin/mc policy get myminio/test-bucket
+
+        /usr/bin/mc rb --force myminio/test-bucket-2
+        /usr/bin/mc mb myminio/test-bucket-2
+        /usr/bin/mc policy get myminio/test-bucket-2
+
+        /usr/bin/mc rb --force myminio/test-bucket-public
+        /usr/bin/mc mb myminio/test-bucket-public
+        /usr/bin/mc policy set download myminio/test-bucket-public
+        /usr/bin/mc policy get myminio/test-bucket-public
+
+        # This is for the test of presigned URLs
+        # !!! When missing, be sure that you have ran 'scripts/generate_presigned_url.sh' !!!
+
+        # small file upload
+        /usr/bin/mc cp /duckdb/data/csv/phonenumbers.csv myminio/test-bucket/presigned/phonenumbers.csv
+        /usr/bin/mc cp /duckdb/data/parquet-testing/glob/t1.parquet myminio/test-bucket/presigned/t1.parquet
+
+        # large file upload
+        /usr/bin/mc cp /duckdb/test_data/presigned-url-lineitem.parquet myminio/test-bucket/presigned/lineitem_large.parquet
+
+        # Upload the db for the attach 
+        /usr/bin/mc cp /duckdb/test_data/attach.db myminio/test-bucket/presigned/attach.db
+        /usr/bin/mc cp /duckdb/test_data/lineitem_sf1.db myminio/test-bucket/presigned/lineitem_sf1.db
+
+        /usr/bin/mc share download myminio/test-bucket/presigned/phonenumbers.csv
+        /usr/bin/mc share download myminio/test-bucket/presigned/t1.parquet
+        /usr/bin/mc share download myminio/test-bucket/presigned/lineitem_large.parquet
+        /usr/bin/mc share download myminio/test-bucket/presigned/attach.db
+
+        echo 'FINISHED SETTING UP MINIO'
+        exit 0;
+      "

scripts/run_minio_aws_env

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+MINIO_CONFIG="$HOME/.minio"
+exec env \
+    "MINIO_CONFIG=$MINIO_CONFIG" \
+    "AWS_CONFIG_FILE=$MINIO_CONFIG/config" \
+    "AWS_SHARED_CREDENTIALS_FILE=$MINIO_CONFIG/credentials" \
+    "$@"

scripts/run_s3_test_server.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+#Note: DONT run as root
+
+if [ ! -f test/test_data/attach.db ]; then
+    echo "File test/test_data/attach.db not found, run ./scripts/generate_presigned_url.sh to generate"
+else
+  rm -rf /tmp/minio_test_data
+  rm -rf /tmp/minio_root_data
+  mkdir -p /tmp/minio_test_data
+  mkdir -p /tmp/minio_root_data
+  docker compose -f scripts/minio_s3.yml -p duckdb-minio up -d
+
+  # for testing presigned url
+  container_name=$(docker ps -a --format '{{.Names}}' | grep -m 1 "duckdb-minio")
+  echo $container_name
+
+  for i in $(seq 1 360);
+  do
+    docker_finish_logs=$(docker logs $container_name 2>/dev/null | grep -m 1 'FINISHED SETTING UP MINIO' || echo '')
+    if [ ! -z "${docker_finish_logs}" ]; then
+      break
+    fi
+    sleep 1
+  done
+
+
+  export S3_SMALL_CSV_PRESIGNED_URL=$(docker logs $container_name 2>/dev/null | grep -m 1 'Share:.*phonenumbers\.csv' | grep -o 'http[s]\?://[^ ]\+')
+  echo $S3_SMALL_CSV_PRESIGNED_URL
+
+  export S3_SMALL_PARQUET_PRESIGNED_URL=$(docker logs $container_name 2>/dev/null | grep -m 1 'Share:.*t1\.parquet' | grep -o 'http[s]\?://[^ ]\+')
+  echo $S3_SMALL_PARQUET_PRESIGNED_URL
+
+  export S3_LARGE_PARQUET_PRESIGNED_URL=$(docker logs $container_name 2>/dev/null | grep -m 1 'Share:.*lineitem_large\.parquet' | grep -o 'http[s]\?://[^ ]\+')
+  echo $S3_LARGE_PARQUET_PRESIGNED_URL
+
+  export S3_ATTACH_DB_PRESIGNED_URL=$(docker logs $container_name 2>/dev/null | grep -m 1 'Share:.*attach\.db' | grep -o 'http[s]\?://[^ ]\+')
+  echo $S3_ATTACH_DB_PRESIGNED_URL
+
+  export S3_ATTACH_DB="s3://test-bucket/presigned/attach.db"
+fi

scripts/set_s3_test_server_variables.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+# Run this script with 'source' or the shorthand: '.':
+# i.e: source scripts/set_s3_test_server_variables.sh
+
+# Enable the S3 tests to run
+export S3_TEST_SERVER_AVAILABLE=1
+
+export AWS_DEFAULT_REGION=eu-west-1
+export AWS_ACCESS_KEY_ID=minio_duckdb_user
+export AWS_SECRET_ACCESS_KEY=minio_duckdb_user_password
+export DUCKDB_S3_ENDPOINT=duckdb-minio.com:9000
+export DUCKDB_S3_USE_SSL=false