Use R2 bucket for duckdb libraries

.github/workflows/ci.yml

@@ -25,6 +25,25 @@ env:
   NIGHTLY_TOOLCHAIN: nightly-2026-02-05
 
 jobs:
+  duckdb-mirror:
+    name: "Mirror DuckDB to R2"
+    if: github.event_name == 'pull_request'
+    uses: ./.github/workflows/duckdb-r2.yml
+    secrets: inherit
+
+  duckdb-ready:
+    name: "DuckDB libraries available in R2"
+    needs: duckdb-mirror
+    if: ${{ !cancelled() }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - name: Verify DuckDB mirror
+        if: ${{ needs.duckdb-mirror.result == 'failure' }}
+        run: |
+          echo "DuckDB mirror failed; downstream builds would 404"
+          exit 1
+
   lint-toml:
     runs-on: ubuntu-latest
     timeout-minutes: 10
@@ -115,6 +134,7 @@ jobs:
 
   rust-docs:
     name: "Rust (docs)"
+    needs: duckdb-ready
     timeout-minutes: 30
     runs-on: >-
       ${{ github.repository == 'vortex-data/vortex'
@@ -204,6 +224,7 @@ jobs:
 
   rust-lint:
     name: "Rust (lint)"
+    needs: duckdb-ready
     timeout-minutes: 30
     runs-on: >-
       ${{ github.repository == 'vortex-data/vortex'
@@ -301,6 +322,7 @@ jobs:
 
   rust-test-other:
     name: "Rust tests (${{ matrix.os }})"
+    needs: duckdb-ready
     timeout-minutes: 30
     strategy:
       fail-fast: false
@@ -422,6 +444,7 @@ jobs:
 
   sqllogic-test:
     name: "SQL logic tests"
+    needs: duckdb-ready
     runs-on: >-
       ${{ github.repository == 'vortex-data/vortex'
           && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=sql-logic-test', github.run_id)

.github/workflows/duckdb-r2.yml

@@ -0,0 +1,192 @@
+name: DuckDB R2 mirror
+
+# Mirror DuckDB libraries referenced by vortex-duckdb/build.rs to R2 when they
+# are not present yet. Download tagged archives or build commits from source.
+on:
+  workflow_call: { }
+
+concurrency:
+  group: duckdb-r2-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: false
+
+permissions:
+  contents: read
+
+env:
+  PUBLIC_BASE_URL: "https://ci-builds.vortex.dev"
+  R2_BUCKET: "duckdb-builds"
+  R2_ENDPOINT_URL: "https://52bdeab5651e1584747feefd051fd566.r2.cloudflarestorage.com"
+
+jobs:
+  check:
+    name: "Resolve DuckDB version and check R2"
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    outputs:
+      version: ${{ steps.resolve.outputs.version }}
+      ref_dir: ${{ steps.resolve.outputs.ref_dir }}
+      release: ${{ steps.resolve.outputs.release }}
+      matrix: ${{ steps.resolve.outputs.matrix }}
+      any_missing: ${{ steps.resolve.outputs.any_missing }}
+    steps:
+      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10  # v6
+      - name: Resolve version and check R2
+        id: resolve
+        run: |
+          set -Eeuo pipefail
+
+          version=$(grep -oP 'DEFAULT_DUCKDB_VERSION:\s*&str\s*=\s*"\K[^"]+' \
+            vortex-duckdb/build.rs)
+
+          # Same as in vortex-duckdb/build.rs: >=2 dot-separated numeric
+          # components is a tagged release (ref dir "vX.Y.Z"), anything
+          # else is a commit.
+          ref="${version#v}"
+          if [[ "$ref" =~ ^[0-9]+(\.[0-9]+)+$ ]]; then
+            release=true
+            ref_dir="v$ref"
+          else
+            release=false
+            ref_dir="$ref"
+          fi
+
+          echo "DuckDB $version release=$release"
+
+          entries=()
+          for archive in \
+            libduckdb-linux-amd64.zip \
+            libduckdb-linux-arm64.zip \
+            libduckdb-osx-universal.zip; do
+
+            url="${PUBLIC_BASE_URL}/${ref_dir}/${archive}"
+            code=$(curl -o /dev/null -s -w '%{http_code}' --head "$url" || echo 000)
+            if [ "$code" = "200" ]; then
+              echo "present in R2: $archive"
+              continue
+            fi
+
+            echo "missing in R2 (HTTP $code): $archive"
+            case "$archive" in
+              *linux-amd64*) runner="ubuntu-latest"; os="linux"; arch="amd64" ;;
+              *linux-arm64*) runner="ubuntu-24.04-arm"; os="linux"; arch="arm64" ;;
+              *osx-universal*) runner="macos-14"; os="osx"; arch="universal" ;;
+            esac
+            entries+=("$(jq -nc \
+              --arg archive "$archive" \
+              --arg runner "$runner" \
+              --arg os "$os" \
+              --arg arch "$arch" \
+              '{archive: $archive, runner: $runner, os: $os, arch: $arch}')")
+          done
+
+          if [ "${#entries[@]}" -eq 0 ]; then
+            matrix='{"include":[]}'
+            any_missing=false
+          else
+            include=$(printf '%s\n' "${entries[@]}" | jq -sc '.')
+            matrix=$(jq -nc --argjson include "$include" '{include: $include}')
+            any_missing=true
+          fi
+
+          echo "any_missing=$any_missing"
+
+          {
+            echo "version=$version"
+            echo "ref_dir=$ref_dir"
+            echo "release=$release"
+            echo "matrix=$matrix"
+            echo "any_missing=$any_missing"
+          } >> "$GITHUB_OUTPUT"
+
+  mirror:
+    name: "Mirror DuckDB ${{ matrix.archive }} to R2"
+    needs: check
+    if: >-
+      needs.check.outputs.any_missing == 'true' &&
+      github.repository == 'vortex-data/vortex' &&
+      github.event.pull_request.head.repo.full_name == github.repository
+    environment: duckdb-build
+    timeout-minutes: 120
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(needs.check.outputs.matrix) }}
+    runs-on: ${{ matrix.runner }}
+    steps:
+      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10  # v6
+
+      - name: Install build dependencies (Linux)
+        if: needs.check.outputs.release != 'true' && runner.os == 'Linux'
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y ninja-build libcurl4-openssl-dev zip unzip
+
+      - name: Install build dependencies (macOS)
+        if: needs.check.outputs.release != 'true' && runner.os == 'macOS'
+        run: brew install ninja
+
+      - name: Prepare ${{ matrix.archive }}
+        env:
+          ARCHIVE: ${{ matrix.archive }}
+          REF_DIR: ${{ needs.check.outputs.ref_dir }}
+          RELEASE: ${{ needs.check.outputs.release }}
+          PLATFORM_OS: ${{ matrix.os }}
+        run: |
+          set -Eeuo pipefail
+
+          if [ "$RELEASE" = "true" ]; then
+            echo "Mirroring DuckDB release ${REF_DIR}/${ARCHIVE}"
+            curl -fSL --retry 3 -o "$ARCHIVE" \
+              "https://github.com/duckdb/duckdb/releases/download/${REF_DIR}/${ARCHIVE}"
+          else
+            echo "Building DuckDB commit ${REF_DIR} from source"
+
+            curl -fSL --retry 3 -o duckdb-src.zip \
+              "https://github.com/duckdb/duckdb/archive/${REF_DIR}.zip"
+            unzip -q duckdb-src.zip
+
+            src_dir="duckdb-${REF_DIR}"
+            extra=""
+            if [ "$PLATFORM_OS" = "osx" ]; then
+              extra="OSX_BUILD_UNIVERSAL=1"
+            fi
+
+            # Same as in build.rs
+            make -C "$src_dir" \
+              GEN=ninja \
+              DISABLE_SANITIZER=1 \
+              THREADSAN=0 \
+              BUILD_SHELL=false \
+              BUILD_UNITTESTS=false \
+              ENABLE_UNITTEST_CPP_TESTS=false \
+              BUILD_EXTENSIONS="parquet;jemalloc;httpfs;tpch;tpcds" \
+              $extra
+
+            lib_dir="${src_dir}/build/release/src"
+            stage="stage"
+            rm -rf "$stage"
+            mkdir -p "$stage"
+
+            cp -a "${lib_dir}/libduckdb.so" "$stage/" 2>/dev/null || true
+            cp -a "${lib_dir}/libduckdb.dylib" "$stage/" 2>/dev/null || true
+            cp -a "${lib_dir}/libduckdb_static.a" "$stage/"
+            cp -a "${src_dir}/src/include/duckdb.h" "$stage/" 2>/dev/null || true
+            cp -a "${src_dir}/src/include/duckdb.hpp" "$stage/" 2>/dev/null || true
+
+            ( cd "$stage" && zip -r "../${ARCHIVE}" . )
+          fi
+
+          ls -la "$ARCHIVE"
+
+      - name: Upload to R2
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.DUCKDB_R2_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.DUCKDB_R2_SECRET_ACCESS_KEY }}
+          AWS_REGION: "us-east-1"
+          AWS_ENDPOINT_URL: ${{ env.R2_ENDPOINT_URL }}
+        run: |
+          set -Eeuo pipefail
+          python3 scripts/s3-upload.py \
+            --bucket "$R2_BUCKET" \
+            --key "${{ needs.check.outputs.ref_dir }}/${{ matrix.archive }}" \
+            --body "${{ matrix.archive }}" \
+            --checksum-algorithm CRC32

.github/workflows/rust-instrumented.yml

@@ -22,8 +22,28 @@ env:
   NIGHTLY_TOOLCHAIN: nightly-2026-02-05
 
 jobs:
+  duckdb-mirror:
+    name: "Mirror DuckDB to R2"
+    if: github.event_name == 'pull_request'
+    uses: ./.github/workflows/duckdb-r2.yml
+    secrets: inherit
+
+  duckdb-ready:
+    name: "DuckDB libraries available in R2"
+    needs: duckdb-mirror
+    if: ${{ !cancelled() }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - name: Verify DuckDB mirror
+        if: ${{ needs.duckdb-mirror.result == 'failure' }}
+        run: |
+          echo "DuckDB mirror failed"
+          exit 1
+
   rust-coverage:
     name: "Rust tests (coverage) (${{ matrix.suite }})"
+    needs: duckdb-ready
     timeout-minutes: 30
     permissions:
       id-token: write

vortex-duckdb/build.rs

@@ -17,7 +17,10 @@
 use bindgen::Abi;
 use bindgen::callbacks::ParseCallbacks;
 
-const DUCKDB_RELEASES_URL: &str = "https://github.com/duckdb/duckdb/releases/download";
+// You can subsitute this URL for https://github.com/duckdb/duckdb/releases/download
+// We want own infrastructure for testing pre-release builds
+const DUCKDB_RELEASES_URL: &str = "https://ci-builds.vortex.dev";
+
 const DUCKDB_SOURCE_RELEASE_URL: &str = "https://github.com/duckdb/duckdb/archive/refs/tags";
 const DUCKDB_SOURCE_COMMIT_URL: &str = "https://github.com/duckdb/duckdb/archive";
 const DEFAULT_DUCKDB_VERSION: &str = "1.5.3";