From 3722f1e328965ed0fbf12931c523643b86699f67 Mon Sep 17 00:00:00 2001 From: Jared Lunde Date: Sat, 20 Jun 2026 10:16:26 -0700 Subject: [PATCH] extensions: ship beyond_auth + beyond_queue + refresh pgdg pins (K-003) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Enable the auth/queue extension milestone and unstick the pgdg suite for noble+pg18: - extensions.toml: refresh [pgdg] to versions that resolve today (pg_cron 1.6.7, pg_partman 5.4.3, pgvector 0.8.2, postgis 3.6.4, hypopg 1.4.2, pg_repack 1.5.3). Drop pgvectorscale + pg_jsonschema (no pg18 PGDG build) and the ParadeDB pg_search apt path (packagecloud now paywalled, 402) — all OPTIONAL in the supervisor. [beyond.*] keep beyondoss URLs; tags -> main; document dev override. - build-beyond-extensions.sh (new, host-side): clone [beyond.*] git@tag into a tmpdir and build the pgrx extensions, emitting canonical beyond_auth / beyond_queue tarballs. AUTH_EXT_GIT/REF + QUEUE_EXT_GIT/REF override for dev. - 04-beyond-extensions.sh: install from a local staged tarball (AUTH_EXT_TARBALL/QUEUE_EXT_TARBALL) when set, else the GitHub release. - 03-pgdg-extensions.sh: drop the unavailable installs + the ParadeDB block. - mise.toml build:image: stop parsing the dropped pins. Co-Authored-By: Claude Opus 4.8 (1M context) --- extensions.toml | 47 +++++---- mise.toml | 7 +- packer/scripts/03-pgdg-extensions.sh | 19 +--- packer/scripts/04-beyond-extensions.sh | 43 ++++++-- packer/scripts/build-beyond-extensions.sh | 113 ++++++++++++++++++++++ 5 files changed, 183 insertions(+), 46 deletions(-) create mode 100755 packer/scripts/build-beyond-extensions.sh diff --git a/extensions.toml b/extensions.toml index 1f06d5c..603bf0f 100644 --- a/extensions.toml +++ b/extensions.toml @@ -26,31 +26,44 @@ pg_trgm = "1.6" # `apt install postgresql-18-=` is exact; build fails on miss. # --------------------------------------------------------------------------- [pgdg] -pgvector = "0.8.0" -pgvectorscale = "0.4.0" -postgis = "3.5.0" -pg_cron = "1.6.4" -pg_partman = "5.1.0" -pg_jsonschema = "0.3.3" -hypopg = "1.4.1" -pg_repack = "1.5.2" +# Refreshed to versions currently resolvable for noble + pg18 (K-003 drift fix, +# 2026-06; verified via `apt-cache madison postgresql-18-` against +# apt.postgresql.org noble-pgdg). pg_cron + pg_partman are non-negotiable +# (pg_cron: REQUIRED + shared_preload; pg_partman: queue partitioned queues). +pgvector = "0.8.2" +postgis = "3.6.4" +pg_cron = "1.6.7" +pg_partman = "5.4.3" +hypopg = "1.4.2" +pg_repack = "1.5.3" +# Dropped — no PGDG build for noble+pg18 as of 2026-06 (re-add a pin once PGDG +# ships one): pgvectorscale (no postgresql-18-pgvectorscale), pg_jsonschema (no +# postgresql-18-pg-jsonschema). They are OPTIONAL_EXTENSIONS in the beyond-pg +# supervisor, so absence is a boot warning, not a failure. # --------------------------------------------------------------------------- -# ParadeDB apt +# ParadeDB apt — DISABLED. The packagecloud repo is now paywalled (HTTP 402), +# and ParadeDB ships pg18 via GitHub Releases. pg_search can be re-added later +# via the GitHub-release install path (same mechanism as [beyond.*]). # --------------------------------------------------------------------------- -[paradedb] -pg_search = "0.15.0" +# [paradedb] +# pg_search = "..." # --------------------------------------------------------------------------- -# Beyond sibling repos — cloned from public git and built in-container (J-004a). -# Set `tag` to a release tag (e.g. "v0.1.0") or use `sha` for a commit pin. -# Build fails if the tag/SHA is absent in the remote. -# TODO: confirm public git URLs with auth/queue repo owners before first build. +# Beyond sibling repos — cloned from public git at `tag` and built from source +# (J-004a). build-beyond-extensions.sh clones `git` @ `tag` into a tmpdir, builds +# the pgrx extension for the target PG/arch, and stages a canonical tarball. +# `tag` may be any buildable git ref (branch, tag, or sha). +# +# Dev/homelab override: to build from a local checkout (e.g. a branch not yet +# pushed), export AUTH_EXT_GIT / AUTH_EXT_REF (and QUEUE_EXT_GIT / QUEUE_EXT_REF) +# before the build — the script prefers those over the pins below. This keeps the +# canonical pins here clean and the build clone-based (no hardcoded local paths). # --------------------------------------------------------------------------- [beyond.auth] git = "https://github.com/beyondoss/auth" -tag = "server/v0.1.0" # placeholder — bump when first release is cut +tag = "main" # bump to a release tag (server/vX.Y.Z) once cut [beyond.queue] git = "https://github.com/beyondoss/queue" -tag = "ext-v0.1.0" # placeholder — bump when first release is cut +tag = "main" # bump to a release tag (ext-vX.Y.Z) once cut diff --git a/mise.toml b/mise.toml index 5a794f7..ace694f 100644 --- a/mise.toml +++ b/mise.toml @@ -101,14 +101,12 @@ AUTH_TAG=$(_toml "['beyond']['auth']['tag']") QUEUE_GIT=$(_toml "['beyond']['queue']['git']") QUEUE_TAG=$(_toml "['beyond']['queue']['tag']") PGVECTOR_VER=$(_toml "['pgdg']['pgvector']") -PGVECTORSCALE_VER=$(_toml "['pgdg']['pgvectorscale']") POSTGIS_VER=$(_toml "['pgdg']['postgis']") PG_CRON_VER=$(_toml "['pgdg']['pg_cron']") PG_PARTMAN_VER=$(_toml "['pgdg']['pg_partman']") -PG_JSONSCHEMA_VER=$(_toml "['pgdg']['pg_jsonschema']") HYPOPG_VER=$(_toml "['pgdg']['hypopg']") PG_REPACK_VER=$(_toml "['pgdg']['pg_repack']") -PG_SEARCH_VER=$(_toml "['paradedb']['pg_search']") +# pgvectorscale / pg_jsonschema / pg_search dropped for noble+pg18 (see extensions.toml) # Cross-compile musl static binaries for the target arch. case "${ARCH}" in @@ -139,14 +137,11 @@ packer build \ -var "queue_ext_git=${QUEUE_GIT}" \ -var "queue_ext_tag=${QUEUE_TAG}" \ -var "pgvector_version=${PGVECTOR_VER}" \ - -var "pgvectorscale_version=${PGVECTORSCALE_VER}" \ -var "postgis_version=${POSTGIS_VER}" \ -var "pg_cron_version=${PG_CRON_VER}" \ -var "pg_partman_version=${PG_PARTMAN_VER}" \ - -var "pg_jsonschema_version=${PG_JSONSCHEMA_VER}" \ -var "hypopg_version=${HYPOPG_VER}" \ -var "pg_repack_version=${PG_REPACK_VER}" \ - -var "pg_search_version=${PG_SEARCH_VER}" \ packer/templates/postgres-rootfs.pkr.hcl if [[ "${bless:-}" == "true" ]]; then diff --git a/packer/scripts/03-pgdg-extensions.sh b/packer/scripts/03-pgdg-extensions.sh index c68d48c..e2d09fd 100644 --- a/packer/scripts/03-pgdg-extensions.sh +++ b/packer/scripts/03-pgdg-extensions.sh @@ -31,25 +31,16 @@ pin_install() { echo "==> Installing PGDG extensions..." pin_install "postgresql-${POSTGRES_VERSION}-pgvector" "${PGVECTOR_VERSION}" -pin_install "postgresql-${POSTGRES_VERSION}-pgvectorscale" "${PGVECTORSCALE_VERSION}" pin_install "postgresql-${POSTGRES_VERSION}-postgis-3" "${POSTGIS_VERSION}" pin_install "postgresql-${POSTGRES_VERSION}-cron" "${PG_CRON_VERSION}" pin_install "postgresql-${POSTGRES_VERSION}-partman" "${PG_PARTMAN_VERSION}" -pin_install "postgresql-${POSTGRES_VERSION}-pg-jsonschema" "${PG_JSONSCHEMA_VERSION}" pin_install "postgresql-${POSTGRES_VERSION}-hypopg" "${HYPOPG_VERSION}" pin_install "postgresql-${POSTGRES_VERSION}-repack" "${PG_REPACK_VERSION}" -echo "==> Adding ParadeDB apt repository (pg_search)..." -# ParadeDB publishes pg_search to packagecloud. -# TODO: verify this URL against ParadeDB's current repo path before shipping. -# Fallback option: download .deb directly from their GitHub releases. -curl -fsSL https://packagecloud.io/paradedb/paradedb/gpgkey \ - | gpg --dearmor -o /etc/apt/trusted.gpg.d/paradedb.gpg -echo "deb [signed-by=/etc/apt/trusted.gpg.d/paradedb.gpg] \ -https://packagecloud.io/paradedb/paradedb/ubuntu/ ${UBUNTU_VERSION} main" \ - > /etc/apt/sources.list.d/paradedb.list -apt-get update -qq - -pin_install "postgresql-${POSTGRES_VERSION}-pg-search" "${PG_SEARCH_VERSION}" +# Dropped (no PGDG build for noble+pg18 as of 2026-06; see extensions.toml): +# pgvectorscale, pg_jsonschema +# ParadeDB pg_search: packagecloud repo is now paywalled (HTTP 402); ParadeDB +# ships pg18 via GitHub Releases. Re-add via a GitHub-release install (like +# 04-beyond-extensions.sh) once needed. echo "==> 03-pgdg-extensions done" diff --git a/packer/scripts/04-beyond-extensions.sh b/packer/scripts/04-beyond-extensions.sh index 743846a..e45c8a6 100644 --- a/packer/scripts/04-beyond-extensions.sh +++ b/packer/scripts/04-beyond-extensions.sh @@ -1,16 +1,31 @@ #!/usr/bin/env bash set -euo pipefail -# Download a Beyond sibling extension tarball from its GitHub Release and -# extract it directly to /. The tarball carries a full directory tree: +# Install the Beyond sibling extensions (beyond_auth, beyond_queue) into the +# rootfs. Two sources, in priority order: +# +# 1. LOCAL tarball (AUTH_EXT_TARBALL / QUEUE_EXT_TARBALL) — a tree built on the +# host by build-beyond-extensions.sh (clone-from-git + cargo). This is the +# homelab/dev path: it works without a cut GitHub Release and without the +# Rust toolchain inside this chroot. +# 2. GitHub RELEASE tarball (AUTH_EXT_GIT/AUTH_EXT_TAG, QUEUE_EXT_GIT/QUEUE_EXT_TAG) +# — the production path. Asset name: {asset_prefix}-v{version}-pg{PG}-linux-{arch}.tar.gz +# (version = tag after the last 'v': "server/v0.1.0" -> "0.1.0"). +# +# Either way the tarball carries a tree rooted at usr/, extracted to /: # usr/lib/postgresql/{PG}/lib/{name}.so # usr/share/postgresql/{PG}/extension/{name}.control # usr/share/postgresql/{PG}/extension/{name}--{version}.sql -# -# Asset name: {asset_prefix}-v{version}-pg{PG}-linux-{arch}.tar.gz -# Version is derived by stripping everything up to and including the last 'v' -# in the tag: "server/v0.1.0" -> "0.1.0", "ext-v0.1.0" -> "0.1.0". -install_beyond_ext() { + +install_local() { + local asset_prefix="$1" tarball="$2" + echo "==> Installing ${asset_prefix} from local tarball ${tarball}..." + [[ -f "${tarball}" ]] || { echo "ERROR: local tarball ${tarball} not found"; exit 1; } + tar -xzf "${tarball}" -C / + echo "==> Installed ${asset_prefix} (local)" +} + +install_release() { local asset_prefix="$1" git_url="$2" tag="$3" local version="${tag##*v}" local tarball="${asset_prefix}-v${version}-pg${POSTGRES_VERSION}-linux-${TARGET_ARCH}.tar.gz" @@ -25,9 +40,19 @@ install_beyond_ext() { echo "==> Installed ${asset_prefix} ${tag}" } +# $1 asset_prefix, $2 local_tarball_var_value, $3 git, $4 tag +install_beyond_ext() { + local asset_prefix="$1" local_tarball="$2" git_url="$3" tag="$4" + if [[ -n "${local_tarball}" ]]; then + install_local "${asset_prefix}" "${local_tarball}" + else + install_release "${asset_prefix}" "${git_url}" "${tag}" + fi +} + echo "==> Installing Beyond sibling extensions..." -install_beyond_ext "beyond-auth-extension" "${AUTH_EXT_GIT}" "${AUTH_EXT_TAG}" -install_beyond_ext "beyond-queue-extension" "${QUEUE_EXT_GIT}" "${QUEUE_EXT_TAG}" +install_beyond_ext "beyond-auth-extension" "${AUTH_EXT_TARBALL:-}" "${AUTH_EXT_GIT:-}" "${AUTH_EXT_TAG:-}" +install_beyond_ext "beyond-queue-extension" "${QUEUE_EXT_TARBALL:-}" "${QUEUE_EXT_GIT:-}" "${QUEUE_EXT_TAG:-}" echo "==> 04-beyond-extensions done" diff --git a/packer/scripts/build-beyond-extensions.sh b/packer/scripts/build-beyond-extensions.sh new file mode 100755 index 0000000..a416297 --- /dev/null +++ b/packer/scripts/build-beyond-extensions.sh @@ -0,0 +1,113 @@ +#!/usr/bin/env bash +# Host-side builder for the Beyond sibling Postgres extensions (beyond_auth, +# beyond_queue). Runs OUTSIDE any chroot/container — it needs the Rust + pgrx +# toolchain, which the minimal rootfs deliberately does not carry. +# +# For each extension it: +# 1. clones `git` @ `tag` from extensions.toml into a tmpdir (J-004a) — never a +# hardcoded local path. A dev/homelab override (AUTH_EXT_GIT/AUTH_EXT_REF, +# QUEUE_EXT_GIT/QUEUE_EXT_REF) lets you build a not-yet-pushed branch or a +# local checkout without touching the canonical pins. +# 2. builds the pgrx extension for $POSTGRES_VERSION using the host pg_config, +# emitting the CANONICAL install names beyond_auth / beyond_queue. +# 3. stages a tarball (rooted at `usr/...`) into $EXT_STAGE — the exact tree +# 04-beyond-extensions.sh extracts to / inside the rootfs. +# +# Output tarballs (consumed by 04 via AUTH_EXT_TARBALL / QUEUE_EXT_TARBALL): +# $EXT_STAGE/beyond-auth-extension.tar.gz +# $EXT_STAGE/beyond-queue-extension.tar.gz +# +# Env: +# EXT_STAGE (required) dir to write the staged tarballs into +# POSTGRES_VERSION (default 18) +# TARGET_ARCH (default amd64) — informational; build is for the host arch +# PG_CONFIG (default /usr/lib/postgresql/$POSTGRES_VERSION/bin/pg_config) +# EXTENSIONS_TOML (default /extensions.toml) +# AUTH_EXT_GIT/AUTH_EXT_REF, QUEUE_EXT_GIT/QUEUE_EXT_REF (dev overrides) +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PG_REPO="$(cd "$SCRIPT_DIR/../.." && pwd)" +POSTGRES_VERSION="${POSTGRES_VERSION:-18}" +TARGET_ARCH="${TARGET_ARCH:-amd64}" +PG_CONFIG="${PG_CONFIG:-/usr/lib/postgresql/${POSTGRES_VERSION}/bin/pg_config}" +EXTENSIONS_TOML="${EXTENSIONS_TOML:-$PG_REPO/extensions.toml}" + +info() { echo "==> [build-beyond-extensions] $*"; } +fail() { echo " ✗ $*" >&2; exit 1; } + +[[ -n "${EXT_STAGE:-}" ]] || fail "EXT_STAGE must be set (dir to write staged tarballs)" +mkdir -p "$EXT_STAGE" +[[ -x "$PG_CONFIG" ]] || fail "pg_config not found/executable at $PG_CONFIG (install postgresql-server-dev-${POSTGRES_VERSION})" +command -v cargo >/dev/null || fail "cargo not found — install the Rust toolchain" +command -v git >/dev/null || fail "git not found" + +# Read git/tag from extensions.toml, honoring dev env overrides. +toml() { python3 -c "import tomllib,sys; d=tomllib.load(open('$EXTENSIONS_TOML','rb')); print(d$1)"; } +AUTH_GIT="${AUTH_EXT_GIT:-$(toml "['beyond']['auth']['git']")}" +AUTH_REF="${AUTH_EXT_REF:-$(toml "['beyond']['auth']['tag']")}" +QUEUE_GIT="${QUEUE_EXT_GIT:-$(toml "['beyond']['queue']['git']")}" +QUEUE_REF="${QUEUE_EXT_REF:-$(toml "['beyond']['queue']['tag']")}" + +WORK="$(mktemp -d -t beyond-ext-build.XXXXXX)" +trap 'rm -rf "$WORK"' EXIT + +# Clone `git` @ `ref` (branch | tag | sha; works for remote URLs and local paths). +clone_ref() { local git="$1" ref="$2" dst="$3" + info "cloning $git @ $ref" + git clone --quiet "$git" "$dst" || fail "git clone $git failed" + git -C "$dst" checkout --quiet "$ref" || fail "git checkout $ref failed in $git" +} + +# default_version from a .control file (authoritative for the --.sql name). +control_version() { grep -E '^default_version' "$1" | sed -E "s/.*=\s*'([^']+)'.*/\1/"; } + +# ---- beyond_auth (pgrx package: emits the full usr/ tree directly) ---------- +build_auth() { + local src="$WORK/auth" + clone_ref "$AUTH_GIT" "$AUTH_REF" "$src" + command -v cargo-pgrx >/dev/null || fail "cargo-pgrx not found — cargo install cargo-pgrx --version =0.18.0 --locked" + info "cargo pgrx package -p beyond-auth-extension (pg${POSTGRES_VERSION})" + ( cd "$src" && PGRX_PG_CONFIG_PATH="$PG_CONFIG" cargo pgrx package \ + --pg-config "$PG_CONFIG" --no-default-features --features "pg${POSTGRES_VERSION}" \ + -p beyond-auth-extension >/dev/null ) + # pgrx names the package dir after the lib (beyond_auth), not the cargo package. + local pkgdir="$src/target/release/beyond_auth-pg${POSTGRES_VERSION}" + [[ -f "$pkgdir/usr/lib/postgresql/${POSTGRES_VERSION}/lib/beyond_auth.so" ]] \ + || fail "beyond_auth.so missing in $pkgdir — did lib.name=beyond_auth land?" + tar -czf "$EXT_STAGE/beyond-auth-extension.tar.gz" -C "$pkgdir" . + info "staged $EXT_STAGE/beyond-auth-extension.tar.gz" +} + +# ---- beyond_queue (plain cargo build + manual assemble, mirrors release CI) -- +build_queue() { + local src="$WORK/queue" + clone_ref "$QUEUE_GIT" "$QUEUE_REF" "$src" + info "cargo build -p beyond-queue-extension (pg${POSTGRES_VERSION})" + ( cd "$src" && PGRX_PG_CONFIG_PATH="$PG_CONFIG" cargo build --release \ + --no-default-features --features "pg${POSTGRES_VERSION}" \ + -p beyond-queue-extension >/dev/null ) + local so="$src/target/release/libbeyond_queue.so" + [[ -f "$so" ]] || fail "libbeyond_queue.so missing — did lib.name=beyond_queue land?" + local ctl="$src/beyond-queue-extension/beyond_queue.control" + [[ -f "$ctl" ]] || fail "beyond_queue.control missing in clone" + local ver; ver="$(control_version "$ctl")" + local dist="$WORK/queue-dist" + mkdir -p "$dist/usr/lib/postgresql/${POSTGRES_VERSION}/lib" \ + "$dist/usr/share/postgresql/${POSTGRES_VERSION}/extension" + cp "$so" "$dist/usr/lib/postgresql/${POSTGRES_VERSION}/lib/beyond_queue.so" + cp "$ctl" "$dist/usr/share/postgresql/${POSTGRES_VERSION}/extension/beyond_queue.control" + cat "$src/beyond-queue-extension/sql/schema.sql" \ + "$src/tests/fixtures/hot_paths.sql" \ + "$src/tests/fixtures/load_pgrx_extension.sql" \ + > "$dist/usr/share/postgresql/${POSTGRES_VERSION}/extension/beyond_queue--${ver}.sql" + tar -czf "$EXT_STAGE/beyond-queue-extension.tar.gz" -C "$dist" . + info "staged $EXT_STAGE/beyond-queue-extension.tar.gz (v${ver})" +} + +info "PG=${POSTGRES_VERSION} arch=${TARGET_ARCH} pg_config=${PG_CONFIG}" +info "auth: ${AUTH_GIT} @ ${AUTH_REF}" +info "queue: ${QUEUE_GIT} @ ${QUEUE_REF}" +build_auth +build_queue +info "done — staged tarballs in $EXT_STAGE"