extensions: ship beyond_auth + beyond_queue + refresh pgdg pins

extensions.toml

@@ -26,31 +26,44 @@ pg_trgm = "1.6"
 # `apt install postgresql-18-<name>=<version>` is exact; build fails on miss.
 # ---------------------------------------------------------------------------
 [pgdg]
-pgvector = "0.8.0"
-pgvectorscale = "0.4.0"
-postgis = "3.5.0"
-pg_cron = "1.6.4"
-pg_partman = "5.1.0"
-pg_jsonschema = "0.3.3"
-hypopg = "1.4.1"
-pg_repack = "1.5.2"
+# Refreshed to versions currently resolvable for noble + pg18 (K-003 drift fix,
+# 2026-06; verified via `apt-cache madison postgresql-18-<name>` against
+# apt.postgresql.org noble-pgdg). pg_cron + pg_partman are non-negotiable
+# (pg_cron: REQUIRED + shared_preload; pg_partman: queue partitioned queues).
+pgvector = "0.8.2"
+postgis = "3.6.4"
+pg_cron = "1.6.7"
+pg_partman = "5.4.3"
+hypopg = "1.4.2"
+pg_repack = "1.5.3"
+# Dropped — no PGDG build for noble+pg18 as of 2026-06 (re-add a pin once PGDG
+# ships one): pgvectorscale (no postgresql-18-pgvectorscale), pg_jsonschema (no
+# postgresql-18-pg-jsonschema). They are OPTIONAL_EXTENSIONS in the beyond-pg
+# supervisor, so absence is a boot warning, not a failure.
 
 # ---------------------------------------------------------------------------
-# ParadeDB apt
+# ParadeDB apt — DISABLED. The packagecloud repo is now paywalled (HTTP 402),
+# and ParadeDB ships pg18 via GitHub Releases. pg_search can be re-added later
+# via the GitHub-release install path (same mechanism as [beyond.*]).
 # ---------------------------------------------------------------------------
-[paradedb]
-pg_search = "0.15.0"
+# [paradedb]
+# pg_search = "..."
 
 # ---------------------------------------------------------------------------
-# Beyond sibling repos — cloned from public git and built in-container (J-004a).
-# Set `tag` to a release tag (e.g. "v0.1.0") or use `sha` for a commit pin.
-# Build fails if the tag/SHA is absent in the remote.
-# TODO: confirm public git URLs with auth/queue repo owners before first build.
+# Beyond sibling repos — cloned from public git at `tag` and built from source
+# (J-004a). build-beyond-extensions.sh clones `git` @ `tag` into a tmpdir, builds
+# the pgrx extension for the target PG/arch, and stages a canonical tarball.
+# `tag` may be any buildable git ref (branch, tag, or sha).
+#
+# Dev/homelab override: to build from a local checkout (e.g. a branch not yet
+# pushed), export AUTH_EXT_GIT / AUTH_EXT_REF (and QUEUE_EXT_GIT / QUEUE_EXT_REF)
+# before the build — the script prefers those over the pins below. This keeps the
+# canonical pins here clean and the build clone-based (no hardcoded local paths).
 # ---------------------------------------------------------------------------
 [beyond.auth]
 git = "https://github.com/beyondoss/auth"
-tag = "server/v0.1.0" # placeholder — bump when first release is cut
+tag = "main" # bump to a release tag (server/vX.Y.Z) once cut
 
 [beyond.queue]
 git = "https://github.com/beyondoss/queue"
-tag = "ext-v0.1.0" # placeholder — bump when first release is cut
+tag = "main" # bump to a release tag (ext-vX.Y.Z) once cut

mise.toml

@@ -101,14 +101,12 @@ AUTH_TAG=$(_toml "['beyond']['auth']['tag']")
 QUEUE_GIT=$(_toml "['beyond']['queue']['git']")
 QUEUE_TAG=$(_toml "['beyond']['queue']['tag']")
 PGVECTOR_VER=$(_toml "['pgdg']['pgvector']")
-PGVECTORSCALE_VER=$(_toml "['pgdg']['pgvectorscale']")
 POSTGIS_VER=$(_toml "['pgdg']['postgis']")
 PG_CRON_VER=$(_toml "['pgdg']['pg_cron']")
 PG_PARTMAN_VER=$(_toml "['pgdg']['pg_partman']")
-PG_JSONSCHEMA_VER=$(_toml "['pgdg']['pg_jsonschema']")
 HYPOPG_VER=$(_toml "['pgdg']['hypopg']")
 PG_REPACK_VER=$(_toml "['pgdg']['pg_repack']")
-PG_SEARCH_VER=$(_toml "['paradedb']['pg_search']")
+# pgvectorscale / pg_jsonschema / pg_search dropped for noble+pg18 (see extensions.toml)
 
 # Cross-compile musl static binaries for the target arch.
 case "${ARCH}" in
@@ -139,14 +137,11 @@ packer build \
   -var "queue_ext_git=${QUEUE_GIT}" \
   -var "queue_ext_tag=${QUEUE_TAG}" \
   -var "pgvector_version=${PGVECTOR_VER}" \
-  -var "pgvectorscale_version=${PGVECTORSCALE_VER}" \
   -var "postgis_version=${POSTGIS_VER}" \
   -var "pg_cron_version=${PG_CRON_VER}" \
   -var "pg_partman_version=${PG_PARTMAN_VER}" \
-  -var "pg_jsonschema_version=${PG_JSONSCHEMA_VER}" \
   -var "hypopg_version=${HYPOPG_VER}" \
   -var "pg_repack_version=${PG_REPACK_VER}" \
-  -var "pg_search_version=${PG_SEARCH_VER}" \
   packer/templates/postgres-rootfs.pkr.hcl
 
 if [[ "${bless:-}" == "true" ]]; then

packer/scripts/03-pgdg-extensions.sh

@@ -31,25 +31,16 @@ pin_install() {
 echo "==> Installing PGDG extensions..."
 
 pin_install "postgresql-${POSTGRES_VERSION}-pgvector"      "${PGVECTOR_VERSION}"
-pin_install "postgresql-${POSTGRES_VERSION}-pgvectorscale" "${PGVECTORSCALE_VERSION}"
 pin_install "postgresql-${POSTGRES_VERSION}-postgis-3"     "${POSTGIS_VERSION}"
 pin_install "postgresql-${POSTGRES_VERSION}-cron"          "${PG_CRON_VERSION}"
 pin_install "postgresql-${POSTGRES_VERSION}-partman"       "${PG_PARTMAN_VERSION}"
-pin_install "postgresql-${POSTGRES_VERSION}-pg-jsonschema" "${PG_JSONSCHEMA_VERSION}"
 pin_install "postgresql-${POSTGRES_VERSION}-hypopg"        "${HYPOPG_VERSION}"
 pin_install "postgresql-${POSTGRES_VERSION}-repack"        "${PG_REPACK_VERSION}"
 
-echo "==> Adding ParadeDB apt repository (pg_search)..."
-# ParadeDB publishes pg_search to packagecloud.
-# TODO: verify this URL against ParadeDB's current repo path before shipping.
-# Fallback option: download .deb directly from their GitHub releases.
-curl -fsSL https://packagecloud.io/paradedb/paradedb/gpgkey \
-  | gpg --dearmor -o /etc/apt/trusted.gpg.d/paradedb.gpg
-echo "deb [signed-by=/etc/apt/trusted.gpg.d/paradedb.gpg] \
-https://packagecloud.io/paradedb/paradedb/ubuntu/ ${UBUNTU_VERSION} main" \
-  > /etc/apt/sources.list.d/paradedb.list
-apt-get update -qq
-
-pin_install "postgresql-${POSTGRES_VERSION}-pg-search" "${PG_SEARCH_VERSION}"
+# Dropped (no PGDG build for noble+pg18 as of 2026-06; see extensions.toml):
+#   pgvectorscale, pg_jsonschema
+# ParadeDB pg_search: packagecloud repo is now paywalled (HTTP 402); ParadeDB
+# ships pg18 via GitHub Releases. Re-add via a GitHub-release install (like
+# 04-beyond-extensions.sh) once needed.
 
 echo "==> 03-pgdg-extensions done"

packer/scripts/04-beyond-extensions.sh

@@ -1,16 +1,31 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-# Download a Beyond sibling extension tarball from its GitHub Release and
-# extract it directly to /. The tarball carries a full directory tree:
+# Install the Beyond sibling extensions (beyond_auth, beyond_queue) into the
+# rootfs. Two sources, in priority order:
+#
+#   1. LOCAL tarball (AUTH_EXT_TARBALL / QUEUE_EXT_TARBALL) — a tree built on the
+#      host by build-beyond-extensions.sh (clone-from-git + cargo). This is the
+#      homelab/dev path: it works without a cut GitHub Release and without the
+#      Rust toolchain inside this chroot.
+#   2. GitHub RELEASE tarball (AUTH_EXT_GIT/AUTH_EXT_TAG, QUEUE_EXT_GIT/QUEUE_EXT_TAG)
+#      — the production path. Asset name: {asset_prefix}-v{version}-pg{PG}-linux-{arch}.tar.gz
+#      (version = tag after the last 'v': "server/v0.1.0" -> "0.1.0").
+#
+# Either way the tarball carries a tree rooted at usr/, extracted to /:
 #   usr/lib/postgresql/{PG}/lib/{name}.so
 #   usr/share/postgresql/{PG}/extension/{name}.control
 #   usr/share/postgresql/{PG}/extension/{name}--{version}.sql
-#
-# Asset name: {asset_prefix}-v{version}-pg{PG}-linux-{arch}.tar.gz
-# Version is derived by stripping everything up to and including the last 'v'
-# in the tag: "server/v0.1.0" -> "0.1.0", "ext-v0.1.0" -> "0.1.0".
-install_beyond_ext() {
+
+install_local() {
+  local asset_prefix="$1" tarball="$2"
+  echo "==> Installing ${asset_prefix} from local tarball ${tarball}..."
+  [[ -f "${tarball}" ]] || { echo "ERROR: local tarball ${tarball} not found"; exit 1; }
+  tar -xzf "${tarball}" -C /
+  echo "==> Installed ${asset_prefix} (local)"
+}
+
+install_release() {
   local asset_prefix="$1" git_url="$2" tag="$3"
   local version="${tag##*v}"
   local tarball="${asset_prefix}-v${version}-pg${POSTGRES_VERSION}-linux-${TARGET_ARCH}.tar.gz"
@@ -25,9 +40,19 @@ install_beyond_ext() {
   echo "==> Installed ${asset_prefix} ${tag}"
 }
 
+# $1 asset_prefix, $2 local_tarball_var_value, $3 git, $4 tag
+install_beyond_ext() {
+  local asset_prefix="$1" local_tarball="$2" git_url="$3" tag="$4"
+  if [[ -n "${local_tarball}" ]]; then
+    install_local "${asset_prefix}" "${local_tarball}"
+  else
+    install_release "${asset_prefix}" "${git_url}" "${tag}"
+  fi
+}
+
 echo "==> Installing Beyond sibling extensions..."
 
-install_beyond_ext "beyond-auth-extension"  "${AUTH_EXT_GIT}"  "${AUTH_EXT_TAG}"
-install_beyond_ext "beyond-queue-extension" "${QUEUE_EXT_GIT}" "${QUEUE_EXT_TAG}"
+install_beyond_ext "beyond-auth-extension"  "${AUTH_EXT_TARBALL:-}"  "${AUTH_EXT_GIT:-}"  "${AUTH_EXT_TAG:-}"
+install_beyond_ext "beyond-queue-extension" "${QUEUE_EXT_TARBALL:-}" "${QUEUE_EXT_GIT:-}" "${QUEUE_EXT_TAG:-}"
 
 echo "==> 04-beyond-extensions done"

packer/scripts/build-beyond-extensions.sh

@@ -0,0 +1,113 @@
+#!/usr/bin/env bash
+# Host-side builder for the Beyond sibling Postgres extensions (beyond_auth,
+# beyond_queue). Runs OUTSIDE any chroot/container — it needs the Rust + pgrx
+# toolchain, which the minimal rootfs deliberately does not carry.
+#
+# For each extension it:
+#   1. clones `git` @ `tag` from extensions.toml into a tmpdir (J-004a) — never a
+#      hardcoded local path. A dev/homelab override (AUTH_EXT_GIT/AUTH_EXT_REF,
+#      QUEUE_EXT_GIT/QUEUE_EXT_REF) lets you build a not-yet-pushed branch or a
+#      local checkout without touching the canonical pins.
+#   2. builds the pgrx extension for $POSTGRES_VERSION using the host pg_config,
+#      emitting the CANONICAL install names beyond_auth / beyond_queue.
+#   3. stages a tarball (rooted at `usr/...`) into $EXT_STAGE — the exact tree
+#      04-beyond-extensions.sh extracts to / inside the rootfs.
+#
+# Output tarballs (consumed by 04 via AUTH_EXT_TARBALL / QUEUE_EXT_TARBALL):
+#   $EXT_STAGE/beyond-auth-extension.tar.gz
+#   $EXT_STAGE/beyond-queue-extension.tar.gz
+#
+# Env:
+#   EXT_STAGE          (required) dir to write the staged tarballs into
+#   POSTGRES_VERSION   (default 18)
+#   TARGET_ARCH        (default amd64) — informational; build is for the host arch
+#   PG_CONFIG          (default /usr/lib/postgresql/$POSTGRES_VERSION/bin/pg_config)
+#   EXTENSIONS_TOML    (default <repo>/extensions.toml)
+#   AUTH_EXT_GIT/AUTH_EXT_REF, QUEUE_EXT_GIT/QUEUE_EXT_REF  (dev overrides)
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PG_REPO="$(cd "$SCRIPT_DIR/../.." && pwd)"
+POSTGRES_VERSION="${POSTGRES_VERSION:-18}"
+TARGET_ARCH="${TARGET_ARCH:-amd64}"
+PG_CONFIG="${PG_CONFIG:-/usr/lib/postgresql/${POSTGRES_VERSION}/bin/pg_config}"
+EXTENSIONS_TOML="${EXTENSIONS_TOML:-$PG_REPO/extensions.toml}"
+
+info() { echo "==> [build-beyond-extensions] $*"; }
+fail() { echo "  ✗ $*" >&2; exit 1; }
+
+[[ -n "${EXT_STAGE:-}" ]] || fail "EXT_STAGE must be set (dir to write staged tarballs)"
+mkdir -p "$EXT_STAGE"
+[[ -x "$PG_CONFIG" ]] || fail "pg_config not found/executable at $PG_CONFIG (install postgresql-server-dev-${POSTGRES_VERSION})"
+command -v cargo >/dev/null || fail "cargo not found — install the Rust toolchain"
+command -v git >/dev/null || fail "git not found"
+
+# Read git/tag from extensions.toml, honoring dev env overrides.
+toml() { python3 -c "import tomllib,sys; d=tomllib.load(open('$EXTENSIONS_TOML','rb')); print(d$1)"; }
+AUTH_GIT="${AUTH_EXT_GIT:-$(toml "['beyond']['auth']['git']")}"
+AUTH_REF="${AUTH_EXT_REF:-$(toml "['beyond']['auth']['tag']")}"
+QUEUE_GIT="${QUEUE_EXT_GIT:-$(toml "['beyond']['queue']['git']")}"
+QUEUE_REF="${QUEUE_EXT_REF:-$(toml "['beyond']['queue']['tag']")}"
+
+WORK="$(mktemp -d -t beyond-ext-build.XXXXXX)"
+trap 'rm -rf "$WORK"' EXIT
+
+# Clone `git` @ `ref` (branch | tag | sha; works for remote URLs and local paths).
+clone_ref() { local git="$1" ref="$2" dst="$3"
+  info "cloning $git @ $ref"
+  git clone --quiet "$git" "$dst" || fail "git clone $git failed"
+  git -C "$dst" checkout --quiet "$ref" || fail "git checkout $ref failed in $git"
+}
+
+# default_version from a .control file (authoritative for the --<ver>.sql name).
+control_version() { grep -E '^default_version' "$1" | sed -E "s/.*=\s*'([^']+)'.*/\1/"; }
+
+# ---- beyond_auth (pgrx package: emits the full usr/ tree directly) ----------
+build_auth() {
+  local src="$WORK/auth"
+  clone_ref "$AUTH_GIT" "$AUTH_REF" "$src"
+  command -v cargo-pgrx >/dev/null || fail "cargo-pgrx not found — cargo install cargo-pgrx --version =0.18.0 --locked"
+  info "cargo pgrx package -p beyond-auth-extension (pg${POSTGRES_VERSION})"
+  ( cd "$src" && PGRX_PG_CONFIG_PATH="$PG_CONFIG" cargo pgrx package \
+      --pg-config "$PG_CONFIG" --no-default-features --features "pg${POSTGRES_VERSION}" \
+      -p beyond-auth-extension >/dev/null )
+  # pgrx names the package dir after the lib (beyond_auth), not the cargo package.
+  local pkgdir="$src/target/release/beyond_auth-pg${POSTGRES_VERSION}"
+  [[ -f "$pkgdir/usr/lib/postgresql/${POSTGRES_VERSION}/lib/beyond_auth.so" ]] \
+    || fail "beyond_auth.so missing in $pkgdir — did lib.name=beyond_auth land?"
+  tar -czf "$EXT_STAGE/beyond-auth-extension.tar.gz" -C "$pkgdir" .
+  info "staged $EXT_STAGE/beyond-auth-extension.tar.gz"
+}
+
+# ---- beyond_queue (plain cargo build + manual assemble, mirrors release CI) --
+build_queue() {
+  local src="$WORK/queue"
+  clone_ref "$QUEUE_GIT" "$QUEUE_REF" "$src"
+  info "cargo build -p beyond-queue-extension (pg${POSTGRES_VERSION})"
+  ( cd "$src" && PGRX_PG_CONFIG_PATH="$PG_CONFIG" cargo build --release \
+      --no-default-features --features "pg${POSTGRES_VERSION}" \
+      -p beyond-queue-extension >/dev/null )
+  local so="$src/target/release/libbeyond_queue.so"
+  [[ -f "$so" ]] || fail "libbeyond_queue.so missing — did lib.name=beyond_queue land?"
+  local ctl="$src/beyond-queue-extension/beyond_queue.control"
+  [[ -f "$ctl" ]] || fail "beyond_queue.control missing in clone"
+  local ver; ver="$(control_version "$ctl")"
+  local dist="$WORK/queue-dist"
+  mkdir -p "$dist/usr/lib/postgresql/${POSTGRES_VERSION}/lib" \
+           "$dist/usr/share/postgresql/${POSTGRES_VERSION}/extension"
+  cp "$so" "$dist/usr/lib/postgresql/${POSTGRES_VERSION}/lib/beyond_queue.so"
+  cp "$ctl" "$dist/usr/share/postgresql/${POSTGRES_VERSION}/extension/beyond_queue.control"
+  cat "$src/beyond-queue-extension/sql/schema.sql" \
+      "$src/tests/fixtures/hot_paths.sql" \
+      "$src/tests/fixtures/load_pgrx_extension.sql" \
+      > "$dist/usr/share/postgresql/${POSTGRES_VERSION}/extension/beyond_queue--${ver}.sql"
+  tar -czf "$EXT_STAGE/beyond-queue-extension.tar.gz" -C "$dist" .
+  info "staged $EXT_STAGE/beyond-queue-extension.tar.gz (v${ver})"
+}
+
+info "PG=${POSTGRES_VERSION} arch=${TARGET_ARCH} pg_config=${PG_CONFIG}"
+info "auth: ${AUTH_GIT} @ ${AUTH_REF}"
+info "queue: ${QUEUE_GIT} @ ${QUEUE_REF}"
+build_auth
+build_queue
+info "done — staged tarballs in $EXT_STAGE"