Automate production deployment after releases

.github/scripts/deploy-release.sh

@@ -0,0 +1,155 @@
+#!/usr/bin/env bash
+
+set -Eeuo pipefail
+
+release_tag=${1:?release tag is required}
+deploy_dir=${2:?deployment directory is required}
+source_dir=${3:?source directory is required}
+pip_index_url=${4:-https://mirrors.aliyun.com/pypi/simple/}
+pip_trusted_host=${5:-}
+
+case "$deploy_dir" in
+    /*) ;;
+    *) deploy_dir="$PWD/$deploy_dir" ;;
+esac
+case "$source_dir" in
+    /*) ;;
+    *) source_dir="$PWD/$source_dir" ;;
+esac
+
+if ! [[ "$release_tag" =~ ^v[0-9]+\.[0-9]+\.[0-9]+([-.][0-9A-Za-z.]+)?$ ]]; then
+    echo "Invalid release tag: $release_tag" >&2
+    exit 1
+fi
+
+cd "$source_dir"
+git rev-parse --is-inside-work-tree >/dev/null
+
+if [ -n "$(git status --porcelain --untracked-files=no)" ]; then
+    echo "Source contains tracked local changes; refusing to overwrite production." >&2
+    git status --short --untracked-files=no >&2
+    exit 1
+fi
+
+if docker version >/dev/null 2>&1; then
+    docker_cmd=(docker)
+elif sudo -n docker version >/dev/null 2>&1; then
+    docker_cmd=(sudo -n docker)
+else
+    echo "Docker is unavailable to the deployment user." >&2
+    exit 1
+fi
+compose=("${docker_cmd[@]}" compose)
+app_services=(backend-api backend-worker frontend)
+
+cd "$deploy_dir"
+test -f .env
+"${compose[@]}" config --quiet
+
+backend_container=$("${compose[@]}" ps -q backend-api)
+worker_container=$("${compose[@]}" ps -q backend-worker)
+frontend_container=$("${compose[@]}" ps -q frontend)
+test -n "$backend_container"
+test -n "$worker_container"
+test -n "$frontend_container"
+
+old_backend_image=$("${docker_cmd[@]}" inspect -f '{{.Image}}' "$backend_container")
+old_worker_image=$("${docker_cmd[@]}" inspect -f '{{.Image}}' "$worker_container")
+old_frontend_image=$("${docker_cmd[@]}" inspect -f '{{.Image}}' "$frontend_container")
+if [ "$old_backend_image" != "$old_worker_image" ]; then
+    echo "Backend API and worker do not use the same image; refusing deployment." >&2
+    exit 1
+fi
+
+cd "$source_dir"
+previous_commit=$(git rev-parse HEAD)
+activation_started=false
+
+rollback() {
+    status=$?
+    trap - EXIT
+    if [ "$status" -ne 0 ]; then
+        echo "Deployment failed; restoring source $previous_commit." >&2
+        set +e
+        cd "$source_dir"
+        git checkout --detach "$previous_commit"
+        if [ "$activation_started" = true ]; then
+            "${docker_cmd[@]}" tag "$old_backend_image" clawith-backend:latest
+            "${docker_cmd[@]}" tag "$old_frontend_image" clawith-frontend:latest
+            cd "$deploy_dir"
+            "${compose[@]}" up -d --no-deps --force-recreate "${app_services[@]}"
+        fi
+    fi
+    exit "$status"
+}
+trap rollback EXIT
+
+git fetch origin "refs/heads/main:refs/remotes/origin/main"
+git fetch origin "refs/tags/$release_tag:refs/tags/$release_tag"
+
+release_commit=$(git rev-parse "$release_tag^{commit}")
+if ! git merge-base --is-ancestor "$release_commit" origin/main; then
+    echo "Release $release_tag is not contained in origin/main." >&2
+    exit 1
+fi
+
+git checkout --detach "$release_tag"
+
+backend_release_image="clawith-backend:$release_tag"
+frontend_release_image="clawith-frontend:$release_tag"
+
+"${docker_cmd[@]}" build \
+    --build-arg "CLAWITH_PIP_INDEX_URL=$pip_index_url" \
+    --build-arg "CLAWITH_PIP_TRUSTED_HOST=$pip_trusted_host" \
+    -t "$backend_release_image" \
+    backend
+"${docker_cmd[@]}" build -t "$frontend_release_image" frontend
+
+activation_started=true
+"${docker_cmd[@]}" tag "$backend_release_image" clawith-backend:latest
+"${docker_cmd[@]}" tag "$frontend_release_image" clawith-frontend:latest
+
+cd "$deploy_dir"
+"${compose[@]}" up -d --no-deps --force-recreate "${app_services[@]}"
+
+expected_backend_id=$("${docker_cmd[@]}" image inspect -f '{{.Id}}' "$backend_release_image")
+expected_frontend_id=$("${docker_cmd[@]}" image inspect -f '{{.Id}}' "$frontend_release_image")
+running_api_id=$("${docker_cmd[@]}" inspect -f '{{.Image}}' "$("${compose[@]}" ps -q backend-api)")
+running_worker_id=$("${docker_cmd[@]}" inspect -f '{{.Image}}' "$("${compose[@]}" ps -q backend-worker)")
+running_frontend_id=$("${docker_cmd[@]}" inspect -f '{{.Image}}' "$("${compose[@]}" ps -q frontend)")
+
+if [ "$running_api_id" != "$expected_backend_id" ] ||
+   [ "$running_worker_id" != "$expected_backend_id" ] ||
+   [ "$running_frontend_id" != "$expected_frontend_id" ]; then
+    echo "Running containers do not use the newly built release images." >&2
+    exit 1
+fi
+
+published_address=$("${compose[@]}" port frontend 3000 | tail -n 1)
+frontend_port=${published_address##*:}
+if ! [[ "$frontend_port" =~ ^[0-9]+$ ]]; then
+    echo "Unable to resolve the published frontend port." >&2
+    exit 1
+fi
+
+expected_version=${release_tag#v}
+health_response=
+for attempt in $(seq 1 24); do
+    if health_response=$(curl -fsS --max-time 5 "http://127.0.0.1:$frontend_port/api/health"); then
+        health_status=$(printf '%s' "$health_response" | sed -n 's/.*"status"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p')
+        running_version=$(printf '%s' "$health_response" | sed -n 's/.*"version"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p')
+        if [ "$health_status" = ok ] && [ "$running_version" = "$expected_version" ]; then
+            echo "Successfully deployed $release_tag ($release_commit)."
+            "${compose[@]}" ps
+            activation_started=false
+            exit 0
+        fi
+    fi
+    sleep 5
+done
+
+echo "Production health check failed after 120 seconds." >&2
+echo "Last health response: ${health_response:-<none>}" >&2
+"${compose[@]}" ps >&2
+"${compose[@]}" logs --tail 200 backend frontend >&2
+exit 1

.github/workflows/release.yml

@@ -443,6 +443,8 @@ jobs:
     name: Publish release
     runs-on: ubuntu-latest
     if: github.event_name == 'pull_request' && github.event.pull_request.merged == true && startsWith(github.event.pull_request.head.ref, 'release/v')
+    outputs:
+      tag: ${{ steps.release_info.outputs.tag }}
     steps:
       - name: Checkout source
         uses: actions/checkout@v7
@@ -497,3 +499,77 @@ jobs:
           body_path: release-notes.extracted.md
           prerelease: ${{ github.event.pull_request.draft }}
           generate_release_notes: false
+
+  deploy_release:
+    name: Deploy release to production
+    needs: publish_release
+    if: ${{ needs.publish_release.result == 'success' }}
+    runs-on: ubuntu-latest
+    environment: production
+    env:
+      DEPLOY_HOST: ${{ vars.CLAWITH_DEPLOY_HOST }}
+      DEPLOY_USER: ${{ vars.CLAWITH_DEPLOY_USER }}
+      DEPLOY_PORT: ${{ vars.CLAWITH_DEPLOY_PORT || '10022' }}
+      DEPLOY_PATH: ${{ vars.CLAWITH_DEPLOY_PATH || 'clawith_new' }}
+      SOURCE_PATH: ${{ vars.CLAWITH_SOURCE_PATH || 'Clawith' }}
+      PIP_INDEX_URL: ${{ vars.CLAWITH_PIP_INDEX_URL || 'https://mirrors.aliyun.com/pypi/simple/' }}
+      PIP_TRUSTED_HOST: ${{ vars.CLAWITH_PIP_TRUSTED_HOST }}
+      RELEASE_TAG: ${{ needs.publish_release.outputs.tag }}
+      SSH_PRIVATE_KEY: ${{ secrets.CLAWITH_DEPLOY_SSH_KEY }}
+      SSH_KNOWN_HOSTS: ${{ secrets.CLAWITH_DEPLOY_KNOWN_HOSTS }}
+    steps:
+      - name: Checkout deployment tooling
+        uses: actions/checkout@v7
+
+      - name: Validate deployment configuration
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          required=(
+            DEPLOY_HOST DEPLOY_USER DEPLOY_PORT DEPLOY_PATH SOURCE_PATH RELEASE_TAG
+            SSH_PRIVATE_KEY SSH_KNOWN_HOSTS
+          )
+          for name in "${required[@]}"; do
+            if [ -z "${!name}" ]; then
+              echo "::error::Missing required production deployment setting: $name"
+              exit 1
+            fi
+          done
+
+          if ! [[ "$DEPLOY_PORT" =~ ^[0-9]+$ ]] || [ "$DEPLOY_PORT" -lt 1 ] || [ "$DEPLOY_PORT" -gt 65535 ]; then
+            echo "::error::CLAWITH_DEPLOY_PORT must be a valid TCP port"
+            exit 1
+          fi
+
+      - name: Configure SSH
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          install -d -m 0700 "$HOME/.ssh"
+          printf '%s\n' "$SSH_KNOWN_HOSTS" > "$HOME/.ssh/known_hosts"
+          chmod 0600 "$HOME/.ssh/known_hosts"
+
+          key_file="$RUNNER_TEMP/clawith-deploy-key"
+          printf '%s\n' "$SSH_PRIVATE_KEY" > "$key_file"
+          chmod 0600 "$key_file"
+          echo "SSH_KEY_FILE=$key_file" >> "$GITHUB_ENV"
+
+      - name: Update, build, and restart production
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          printf -v remote_command 'bash -s -- %q %q %q %q %q' \
+            "$RELEASE_TAG" "$DEPLOY_PATH" "$SOURCE_PATH" \
+            "$PIP_INDEX_URL" "$PIP_TRUSTED_HOST"
+          ssh \
+            -i "$SSH_KEY_FILE" \
+            -p "$DEPLOY_PORT" \
+            -o BatchMode=yes \
+            -o IdentitiesOnly=yes \
+            -o StrictHostKeyChecking=yes \
+            "$DEPLOY_USER@$DEPLOY_HOST" \
+            "$remote_command" \
+            < .github/scripts/deploy-release.sh

deploy/RELEASE_DEPLOYMENT.md

@@ -0,0 +1,81 @@
+# Production release deployment
+
+Merging an automated `release/vX.Y.Z` pull request publishes the GitHub Release
+and then deploys that exact tag to production. GitHub Actions does not build or
+upload application artifacts. The production server fetches the tag in its
+source checkout, builds versioned backend and frontend images, points the
+deployment's `latest` image tags to them, and restarts the existing Compose
+project. Success requires the API and worker containers to use the newly built
+backend image, the frontend container to use the newly built frontend image,
+and `/api/health` to report both `status: ok` and the released version.
+
+The deployment keeps the server's ignored `.env`, `ss-nodes.json`,
+`backend/agent_data`, and Docker volumes in place. It refuses to overwrite
+tracked local changes. If the build, restart, or health check fails after the
+checkout, it checks out the previous commit and rebuilds the previous version.
+
+## GitHub production configuration
+
+Configure these under **Settings > Environments > production**. Environment
+protection rules can require approval before the deployment job starts.
+
+### Variables
+
+| Name | Current production value |
+| --- | --- |
+| `CLAWITH_DEPLOY_HOST` | `82.156.53.84` |
+| `CLAWITH_DEPLOY_USER` | `qinrui` |
+| `CLAWITH_DEPLOY_PORT` | `10022` |
+| `CLAWITH_DEPLOY_PATH` | `clawith_new` |
+| `CLAWITH_SOURCE_PATH` | `Clawith` |
+| `CLAWITH_PIP_INDEX_URL` | `https://mirrors.aliyun.com/pypi/simple/` (optional default) |
+| `CLAWITH_PIP_TRUSTED_HOST` | Optional; only for a mirror that requires pip trusted-host handling |
+
+### Secrets
+
+| Name | Value |
+| --- | --- |
+| `CLAWITH_DEPLOY_SSH_KEY` | Private key for a dedicated deployment identity |
+| `CLAWITH_DEPLOY_KNOWN_HOSTS` | Verified OpenSSH `known_hosts` entry for the production server |
+
+The SSH key must be authorized for the deployment user and should be dedicated
+to GitHub Actions. Verify the server fingerprint through a trusted channel
+before storing the `known_hosts` entry; do not blindly trust an `ssh-keyscan`
+result.
+
+## Server prerequisites
+
+The source directory must be a Git checkout whose `origin` points to this
+repository. The deployment directory keeps the production `.env`, Compose
+file, Nginx configuration, and other environment-specific files. The server
+also needs:
+
+- Docker with the Compose plugin
+- `curl`
+- read access to the repository
+- permission for the deployment user to run Docker directly or with
+  passwordless `sudo`
+- an existing `.env` in `clawith_new`
+
+Validate the server once before enabling automatic deployments:
+
+```bash
+ssh clawith
+cd Clawith
+git status --short
+git remote -v
+cd ../clawith_new
+test -f .env
+docker compose config --quiet
+docker compose ps
+curl -fsS http://127.0.0.1:3008/api/health
+```
+
+The PyPI mirror defaults to Aliyun and can be overridden with the two optional
+production Environment variables above. Docker's existing image-layer cache
+continues to skip dependency installation when `backend/pyproject.toml` has not
+changed.
+
+The server checkout is intentionally left at a detached release tag after a
+successful deployment. Never move or reuse a published tag; publish a new
+version to roll forward.