From e166658a834bf34adb33ec926ab7bb4e562f1e95 Mon Sep 17 00:00:00 2001
From: Richard Wall <richard.wall@cyberark.com>
Date: Wed, 8 Jul 2026 20:42:21 +0000
Subject: [PATCH] Document how to manually rerun periodic Prow jobs

Signed-off-by: Richard Wall <richard.wall@venafi.com>
Signed-off-by: Richard Wall <richard.wall@cyberark.com>
---
 Makefile              | 14 +++++++++
 prow/README.md        | 44 +++++++++++++++++++++++++++
 prow/periodic-jobs.sh | 71 +++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 129 insertions(+)
 create mode 100755 prow/periodic-jobs.sh

diff --git a/Makefile b/Makefile
index c34f483c5..2c99b1c04 100644
--- a/Makefile
+++ b/Makefile
@@ -45,6 +45,20 @@ verify: verify-boilerplate verify-prowgen
 test:
 	cd ./config/prowgen/ && go test ./...
 
+.PHONY: list-periodics
+list-periodics:
+ifndef BRANCH
+	$(error BRANCH is required, e.g. make list-periodics BRANCH=release-1.21)
+endif
+	@$(CURDIR)/prow/periodic-jobs.sh list $(BRANCH)
+
+.PHONY: rerun-failed-periodics
+rerun-failed-periodics:
+ifndef BRANCH
+	$(error BRANCH is required, e.g. make rerun-failed-periodics BRANCH=release-1.21)
+endif
+	@$(CURDIR)/prow/periodic-jobs.sh rerun $(BRANCH)
+
 # Run checkconfig locally to verify the Prow configuration, CI runs this
 # directly in the Prow cluster.
 local-checkconfig:
diff --git a/prow/README.md b/prow/README.md
index 651f7997a..0b9a886f7 100644
--- a/prow/README.md
+++ b/prow/README.md
@@ -62,3 +62,47 @@ make deploy-prow
 - Ensure you can access `https://prow.infra.cert-manager.io/` (and see logs for the tests there) and `https://triage.infra.cert-manager.io/s/daily`
 
 11. Commit and PR in your change
+
+## Rerunning periodic jobs
+
+Periodic jobs run on a cron schedule and cannot be retriggered with `/retest`.
+Sometimes you need to rerun them manually — for example, shortly before a
+release after merging last-minute dependency updates (Go version bumps, trivy
+ignore list changes), when you want to see the periodic trivy scans go green
+for reassurance before proceeding with the release.
+
+### Prerequisites
+
+You need `roles/container.developer` on the `cert-manager-tests-trusted` GCP
+project (see the
+[cluster definitions in cert-manager/infrastructure](https://github.com/cert-manager/infrastructure/blob/7b45ed95c68919c1d3cb14b8ff35fa5de46275be/gcp/clusters.tf#L3-L24))
+and kubectl access to the `prow-trusted` cluster:
+
+```sh
+gcloud container clusters get-credentials \
+  prow-trusted \
+  --zone europe-west1-b \
+  --project cert-manager-tests-trusted
+```
+
+### Listing periodic job status
+
+List the latest run of each periodic job matching a search term:
+
+```sh
+make list-periodics BRANCH=release-1.21
+```
+
+### Rerunning all failed periodic jobs
+
+Rerun every non-success periodic job for a release branch:
+
+```sh
+make rerun-failed-periodics BRANCH=release-1.21
+```
+
+This creates new ProwJob instances; any already-running instances are not
+interrupted. Results will appear on the
+[Prow dashboard](https://prow.infra.cert-manager.io/) immediately and on
+[testgrid](https://testgrid.k8s.io/) after its next sync (typically within an
+hour).
diff --git a/prow/periodic-jobs.sh b/prow/periodic-jobs.sh
new file mode 100755
index 000000000..3547fb66d
--- /dev/null
+++ b/prow/periodic-jobs.sh
@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+
+# Copyright 2026 The cert-manager Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -euo pipefail
+
+PROW_URL="https://prow.infra.cert-manager.io"
+PROW_JOBS_URL="${PROW_URL}/prowjobs.js?omit=annotations,labels,decoration_config,pod_spec"
+
+usage() {
+    echo "Usage: $0 <list|rerun> <search-term>"
+    echo
+    echo "Commands:"
+    echo "  list   List the latest run of each periodic job matching <search-term>"
+    echo "  rerun  Rerun all non-success periodic jobs matching <search-term>"
+    echo
+    echo "Examples:"
+    echo "  $0 list release-1.21"
+    echo "  $0 rerun release-1.21"
+    exit 1
+}
+
+if [[ $# -ne 2 ]]; then
+    usage
+fi
+
+command="$1"
+search_term="$2"
+
+fetch_latest_periodics() {
+    curl -s "${PROW_JOBS_URL}" \
+        | jq -r --arg q "${search_term}" '
+            [.items[] | select(.spec.type == "periodic" and (.spec.job | contains($q)))]
+            | group_by(.spec.job)
+            | map(sort_by(.status.startTime) | last)
+        '
+}
+
+case "${command}" in
+    list)
+        fetch_latest_periodics | jq -r '
+            sort_by(.spec.job)[]
+            | "\(.spec.job): \(.status.state) (started: \(.status.startTime), id: \(.metadata.name))"
+        '
+        ;;
+    rerun)
+        fetch_latest_periodics | jq -r '
+            map(select(.status.state != "success"))[]
+            | "\(.spec.job)\t\(.metadata.name)"
+        ' \
+        | while IFS=$'\t' read -r job id; do
+            echo "Rerunning: ${job}"
+            kubectl create -f "${PROW_URL}/rerun?mode=original&prowjob=${id}"
+        done
+        ;;
+    *)
+        usage
+        ;;
+esac