diff --git a/.config/.copier-answers.yml b/.config/.copier-answers.yml index 35a14f57..99511a81 100644 --- a/.config/.copier-answers.yml +++ b/.config/.copier-answers.yml @@ -15,7 +15,7 @@ ssh_port_number: 55874 template_might_want_to_install_aws_ssm_port_forwarding_plugin: true template_might_want_to_use_python_asyncio: true template_might_want_to_use_vcrpy: true -template_publishes_releases: false +template_publishes_releases: true template_uses_pulumi: false template_uses_python: true template_uses_typescript: false diff --git a/.config/.copier-managed-files.json b/.config/.copier-managed-files.json index 617f5980..a050fb85 100644 --- a/.config/.copier-managed-files.json +++ b/.config/.copier-managed-files.json @@ -183,6 +183,7 @@ "template/.github/actions/update-devcontainer-hash/action.yml", "template/.github/pull_request_template.md", "template/.github/workflows/confirm-on-tagged-copier-template.yaml", + "template/.github/workflows/extract_project_version.py", "template/.github/workflows/get-values.yaml", "template/.github/workflows/git_tag.py", "template/.github/workflows/hash_git_files.py", diff --git a/copier_template_resources/git_tag.py b/copier_template_resources/git_tag.py deleted file mode 100644 index 3bbede74..00000000 --- a/copier_template_resources/git_tag.py +++ /dev/null @@ -1,102 +0,0 @@ -# ============== WARNING ============================================================================== -# File is managed by copier template: gh:LabAutomationAndScreening/copier-base-template.git -# See .config/.copier-managed-files.json for details. -# -# You are welcome to make changes to this file in your repo if they are custom to your project, -# but if the change should be shared with other projects, please backport it to the template repo. -# ===================================================================================================== -import argparse -import subprocess -import tomllib -from pathlib import Path - - -def extract_version(toml_path: Path | str) -> str: - """Load toml_path and return the version string. - - Checks [project].version (PEP 621) first, then [tool.poetry].version. Raises KeyError if no version field is found. - """ - path = Path(toml_path) - with path.open("rb") as f: - data = tomllib.load(f) - - project = data.get("project", {}) - if version := project.get("version"): - return version - - tool = data.get("tool", {}) - poetry = tool.get("poetry", {}) - if version := poetry.get("version"): - return version - - raise KeyError(f"No version field found in {path!r}") - - -def ensure_tag_not_present(tag: str, remote: str) -> None: - try: - _ = subprocess.run( # noqa: S603 # this is trusted input, it's our own arguments being passed in - ["git", "ls-remote", "--exit-code", "--tags", remote, f"refs/tags/{tag}"], # noqa: S607 # if `git` isn't in PATH already, then there are bigger problems to solve - check=True, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - ) - raise Exception(f"Error: tag '{tag}' exists on remote '{remote}'") # noqa: TRY002 # not worth a custom exception - except subprocess.CalledProcessError: - # tag not present, continue - return - - -def main(): - parser = argparse.ArgumentParser( - description=( - "Extract the version from a pyproject.toml file, " - "confirm that git tag v is not present, or " - "create and push the tag to a remote." - ) - ) - _ = parser.add_argument( - "file", - nargs="?", - default="pyproject.toml", - help="Path to pyproject.toml (default: pyproject.toml)", - ) - _ = parser.add_argument( - "--confirm-tag-not-present", - action="store_true", - help=("Check that git tag v is NOT present on the remote. If the tag exists, exit with an error."), - ) - _ = parser.add_argument( - "--push-tag-to-remote", - action="store_true", - help=( - "Create git tag v locally and push it to the remote. " - "Internally confirms the tag is not already present." - ), - ) - _ = parser.add_argument( - "--remote", - default="origin", - help="Name of git remote to query/push (default: origin)", - ) - args = parser.parse_args() - - ver = extract_version(args.file) - - tag = f"v{ver}" - - if args.push_tag_to_remote: - ensure_tag_not_present(tag, args.remote) - _ = subprocess.run(["git", "tag", tag], check=True) # noqa: S603,S607 # this is trusted input, it's our own pyproject.toml file. and if `git` isn't in PATH, then there are larger problems anyway - _ = subprocess.run(["git", "push", args.remote, tag], check=True) # noqa: S603,S607 # this is trusted input, it's our own pyproject.toml file. and if `git` isn't in PATH, then there are larger problems anyway - return - - if args.confirm_tag_not_present: - ensure_tag_not_present(tag, args.remote) - return - - # Default behavior: just print the version - print(ver) # noqa: T201 # specifically printing this out so CI pipelines can read the value from stdout - - -if __name__ == "__main__": - main() diff --git a/template/.github/workflows/ci.yaml.jinja b/template/.github/workflows/ci.yaml.jinja index 431add41..77ad06c4 100644 --- a/template/.github/workflows/ci.yaml.jinja +++ b/template/.github/workflows/ci.yaml.jinja @@ -107,6 +107,50 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} slug: {% endraw %}{{ full_repo_url | replace("https://github.com/", "") }}{% raw %}{% endraw %}{% endif %}{% raw %} +{% endraw %}{% if not deploy_as_executable %}{% raw %} + build: + needs: + - test + - check-skip-duplicate + if: needs.check-skip-duplicate.outputs.should-run == 'true' + timeout-minutes: {% endraw %}{{ gha_medium_timeout_minutes }}{% raw %} + runs-on: {% endraw %}{{ gha_linux_runner }}{% raw %} + + steps: + - name: Checkout code + uses: actions/checkout@{% endraw %}{{ gha_checkout }}{% raw %} + + - name: Install python tooling + uses: ./.github/actions/install_deps + with: + skip-installing-ssm-plugin-manager: true + skip-installing-pulumi-cli: true + python-version: {% endraw %}{{ python_version }}{% if python_package_registry == "AWS CodeArtifact" %}{% raw %} + code-artifact-auth-role-name: CoreInfraBaseAccess + code-artifact-auth-role-account-id: {% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %} + code-artifact-auth-region: {% endraw %}{{ aws_org_home_region }}{% endif %}{% raw %} + +{% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}{% raw %} + - name: OIDC Auth for Installing any dependencies that uv may need for build (sometimes it likes to install setuptools...even if it's already in the package dependencies) + uses: aws-actions/configure-aws-credentials@{% endraw %}{{ gha_configure_aws_credentials }}{% raw %} + with: + role-to-assume: arn:aws:iam::{% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %}:role/CoreInfraBaseAccess + aws-region: {% endraw %}{{ aws_org_home_region }}{% raw %} + +{% endraw %}{% endif %}{% raw %} + - name: Build package + run: | +{% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}{% raw %} . .devcontainer/code-artifact-auth.sh{% endraw %}{% endif %}{% raw %} + uv build --no-sources + + - name: Upload build package + uses: actions/upload-artifact@{% endraw %}{{ gha_upload_artifact }}{% raw %} + with: + name: python-package-distributions + path: dist/ + if-no-files-found: error +{% endraw %}{% endif %}{% raw %} + {% endraw %}{% if deploy_as_executable %}{% raw %} executable: needs: - test @@ -193,7 +237,8 @@ jobs: - get-values - check-skip-duplicate - lint - - test{% endraw %}{% if create_docs %} + - test{% endraw %}{% if not deploy_as_executable %} + - build{% endif %}{% if create_docs %} - build-docs{% endif %}{% if deploy_as_executable %} - executable{% endif %}{% raw %} - confirm-on-tagged-copier-template @@ -207,7 +252,8 @@ jobs: if [[ ! "${{ needs.get-values.result }}" =~ $success_pattern ]] || [[ ! "${{ needs.check-skip-duplicate.result }}" =~ $success_pattern ]] || - [[ ! "${{ needs.lint.result }}" =~ $success_pattern ]] ||{% endraw %}{% if create_docs %}{% raw %} + [[ ! "${{ needs.lint.result }}" =~ $success_pattern ]] ||{% endraw %}{% if not deploy_as_executable %}{% raw %} + [[ ! "${{ needs.build.result }}" =~ $success_pattern ]] ||{% endraw %}{% endif %}{% if create_docs %}{% raw %} [[ ! "${{ needs.build-docs.result }}" =~ $success_pattern ]] ||{% endraw %}{% endif %}{% raw %}{% endraw %}{% if deploy_as_executable %}{% raw %} [[ ! "${{ needs.executable.result }}" =~ $success_pattern ]] ||{% endraw %}{% endif %}{% raw %} [[ ! "${{ needs.test.result }}" =~ $success_pattern ]] || diff --git a/template/.github/workflows/extract_project_version.py b/template/.github/workflows/extract_project_version.py new file mode 100644 index 00000000..e5404ee3 --- /dev/null +++ b/template/.github/workflows/extract_project_version.py @@ -0,0 +1,53 @@ +# ============== WARNING ============================================================================== +# File is managed by copier template: gh:LabAutomationAndScreening/copier-base-template.git +# See .config/.copier-managed-files.json for details. +# +# You are welcome to make changes to this file in your repo if they are custom to your project, +# but if the change should be shared with other projects, please backport it to the template repo. +# ===================================================================================================== +import argparse +import json +import tomllib +from pathlib import Path + + +def extract_version(file_path: Path | str) -> str: + path = Path(file_path) + + if path.name == "package.json": + data = json.loads(path.read_text()) + if version := data.get("version"): + return version + raise KeyError(f"No version field found in {path!r}") + + if path.name == "pyproject.toml": + with path.open("rb") as f: + data = tomllib.load(f) + project = data.get("project", {}) + if version := project.get("version"): + return version + tool = data.get("tool", {}) + if version := tool.get("poetry", {}).get("version"): + return version + raise KeyError(f"No version field found in {path!r}") + + raise ValueError(f"Unsupported file type {path.name!r}; expected pyproject.toml or package.json") + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Extract the version from a pyproject.toml or package.json file and print it." + ) + _ = parser.add_argument( + "file", + nargs="?", + default="pyproject.toml", + help="Path to pyproject.toml or package.json (default: pyproject.toml)", + ) + args = parser.parse_args() + + print(extract_version(args.file)) # noqa: T201 # specifically printing this out so CI pipelines can read the value from stdout + + +if __name__ == "__main__": + main() diff --git a/template/.github/workflows/git_tag.py b/template/.github/workflows/git_tag.py deleted file mode 120000 index 6a9ce511..00000000 --- a/template/.github/workflows/git_tag.py +++ /dev/null @@ -1 +0,0 @@ -../../../copier_template_resources/git_tag.py \ No newline at end of file diff --git a/template/.github/workflows/git_tag.py b/template/.github/workflows/git_tag.py new file mode 100644 index 00000000..9f99df3a --- /dev/null +++ b/template/.github/workflows/git_tag.py @@ -0,0 +1,67 @@ +# ============== WARNING ============================================================================== +# File is managed by copier template: gh:LabAutomationAndScreening/copier-base-template.git +# See .config/.copier-managed-files.json for details. +# +# You are welcome to make changes to this file in your repo if they are custom to your project, +# but if the change should be shared with other projects, please backport it to the template repo. +# ===================================================================================================== +import argparse +import subprocess + + +def ensure_tag_not_present(tag: str, remote: str) -> None: + no_matching_refs_return_code = 2 + result = subprocess.run( # noqa: S603 # this is trusted input, it's our own arguments being passed in + ["git", "ls-remote", "--exit-code", "--tags", remote, f"refs/tags/{tag}"], # noqa: S607 # if `git` isn't in PATH already, then there are bigger problems to solve + stdout=subprocess.DEVNULL, + check=False, + ) + if result.returncode == 0: + raise Exception(f"Error: tag '{tag}' exists on remote '{remote}'") # noqa: TRY002 # not worth a custom exception + if ( + result.returncode != no_matching_refs_return_code + ): # anything else is a real error (bad remote, auth failure, network) + raise Exception(f"git ls-remote exited with code {result.returncode} (remote={remote!r})") # noqa: TRY002 # not worth a custom exception + + +def main() -> None: + parser = argparse.ArgumentParser( + description=("Confirm that git tag v is not present on a remote, or create and push the tag.") + ) + _ = parser.add_argument( + "--version", + required=True, + help="Version string (e.g. 1.0.6 or v1.0.6); the tag will always be v", + ) + mode = parser.add_mutually_exclusive_group(required=True) + _ = mode.add_argument( + "--confirm-tag-not-present", + action="store_true", + help="Check that git tag v is NOT present on the remote. If the tag exists, exit with an error.", + ) + _ = mode.add_argument( + "--push-tag-to-remote", + action="store_true", + help="Create git tag v locally and push it to the remote. Internally confirms the tag is not already present.", + ) + _ = parser.add_argument( + "--remote", + default="origin", + help="Name of git remote to query/push (default: origin)", + ) + args = parser.parse_args() + + tag = args.version if args.version.startswith("v") else f"v{args.version}" + + if args.push_tag_to_remote: + ensure_tag_not_present(tag, args.remote) + _ = subprocess.run(["git", "tag", tag], check=True) # noqa: S603,S607 # this is trusted input, it's our own version string. and if `git` isn't in PATH, then there are larger problems anyway + _ = subprocess.run(["git", "push", args.remote, tag], check=True) # noqa: S603,S607 # this is trusted input, it's our own version string. and if `git` isn't in PATH, then there are larger problems anyway + return + + if args.confirm_tag_not_present: + ensure_tag_not_present(tag, args.remote) + + +if __name__ == "__main__": + main() diff --git a/template/.github/workflows/release.yaml.jinja b/template/.github/workflows/release.yaml.jinja new file mode 100644 index 00000000..997534d9 --- /dev/null +++ b/template/.github/workflows/release.yaml.jinja @@ -0,0 +1,413 @@ +{% raw %}name: Release + +on: + workflow_dispatch: + inputs: + dry_run: + description: "Dry run (test the workflow, skip actual release)" + type: boolean + default: false + +env: + PYTHONUNBUFFERED: True + +concurrency: + group: release + cancel-in-progress: false + +permissions: + contents: read # need to explicitly provide this whenever defining permissions because the default value is 'none' for anything not explicitly set when permissions are defined + +jobs: + guard: + name: Guard on a passing CI run + runs-on: {% endraw %}{{ gha_linux_runner }}{% raw %} + timeout-minutes: {% endraw %}{{ gha_short_timeout_minutes }}{% raw %} + permissions: + contents: read # needed for checkout + actions: read # needed to query CI workflow runs + outputs: + version: ${{ steps.get-version.outputs.version }} + ci-run-id: ${{ steps.get-ci-run-id.outputs.ci-run-id }} + steps: + - name: Fail if real release is not on main + if: ${{ !inputs.dry_run && github.ref != 'refs/heads/main' }} + run: | + echo "Real releases must run from main. Current ref: ${{ github.ref }}" + exit 1 + - name: Get CI run ID for this commit + id: get-ci-run-id + env: + GH_TOKEN: ${{ github.token }} + run: | + ALL_RUNS=$(gh api "/repos/${{ github.repository }}/actions/workflows/ci.yaml/runs?head_sha=${{ github.sha }}") + echo "All ci.yaml runs for ${{ github.sha }}:" + echo "$ALL_RUNS" | jq '[.workflow_runs[] | {id, status, conclusion, created_at}]' + LATEST_RUN=$(echo "$ALL_RUNS" | jq '.workflow_runs | first') + RUN_ID=$(echo "$LATEST_RUN" | jq -r '.id') + STATUS=$(echo "$LATEST_RUN" | jq -r '.status') + CONCLUSION=$(echo "$LATEST_RUN" | jq -r '.conclusion') + echo "Most recent ci.yaml run: id=$RUN_ID status=$STATUS conclusion=$CONCLUSION" + if [ "$STATUS" != "completed" ] || [ "$CONCLUSION" != "success" ]; then + echo "Most recent ci.yaml run has not completed successfully. Wait for CI to pass before releasing." + exit 1 + fi + echo "ci-run-id=$RUN_ID" >> $GITHUB_OUTPUT + - name: Verify workflow-summary passed in CI run + env: + GH_TOKEN: ${{ github.token }} + run: | + JOBS=$(gh api "/repos/${{ github.repository }}/actions/runs/${{ steps.get-ci-run-id.outputs.ci-run-id }}/jobs") + echo "Jobs in CI run ${{ steps.get-ci-run-id.outputs.ci-run-id }}:" + echo "$JOBS" | jq '[.jobs[] | {name, conclusion}]' + CONCLUSION=$(echo "$JOBS" | jq -r '.jobs[] | select(.name == "workflow-summary") | .conclusion') + echo "workflow-summary conclusion: $CONCLUSION" + if [ "$CONCLUSION" != "success" ] && [ "${{ inputs.dry_run }}" != "true" ]; then + echo "workflow-summary did not succeed. Wait for CI to pass before releasing." + exit 1 + fi + - name: Checkout code + uses: actions/checkout@{% endraw %}{{ gha_checkout }}{% raw %} + - name: Setup python + uses: actions/setup-python@{% endraw %}{{ gha_setup_python }}{% raw %} + with: + python-version: {% endraw %}{{ python_version }}{% raw %} + - name: Extract project version + id: get-version + run: | + VERSION=$(python3 .github/workflows/extract_project_version.py pyproject.toml) + echo "Project version: $VERSION" + echo "version=$VERSION" >> $GITHUB_OUTPUT + - name: Check version tag does not already exist + run: | + if python3 .github/workflows/git_tag.py --version "${{ steps.get-version.outputs.version }}" --confirm-tag-not-present; then + echo "Tag v${{ steps.get-version.outputs.version }} does not exist yet." + else + if [ "${{ inputs.dry_run }}" = "true" ]; then + echo "Tag v${{ steps.get-version.outputs.version }} already exists (dry run, continuing)." + else + echo "Tag v${{ steps.get-version.outputs.version }} already exists. Bump the version before releasing." + exit 1 + fi + fi +{% endraw %}{% if not deploy_as_executable %}{% raw %} + + publish-to-staging: + name: Publish Python distribution to Staging Package Registry + timeout-minutes: {% endraw %}{{ gha_medium_timeout_minutes }}{% raw %} + needs: + - guard + runs-on: {% endraw %}{{ gha_linux_runner }}{% raw %} + environment: + name: testpypi + url: https://test.pypi.org/p/{% endraw %}{{ package_name | replace('_', '-') }}{% raw %} + permissions: + attestations: write + id-token: write + actions: read # needed to download the distribution artifact from the CI run + steps: + - name: Download all the dists + uses: actions/download-artifact@{% endraw %}{{ gha_download_artifact }}{% raw %} + with: + name: python-package-distributions + path: dist/ + run-id: ${{ needs.guard.outputs.ci-run-id }} + github-token: ${{ github.token }} +{% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}{% raw %} + - name: OIDC Auth for Publishing to CodeArtifact + uses: aws-actions/configure-aws-credentials@{% endraw %}{{ gha_configure_aws_credentials }}{% raw %} + with: + role-to-assume: arn:aws:iam::{% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %}:role/GHA-CA-Primary-{% endraw %}{{ repo_name }}{% raw %} + aws-region: {% endraw %}{{ aws_org_home_region }}{% raw %} + + - name: Publish distribution to Code Artifact + run: | + . .devcontainer/code-artifact-auth.sh + uv publish --verbose --index code-artifact-primary --username aws --password "$TWINE_PASSWORD" + +{% endraw %}{% else %}{% raw %} + - name: Publish distribution to Test PyPI + uses: pypa/gh-action-pypi-publish@{% endraw %}{{ gha_pypi_publish }}{% raw %} + with: + attestations: false + repository-url: https://test.pypi.org/legacy/ +{% endraw %}{% endif %}{% raw %} + + install-from-staging: + name: Install package from staging registry + timeout-minutes: {% endraw %}{{ gha_medium_timeout_minutes }}{% raw %} + needs: + - publish-to-staging + - guard + strategy: + matrix: + os: + - "{% endraw %}{{ gha_linux_runner }}{% raw %}" +{% endraw %}{% if use_windows_in_ci %} - {{ gha_windows_runner }}{% endif %}{% raw %} + python-version: +{% endraw %}{% for item in python_ci_versions %} + - {{ item }} +{% endfor %}{% raw %} + include: + - os: "{% endraw %}{{ gha_linux_runner }}{% raw %}" + python-version: "{% endraw %}{{ python_ci_versions[0] }}{% raw %}" + JOB_MATCHING_DEV_ENV: true + + runs-on: ${{ matrix.os }} + env: + UV_PYTHON: ${{ matrix.python-version }} + + steps: + - name: Checkout code + uses: actions/checkout@{% endraw %}{{ gha_checkout }}{% raw %} + - name: Setup python + uses: actions/setup-python@{% endraw %}{{ gha_setup_python }}{% raw %} + with: + python-version: ${{ matrix.python-version }} +{% endraw %}{% if python_package_registry == "PyPI" %}{% raw %} + - name: Wait for Test PyPI to serve the new version + shell: bash + run: | + set -euo pipefail + PKG="{% endraw %}{{ package_name | replace('_', '-') }}{% raw %}" + VER="${{ needs.guard.outputs.version }}" + for i in $(seq 1 60); do + code="$(curl -fsS -o /dev/null -w '%{http_code}' "https://test.pypi.org/pypi/${PKG}/${VER}/json" || true)" + if [ "$code" = "200" ]; then + echo "Found ${PKG}==${VER} on Test PyPI." + break + fi + echo "Not yet available; sleeping 5s..." + sleep 5 + done + + if [ "$code" != "200" ]; then + echo "Timeout waiting for ${PKG}==${VER} on Test PyPI API." + exit 1 + fi + + # Then try to install with retries to ensure it's actually available in the index + for i in $(seq 1 12); do + if pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://www.pypi.org/simple "${PKG}==${VER}" --no-cache-dir --quiet; then + echo "Successfully installed ${PKG}==${VER}" + exit 0 + fi + echo "Package not yet installable; sleeping 10s... (attempt $i/12)" + sleep 10 + done + + echo "Timeout waiting for ${PKG}==${VER} to be installable." + exit 1{% endraw %}{% endif %}{% raw %} + - name: Display dependencies + run: pip list + - name: Confirm library can be imported successfully + env: + PYTHONPATH: "" # avoid picking up local sources + shell: bash + run: | + python - <<'PY' + import sys, importlib, importlib.util, importlib.metadata as md, pathlib + print(f"Python version {sys.version}") + mod_name = "{% endraw %}{{ package_name | replace('-', '_') }}{% raw %}" + dist_name = mod_name + m = importlib.import_module(mod_name) + spec = importlib.util.find_spec(mod_name) + origin = (getattr(spec, "origin", None) or getattr(m, "__file__", "")) or "" + p = pathlib.Path(origin) + print(f"Imported from: {p}") + assert any(s in str(p) for s in ("site-packages", "dist-packages")), f"Expected site/dist-packages, got {p}" + expected = "${{ needs.guard.outputs.version }}" + installed = md.version(dist_name) + print(f"Installed distribution version: {installed} (expected {expected})") + assert installed == expected, f"Version mismatch: expected {expected}, got {installed}" + PY +{% endraw %}{% endif %}{% raw %} + + create-tag: + name: Create the git tag + if: ${{ !inputs.dry_run }} + timeout-minutes: {% endraw %}{{ gha_medium_timeout_minutes }}{% raw %} + needs: +{% endraw %}{% if not deploy_as_executable %}{% raw %} - install-from-staging +{% endraw %}{% endif %}{% raw %} - guard + permissions: + contents: write # needed to push the tag + runs-on: {% endraw %}{{ gha_linux_runner }}{% raw %} + steps: + - name: Checkout code + uses: actions/checkout@{% endraw %}{{ gha_checkout }}{% raw %} + - name: Setup python + uses: actions/setup-python@{% endraw %}{{ gha_setup_python }}{% raw %} + with: + python-version: {% endraw %}{{ python_version }}{% raw %} + - name: Confirm tag not already present + run: python3 ./.github/workflows/git_tag.py --version "${{ needs.guard.outputs.version }}" --confirm-tag-not-present + - name: Create tag + run: python3 ./.github/workflows/git_tag.py --version "${{ needs.guard.outputs.version }}" --push-tag-to-remote +{% endraw %}{% if not deploy_as_executable %}{% raw %} + + publish-to-primary: + name: Publish Python distribution to Primary Package Registry + if: ${{ !inputs.dry_run }} + needs: + - create-tag + - guard + timeout-minutes: {% endraw %}{{ gha_medium_timeout_minutes }}{% raw %} + runs-on: {% endraw %}{{ gha_linux_runner }}{% raw %} + environment: + name: pypi + url: https://pypi.org/p/{% endraw %}{{ package_name | replace('_', '-') }}{% raw %} + permissions: + attestations: write + id-token: write + actions: read # needed to download the distribution artifact from the CI run + steps: + - name: Download all the dists + uses: actions/download-artifact@{% endraw %}{{ gha_download_artifact }}{% raw %} + with: + name: python-package-distributions + path: dist/ + run-id: ${{ needs.guard.outputs.ci-run-id }} + github-token: ${{ github.token }} +{% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}{% raw %} + - name: OIDC Auth for Publishing to CodeArtifact + uses: aws-actions/configure-aws-credentials@{% endraw %}{{ gha_configure_aws_credentials }}{% raw %} + with: + role-to-assume: arn:aws:iam::{% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %}:role/GHA-CA-Primary-{% endraw %}{{ repo_name }}{% raw %} + aws-region: {% endraw %}{{ aws_org_home_region }}{% raw %} + + - name: Publish distribution to Code Artifact + run: | + . .devcontainer/code-artifact-auth.sh + uv publish --verbose --index code-artifact-primary --username aws --password "$TWINE_PASSWORD" + +{% endraw %}{% else %}{% raw %} + - name: Publish distribution to PyPI + uses: pypa/gh-action-pypi-publish@{% endraw %}{{ gha_pypi_publish }}{% raw %} + with: + attestations: false{% endraw %}{% endif %}{% raw %} + + install-from-primary: + name: Install package from primary registry + if: ${{ !inputs.dry_run }} + needs: + - publish-to-primary + - guard + strategy: + matrix: + os: + - "{% endraw %}{{ gha_linux_runner }}{% raw %}" +{% endraw %}{% if use_windows_in_ci %} - {{ gha_windows_runner }}{% endif %}{% raw %} + python-version: +{% endraw %}{% for item in python_ci_versions %} + - {{ item }} +{% endfor %}{% raw %} + include: + - os: "{% endraw %}{{ gha_linux_runner }}{% raw %}" + python-version: "{% endraw %}{{ python_ci_versions[0] }}{% raw %}" + JOB_MATCHING_DEV_ENV: true + + runs-on: ${{ matrix.os }} + timeout-minutes: {% endraw %}{{ gha_medium_timeout_minutes }}{% raw %} + env: + UV_PYTHON: ${{ matrix.python-version }} + + steps: + - name: Checkout code + uses: actions/checkout@{% endraw %}{{ gha_checkout }}{% raw %} + - name: Setup python + uses: actions/setup-python@{% endraw %}{{ gha_setup_python }}{% raw %} + with: + python-version: ${{ matrix.python-version }} +{% endraw %}{% if python_package_registry == "PyPI" %}{% raw %} + - name: Wait for PyPI to serve the new version + shell: bash + run: | + set -euo pipefail + PKG="{% endraw %}{{ package_name | replace('_', '-') }}{% raw %}" + VER="${{ needs.guard.outputs.version }}" + for i in $(seq 1 60); do + code="$(curl -fsS -o /dev/null -w '%{http_code}' "https://pypi.org/pypi/${PKG}/${VER}/json" || true)" + if [ "$code" = "200" ]; then + echo "Found ${PKG}==${VER} on PyPI." + break + fi + echo "Not yet available; sleeping 5s..." + sleep 5 + done + + if [ "$code" != "200" ]; then + echo "Timeout waiting for ${PKG}==${VER} on PyPI API." + exit 1 + fi + + # Then try to install with retries to ensure it's actually available in the index + for i in $(seq 1 12); do + if pip install --index-url https://pypi.org/simple/ "${PKG}==${VER}" --no-cache-dir --quiet; then + echo "Successfully installed ${PKG}==${VER}" + exit 0 + fi + echo "Package not yet installable; sleeping 10s... (attempt $i/12)" + sleep 10 + done + + echo "Timeout waiting for ${PKG}==${VER} to be installable." + exit 1{% endraw %}{% endif %}{% raw %} + - name: Display dependencies + run: pip list + - name: Confirm library can be imported successfully + env: + PYTHONPATH: "" # avoid picking up local sources + shell: bash + run: | + python - <<'PY' + import sys, importlib, importlib.util, importlib.metadata as md, pathlib + print(f"Python version {sys.version}") + mod_name = "{% endraw %}{{ package_name | replace('-', '_') }}{% raw %}" + dist_name = mod_name + m = importlib.import_module(mod_name) + spec = importlib.util.find_spec(mod_name) + origin = (getattr(spec, "origin", None) or getattr(m, "__file__", "")) or "" + p = pathlib.Path(origin) + print(f"Imported from: {p}") + assert any(s in str(p) for s in ("site-packages", "dist-packages")), f"Expected site/dist-packages, got {p}" + expected = "${{ needs.guard.outputs.version }}" + installed = md.version(dist_name) + print(f"Installed distribution version: {installed} (expected {expected})") + assert installed == expected, f"Version mismatch: expected {expected}, got {installed}" + PY +{% endraw %}{% endif %}{% raw %} + + release: + name: Create GitHub Release + if: ${{ !inputs.dry_run }} + runs-on: {% endraw %}{{ gha_linux_runner }}{% raw %} + timeout-minutes: {% endraw %}{{ gha_short_timeout_minutes }}{% raw %} + needs: +{% endraw %}{% if not deploy_as_executable %}{% raw %} - install-from-primary +{% endraw %}{% else %}{% raw %} - create-tag +{% endraw %}{% endif %}{% raw %} - guard + permissions: + contents: write # needed to create GitHub releases{% endraw %}{% if deploy_as_executable %}{% raw %} + actions: read # needed to download the executable artifacts from the CI run{% endraw %}{% endif %}{% raw %} + steps:{% endraw %}{% if deploy_as_executable %}{% set exe_os_list = [gha_linux_runner] + ([gha_windows_runner] if use_windows_in_exe_ci else []) %}{% for exe_os in exe_os_list %}{% for exe_py in python_ci_versions %}{% set ext = "tar" if exe_os == gha_linux_runner else "zip" %}{% raw %} + - name: Download executable artifact ({% endraw %}{{ exe_os }} py{{ exe_py }}{% raw %}) + uses: actions/download-artifact@{% endraw %}{{ gha_download_artifact }}{% raw %} + with: + name: exe-{% endraw %}{{ exe_os }}-{{ exe_py }}{% raw %} + path: artifacts/{% endraw %}{{ exe_os }}-{{ exe_py }}{% raw %} + run-id: ${{ needs.guard.outputs.ci-run-id }} + github-token: ${{ github.token }} + - name: Package executable ({% endraw %}{{ exe_os }} py{{ exe_py }}{% raw %}) + shell: bash + run: {% endraw %}{% if exe_os == gha_linux_runner %}tar -cf "{{ repo_name }}-{{ exe_os }}-{{ exe_py }}-v{% raw %}${{ needs.guard.outputs.version }}{% endraw %}.{{ ext }}" -C "artifacts/{{ exe_os }}-{{ exe_py }}" .{% else %}(cd "artifacts/{{ exe_os }}-{{ exe_py }}" && zip -r "../../{{ repo_name }}-{{ exe_os }}-{{ exe_py }}-v{% raw %}${{ needs.guard.outputs.version }}{% endraw %}.{{ ext }}" .){% endif %}{% raw %} +{% endraw %}{% endfor %}{% endfor %}{% endif %}{% raw %} + - name: Create GitHub Release + uses: softprops/action-gh-release@{% endraw %}{{ gha_action_gh_release }}{% raw %} + with: + tag_name: v${{ needs.guard.outputs.version }} + name: {% endraw %}{{ repo_name }}{% raw %} v${{ needs.guard.outputs.version }} + generate_release_notes: true{% endraw %}{% if deploy_as_executable %}{% raw %} + fail_on_unmatched_files: true + files: |{% endraw %}{% set exe_os_list = [gha_linux_runner] + ([gha_windows_runner] if use_windows_in_exe_ci else []) %}{% for exe_os in exe_os_list %}{% for exe_py in python_ci_versions %}{% set ext = "tar" if exe_os == gha_linux_runner else "zip" %} + {{ repo_name }}-{{ exe_os }}-{{ exe_py }}-v{% raw %}${{ needs.guard.outputs.version }}{% endraw %}.{{ ext }}{% endfor %}{% endfor %}{% endif %}{% raw %}{% endraw %} diff --git a/template/.github/workflows/{% if not is_frozen_executable %}publish.yaml{% endif %}.jinja b/template/.github/workflows/{% if not deploy_as_executable %}publish.yaml{% endif %}.jinja similarity index 97% rename from template/.github/workflows/{% if not is_frozen_executable %}publish.yaml{% endif %}.jinja rename to template/.github/workflows/{% if not deploy_as_executable %}publish.yaml{% endif %}.jinja index 690346c2..3aee9a26 100644 --- a/template/.github/workflows/{% if not is_frozen_executable %}publish.yaml{% endif %}.jinja +++ b/template/.github/workflows/{% if not deploy_as_executable %}publish.yaml{% endif %}.jinja @@ -33,7 +33,7 @@ jobs: - name: Extract package version id: extract-package-version run: | - VERSION=$(python3 ./.github/workflows/git_tag.py) + VERSION=$(python3 ./.github/workflows/extract_project_version.py pyproject.toml) echo "Extracted version: $VERSION" echo "package_version=$VERSION" >> $GITHUB_OUTPUT @@ -249,7 +249,9 @@ jobs: name: Create the git tag if: ${{ fromJSON(github.event.inputs.publish_to_primary) }} timeout-minutes: {% endraw %}{{ gha_medium_timeout_minutes }}{% raw %} - needs: [ install-from-staging ] + needs: + - install-from-staging + - get-values permissions: contents: write # needed to push the tag runs-on: {% endraw %}{{ gha_linux_runner }}{% raw %} @@ -261,9 +263,9 @@ jobs: with: python-version: {% endraw %}{{ python_version }}{% raw %} - name: Confirm tag not already present - run: python3 ./.github/workflows/git_tag.py --confirm-tag-not-present + run: python3 ./.github/workflows/git_tag.py --version "${{ needs.get-values.outputs.package-version }}" --confirm-tag-not-present - name: Create tag - run: python3 ./.github/workflows/git_tag.py --push-tag-to-remote + run: python3 ./.github/workflows/git_tag.py --version "${{ needs.get-values.outputs.package-version }}" --push-tag-to-remote publish-to-primary: name: Publish Python distribution to Primary Package Registry diff --git a/template/.github/workflows/{% if not is_frozen_executable %}publish_to_staging.yaml{% endif %}.jinja b/template/.github/workflows/{% if not deploy_as_executable %}publish_to_staging.yaml{% endif %}.jinja similarity index 100% rename from template/.github/workflows/{% if not is_frozen_executable %}publish_to_staging.yaml{% endif %}.jinja rename to template/.github/workflows/{% if not deploy_as_executable %}publish_to_staging.yaml{% endif %}.jinja diff --git a/template/README.md.jinja b/template/README.md.jinja index 93966dbd..21a3ded1 100644 --- a/template/README.md.jinja +++ b/template/README.md.jinja @@ -25,7 +25,12 @@ Documentation is hosted on [ReadTheDocs](https://{% endraw %}{{ package_name }}{ # Development This project has a dev container. If you already have VS Code and Docker installed, you can click the badge above or [here](https://vscode.dev/redirect?url=vscode://ms-vscode-remote.remote-containers/cloneInVolume?url={% endraw %}{{ full_repo_url }}{% raw %}) to get started. Clicking these links will cause VS Code to automatically install the Dev Containers extension if needed, clone the source code into a container volume, and spin up a dev container for use. -To publish a new version of the repository, you can run the `Publish` workflow manually and publish to the staging registry from any branch, and you can check the 'Publish to Primary' option when on `main` to publish to the primary registry and create a git tag. +{% endraw %}{% if not deploy_as_executable %}{% raw %}To publish a new version of the repository, you can run the `Publish` workflow manually and publish to the staging registry from any branch, and you can check the 'Publish to Primary' option when on `main` to publish to the primary registry and create a git tag. + +{% endraw %}{% endif %}{% raw %}The `Release` workflow reuses the checks from CI instead of re-running them: it requires that the `CI` workflow has already completed successfully for the current commit (including the `workflow-summary` job) and reuses the artifacts CI built rather than rebuilding. Trigger it manually with `workflow_dispatch`; a real release must be run from `main`, while the `dry_run` option lets you exercise the workflow from any branch without tagging or releasing. {% endraw %}{% if deploy_as_executable %}{% raw %}A real run pushes the `v` git tag and creates a GitHub Release, attaching the executable that CI built for every OS and Python version.{% endraw %}{% else %}{% raw %}A real run publishes the CI-built distribution to the staging registry and verifies a fresh install, then pushes the `v` git tag, publishes to the primary registry, verifies the install again, and finally creates a GitHub Release with auto-generated notes.{% endraw %}{% endif %}{% raw %} +{% endraw %}{% if not deploy_as_executable %}{% raw %} + +{% endraw %}{% endif %}{% raw %} diff --git a/template/src/{% if is_frozen_executable %}entrypoint.py{% endif %}.jinja b/template/src/{% if deploy_as_executable %}entrypoint.py{% endif %}.jinja similarity index 100% rename from template/src/{% if is_frozen_executable %}entrypoint.py{% endif %}.jinja rename to template/src/{% if deploy_as_executable %}entrypoint.py{% endif %}.jinja diff --git a/template/src/{{ package_name.replace('-', '_') }}/{% if is_frozen_executable %}main.py{% endif %}.jinja b/template/src/{{ package_name.replace('-', '_') }}/{% if deploy_as_executable %}main.py{% endif %}.jinja similarity index 100% rename from template/src/{{ package_name.replace('-', '_') }}/{% if is_frozen_executable %}main.py{% endif %}.jinja rename to template/src/{{ package_name.replace('-', '_') }}/{% if deploy_as_executable %}main.py{% endif %}.jinja