[typespec-go] Refine SDK regenerate pipeline

docs/sdk-regeneration-pipeline.md

@@ -37,6 +37,7 @@ The SDK regeneration pipeline automates the process of updating Azure SDK packag
 #### 4. Results & PR Creation
 - Generates results report (`regenerate-sdk-result.json`)
 - Creates draft PR in azure-sdk-for-go with all changes, PR title: `[Automation] Regenerate SDK based on typespec-go branch {branch-name}`
+- If `CreateSpecPR` is enabled and a module version bump was detected, creates a draft PR in azure-rest-api-specs to bump the go module suffix in `tspconfig.yaml`
 
 ### Pipeline Parameters
 
@@ -47,6 +48,7 @@ The SDK regeneration pipeline automates the process of updating Azure SDK packag
 | `UseLatestSpec` | boolean | `false` | Whether to use the latest API specifications from [azure-rest-api-specs](https://github.com/Azure/azure-rest-api-specs) or the original commit of `tsp-location.yml` |
 | `ServiceFilter` | string | `.*` | Regex pattern to filter which services to regenerate. Matches against the service package name (e.g., `armcompute`, `armstorage`) |
 | `UseDevPackage` | boolean | `false` | Whether to use dev package (.tgz) from current branch or the recent released package from npm registry |
+| `CreateSpecPR` | boolean | `false` | Whether to create a PR in azure-rest-api-specs to bump go module suffixes in `tspconfig.yaml` when module versions are detected |
 
 #### Usage Examples
 
@@ -126,6 +128,9 @@ You can find the generated SDK pull request link from pipeline logs
   "succeed_to_regenerate": ["package1", "package2"],
   "fail_to_regenerate": ["package3"],
   "not_found_api_version": ["package4"],
+  "module_version_changed": {
+    "specification/advisor/Advisor.Management": "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/advisor/armadvisor/v2"
+  },
   "time_to_regenerate": "2025-09-25 10:30:45.123456",
   "typespec_go_commit_hash": "abc123def456789..."
 }
@@ -161,7 +166,7 @@ After each TypeSpec or Go emitter version release:
 3. **Merge Regeneration PR**: Merge the refresh PR to keep SDK up-to-date
 
 ### Quality Gates
-- All pipelines must be pass
+- All pipelines must pass
 - All SDK code changes are made by either TypeSpec changes or Go emitter changes
 - Module versions must not be changed
 - API versions must not be changed

eng/pipelines/sdk-regenerate.yml

@@ -18,6 +18,9 @@ parameters:
 - name: UseDevPackage
   default: false
   type: boolean
+- name: CreateSpecPR
+  default: false
+  type: boolean
 
 variables:
   - template: /eng/pipelines/templates/variables/globals.yml
@@ -35,6 +38,11 @@ resources:
       name: Azure/azure-sdk-for-go
       endpoint: azure
       ref: main
+    - repository: azure-rest-api-specs
+      type: github
+      name: Azure/azure-rest-api-specs
+      endpoint: azure
+      ref: main
 
 jobs:
 - job: Generate_SDK
@@ -45,6 +53,8 @@ jobs:
   - checkout: self
     fetchDepth: 1
   - checkout: azure-sdk-for-go
+  - ${{ if eq(parameters.CreateSpecPR, true) }}:
+    - checkout: azure-rest-api-specs
 
   - task: NodeTool@0
     displayName: 'Install Node.js $(NodeVersion)'
@@ -73,10 +83,14 @@ jobs:
     displayName: 'Install tsp-client'
 
   - script: |
-      python3 $(Build.SourcesDirectory)/autorest.go/eng/scripts/sdk_regenerate.py --sdk-root=$(Build.SourcesDirectory)/azure-sdk-for-go --typespec-go-root=$(Build.SourcesDirectory)/autorest.go/packages/typespec-go --typespec-go-branch=$(Build.SourceBranchName) --use-latest-spec=${{ parameters.UseLatestSpec }} --service-filter="${{ parameters.ServiceFilter }}" --use-dev-package=${{ parameters.UseDevPackage }}
+      python3 $(Build.SourcesDirectory)/autorest.go/eng/scripts/sdk_regenerate.py --sdk-root=$(Build.SourcesDirectory)/azure-sdk-for-go --typespec-go-root=$(Build.SourcesDirectory)/autorest.go/packages/typespec-go --typespec-go-branch=$(Build.SourceBranchName) --use-latest-spec=${{ parameters.UseLatestSpec }} --service-filter="${{ parameters.ServiceFilter }}" --use-dev-package=${{ parameters.UseDevPackage }} --create-spec-pr=${{ parameters.CreateSpecPR }} --spec-root=$(Build.SourcesDirectory)/azure-rest-api-specs
     displayName: 'Generate SDK'
     workingDirectory: $(Build.SourcesDirectory)/azure-sdk-for-go
 
+  - publish: $(Build.ArtifactStagingDirectory)/regenerate-sdk-result.json
+    artifact: regenerate-sdk-result
+    displayName: 'Publish regenerate SDK result'
+
   - template: /eng/common/pipelines/templates/steps/login-to-github.yml@azure-sdk-for-go
     parameters:
       ScriptDirectory: $(Build.SourcesDirectory)/azure-sdk-for-go/eng/common/scripts
@@ -94,3 +108,18 @@ jobs:
       OpenAsDraft: 'true'
       PushArgs: '--force'
       AuthToken: $(GH_TOKEN)
+
+  - ${{ if parameters.CreateSpecPR }}:
+    - template: /eng/common/pipelines/templates/steps//create-pull-request.yml@azure-sdk-for-go
+      parameters:
+        WorkingDirectory: $(Build.SourcesDirectory)/azure-rest-api-specs
+        ScriptDirectory: $(Build.SourcesDirectory)/azure-sdk-for-go/eng/common/scripts
+        RepoName: azure-rest-api-specs
+        PROwner: Azure
+        BaseBranchName: 'refs/heads/main'
+        PRBranchName: typespec-go-module-suffix-$(Build.SourceBranchName)
+        CommitMsg: 'Bump go module suffix in tspconfig.yaml'
+        PRTitle: '[Automation] Bump go module suffix in tspconfig.yaml'
+        OpenAsDraft: 'true'
+        PushArgs: '--force'
+        AuthToken: $(GH_TOKEN)

eng/scripts/sdk_regenerate.py

@@ -4,14 +4,15 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the MIT License. See License.txt in the project root for license information.
 # --------------------------------------------------------------------------------------------
-from typing import Dict, List, Optional
+from typing import Any, Dict, List, Optional
 from pathlib import Path
 import subprocess
 from datetime import datetime
 from subprocess import check_call, check_output, call
 import argparse
 import logging
 import json
+import os
 import re
 import glob
 import urllib.request
@@ -165,22 +166,52 @@ def update_commit_id(file: Path, commit_id: str):
 
 
 def get_api_version_from_metadata(package_folder: Path) -> Optional[str]:
-    """Extract API version from metadata.json file if it exists."""
+    """Return the api-version emitter option value from metadata.json, if any.
+
+    Supports the legacy ``{"apiVersion": "..."}`` string and the current
+    ``{"apiVersions": {namespace: version}}`` map. Returns a plain version when
+    all namespaces share one version, otherwise a compact JSON namespace->version
+    map that the emitter uses to regenerate each namespace at its own version.
+    """
     # Construct the metadata.json path based on the package folder structure
     # {package_folder}/testdata/_metadata.json
     metadata_path = package_folder / "testdata" / "_metadata.json"
-
-    if metadata_path.exists():
-        try:
-            with open(metadata_path, "r") as f:
-                metadata = json.load(f)
-                api_version = metadata.get("apiVersion")
-                if api_version:
-                    logging.info(f"Found API version {api_version} in metadata.json for {package_folder.name}")
-                    return api_version
-        except (json.JSONDecodeError, FileNotFoundError) as e:
-            logging.warning(f"Failed to read metadata.json for {package_folder.name}: {e}")
-
+
+    if not metadata_path.exists():
+        return None
+
+    try:
+        with open(metadata_path, "r") as f:
+            metadata = json.load(f)
+    except (json.JSONDecodeError, FileNotFoundError) as e:
+        logging.warning(f"Failed to read metadata.json for {package_folder.name}: {e}")
+        return None
+
+    # current format: a map of namespace -> API version
+    api_versions = metadata.get("apiVersions")
+    if isinstance(api_versions, dict):
+        # drop namespaces without a version, keeping the namespace->version map
+        versions = {ns: ver for ns, ver in api_versions.items() if ver}
+        if versions:
+            distinct = set(versions.values())
+            if len(distinct) == 1:
+                api_version = next(iter(distinct))
+                logging.info(f"Found API version {api_version} in metadata.json for {package_folder.name}")
+                return api_version
+            # multiple namespaces at different versions: pass the whole map so the
+            # emitter regenerates each service namespace at its recorded version
+            api_version = json.dumps(versions, separators=(",", ":"))
+            logging.info(
+                f"Found multiple API versions {versions} in metadata.json for "
+                f"{package_folder.name}, passing per-namespace versions"
+            )
+            return api_version
+
+    # legacy format: a single API version string
+    api_version = metadata.get("apiVersion")
+    if api_version:
+        logging.info(f"Found API version {api_version} in metadata.json for {package_folder.name}")
+        return api_version
     return None
 
 
@@ -223,13 +254,65 @@ def get_api_version(package_folder: Path) -> Optional[str]:
 
     return api_version
 
-def regenerate_sdk(use_latest_spec: bool, service_filter: str, sdk_root: str, typespec_go_root: str) -> Dict[str, List[str]]:
+
+def get_module_name(package_folder: Path) -> Optional[str]:
+    """Read the module name from go.mod in the package folder."""
+    go_mod_path = package_folder / "go.mod"
+    if not go_mod_path.exists():
+        return None
+    try:
+        with open(go_mod_path, "r", encoding="utf-8") as f:
+            for line in f:
+                stripped = line.strip()
+                if stripped.startswith("module "):
+                    return stripped[len("module "):].strip()
+    except FileNotFoundError as e:
+        logging.warning(f"Failed to read go.mod for {package_folder.name}: {e}")
+    return None
+
+
+def detect_module_version_change(package_folder: Path, original_module: str) -> Optional[str]:
+    """Detect if regen dropped the module version suffix. If so, revert all changes in the
+    package and return the original suffixed module so the spec PR can bump tspconfig."""
+    if not original_module:
+        return None
+    suffix_match = re.search(r"/v\d+$", original_module)
+    if not suffix_match:
+        return None
+    current_module = get_module_name(package_folder)
+    if current_module == original_module:
+        return None
+    # Module suffix was dropped during regen; revert the whole package and bump the spec instead.
+    check_call(["git", "checkout", "--", str(package_folder)])
+    logging.info(f"Reverted {package_folder.name}; module version bump will be handled in spec repo")
+    return original_module
+
+
+def get_spec_directory(package_folder: Path) -> Optional[str]:
+    """Read the spec repo directory (containing tspconfig.yaml) from tsp-location.yaml."""
+    tsp_location = package_folder / "tsp-location.yaml"
+    if not tsp_location.exists():
+        return None
+    try:
+        with open(tsp_location, "r", encoding="utf-8") as f:
+            for line in f:
+                stripped = line.strip()
+                if stripped.startswith("directory:"):
+                    return stripped[len("directory:"):].strip().strip('"')
+    except FileNotFoundError as e:
+        logging.warning(f"Failed to read tsp-location.yaml for {package_folder.name}: {e}")
+    return None
+
+
+
+def regenerate_sdk(use_latest_spec: bool, service_filter: str, sdk_root: str, typespec_go_root: str) -> Dict[str, Any]:
     result = {
-        "succeed_to_regenerate": [], 
-        "fail_to_regenerate": [], 
-        "not_found_api_version": [], 
+        "succeed_to_regenerate": [],
+        "fail_to_regenerate": [],
+        "not_found_api_version": [],
+        "module_version_changed": {},
         "time_to_regenerate": str(datetime.now()),
-        "typespec_go_commit_hash": get_typespec_go_commit_hash(typespec_go_root)
+        "typespec_go_commit_hash": get_typespec_go_commit_hash(typespec_go_root),
     }
     # get all tsp-location.yaml
     commit_id = get_latest_commit_id()
@@ -242,6 +325,8 @@ def regenerate_sdk(use_latest_spec: bool, service_filter: str, sdk_root: str, ty
         if use_latest_spec:
             logging.info("Using latest spec")
             update_commit_id(item, commit_id)
+        # Record the original module name so it is not changed by regeneration
+        original_module = get_module_name(package_folder)
         try:
             # Get API version for this package
             api_version = get_api_version(package_folder)
@@ -299,7 +384,14 @@ def regenerate_sdk(use_latest_spec: bool, service_filter: str, sdk_root: str, ty
         else:
             logging.info(f"Successfully regenerated {package_folder.name}")
             result["succeed_to_regenerate"].append(package_folder.name)
-
+        finally:
+            # If regen dropped the module version suffix, revert the package and bump the spec instead
+            if original_module:
+                bumped_module = detect_module_version_change(package_folder, original_module)
+                if bumped_module:
+                    spec_directory = get_spec_directory(package_folder)
+                    if spec_directory:
+                        result["module_version_changed"][spec_directory] = bumped_module
     result["succeed_to_regenerate"].sort()
     result["fail_to_regenerate"].sort()
     result["not_found_api_version"].sort()
@@ -329,7 +421,56 @@ def git_add():
     check_call("git add .", shell=True)
 
 
-def main(sdk_root: str, typespec_go_root: str, typespec_go_branch: str, use_latest_spec: bool, service_filter: str, use_dev_package: bool):
+def bump_tspconfig_module(spec_root: str, spec_directory: str, bumped_module: str) -> bool:
+    """Update only the go module version suffix in tspconfig.yaml.
+
+    Returns True if the file was changed.
+    """
+    tspconfig_path = Path(spec_root) / spec_directory / "tspconfig.yaml"
+    if not tspconfig_path.exists():
+        logging.warning(f"tspconfig.yaml not found at {tspconfig_path}")
+        return False
+    suffix_match = re.search(r"/(v\d+)$", bumped_module)
+    new_suffix = f"/{suffix_match.group(1)}" if suffix_match else ""
+    with open(tspconfig_path, "r", encoding="utf-8") as f:
+        content = f.readlines()
+    changed = False
+    for idx in range(len(content)):
+        match = re.match(r"^(\s*module:\s*\"?)(\S+?)(/v\d+)?(\"?\s*)$", content[idx])
+        if match:
+            updated = f"{match.group(1)}{match.group(2)}{new_suffix}{match.group(4)}"
+            if updated != content[idx]:
+                content[idx] = updated
+                changed = True
+                logging.info(f"Updated module suffix in {tspconfig_path} to '{new_suffix}'")
+            break
+    if changed:
+        with open(tspconfig_path, "w", encoding="utf-8") as f:
+            f.writelines(content)
+    return changed
+
+
+def apply_tspconfig_module_bumps(spec_root: str, module_version_changed: dict) -> bool:
+    """Bump go module suffixes in tspconfig.yaml for changed packages.
+
+    Returns True if any tspconfig.yaml was modified. The pipeline is responsible for
+    committing and opening the PR against the spec repo.
+    """
+    if not module_version_changed:
+        logging.info("No module version changes; nothing to update in spec repo")
+        return False
+    if not spec_root or not Path(spec_root).exists():
+        logging.warning("spec-root not provided or does not exist; skipping tspconfig update")
+        return False
+
+    changed_any = False
+    for spec_directory, bumped_module in module_version_changed.items():
+        if bump_tspconfig_module(spec_root, spec_directory, bumped_module):
+            changed_any = True
+    return changed_any
+
+
+def main(sdk_root: str, typespec_go_root: str, typespec_go_branch: str, use_latest_spec: bool, service_filter: str, use_dev_package: bool, create_spec_pr_flag: bool, spec_root: str):
     # Configure logging for better pipeline visibility
     logging.basicConfig(
         level=logging.INFO,
@@ -342,10 +483,24 @@ def main(sdk_root: str, typespec_go_root: str, typespec_go_branch: str, use_late
     prepare_branch(typespec_go_branch)
     update_emitter_package(sdk_root, typespec_go_root, use_dev_package)
     result = regenerate_sdk(use_latest_spec, service_filter, sdk_root, typespec_go_root)
-    with open("regenerate-sdk-result.json", "w") as f:
-        json.dump(result, f, indent=2)
+
+    # Print the result instead of committing it to the repo
+    result_json = json.dumps(result, indent=2)
+    logging.info("Regenerate SDK result:\n%s", result_json)
+
+    # Write the result to the artifact staging directory so it can be published as a pipeline artifact
+    staging_dir = os.environ.get("BUILD_ARTIFACTSTAGINGDIRECTORY")
+    if staging_dir:
+        result_path = Path(staging_dir) / "regenerate-sdk-result.json"
+        with open(result_path, "w") as f:
+            f.write(result_json)
+        logging.info(f"Wrote regenerate-sdk-result.json to {result_path}")
+
     git_add()
 
+    if create_spec_pr_flag:
+        apply_tspconfig_module_bumps(spec_root, result.get("module_version_changed", {}))
+
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="SDK regeneration")
@@ -388,6 +543,20 @@ def main(sdk_root: str, typespec_go_root: str, typespec_go_branch: str, use_late
         default=False,
     )
 
+    parser.add_argument(
+        "--create-spec-pr",
+        help="Whether to create a PR in the spec repo to bump go module suffixes in tspconfig.yaml",
+        type=lambda x: x.lower() == 'true',
+        default=False,
+    )
+
+    parser.add_argument(
+        "--spec-root",
+        help="azure-rest-api-specs repo root folder (required when --create-spec-pr is true)",
+        type=str,
+        default="",
+    )
+
     args = parser.parse_args()
 
-    main(args.sdk_root, args.typespec_go_root, args.typespec_go_branch, args.use_latest_spec, args.service_filter, args.use_dev_package)
+    main(args.sdk_root, args.typespec_go_root, args.typespec_go_branch, args.use_latest_spec, args.service_filter, args.use_dev_package, args.create_spec_pr, args.spec_root)