From 248e2dee834dbe42ab86d1bdec00161141a82e82 Mon Sep 17 00:00:00 2001 From: Sandhya Dasu Date: Thu, 16 Jul 2026 14:48:02 -0400 Subject: [PATCH 1/2] Update featuregate-test-analyzer output for Install features Augment `verify-feature-promotion` output to indicate pass percentage for `install should succeed` tests for featuregates that include "Install" in their name. This update gives a better indication of whether Install features are failing at installation or later during execution of e2e conformance tests. This update does not change the criteria for reporting success but adds more information in the output for easier analysis of feature state. --- .../codegen/cmd/featuregate-test-analyzer.go | 237 ++++++++++++++++-- .../cmd/featuregate-test-analyzer_test.go | 185 +++++++++++++- tools/codegen/pkg/sippy/json_types.go | 98 ++++++++ 3 files changed, 499 insertions(+), 21 deletions(-) diff --git a/tools/codegen/cmd/featuregate-test-analyzer.go b/tools/codegen/cmd/featuregate-test-analyzer.go index 9eda83a226b..175fcebae7b 100644 --- a/tools/codegen/cmd/featuregate-test-analyzer.go +++ b/tools/codegen/cmd/featuregate-test-analyzer.go @@ -35,6 +35,9 @@ const ( // required pass rate. // nearly all current tests pass 99% of the time, but in a two week window we lack enough data to say. requiredPassRateOfTestsPerVariant = 0.95 + + // required pass rate for "install should succeed" test + requiredPassRateForInstallTest = 1.0 ) type FeatureGateTestAnalyzerOptions struct { @@ -181,7 +184,7 @@ func (o *FeatureGateTestAnalyzerOptions) Run(ctx context.Context) error { clusterProfiles := recentlyEnabledFeatureGatesToClusterProfiles[enabledFeatureGate] md.Title(1, enabledFeatureGate) - testingResults, err := listTestResultFor(enabledFeatureGate, clusterProfiles) + testingResults, installTestLevelData, err := listTestResultFor(enabledFeatureGate, clusterProfiles) if err != nil { return err } @@ -190,7 +193,7 @@ func (o *FeatureGateTestAnalyzerOptions) Run(ctx context.Context) error { validationResults := checkIfTestingIsSufficient(enabledFeatureGate, testingResults) - // Separate warnings from blocking errors + // Separate warnings and blocking errors blockingErrors := []error{} warnings := []error{} for _, vr := range validationResults { @@ -201,6 +204,38 @@ func (o *FeatureGateTestAnalyzerOptions) Run(ctx context.Context) error { } } + // For Install feature gates, report "install should succeed: overall" test statistics first + if strings.Contains(enabledFeatureGate, "Install") { + md.Text("") + fmt.Fprintf(o.Out, "\n") + md.Textf("**Install test statistics for \"install should succeed: overall\":**\n") + fmt.Fprintf(o.Out, "Install test statistics for \"install should succeed: overall\":\n") + jobVariants := make([]JobVariant, 0, len(testingResults)) + for jobVariant := range testingResults { + jobVariants = append(jobVariants, jobVariant) + } + sort.Slice(jobVariants, func(i, j int) bool { + return jobVariants[i].String() < jobVariants[j].String() + }) + for _, jobVariant := range jobVariants { + installTest := installTestLevelData[jobVariant] + if installTest == nil { + md.Textf(" - %v: test not found\n", jobVariant) + fmt.Fprintf(o.Out, " %v: test not found\n", jobVariant) + } else if installTest.TotalRuns > 0 { + passPercent := float32(installTest.SuccessfulRuns) / float32(installTest.TotalRuns) + displayActual := int(passPercent * 100) + md.Textf(" - %v: passed %d%% (%d/%d runs)\n", jobVariant, displayActual, installTest.SuccessfulRuns, installTest.TotalRuns) + fmt.Fprintf(o.Out, " %v: passed %d%% (%d/%d runs)\n", jobVariant, displayActual, installTest.SuccessfulRuns, installTest.TotalRuns) + } else { + md.Textf(" - %v: 0 runs\n", jobVariant) + fmt.Fprintf(o.Out, " %v: 0 runs\n", jobVariant) + } + } + md.Text("") + fmt.Fprintf(o.Out, "\n") + } + if len(validationResults) == 0 { md.Textf("Sufficient CI testing for %q.\n", enabledFeatureGate) fmt.Fprintf(o.Out, "Sufficient CI testing for %q.\n", enabledFeatureGate) @@ -208,17 +243,23 @@ func (o *FeatureGateTestAnalyzerOptions) Run(ctx context.Context) error { if len(blockingErrors) > 0 { md.Textf("INSUFFICIENT CI testing for %q.\n", enabledFeatureGate) fmt.Fprintf(o.Out, "INSUFFICIENT CI testing for %q.\n", enabledFeatureGate) - } else { + } else if len(warnings) > 0 { md.Textf("CI testing issues found for %q (non-blocking warnings).\n", enabledFeatureGate) fmt.Fprintf(o.Out, "CI testing issues found for %q (non-blocking warnings).\n", enabledFeatureGate) + } else { + md.Textf("Sufficient CI testing for %q.\n", enabledFeatureGate) + fmt.Fprintf(o.Out, "Sufficient CI testing for %q.\n", enabledFeatureGate) } - md.Textf("* At least five tests are expected for a feature\n") - md.Textf("* Tests must be be run on every TechPreview platform (ask for an exception if your feature doesn't support a variant)") - md.Textf("* All tests must run at least 14 times on every platform") - md.Textf("* All tests must pass at least 95%% of the time") - md.Textf("* JobTier must be one of: standard, informing, blocking, candidate (candidate is allowed but produces a warning as it is not covered by Component Readiness)\n") - md.Text("") + if len(blockingErrors) > 0 || len(warnings) > 0 { + md.Textf("* At least five tests are expected for a feature\n") + md.Textf("* Tests must be be run on every TechPreview platform (ask for an exception if your feature doesn't support a variant)") + md.Textf("* All tests must run at least 14 times on every platform") + md.Textf("* All tests must pass at least 95%% of the time") + md.Textf("* For Install feature gates, the \"install should succeed: overall\" test must pass at least 100%% of the time") + md.Textf("* JobTier must be one of: standard, informing, blocking, candidate (candidate is allowed but produces a warning as it is not covered by Component Readiness)\n") + md.Text("") + } if len(warnings) > 0 { md.Textf("**Non-blocking warnings (optional variants):**\n") @@ -394,6 +435,11 @@ func checkIfTestingIsSufficient(featureGate string, testingResults map[JobVarian } for _, testResults := range testedVariant.TestResults { + // Skip "install should succeed: overall" for Install feature gates - it has special validation below + if strings.Contains(featureGate, "Install") && testResults.TestName == "install should succeed: overall" { + continue + } + if testResults.TotalRuns < requiredNumberOfTestRunsPerVariant { results = append(results, ValidationResult{ Error: fmt.Errorf("error: %q only has %d runs, need at least %d runs for %q on %v", @@ -415,6 +461,38 @@ func checkIfTestingIsSufficient(featureGate string, testingResults map[JobVarian }) } } + + // For Install feature gates, validate "install should succeed: overall" test + if strings.Contains(featureGate, "Install") { + installTest := testResultByName(testedVariant.TestResults, "install should succeed: overall") + if installTest == nil { + results = append(results, ValidationResult{ + Error: fmt.Errorf("warning: \"install should succeed: overall\" test not found for Install feature gate %q on %v", + featureGate, jobVariant), + IsWarning: true, + }) + } else { + if installTest.TotalRuns < requiredNumberOfTestRunsPerVariant { + results = append(results, ValidationResult{ + Error: fmt.Errorf("error: \"install should succeed: overall\" only has %d runs, need at least %d runs for %q on %v", + installTest.TotalRuns, requiredNumberOfTestRunsPerVariant, featureGate, jobVariant), + IsWarning: isOptional, + }) + } + if installTest.TotalRuns > 0 { + passPercent := float32(installTest.SuccessfulRuns) / float32(installTest.TotalRuns) + if passPercent < requiredPassRateForInstallTest { + displayExpected := int(requiredPassRateForInstallTest * 100) + displayActual := int(passPercent * 100) + results = append(results, ValidationResult{ + Error: fmt.Errorf("error: \"install should succeed: overall\" only passed %d%%, need at least %d%% for %q on %v", + displayActual, displayExpected, featureGate, jobVariant), + IsWarning: isOptional, + }) + } + } + } + } } return results @@ -622,6 +700,20 @@ type JobVariant struct { Optional bool // If true, validation failures for this variant are non-blocking warnings } +func (jv JobVariant) String() string { + result := fmt.Sprintf("cloud=%s arch=%s topology=%s", jv.Cloud, jv.Architecture, jv.Topology) + if jv.NetworkStack != "" { + result += fmt.Sprintf(" network=%s", jv.NetworkStack) + } + if jv.OS != "" { + result += fmt.Sprintf(" os=%s", jv.OS) + } + if jv.Optional { + result += " optional=true" + } + return result +} + type OrderedJobVariants []JobVariant func (a OrderedJobVariants) Len() int { return len(a) } @@ -694,6 +786,7 @@ type TestResults struct { type ValidationResult struct { Error error IsWarning bool // if true, this is a non-blocking warning (for optional variants) + IsInfo bool // if true, this is informational telemetry (e.g., install test pass percentage) } func testResultByName(results []TestResults, testName string) *TestResults { @@ -736,10 +829,11 @@ func validateJobTiers(jobVariant JobVariant) error { return nil } -func listTestResultFor(featureGate string, clusterProfiles sets.Set[string]) (map[JobVariant]*TestingResults, error) { +func listTestResultFor(featureGate string, clusterProfiles sets.Set[string]) (map[JobVariant]*TestingResults, map[JobVariant]*TestResults, error) { fmt.Printf("Query sippy for all test run results for feature gate %q on clusterProfile %q\n", featureGate, sets.List(clusterProfiles)) results := map[JobVariant]*TestingResults{} + installTestLevelData := map[JobVariant]*TestResults{} var jobVariantsToCheck []JobVariant if clusterProfiles.Has("Hypershift") && !nonHypershiftPlatforms.MatchString(featureGate) { @@ -760,19 +854,28 @@ func listTestResultFor(featureGate string, clusterProfiles sets.Set[string]) (ma // Validate all variants before making expensive API calls for _, jobVariant := range jobVariantsToCheck { if err := validateJobTiers(jobVariant); err != nil { - return nil, err + return nil, nil, err } } for _, jobVariant := range jobVariantsToCheck { jobVariantResults, err := listTestResultForVariant(featureGate, jobVariant) if err != nil { - return nil, err + return nil, nil, err } results[jobVariant] = jobVariantResults + + // For Install feature gates, also get test-level data for "install should succeed: overall" + if strings.Contains(featureGate, "Install") { + installTestData, err := getInstallTestLevelData(featureGate, jobVariant) + if err != nil { + return nil, nil, err + } + installTestLevelData[jobVariant] = installTestData + } } - return results, nil + return results, installTestLevelData, nil } func filterVariants(featureGate string, variantsList ...[]JobVariant) []JobVariant { @@ -862,15 +965,111 @@ func getRelease() (string, error) { return getLatestRelease() } -func listTestResultForVariant(featureGate string, jobVariant JobVariant) (*TestingResults, error) { - // Substring here matches for both [OCPFeatureGate:...] and [FeatureGate:...] - testPattern := fmt.Sprintf("FeatureGate:%s]", featureGate) +func getInstallTestLevelData(featureGate string, jobVariant JobVariant) (*TestResults, error) { + testPattern := "install should succeed: overall" + queries := sippy.QueriesForWithCapability(jobVariant.Cloud, jobVariant.Architecture, jobVariant.Topology, + jobVariant.NetworkStack, jobVariant.OS, jobVariant.JobTiers, testPattern, featureGate) + + defaultTransport := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + ForceAttemptHTTP2: true, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, + }, + } + + sippyClient := &http.Client{ + Timeout: 2 * time.Minute, + Transport: defaultTransport, + } + + release, err := getRelease() + if err != nil { + return nil, fmt.Errorf("couldn't fetch latest release version: %w", err) + } + + var installTestResult *TestResults + for _, currQuery := range queries { + currURL := &url.URL{ + Scheme: "https", + Host: "sippy.dptools.openshift.org", + Path: "api/tests", + } + queryParams := currURL.Query() + queryParams.Add("release", release) + queryParams.Add("period", "default") + filterJSON, err := json.Marshal(currQuery) + if err != nil { + return nil, err + } + queryParams.Add("filter", string(filterJSON)) + currURL.RawQuery = queryParams.Encode() + + req, err := http.NewRequest(http.MethodGet, currURL.String(), nil) + if err != nil { + return nil, err + } + + response, err := sippyClient.Do(req) + if err != nil { + return nil, err + } + if response.StatusCode < 200 || response.StatusCode > 299 { + return nil, fmt.Errorf("error getting sippy results (status=%d) for: %v", response.StatusCode, currURL.String()) + } + queryResultBytes, err := io.ReadAll(response.Body) + if err != nil { + return nil, err + } + response.Body.Close() + testInfos := []sippy.SippyTestInfo{} + if err := json.Unmarshal(queryResultBytes, &testInfos); err != nil { + return nil, err + } + + for _, currTest := range testInfos { + if installTestResult == nil { + installTestResult = &TestResults{ + TestName: currTest.Name, + } + } + + // Accumulate results across multiple JobTier queries + if currTest.CurrentRuns >= requiredNumberOfTestRunsPerVariant { + installTestResult.TotalRuns += currTest.CurrentRuns + installTestResult.SuccessfulRuns += currTest.CurrentSuccesses + installTestResult.FailedRuns += currTest.CurrentFailures + installTestResult.FlakedRuns += currTest.CurrentFlakes + } else { + installTestResult.TotalRuns += currTest.CurrentRuns + currTest.PreviousRuns + installTestResult.SuccessfulRuns += currTest.CurrentSuccesses + currTest.PreviousSuccesses + installTestResult.FailedRuns += currTest.CurrentFailures + currTest.PreviousFailures + installTestResult.FlakedRuns += currTest.CurrentFlakes + currTest.PreviousFlakes + } + } + } + + return installTestResult, nil +} + +func listTestResultForVariant(featureGate string, jobVariant JobVariant) (*TestingResults, error) { // Feature gates used by the installer don't need separate tests, use the overall install tests if strings.Contains(featureGate, "Install") { return verifyJobBasedFeatureGatePromotion(featureGate, jobVariant) } + var testPattern string + var queries []*sippy.SippyQueryStruct + + // Substring here matches for both [OCPFeatureGate:...] and [FeatureGate:...] + testPattern = fmt.Sprintf("FeatureGate:%s]", featureGate) + queries = sippy.QueriesFor(jobVariant.Cloud, jobVariant.Architecture, jobVariant.Topology, + jobVariant.NetworkStack, jobVariant.OS, jobVariant.JobTiers, testPattern) fmt.Printf("Query sippy for all test run results for pattern %q on variant %#v\n", testPattern, jobVariant) defaultTransport := &http.Transport{ @@ -892,12 +1091,10 @@ func listTestResultForVariant(featureGate string, jobVariant JobVariant) (*Testi testNameToResults := map[string]*TestResults{} hasCandidateTierResults := false - queries := sippy.QueriesFor(jobVariant.Cloud, jobVariant.Architecture, jobVariant.Topology, jobVariant.NetworkStack, jobVariant.OS, jobVariant.JobTiers, testPattern) release, err := getRelease() if err != nil { return nil, fmt.Errorf("couldn't fetch latest release version: %w", err) } - fmt.Printf("Querying sippy release %s for test run results\n", release) for _, currQuery := range queries { currURL := &url.URL{ @@ -1139,7 +1336,7 @@ func getJobsForFeatureGateFromSippy(client *http.Client, release, featureGate st defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("expected a 200 OK status code but got %s", resp.StatusCode) + return nil, fmt.Errorf("expected a 200 OK status code but got %d", resp.StatusCode) } body, err := io.ReadAll(resp.Body) @@ -1165,7 +1362,7 @@ func getJobRunsFromSippy(client *http.Client, release, jobName string) ([]sippy. defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("expected a 200 OK status code but got %s", resp.StatusCode) + return nil, fmt.Errorf("expected a 200 OK status code but got %d", resp.StatusCode) } body, err := io.ReadAll(resp.Body) diff --git a/tools/codegen/cmd/featuregate-test-analyzer_test.go b/tools/codegen/cmd/featuregate-test-analyzer_test.go index 24d2bad6c20..2c22e5da4e9 100644 --- a/tools/codegen/cmd/featuregate-test-analyzer_test.go +++ b/tools/codegen/cmd/featuregate-test-analyzer_test.go @@ -65,7 +65,7 @@ func Test_listTestResultFor(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Skip("this is for ease of manual testing") - got, err := listTestResultFor(tt.args.featureGate, sets.New[string](tt.args.clusterProfile)) + got, _, err := listTestResultFor(tt.args.featureGate, sets.New[string](tt.args.clusterProfile)) if (err != nil) != tt.wantErr { t.Errorf("listTestResultFor() error = %v, wantErr %v", err, tt.wantErr) return @@ -481,6 +481,189 @@ func Test_checkIfTestingIsSufficient_OptionalVariants(t *testing.T) { } } +func Test_checkIfTestingIsSufficient_InstallFeatureGates(t *testing.T) { + tests := []struct { + name string + featureGate string + testingResults map[JobVariant]*TestingResults + wantBlockingErrors int + wantWarnings int + }{ + { + name: "Install feature gate with install should succeed: overall test failing 100% requirement", + featureGate: "FakeInstallFeature", + testingResults: map[JobVariant]*TestingResults{ + { + Cloud: "metal", + Architecture: "amd64", + Topology: "ha", + }: { + TestResults: []TestResults{ + {TestName: "install should succeed: overall", TotalRuns: 20, SuccessfulRuns: 19}, + {TestName: "test1", TotalRuns: 15, SuccessfulRuns: 15}, + {TestName: "test2", TotalRuns: 15, SuccessfulRuns: 15}, + {TestName: "test3", TotalRuns: 15, SuccessfulRuns: 15}, + {TestName: "test4", TotalRuns: 15, SuccessfulRuns: 15}, + }, + }, + }, + wantBlockingErrors: 1, // Blocking error: install should succeed: overall must pass at 100% + wantWarnings: 0, + }, + { + name: "Install feature gate without install should succeed: overall test - warning reported", + featureGate: "MockInstallGate", + testingResults: map[JobVariant]*TestingResults{ + { + Cloud: "metal", + Architecture: "amd64", + Topology: "ha", + }: { + TestResults: []TestResults{ + {TestName: "test1", TotalRuns: 15, SuccessfulRuns: 15}, + {TestName: "test2", TotalRuns: 15, SuccessfulRuns: 15}, + {TestName: "test3", TotalRuns: 15, SuccessfulRuns: 15}, + {TestName: "test4", TotalRuns: 15, SuccessfulRuns: 15}, + {TestName: "test5", TotalRuns: 15, SuccessfulRuns: 15}, + }, + }, + }, + wantBlockingErrors: 0, + wantWarnings: 1, // Warning about missing "install should succeed: overall" test + }, + { + name: "Non-Install feature gate with install should succeed: overall test - no special validation", + featureGate: "SomeOtherFeature", + testingResults: map[JobVariant]*TestingResults{ + { + Cloud: "aws", + Architecture: "amd64", + Topology: "ha", + }: { + TestResults: []TestResults{ + {TestName: "install should succeed: overall", TotalRuns: 20, SuccessfulRuns: 19}, + {TestName: "test1", TotalRuns: 15, SuccessfulRuns: 15}, + {TestName: "test2", TotalRuns: 15, SuccessfulRuns: 15}, + {TestName: "test3", TotalRuns: 15, SuccessfulRuns: 15}, + {TestName: "test4", TotalRuns: 15, SuccessfulRuns: 15}, + }, + }, + }, + wantBlockingErrors: 0, + wantWarnings: 0, + }, + { + name: "Install feature gate with 100% pass rate - no errors", + featureGate: "FakeInstallFeature", + testingResults: map[JobVariant]*TestingResults{ + { + Cloud: "aws", + Architecture: "amd64", + Topology: "ha", + }: { + TestResults: []TestResults{ + {TestName: "install should succeed: overall", TotalRuns: 20, SuccessfulRuns: 20}, + {TestName: "test1", TotalRuns: 15, SuccessfulRuns: 15}, + {TestName: "test2", TotalRuns: 15, SuccessfulRuns: 15}, + {TestName: "test3", TotalRuns: 15, SuccessfulRuns: 15}, + {TestName: "test4", TotalRuns: 15, SuccessfulRuns: 15}, + }, + }, + }, + wantBlockingErrors: 0, + wantWarnings: 0, + }, + { + name: "Install feature gate with multiple variants - one fails 100% requirement", + featureGate: "FakeInstallFeature", + testingResults: map[JobVariant]*TestingResults{ + { + Cloud: "metal", + Architecture: "amd64", + Topology: "ha", + }: { + TestResults: []TestResults{ + {TestName: "install should succeed: overall", TotalRuns: 20, SuccessfulRuns: 19}, + {TestName: "test1", TotalRuns: 15, SuccessfulRuns: 15}, + {TestName: "test2", TotalRuns: 15, SuccessfulRuns: 15}, + {TestName: "test3", TotalRuns: 15, SuccessfulRuns: 15}, + {TestName: "test4", TotalRuns: 15, SuccessfulRuns: 15}, + }, + }, + { + Cloud: "metal", + Architecture: "amd64", + Topology: "single", + }: { + TestResults: []TestResults{ + {TestName: "install should succeed: overall", TotalRuns: 18, SuccessfulRuns: 18}, + {TestName: "test1", TotalRuns: 15, SuccessfulRuns: 15}, + {TestName: "test2", TotalRuns: 15, SuccessfulRuns: 15}, + {TestName: "test3", TotalRuns: 15, SuccessfulRuns: 15}, + {TestName: "test4", TotalRuns: 15, SuccessfulRuns: 15}, + }, + }, + }, + wantBlockingErrors: 1, // One variant (ha) fails 100% requirement + wantWarnings: 0, + }, + { + name: "Install feature gate with insufficient runs for install test - blocking error", + featureGate: "MockInstallGate", + testingResults: map[JobVariant]*TestingResults{ + { + Cloud: "aws", + Architecture: "amd64", + Topology: "ha", + }: { + TestResults: []TestResults{ + {TestName: "install should succeed: overall", TotalRuns: 10, SuccessfulRuns: 10}, + {TestName: "test1", TotalRuns: 15, SuccessfulRuns: 15}, + {TestName: "test2", TotalRuns: 15, SuccessfulRuns: 15}, + {TestName: "test3", TotalRuns: 15, SuccessfulRuns: 15}, + {TestName: "test4", TotalRuns: 15, SuccessfulRuns: 15}, + }, + }, + }, + wantBlockingErrors: 1, // Blocking error for insufficient runs (< 14) + wantWarnings: 0, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + results := checkIfTestingIsSufficient(tt.featureGate, tt.testingResults) + + blockingErrors := 0 + warnings := 0 + for _, result := range results { + if result.IsWarning { + warnings++ + } else { + blockingErrors++ + } + } + + if blockingErrors != tt.wantBlockingErrors { + t.Errorf("got %d blocking errors, want %d", blockingErrors, tt.wantBlockingErrors) + for _, result := range results { + if !result.IsWarning { + t.Logf(" Blocking error: %v", result.Error) + } + } + } + if warnings != tt.wantWarnings { + t.Errorf("got %d warnings, want %d", warnings, tt.wantWarnings) + for _, result := range results { + if result.IsWarning { + t.Logf(" Warning: %v", result.Error) + } + } + } + }) + } +} + func Test_defaultQueriesIncludeCandidateTier(t *testing.T) { // When JobTiers is empty, QueriesFor should generate queries for all tiers // including candidate. This test is added to prevent regressions for candidate-tier diff --git a/tools/codegen/pkg/sippy/json_types.go b/tools/codegen/pkg/sippy/json_types.go index ad2012a3ef5..2242eab6ac9 100644 --- a/tools/codegen/pkg/sippy/json_types.go +++ b/tools/codegen/pkg/sippy/json_types.go @@ -179,6 +179,104 @@ func QueriesFor(cloud, architecture, topology, networkStack, os, jobTiers, testP return queries } +func QueriesForWithCapability(cloud, architecture, topology, networkStack, os, jobTiers, testPattern, capability string) []*SippyQueryStruct { + // Build base query items that are common to all JobTier queries + baseItems := []SippyQueryItem{ + { + ColumnField: "variants", + Not: false, + OperatorValue: "contains", + Value: fmt.Sprintf("Platform:%s", cloud), + }, + { + ColumnField: "variants", + Not: false, + OperatorValue: "contains", + Value: fmt.Sprintf("Architecture:%s", architecture), + }, + { + ColumnField: "variants", + Not: false, + OperatorValue: "contains", + Value: fmt.Sprintf("Topology:%s", topology), + }, + { + ColumnField: "name", + Not: false, + OperatorValue: "contains", + Value: testPattern, + }, + { + ColumnField: "variants", + Not: false, + OperatorValue: "contains", + Value: fmt.Sprintf("Capability:%s", capability), + }, + } + + if networkStack != "" { + baseItems = append(baseItems, SippyQueryItem{ + ColumnField: "variants", + Not: false, + OperatorValue: "contains", + Value: fmt.Sprintf("NetworkStack:%s", networkStack), + }) + } + + if os != "" { + baseItems = append(baseItems, SippyQueryItem{ + ColumnField: "variants", + Not: false, + OperatorValue: "contains", + Value: fmt.Sprintf("OS:%s", os), + }) + } + + // Parse JobTiers - comma-separated string, default to standard/informing/blocking/candidate if empty + var jobTiersList []string + if jobTiers == "" { + jobTiersList = []string{"standard", "informing", "blocking", "candidate"} + } else { + // Split by comma, trim whitespace, and deduplicate using sets + tierSet := sets.New[string]() + for _, tier := range strings.Split(jobTiers, ",") { + if trimmed := strings.TrimSpace(tier); trimmed != "" { + tierSet.Insert(trimmed) + } + } + // If all tiers were whitespace/empty after trimming, use defaults + if tierSet.Len() == 0 { + jobTiersList = []string{"standard", "informing", "blocking", "candidate"} + } else { + jobTiersList = sets.List(tierSet) + } + } + + // Generate one query per JobTier (to work around API's single LinkOperator limitation) + var queries []*SippyQueryStruct + for _, jobTier := range jobTiersList { + // Copy base items for this query + items := make([]SippyQueryItem, len(baseItems)) + copy(items, baseItems) + + // Add JobTier filter + items = append(items, SippyQueryItem{ + ColumnField: "variants", + Not: false, + OperatorValue: "contains", + Value: fmt.Sprintf("JobTier:%s", jobTier), + }) + + queries = append(queries, &SippyQueryStruct{ + Items: items, + LinkOperator: "and", + TierName: jobTier, + }) + } + + return queries +} + func BuildSippyTestAnalysisURL(release, testName, topology, cloud, architecture, networkStack, os string) string { filterItems := []SippyQueryItem{ { From 085632883a6daeb70341745af5d9f6ee0eea9d1a Mon Sep 17 00:00:00 2001 From: Sandhya Dasu Date: Fri, 17 Jul 2026 13:40:52 -0400 Subject: [PATCH 2/2] Updated feature promotion pass condition for Install features Installer features now require a 100% pass rate for installation success and 80% pass rate for e2e test runs. --- .../codegen/cmd/featuregate-test-analyzer.go | 91 ++++++++++++++++--- 1 file changed, 80 insertions(+), 11 deletions(-) diff --git a/tools/codegen/cmd/featuregate-test-analyzer.go b/tools/codegen/cmd/featuregate-test-analyzer.go index 175fcebae7b..181e44d8b15 100644 --- a/tools/codegen/cmd/featuregate-test-analyzer.go +++ b/tools/codegen/cmd/featuregate-test-analyzer.go @@ -36,6 +36,10 @@ const ( // nearly all current tests pass 99% of the time, but in a two week window we lack enough data to say. requiredPassRateOfTestsPerVariant = 0.95 + // required pass rate for Install feature gates (non-install tests) + // Install features have a lower bar for e2e test pass percentage + requiredPassRateForInstallFeatures = 0.80 + // required pass rate for "install should succeed" test requiredPassRateForInstallTest = 1.0 ) @@ -189,7 +193,7 @@ func (o *FeatureGateTestAnalyzerOptions) Run(ctx context.Context) error { return err } - writeTestingMarkDown(testingResults, md) + writeTestingMarkDown(enabledFeatureGate, testingResults, md) validationResults := checkIfTestingIsSufficient(enabledFeatureGate, testingResults) @@ -206,6 +210,14 @@ func (o *FeatureGateTestAnalyzerOptions) Run(ctx context.Context) error { // For Install feature gates, report "install should succeed: overall" test statistics first if strings.Contains(enabledFeatureGate, "Install") { + md.Text("") + fmt.Fprintf(o.Out, "\n") + md.Textf("**ℹ️ Install Feature Gate Promotion Criteria:**\n") + fmt.Fprintf(o.Out, "ℹ️ Install Feature Gate Promotion Criteria:\n") + md.Textf("- E2E tests must pass at least **80%%** of the time (at least %d runs per variant)\n", requiredNumberOfTestRunsPerVariant) + fmt.Fprintf(o.Out, "- E2E tests must pass at least 80%% of the time (at least %d runs per variant)\n", requiredNumberOfTestRunsPerVariant) + md.Textf("- \"install should succeed: overall\" test must pass **100%%** of the time\n") + fmt.Fprintf(o.Out, "- \"install should succeed: overall\" test must pass 100%% of the time\n") md.Text("") fmt.Fprintf(o.Out, "\n") md.Textf("**Install test statistics for \"install should succeed: overall\":**\n") @@ -234,6 +246,42 @@ func (o *FeatureGateTestAnalyzerOptions) Run(ctx context.Context) error { } md.Text("") fmt.Fprintf(o.Out, "\n") + + // Report additional e2e tests if they exist for Install feature gates + additionalTests := []string{} + for _, variant := range testingResults { + for _, test := range variant.TestResults { + if test.TestName != "install should succeed: overall" { + // Add unique test names + found := false + for _, existing := range additionalTests { + if existing == test.TestName { + found = true + break + } + } + if !found { + additionalTests = append(additionalTests, test.TestName) + } + } + } + } + + if len(additionalTests) > 0 { + md.Textf("**Additional E2E tests found: %d test(s)**\n", len(additionalTests)) + fmt.Fprintf(o.Out, "Additional E2E tests found: %d test(s)\n", len(additionalTests)) + for _, testName := range additionalTests { + md.Textf(" - %s\n", testName) + fmt.Fprintf(o.Out, " - %s\n", testName) + } + md.Text("") + fmt.Fprintf(o.Out, "\n") + } else { + md.Textf("**Additional E2E tests: None found**\n") + fmt.Fprintf(o.Out, "Additional E2E tests: None found\n") + md.Text("") + fmt.Fprintf(o.Out, "\n") + } } if len(validationResults) == 0 { @@ -366,7 +414,12 @@ func buildHTMLFeatureGateData(name string, testingResults map[JobVariant]*Testin cell.SuccessfulRuns = testResults.SuccessfulRuns cell.TotalRuns = testResults.TotalRuns cell.FailedRuns = testResults.FailedRuns - if testResults.TotalRuns < requiredNumberOfTestRunsPerVariant || passPercent < requiredPassRateOfTestsPerVariant { + // Install features have a lower pass rate requirement for e2e tests + requiredPassRate := requiredPassRateOfTestsPerVariant + if strings.Contains(name, "Install") { + requiredPassRate = requiredPassRateForInstallFeatures + } + if testResults.TotalRuns < requiredNumberOfTestRunsPerVariant || passPercent < float32(requiredPassRate) { cell.Failed = true } } @@ -426,7 +479,8 @@ func checkIfTestingIsSufficient(featureGate string, testingResults map[JobVarian }) } - if len(testedVariant.TestResults) < requiredNumberOfTests { + // For Install feature gates, additional tests are optional - only "install should succeed: overall" is required + if !strings.Contains(featureGate, "Install") && len(testedVariant.TestResults) < requiredNumberOfTests { results = append(results, ValidationResult{ Error: fmt.Errorf("error: only %d tests found, need at least %d for %q on %v", len(testedVariant.TestResults), requiredNumberOfTests, featureGate, jobVariant), @@ -451,8 +505,13 @@ func checkIfTestingIsSufficient(featureGate string, testingResults map[JobVarian continue } passPercent := float32(testResults.SuccessfulRuns) / float32(testResults.TotalRuns) - if passPercent < requiredPassRateOfTestsPerVariant { - displayExpected := int(requiredPassRateOfTestsPerVariant * 100) + // Install features have a lower pass rate requirement for e2e tests + requiredPassRate := requiredPassRateOfTestsPerVariant + if strings.Contains(featureGate, "Install") { + requiredPassRate = requiredPassRateForInstallFeatures + } + if passPercent < float32(requiredPassRate) { + displayExpected := int(requiredPassRate * 100) displayActual := int(passPercent * 100) results = append(results, ValidationResult{ Error: fmt.Errorf("error: %q only passed %d%%, need at least %d%% for %q on %v", @@ -498,7 +557,7 @@ func checkIfTestingIsSufficient(featureGate string, testingResults map[JobVarian return results } -func writeTestingMarkDown(testingResults map[JobVariant]*TestingResults, md *utils.Markdown) { +func writeTestingMarkDown(featureGate string, testingResults map[JobVariant]*TestingResults, md *utils.Markdown) { jobVariantsSet := sets.KeySet(testingResults) jobVariants := jobVariantsSet.UnsortedList() sort.Sort(OrderedJobVariants(jobVariants)) @@ -549,10 +608,15 @@ func writeTestingMarkDown(testingResults map[JobVariant]*TestingResults, md *uti } failString := "" passPercent := float32(testResults.SuccessfulRuns) / float32(testResults.TotalRuns) + // Install features have a lower pass rate requirement for e2e tests + requiredPassRate := requiredPassRateOfTestsPerVariant + if strings.Contains(featureGate, "Install") { + requiredPassRate = requiredPassRateForInstallFeatures + } switch { case testResults.TotalRuns < requiredNumberOfTestRunsPerVariant: failString = "FAIL
" - case passPercent < requiredPassRateOfTestsPerVariant: + case passPercent < float32(requiredPassRate): failString = "FAIL
" } cellString := fmt.Sprintf("%s%d%% ( %d / %d ) ", failString, int(passPercent*100), testResults.SuccessfulRuns, testResults.TotalRuns) @@ -1218,7 +1282,7 @@ func verifyJobBasedFeatureGatePromotion(featureGate string, jobVariant JobVarian testResults := []TestResults{} for _, job := range jobs { - results, err := verifyJobPassRate(sippyClient, ocpRelease, job, jobVariant) + results, err := verifyJobPassRate(sippyClient, ocpRelease, job, jobVariant, featureGate) if err != nil { return nil, fmt.Errorf("verifying job pass rate for job %q: %w", job.Name, err) } @@ -1232,7 +1296,7 @@ func verifyJobBasedFeatureGatePromotion(featureGate string, jobVariant JobVarian }, nil } -func verifyJobPassRate(client *http.Client, release string, job sippy.SippyJob, variant JobVariant) (*TestResults, error) { +func verifyJobPassRate(client *http.Client, release string, job sippy.SippyJob, variant JobVariant, featureGate string) (*TestResults, error) { // Do an early check for 95% pass rate with at least 14 runs runs := job.CurrentRuns passes := job.CurrentPasses @@ -1257,12 +1321,17 @@ func verifyJobPassRate(client *http.Client, release string, job sippy.SippyJob, }, nil } - // If we have greater than or equal to 14 runs AND they are passing at a rate of at least 95%, + // If we have greater than or equal to 14 runs AND they are passing at a rate of at least the required pass rate, // we can return early because this job has passed the promotion requirements. // // This saves us from unnecessarily making calls out to Sippy to perform a more nuanced // failures analysis of the job runs to see if failed runs are true failures or known regressions. - if float32(passes) / float32(runs) >= requiredPassRateOfTestsPerVariant { + // Install features have a lower pass rate requirement (80%) for e2e tests + requiredPassRate := requiredPassRateOfTestsPerVariant + if strings.Contains(featureGate, "Install") { + requiredPassRate = requiredPassRateForInstallFeatures + } + if float32(passes) / float32(runs) >= float32(requiredPassRate) { return &TestResults{ TestName: job.Name, TotalRuns: runs,