diff --git a/.github/workflows/unit-test-mmv1.yml b/.github/workflows/unit-test-mmv1.yml index ddcda7b69649..0621df306c20 100644 --- a/.github/workflows/unit-test-mmv1.yml +++ b/.github/workflows/unit-test-mmv1.yml @@ -90,7 +90,7 @@ jobs: cd repo if [ "${{ github.event_name }}" == "pull_request" ]; then # For PRs, get only changed files - yamlfiles=$(git diff --name-only origin/${GITHUB_BASE_REF} -- mmv1/products) + yamlfiles=$(git diff --name-only --diff-filter=d origin/${GITHUB_BASE_REF} -- mmv1/products) if [ ! -z "$yamlfiles" ]; then echo "yamlfiles=${yamlfiles//$'\n'/ }" >> $GITHUB_OUTPUT fi diff --git a/mmv1/products/vertexai/OnlineEvaluator.yaml b/mmv1/products/vertexai/OnlineEvaluator.yaml deleted file mode 100644 index 671d2655c6ee..000000000000 --- a/mmv1/products/vertexai/OnlineEvaluator.yaml +++ /dev/null @@ -1,200 +0,0 @@ ---- -# Copyright 2026 Google Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -name: OnlineEvaluator -description: Description -base_url: projects/{{project}}/locations/{{region}}/onlineEvaluators -min_version: beta -self_link: projects/{{project}}/locations/{{region}}/onlineEvaluators/{{name}} -create_url: projects/{{project}}/locations/{{region}}/onlineEvaluators -update_mask: true -update_verb: PATCH -timeouts: - insert_minutes: 20 - update_minutes: 20 - delete_minutes: 60 -async: - type: OpAsync - operation: - base_url: '{{op_id}}' - actions: - - create - - update - - delete - result: - resource_inside_response: true -autogen_status: T25saW5lRXZhbHVhdG9y -samples: - - name: vertex_ai_online_evaluator_basic - primary_resource_id: evaluator - steps: - - name: vertex_ai_online_evaluator_basic - resource_id_vars: - evaluator_name: my-evaluator - engine_name: my-engine -parameters: - - name: region - type: String - required: true - description: Resource ID segment making up resource `name`. It identifies the resource within its parent collection as described in https://google.aip.dev/122. - immutable: true - url_param_only: true -properties: - - name: agentResource - type: String - required: true - description: The name of the agent that the OnlineEvaluator evaluates periodically. This value is used to filter the traces with a matching cloud.resource_id and link the evaluation results with relevant dashboards/UIs. This field is immutable. Once set, it cannot be changed. - immutable: true - - name: cloudObservability - type: NestedObject - description: Data source for the OnlineEvaluator, based on Google Cloud Observability stack (Cloud Trace & Cloud Logging). - properties: - - name: logView - type: String - description: Optional log view that will be used to query logs. If empty, the `_Default` view will be used. - immutable: true - - name: openTelemetry - type: NestedObject - description: Configuration for data source following OpenTelemetry. - immutable: true - ignore_read: true - properties: - - name: semconvVersion - type: String - required: true - description: Defines which version OTel Semantic Convention the data follows. Can be "1.39.0" or newer. - immutable: true - - name: traceScope - type: NestedObject - description: If chosen, the online evaluator will evaluate single traces matching specified `filter`. - immutable: true - properties: - - name: filter - type: Array - description: A list of predicates to filter traces. Multiple predicates are combined using AND. The maximum number of predicates is 10. - immutable: true - item_type: - type: NestedObject - properties: - - name: duration - type: NestedObject - description: Defines a predicate for filtering based on a numeric value. - immutable: true - properties: - - name: comparisonOperator - type: String - required: true - description: 'The comparison operator to apply. Possible values: LESS LESS_OR_EQUAL EQUAL NOT_EQUAL GREATER_OR_EQUAL GREATER' - immutable: true - - name: value - type: Double - required: true - description: The value to compare against. - immutable: true - - name: totalTokenUsage - type: NestedObject - description: Defines a predicate for filtering based on a numeric value. - immutable: true - properties: - - name: comparisonOperator - type: String - required: true - description: 'The comparison operator to apply. Possible values: LESS LESS_OR_EQUAL EQUAL NOT_EQUAL GREATER_OR_EQUAL GREATER' - immutable: true - - name: value - type: Double - required: true - description: The value to compare against. - immutable: true - - name: traceView - type: String - description: 'Optional trace view that will be used to query traces. If empty, the `_Default` view will be used. NOTE: This field is not supported yet and will be ignored if set.' - immutable: true - required: true - immutable: true - - name: config - type: NestedObject - required: true - description: Configuration for sampling behavior of the OnlineEvaluator. The OnlineEvaluator runs at a fixed interval of 10 minutes. - properties: - - name: maxEvaluatedSamplesPerRun - type: String - description: The maximum number of evaluations to perform per run. If set to 0, the number is unbounded. - - name: randomSampling - type: NestedObject - description: Configuration for random sampling. - properties: - - name: percentage - type: Integer - required: true - description: The percentage of traces to sample for evaluation. Must be an integer between `1` and `100`. - - name: createTime - type: String - description: Timestamp when the OnlineEvaluator was created. - output: true - - name: displayName - type: String - description: Human-readable name for the `OnlineEvaluator`. The name doesn't have to be unique. The name can consist of any UTF-8 characters. The maximum length is `63` characters. If the display name exceeds max characters, an `INVALID_ARGUMENT` error is returned. - - name: metricSources - type: Array - required: true - immutable: true - description: A list of metric sources to be used for evaluating samples. At least one MetricSource must be provided. Right now, only predefined metrics and registered metrics are supported. Every registered metric must have `display_name` (or `title`) and `score_range` defined. Otherwise, the evaluations will fail. The maximum number of `metric_sources` is 25. - item_type: - type: NestedObject - properties: - # TODO: Support structured metric config instead of JSON string. - - name: metric - type: String - description: | - Inline metric config. Provide this field as a JSON-formatted string. - Suggested predefined metricSpecName values: - - `final_response_quality_v1` - - `tool_use_quality_v1` - - `hallucination_v1` - - `safety_v1` - - `multi_turn_task_success_v1` - - `multi_turn_tool_use_quality_v1` - - `multi_turn_trajectory_quality_v1` - immutable: true - custom_expand: templates/terraform/custom_expand/json_value.tmpl - custom_flatten: templates/terraform/custom_flatten/json_schema.tmpl - - name: metricResourceName - type: String - description: Resource name for registered metric. - immutable: true - - name: name - type: String - description: 'Identifier. The resource name of the OnlineEvaluator. Format: projects/{project}/locations/{region}/onlineEvaluators/{id}.' - output: true - custom_flatten: templates/terraform/custom_flatten/name_from_self_link.tmpl - - name: state - type: String - description: 'The state of the OnlineEvaluator. Possible values: ACTIVE SUSPENDED FAILED WARNING' - output: true - - name: stateDetails - type: Array - description: Contains additional information about the state of the OnlineEvaluator. This is used to provide more details in the event of a failure. - output: true - item_type: - type: NestedObject - output: true - properties: - - name: message - type: String - description: Human-readable message describing the state of the OnlineEvaluator. - output: true - - name: updateTime - type: String - description: Timestamp when the OnlineEvaluator was last updated. - output: true diff --git a/mmv1/templates/terraform/samples/services/vertexai/vertex_ai_online_evaluator_basic.tf.tmpl b/mmv1/templates/terraform/samples/services/vertexai/vertex_ai_online_evaluator_basic.tf.tmpl deleted file mode 100644 index 137cce667b7d..000000000000 --- a/mmv1/templates/terraform/samples/services/vertexai/vertex_ai_online_evaluator_basic.tf.tmpl +++ /dev/null @@ -1,71 +0,0 @@ -resource "google_vertex_ai_reasoning_engine" "engine" { - provider = google-beta - display_name = "{{index $.ResourceIdVars "engine_name"}}" - description = "A basic reasoning engine" - labels = { - "key" = "value" - } - region = "us-central1" -} - -resource "google_vertex_ai_online_evaluator" "evaluator" { - provider = google-beta - region = "us-central1" - display_name = "{{index $.ResourceIdVars "evaluator_name"}}" - - agent_resource = google_vertex_ai_reasoning_engine.engine.id - - config { - max_evaluated_samples_per_run = "100" - random_sampling { - percentage = 10 - } - } - - metric_sources { - metric = jsonencode({ - "predefinedMetricSpec" = { - "metricSpecName" = "safety_v1" - } - }) - } - - metric_sources { - metric = jsonencode({ - "predefinedMetricSpec" = { - "metricSpecName" = "hallucination_v1" - } - }) - } - - metric_sources { - metric = jsonencode({ - "predefinedMetricSpec" = { - "metricSpecName" = "final_response_quality_v1" - } - }) - } - - metric_sources { - metric = jsonencode({ - "predefinedMetricSpec" = { - "metricSpecName" = "tool_use_quality_v1" - } - }) - } - - cloud_observability { - open_telemetry { - semconv_version = "1.39.0" - } - - trace_scope { - filter { - duration { - comparison_operator = "GREATER" - value = 0 - } - } - } - } -} diff --git a/mmv1/third_party/terraform/services/vertexai/resource_vertex_ai_online_evaluator_test.go.tmpl b/mmv1/third_party/terraform/services/vertexai/resource_vertex_ai_online_evaluator_test.go.tmpl deleted file mode 100644 index 74425e4914e9..000000000000 --- a/mmv1/third_party/terraform/services/vertexai/resource_vertex_ai_online_evaluator_test.go.tmpl +++ /dev/null @@ -1,180 +0,0 @@ -package vertexai_test - -{{ if ne $.TargetVersionName "ga" -}} -import ( - "testing" - - "github.com/hashicorp/terraform-plugin-testing/helper/resource" - "github.com/hashicorp/terraform-provider-google/google/acctest" -) - -func TestAccVertexAIOnlineEvaluator_update(t *testing.T) { - t.Parallel() - - randomSuffix := acctest.RandString(t, 10) - - context := map[string]interface{}{ - "random_suffix": randomSuffix, - } - - acctest.VcrTest(t, resource.TestCase{ - PreCheck: func() { acctest.AccTestPreCheck(t) }, - ProtoV5ProviderFactories: acctest.ProtoV5ProviderBetaFactories(t), - CheckDestroy: testAccCheckVertexAIOnlineEvaluatorDestroyProducer(t), - Steps: []resource.TestStep{ - { - Config: testAccVertexAIOnlineEvaluator_basic(context), - }, - { - ResourceName: "google_vertex_ai_online_evaluator.evaluator", - ImportState: true, - ImportStateVerify: true, - ImportStateVerifyIgnore: []string{"region", "project", "cloud_observability.0.open_telemetry"}, - }, - { - Config: testAccVertexAIOnlineEvaluator_update(context), - }, - { - ResourceName: "google_vertex_ai_online_evaluator.evaluator", - ImportState: true, - ImportStateVerify: true, - ImportStateVerifyIgnore: []string{"region", "project", "cloud_observability.0.open_telemetry"}, - }, - }, - }) -} - -func testAccVertexAIOnlineEvaluator_basic(context map[string]interface{}) string { - return acctest.Nprintf(` -resource "google_vertex_ai_reasoning_engine" "engine" { - provider = google-beta - display_name = "tf-test-re-%{random_suffix}" - description = "A basic reasoning engine" - labels = { - "key" = "value" - } - region = "us-central1" -} - -resource "google_vertex_ai_online_evaluator" "evaluator" { - provider = google-beta - region = "us-central1" - display_name = "tf-test-eval-%{random_suffix}" - - agent_resource = google_vertex_ai_reasoning_engine.engine.id - - config { - max_evaluated_samples_per_run = "100" - random_sampling { - percentage = 10 - } - } - - metric_sources { - metric = jsonencode({ - "predefinedMetricSpec" = { - "metricSpecName" = "safety_v1" - } - }) - } - - metric_sources { - metric = jsonencode({ - "predefinedMetricSpec" = { - "metricSpecName" = "hallucination_v1" - } - }) - } - - cloud_observability { - open_telemetry { - semconv_version = "1.39.0" - } - - trace_scope { - filter { - duration { - comparison_operator = "GREATER" - value = 0 - } - } - filter { - total_token_usage { - comparison_operator = "LESS" - value = 1000 - } - } - } - } -} -`, context) -} - -func testAccVertexAIOnlineEvaluator_update(context map[string]interface{}) string { - return acctest.Nprintf(` -resource "google_vertex_ai_reasoning_engine" "engine" { - provider = google-beta - display_name = "tf-test-re-%{random_suffix}" - description = "A basic reasoning engine" - labels = { - "key" = "value" - } - region = "us-central1" -} - -resource "google_vertex_ai_online_evaluator" "evaluator" { - provider = google-beta - region = "us-central1" - display_name = "tf-test-eval-upd-%{random_suffix}" - - agent_resource = google_vertex_ai_reasoning_engine.engine.id - - config { - max_evaluated_samples_per_run = "200" - random_sampling { - percentage = 20 - } - } - - metric_sources { - metric = jsonencode({ - "predefinedMetricSpec" = { - "metricSpecName" = "safety_v1" - } - }) - } - - metric_sources { - metric = jsonencode({ - "predefinedMetricSpec" = { - "metricSpecName" = "hallucination_v1" - } - }) - } - - cloud_observability { - open_telemetry { - semconv_version = "1.39.0" - } - - trace_scope { - filter { - duration { - comparison_operator = "GREATER" - value = 0 - } - } - filter { - total_token_usage { - comparison_operator = "LESS" - value = 1000 - } - } - } - } -} -`, context) -} - - -{{- end }} diff --git a/tools/issue-labeler/labeler/enrolled_teams.yml b/tools/issue-labeler/labeler/enrolled_teams.yml index c6502889b1f0..3718120f54a7 100755 --- a/tools/issue-labeler/labeler/enrolled_teams.yml +++ b/tools/issue-labeler/labeler/enrolled_teams.yml @@ -10,7 +10,6 @@ service/accesscontextmanager: service/aiplatform-agent-engine: resources: - google_vertex_ai_reasoning_engine.* - - google_vertex_ai_online_evaluator service/aiplatform-colab-enterprise: resources: - google_colab_.*