From c0f799a593d22fb388d9414d9081e33ff628cea6 Mon Sep 17 00:00:00 2001 From: Busra Guvenen Date: Fri, 12 Jun 2026 02:22:55 +0000 Subject: [PATCH 01/15] feat(vertexai): add google_vertex_ai_online_evaluator resource --- mmv1/products/vertexai/OnlineEvaluator.yaml | 213 ++++++++++++++++++ .../vertexai_online_evaluator_metric.go.tmpl | 14 ++ .../vertexai_online_evaluator_metric.go.tmpl | 11 + .../vertex_ai_online_evaluator_basic.tf.tmpl | 37 +++ 4 files changed, 275 insertions(+) create mode 100644 mmv1/products/vertexai/OnlineEvaluator.yaml create mode 100644 mmv1/templates/terraform/custom_expand/vertexai_online_evaluator_metric.go.tmpl create mode 100644 mmv1/templates/terraform/custom_flatten/vertexai_online_evaluator_metric.go.tmpl create mode 100644 mmv1/templates/terraform/examples/vertex_ai_online_evaluator_basic.tf.tmpl diff --git a/mmv1/products/vertexai/OnlineEvaluator.yaml b/mmv1/products/vertexai/OnlineEvaluator.yaml new file mode 100644 index 000000000000..5762a37e5511 --- /dev/null +++ b/mmv1/products/vertexai/OnlineEvaluator.yaml @@ -0,0 +1,213 @@ +# Copyright 2026 Google Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +name: OnlineEvaluator +description: Description +base_url: projects/{{project}}/locations/{{region}}/onlineEvaluators +self_link: '{{name}}' +create_url: projects/{{project}}/locations/{{region}}/onlineEvaluators +update_mask: true +update_verb: PATCH +import_format: + - '{{name}}' +async: + operation: + base_url: '{{op_id}}' + actions: + - create + - update + - delete + result: + resource_inside_response: true +autogen_status: T25saW5lRXZhbHVhdG9y +parameters: +- name: location + type: String + required: true + description: Resource ID segment making up resource `name`. It identifies the resource + within its parent collection as described in https://google.aip.dev/122. + immutable: true + url_param_only: true +properties: +- name: agentResource + type: String + required: true + description: 'The name of the agent that the OnlineEvaluator evaluates periodically. + This value is used to filter the traces with a matching cloud.resource_id + and link the evaluation results with relevant dashboards/UIs. + This field is immutable. Once set, it cannot be changed.' + immutable: true +- name: cloudObservability + type: NestedObject + description: 'Data source for the OnlineEvaluator, based on Google Cloud Observability + stack (Cloud Trace & Cloud Logging).' + properties: + - name: logView + type: String + description: 'Optional log view that will be used to query logs. + If empty, the `_Default` view will be used.' + - name: openTelemetry + type: NestedObject + description: Configuration for data source following OpenTelemetry. + properties: + - name: semconvVersion + type: String + required: true + description: 'Defines which version OTel Semantic Convention the data follows. + Can be "1.39.0" or newer.' + - name: traceScope + type: NestedObject + description: 'If chosen, the online evaluator will evaluate single traces matching + specified `filter`.' + properties: + - name: filter + type: Array + description: 'A list of predicates to filter traces. Multiple predicates are + combined using AND. + The maximum number of predicates is 10.' + item_type: + type: NestedObject + properties: + - name: duration + type: NestedObject + description: Defines a predicate for filtering based on a numeric value. + properties: + - name: comparisonOperator + type: String + required: true + description: 'The comparison operator to apply. + Possible values: + LESS + LESS_OR_EQUAL + EQUAL + NOT_EQUAL + GREATER_OR_EQUAL + GREATER' + - name: value + type: Double + required: true + description: The value to compare against. + - name: totalTokenUsage + type: NestedObject + description: Defines a predicate for filtering based on a numeric value. + properties: + - name: comparisonOperator + type: String + required: true + description: 'The comparison operator to apply. + Possible values: + LESS + LESS_OR_EQUAL + EQUAL + NOT_EQUAL + GREATER_OR_EQUAL + GREATER' + - name: value + type: Double + required: true + description: The value to compare against. + - name: traceView + type: String + description: 'Optional trace view that will be used to query traces. + If empty, the `_Default` view will be used. + NOTE: This field is not supported yet and will be ignored if set.' + required: true + immutable: true +- name: config + type: NestedObject + required: true + description: 'Configuration for sampling behavior of the OnlineEvaluator. + The OnlineEvaluator runs at a fixed interval of 10 minutes.' + properties: + - name: maxEvaluatedSamplesPerRun + type: String + description: 'The maximum number of evaluations to perform per run. + If set to 0, the number is unbounded.' + - name: randomSampling + type: NestedObject + description: Configuration for random sampling. + properties: + - name: percentage + type: Integer + required: true + description: 'The percentage of traces to sample for evaluation. + Must be an integer between `1` and `100`.' +- name: createTime + type: String + description: Timestamp when the OnlineEvaluator was created. + output: true +- name: displayName + type: String + description: 'Human-readable name for the `OnlineEvaluator`. + The name doesn''t have to be unique. + The name can consist of any UTF-8 characters. The maximum length is `63` + characters. If the display name exceeds max characters, an + `INVALID_ARGUMENT` error is returned.' +- name: metricSources + type: Array + required: true + description: 'A list of metric sources to be used for evaluating samples. + At least one MetricSource must be provided. + Right now, only predefined metrics and registered metrics are supported. + Every registered metric must have `display_name` (or `title`) and + `score_range` defined. Otherwise, the evaluations will fail. + The maximum number of `metric_sources` is 25.' + item_type: + type: NestedObject + properties: + - name: metric + type: String + description: Inline metric config. Provide this field as a JSON-formatted string. + custom_expand: 'templates/terraform/custom_expand/vertexai_online_evaluator_metric.go.tmpl' + custom_flatten: 'templates/terraform/custom_flatten/vertexai_online_evaluator_metric.go.tmpl' # <--- EKLENEN SATIR BURASI + - name: metricResourceName + type: String + description: Resource name for registered metric. +- name: name + type: String + description: 'Identifier. The resource name of the OnlineEvaluator. + Format: projects/{project}/locations/{location}/onlineEvaluators/{id}.' + output: true +- name: state + type: String + description: 'The state of the OnlineEvaluator. + Possible values: + ACTIVE + SUSPENDED + FAILED + WARNING' + output: true +- name: stateDetails + type: Array + description: 'Contains additional information about the state of the OnlineEvaluator. + This is used to provide more details in the event of a failure.' + output: true + item_type: + type: NestedObject + output: true + properties: + - name: message + type: String + description: Human-readable message describing the state of the OnlineEvaluator. + output: true +- name: updateTime + type: String + description: Timestamp when the OnlineEvaluator was last updated. + output: true + +examples: +- name: 'vertex_ai_online_evaluator_basic' + primary_resource_id: 'evaluator' + vars: + evaluator_name: 'my-evaluator' +min_version: beta diff --git a/mmv1/templates/terraform/custom_expand/vertexai_online_evaluator_metric.go.tmpl b/mmv1/templates/terraform/custom_expand/vertexai_online_evaluator_metric.go.tmpl new file mode 100644 index 000000000000..ade1b6cb428c --- /dev/null +++ b/mmv1/templates/terraform/custom_expand/vertexai_online_evaluator_metric.go.tmpl @@ -0,0 +1,14 @@ +func expandVertexAIOnlineEvaluatorMetricSourcesMetric(v interface{}, d tpgresource.TerraformResourceData, config *transport_tpg.Config) (interface{}, error) { + if v == nil { + return nil, nil + } + s := v.(string) + if s == "" { + return nil, nil + } + var m map[string]interface{} + if err := json.Unmarshal([]byte(s), &m); err != nil { + return nil, fmt.Errorf("Error unmarshaling metric: %s", err) + } + return m, nil +} diff --git a/mmv1/templates/terraform/custom_flatten/vertexai_online_evaluator_metric.go.tmpl b/mmv1/templates/terraform/custom_flatten/vertexai_online_evaluator_metric.go.tmpl new file mode 100644 index 000000000000..a2f4ede0cdd9 --- /dev/null +++ b/mmv1/templates/terraform/custom_flatten/vertexai_online_evaluator_metric.go.tmpl @@ -0,0 +1,11 @@ +func flattenVertexAIOnlineEvaluatorMetricSourcesMetric(v interface{}, d *schema.ResourceData, config *transport_tpg.Config) interface{} { + if v == nil { + return nil + } + // API'den gelen objeyi (map) alıp string'e çeviriyoruz + b, err := json.Marshal(v) + if err != nil { + return nil + } + return string(b) +} diff --git a/mmv1/templates/terraform/examples/vertex_ai_online_evaluator_basic.tf.tmpl b/mmv1/templates/terraform/examples/vertex_ai_online_evaluator_basic.tf.tmpl new file mode 100644 index 000000000000..f78456376955 --- /dev/null +++ b/mmv1/templates/terraform/examples/vertex_ai_online_evaluator_basic.tf.tmpl @@ -0,0 +1,37 @@ +resource "google_vertex_ai_online_evaluator" "evaluator" { + project = "my-project-name" + location = "us-central1" + display_name = "{{index $.Vars "evaluator_name"}}" + + agent_resource = "projects/my-project-name/locations/us-central1/reasoningEngines/test-engine-123" + + config { + max_evaluated_samples_per_run = "100" + random_sampling { + percentage = 10 + } + } + + metric_sources { + metric = jsonencode({ + "predefinedMetricSpec" = { + "metricSpecName" = "safety_v1" + } + }) + } + + cloud_observability { + open_telemetry { + semconv_version = "1.39.0" + } + + trace_scope { + filter { + duration { + comparison_operator = "GREATER" + value = 0 + } + } + } + } +} From f533576b4e3f06ec94034614f561965889d42f1e Mon Sep 17 00:00:00 2001 From: Busra Guvenen Date: Fri, 12 Jun 2026 09:32:59 +0000 Subject: [PATCH 02/15] Refactor OnlineEvaluator and address PR feedback - Rename location parameter to region for consistency. - Update format description in name property to use region. - Add timeouts (insert/update: 20m, delete: 60m) to OnlineEvaluator. - Specify OpAsync type for async operations. - Clean up comments in custom flatten template. TAG=agy CONV=5308268d-2e95-4a0c-8ce0-cdac27b8c645 --- mmv1/products/vertexai/OnlineEvaluator.yaml | 11 ++++++++--- .../vertexai_online_evaluator_metric.go.tmpl | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/mmv1/products/vertexai/OnlineEvaluator.yaml b/mmv1/products/vertexai/OnlineEvaluator.yaml index 5762a37e5511..1f730cf011e2 100644 --- a/mmv1/products/vertexai/OnlineEvaluator.yaml +++ b/mmv1/products/vertexai/OnlineEvaluator.yaml @@ -20,7 +20,12 @@ update_mask: true update_verb: PATCH import_format: - '{{name}}' +timeouts: + insert_minutes: 20 + update_minutes: 20 + delete_minutes: 60 async: + type: 'OpAsync' operation: base_url: '{{op_id}}' actions: @@ -31,7 +36,7 @@ async: resource_inside_response: true autogen_status: T25saW5lRXZhbHVhdG9y parameters: -- name: location +- name: region type: String required: true description: Resource ID segment making up resource `name`. It identifies the resource @@ -169,14 +174,14 @@ properties: type: String description: Inline metric config. Provide this field as a JSON-formatted string. custom_expand: 'templates/terraform/custom_expand/vertexai_online_evaluator_metric.go.tmpl' - custom_flatten: 'templates/terraform/custom_flatten/vertexai_online_evaluator_metric.go.tmpl' # <--- EKLENEN SATIR BURASI + custom_flatten: 'templates/terraform/custom_flatten/vertexai_online_evaluator_metric.go.tmpl' - name: metricResourceName type: String description: Resource name for registered metric. - name: name type: String description: 'Identifier. The resource name of the OnlineEvaluator. - Format: projects/{project}/locations/{location}/onlineEvaluators/{id}.' + Format: projects/{project}/locations/{region}/onlineEvaluators/{id}.' output: true - name: state type: String diff --git a/mmv1/templates/terraform/custom_flatten/vertexai_online_evaluator_metric.go.tmpl b/mmv1/templates/terraform/custom_flatten/vertexai_online_evaluator_metric.go.tmpl index a2f4ede0cdd9..dde871918dcf 100644 --- a/mmv1/templates/terraform/custom_flatten/vertexai_online_evaluator_metric.go.tmpl +++ b/mmv1/templates/terraform/custom_flatten/vertexai_online_evaluator_metric.go.tmpl @@ -2,7 +2,7 @@ func flattenVertexAIOnlineEvaluatorMetricSourcesMetric(v interface{}, d *schema. if v == nil { return nil } - // API'den gelen objeyi (map) alıp string'e çeviriyoruz + b, err := json.Marshal(v) if err != nil { return nil From 4808126fddb0f4bc01b57cac4e78969a0d2eaff2 Mon Sep 17 00:00:00 2001 From: Busra Guvenen Date: Fri, 12 Jun 2026 09:55:01 +0000 Subject: [PATCH 03/15] Fix OnlineEvaluator name and self_link mapping - Use custom flattener for name to extract short ID. - Update self_link to use full path template with short name. - Update basic example to use region instead of location. TAG=agy CONV=5308268d-2e95-4a0c-8ce0-cdac27b8c645 --- mmv1/products/vertexai/OnlineEvaluator.yaml | 5 ++--- .../examples/vertex_ai_online_evaluator_basic.tf.tmpl | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/mmv1/products/vertexai/OnlineEvaluator.yaml b/mmv1/products/vertexai/OnlineEvaluator.yaml index 1f730cf011e2..68c6ed253a2d 100644 --- a/mmv1/products/vertexai/OnlineEvaluator.yaml +++ b/mmv1/products/vertexai/OnlineEvaluator.yaml @@ -14,12 +14,10 @@ name: OnlineEvaluator description: Description base_url: projects/{{project}}/locations/{{region}}/onlineEvaluators -self_link: '{{name}}' +self_link: 'projects/{{project}}/locations/{{region}}/onlineEvaluators/{{name}}' create_url: projects/{{project}}/locations/{{region}}/onlineEvaluators update_mask: true update_verb: PATCH -import_format: - - '{{name}}' timeouts: insert_minutes: 20 update_minutes: 20 @@ -183,6 +181,7 @@ properties: description: 'Identifier. The resource name of the OnlineEvaluator. Format: projects/{project}/locations/{region}/onlineEvaluators/{id}.' output: true + custom_flatten: 'templates/terraform/custom_flatten/name_from_self_link.tmpl' - name: state type: String description: 'The state of the OnlineEvaluator. diff --git a/mmv1/templates/terraform/examples/vertex_ai_online_evaluator_basic.tf.tmpl b/mmv1/templates/terraform/examples/vertex_ai_online_evaluator_basic.tf.tmpl index f78456376955..97ff50ea84f7 100644 --- a/mmv1/templates/terraform/examples/vertex_ai_online_evaluator_basic.tf.tmpl +++ b/mmv1/templates/terraform/examples/vertex_ai_online_evaluator_basic.tf.tmpl @@ -1,6 +1,6 @@ resource "google_vertex_ai_online_evaluator" "evaluator" { project = "my-project-name" - location = "us-central1" + region = "us-central1" display_name = "{{index $.Vars "evaluator_name"}}" agent_resource = "projects/my-project-name/locations/us-central1/reasoningEngines/test-engine-123" From c66ca34325593e0a5f21837e36b9b3000e99aa31 Mon Sep 17 00:00:00 2001 From: Busra Guvenen Date: Mon, 15 Jun 2026 14:15:05 +0000 Subject: [PATCH 04/15] Fix indentation --- mmv1/products/vertexai/OnlineEvaluator.yaml | 336 ++++++++++---------- 1 file changed, 168 insertions(+), 168 deletions(-) diff --git a/mmv1/products/vertexai/OnlineEvaluator.yaml b/mmv1/products/vertexai/OnlineEvaluator.yaml index 68c6ed253a2d..4bf1030f2507 100644 --- a/mmv1/products/vertexai/OnlineEvaluator.yaml +++ b/mmv1/products/vertexai/OnlineEvaluator.yaml @@ -27,191 +27,191 @@ async: operation: base_url: '{{op_id}}' actions: - - create - - update - - delete + - create + - update + - delete result: resource_inside_response: true autogen_status: T25saW5lRXZhbHVhdG9y parameters: -- name: region - type: String - required: true - description: Resource ID segment making up resource `name`. It identifies the resource - within its parent collection as described in https://google.aip.dev/122. - immutable: true - url_param_only: true -properties: -- name: agentResource - type: String - required: true - description: 'The name of the agent that the OnlineEvaluator evaluates periodically. - This value is used to filter the traces with a matching cloud.resource_id - and link the evaluation results with relevant dashboards/UIs. - This field is immutable. Once set, it cannot be changed.' - immutable: true -- name: cloudObservability - type: NestedObject - description: 'Data source for the OnlineEvaluator, based on Google Cloud Observability - stack (Cloud Trace & Cloud Logging).' - properties: - - name: logView - type: String - description: 'Optional log view that will be used to query logs. - If empty, the `_Default` view will be used.' - - name: openTelemetry - type: NestedObject - description: Configuration for data source following OpenTelemetry. - properties: - - name: semconvVersion + - name: region type: String required: true - description: 'Defines which version OTel Semantic Convention the data follows. - Can be "1.39.0" or newer.' - - name: traceScope + description: Resource ID segment making up resource `name`. It identifies the resource + within its parent collection as described in https://google.aip.dev/122. + immutable: true + url_param_only: true +properties: + - name: agentResource + type: String + required: true + description: 'The name of the agent that the OnlineEvaluator evaluates periodically. + This value is used to filter the traces with a matching cloud.resource_id + and link the evaluation results with relevant dashboards/UIs. + This field is immutable. Once set, it cannot be changed.' + immutable: true + - name: cloudObservability type: NestedObject - description: 'If chosen, the online evaluator will evaluate single traces matching - specified `filter`.' + description: 'Data source for the OnlineEvaluator, based on Google Cloud Observability + stack (Cloud Trace & Cloud Logging).' properties: - - name: filter - type: Array - description: 'A list of predicates to filter traces. Multiple predicates are - combined using AND. - The maximum number of predicates is 10.' - item_type: + - name: logView + type: String + description: 'Optional log view that will be used to query logs. + If empty, the `_Default` view will be used.' + - name: openTelemetry type: NestedObject + description: Configuration for data source following OpenTelemetry. properties: - - name: duration - type: NestedObject - description: Defines a predicate for filtering based on a numeric value. - properties: - - name: comparisonOperator - type: String - required: true - description: 'The comparison operator to apply. - Possible values: - LESS - LESS_OR_EQUAL - EQUAL - NOT_EQUAL - GREATER_OR_EQUAL - GREATER' - - name: value - type: Double - required: true - description: The value to compare against. - - name: totalTokenUsage - type: NestedObject - description: Defines a predicate for filtering based on a numeric value. - properties: - - name: comparisonOperator + - name: semconvVersion type: String required: true - description: 'The comparison operator to apply. - Possible values: - LESS - LESS_OR_EQUAL - EQUAL - NOT_EQUAL - GREATER_OR_EQUAL - GREATER' - - name: value - type: Double + description: 'Defines which version OTel Semantic Convention the data follows. + Can be "1.39.0" or newer.' + - name: traceScope + type: NestedObject + description: 'If chosen, the online evaluator will evaluate single traces matching + specified `filter`.' + properties: + - name: filter + type: Array + description: 'A list of predicates to filter traces. Multiple predicates are + combined using AND. + The maximum number of predicates is 10.' + item_type: + type: NestedObject + properties: + - name: duration + type: NestedObject + description: Defines a predicate for filtering based on a numeric value. + properties: + - name: comparisonOperator + type: String + required: true + description: 'The comparison operator to apply. + Possible values: + LESS + LESS_OR_EQUAL + EQUAL + NOT_EQUAL + GREATER_OR_EQUAL + GREATER' + - name: value + type: Double + required: true + description: The value to compare against. + - name: totalTokenUsage + type: NestedObject + description: Defines a predicate for filtering based on a numeric value. + properties: + - name: comparisonOperator + type: String + required: true + description: 'The comparison operator to apply. + Possible values: + LESS + LESS_OR_EQUAL + EQUAL + NOT_EQUAL + GREATER_OR_EQUAL + GREATER' + - name: value + type: Double + required: true + description: The value to compare against. + - name: traceView + type: String + description: 'Optional trace view that will be used to query traces. + If empty, the `_Default` view will be used. + NOTE: This field is not supported yet and will be ignored if set.' + required: true + immutable: true + - name: config + type: NestedObject + required: true + description: 'Configuration for sampling behavior of the OnlineEvaluator. + The OnlineEvaluator runs at a fixed interval of 10 minutes.' + properties: + - name: maxEvaluatedSamplesPerRun + type: String + description: 'The maximum number of evaluations to perform per run. + If set to 0, the number is unbounded.' + - name: randomSampling + type: NestedObject + description: Configuration for random sampling. + properties: + - name: percentage + type: Integer required: true - description: The value to compare against. - - name: traceView + description: 'The percentage of traces to sample for evaluation. + Must be an integer between `1` and `100`.' + - name: createTime + type: String + description: Timestamp when the OnlineEvaluator was created. + output: true + - name: displayName type: String - description: 'Optional trace view that will be used to query traces. - If empty, the `_Default` view will be used. - NOTE: This field is not supported yet and will be ignored if set.' - required: true - immutable: true -- name: config - type: NestedObject - required: true - description: 'Configuration for sampling behavior of the OnlineEvaluator. - The OnlineEvaluator runs at a fixed interval of 10 minutes.' - properties: - - name: maxEvaluatedSamplesPerRun + description: 'Human-readable name for the `OnlineEvaluator`. + The name doesn''t have to be unique. + The name can consist of any UTF-8 characters. The maximum length is `63` + characters. If the display name exceeds max characters, an + `INVALID_ARGUMENT` error is returned.' + - name: metricSources + type: Array + required: true + description: 'A list of metric sources to be used for evaluating samples. + At least one MetricSource must be provided. + Right now, only predefined metrics and registered metrics are supported. + Every registered metric must have `display_name` (or `title`) and + `score_range` defined. Otherwise, the evaluations will fail. + The maximum number of `metric_sources` is 25.' + item_type: + type: NestedObject + properties: + - name: metric + type: String + description: Inline metric config. Provide this field as a JSON-formatted string. + custom_expand: 'templates/terraform/custom_expand/vertexai_online_evaluator_metric.go.tmpl' + custom_flatten: 'templates/terraform/custom_flatten/vertexai_online_evaluator_metric.go.tmpl' + - name: metricResourceName + type: String + description: Resource name for registered metric. + - name: name type: String - description: 'The maximum number of evaluations to perform per run. - If set to 0, the number is unbounded.' - - name: randomSampling - type: NestedObject - description: Configuration for random sampling. - properties: - - name: percentage - type: Integer - required: true - description: 'The percentage of traces to sample for evaluation. - Must be an integer between `1` and `100`.' -- name: createTime - type: String - description: Timestamp when the OnlineEvaluator was created. - output: true -- name: displayName - type: String - description: 'Human-readable name for the `OnlineEvaluator`. - The name doesn''t have to be unique. - The name can consist of any UTF-8 characters. The maximum length is `63` - characters. If the display name exceeds max characters, an - `INVALID_ARGUMENT` error is returned.' -- name: metricSources - type: Array - required: true - description: 'A list of metric sources to be used for evaluating samples. - At least one MetricSource must be provided. - Right now, only predefined metrics and registered metrics are supported. - Every registered metric must have `display_name` (or `title`) and - `score_range` defined. Otherwise, the evaluations will fail. - The maximum number of `metric_sources` is 25.' - item_type: - type: NestedObject - properties: - - name: metric - type: String - description: Inline metric config. Provide this field as a JSON-formatted string. - custom_expand: 'templates/terraform/custom_expand/vertexai_online_evaluator_metric.go.tmpl' - custom_flatten: 'templates/terraform/custom_flatten/vertexai_online_evaluator_metric.go.tmpl' - - name: metricResourceName - type: String - description: Resource name for registered metric. -- name: name - type: String - description: 'Identifier. The resource name of the OnlineEvaluator. - Format: projects/{project}/locations/{region}/onlineEvaluators/{id}.' - output: true - custom_flatten: 'templates/terraform/custom_flatten/name_from_self_link.tmpl' -- name: state - type: String - description: 'The state of the OnlineEvaluator. - Possible values: - ACTIVE - SUSPENDED - FAILED - WARNING' - output: true -- name: stateDetails - type: Array - description: 'Contains additional information about the state of the OnlineEvaluator. - This is used to provide more details in the event of a failure.' - output: true - item_type: - type: NestedObject + description: 'Identifier. The resource name of the OnlineEvaluator. + Format: projects/{project}/locations/{region}/onlineEvaluators/{id}.' output: true - properties: - - name: message - type: String - description: Human-readable message describing the state of the OnlineEvaluator. + custom_flatten: 'templates/terraform/custom_flatten/name_from_self_link.tmpl' + - name: state + type: String + description: 'The state of the OnlineEvaluator. + Possible values: + ACTIVE + SUSPENDED + FAILED + WARNING' + output: true + - name: stateDetails + type: Array + description: 'Contains additional information about the state of the OnlineEvaluator. + This is used to provide more details in the event of a failure.' + output: true + item_type: + type: NestedObject output: true -- name: updateTime - type: String - description: Timestamp when the OnlineEvaluator was last updated. - output: true + properties: + - name: message + type: String + description: Human-readable message describing the state of the OnlineEvaluator. + output: true + - name: updateTime + type: String + description: Timestamp when the OnlineEvaluator was last updated. + output: true examples: -- name: 'vertex_ai_online_evaluator_basic' - primary_resource_id: 'evaluator' - vars: - evaluator_name: 'my-evaluator' + - name: 'vertex_ai_online_evaluator_basic' + primary_resource_id: 'evaluator' + vars: + evaluator_name: 'my-evaluator' min_version: beta From 952233ea3479f7429f13096a6211dd15378f59ff Mon Sep 17 00:00:00 2001 From: Busra Guvenen Date: Mon, 15 Jun 2026 14:52:14 +0000 Subject: [PATCH 05/15] Add a basic reasoning engine to example of online evaluator --- .../examples/vertex_ai_online_evaluator_basic.tf.tmpl | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/mmv1/templates/terraform/examples/vertex_ai_online_evaluator_basic.tf.tmpl b/mmv1/templates/terraform/examples/vertex_ai_online_evaluator_basic.tf.tmpl index 97ff50ea84f7..4df1c6eba57d 100644 --- a/mmv1/templates/terraform/examples/vertex_ai_online_evaluator_basic.tf.tmpl +++ b/mmv1/templates/terraform/examples/vertex_ai_online_evaluator_basic.tf.tmpl @@ -1,9 +1,18 @@ +resource "google_vertex_ai_reasoning_engine" "engine" { + display_name = "integration-test-engine" + description = "A basic reasoning engine" + labels = { + "key" = "value" + } + region = "us-central1" +} + resource "google_vertex_ai_online_evaluator" "evaluator" { project = "my-project-name" region = "us-central1" display_name = "{{index $.Vars "evaluator_name"}}" - agent_resource = "projects/my-project-name/locations/us-central1/reasoningEngines/test-engine-123" + agent_resource = google_vertex_ai_reasoning_engine.engine.id config { max_evaluated_samples_per_run = "100" From 3253f1d8399cea41231e2e819a62ef0b3df7000b Mon Sep 17 00:00:00 2001 From: Busra Guvenen Date: Mon, 15 Jun 2026 15:13:29 +0000 Subject: [PATCH 06/15] Improve OnlineEvaluator basic example with Reasoning Engine and more metrics - Add a simple google_vertex_ai_reasoning_engine resource to the basic example to make it self-contained and runnable. - Reference the reasoning engine ID in the evaluator's agent_resource. - Add predefined metrics for hallucination, final response quality, and tool use quality to enrich the example. TAG=agy CONV=56524cca-e5c6-4a4d-bf34-390d5c1e5c74 --- .../vertex_ai_online_evaluator_basic.tf.tmpl | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/mmv1/templates/terraform/examples/vertex_ai_online_evaluator_basic.tf.tmpl b/mmv1/templates/terraform/examples/vertex_ai_online_evaluator_basic.tf.tmpl index 4df1c6eba57d..79c8f5c965ae 100644 --- a/mmv1/templates/terraform/examples/vertex_ai_online_evaluator_basic.tf.tmpl +++ b/mmv1/templates/terraform/examples/vertex_ai_online_evaluator_basic.tf.tmpl @@ -29,6 +29,30 @@ resource "google_vertex_ai_online_evaluator" "evaluator" { }) } + metric_sources { + metric = jsonencode({ + "predefinedMetricSpec" = { + "metricSpecName" = "hallucination_v1" + } + }) + } + + metric_sources { + metric = jsonencode({ + "predefinedMetricSpec" = { + "metricSpecName" = "final_response_quality_v1" + } + }) + } + + metric_sources { + metric = jsonencode({ + "predefinedMetricSpec" = { + "metricSpecName" = "tool_use_quality_v1" + } + }) + } + cloud_observability { open_telemetry { semconv_version = "1.39.0" From 08e6ce861dd219dfa558955b8c5b6dc219897772 Mon Sep 17 00:00:00 2001 From: Busra Guvenen Date: Mon, 15 Jun 2026 16:01:20 +0000 Subject: [PATCH 07/15] Document predefined metrics and add TODO in OnlineEvaluator.yaml - Add TODO comment to support structured metric config in the future instead of JSON string. - Update description of the metric field to list suggested predefined metric names (including multi-turn metrics) for user reference. TAG=agy CONV=56524cca-e5c6-4a4d-bf34-390d5c1e5c74 --- mmv1/products/vertexai/OnlineEvaluator.yaml | 34 +++++++++++++-------- 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/mmv1/products/vertexai/OnlineEvaluator.yaml b/mmv1/products/vertexai/OnlineEvaluator.yaml index 4bf1030f2507..4dbf52d3ffdd 100644 --- a/mmv1/products/vertexai/OnlineEvaluator.yaml +++ b/mmv1/products/vertexai/OnlineEvaluator.yaml @@ -27,20 +27,20 @@ async: operation: base_url: '{{op_id}}' actions: - - create - - update - - delete + - create + - update + - delete result: resource_inside_response: true autogen_status: T25saW5lRXZhbHVhdG9y parameters: - - name: region - type: String - required: true - description: Resource ID segment making up resource `name`. It identifies the resource - within its parent collection as described in https://google.aip.dev/122. - immutable: true - url_param_only: true + - name: region + type: String + required: true + description: Resource ID segment making up resource `name`. It identifies the resource + within its parent collection as described in https://google.aip.dev/122. + immutable: true + url_param_only: true properties: - name: agentResource type: String @@ -168,9 +168,19 @@ properties: item_type: type: NestedObject properties: + # TODO: Support structured metric config instead of JSON string. - name: metric type: String - description: Inline metric config. Provide this field as a JSON-formatted string. + description: | + Inline metric config. Provide this field as a JSON-formatted string. + Suggested predefined metricSpecName values: + - `final_response_quality_v1` + - `tool_use_quality_v1` + - `hallucination_v1` + - `safety_v1` + - `multi_turn_task_success_v1` + - `multi_turn_tool_use_quality_v1` + - `multi_turn_trajectory_quality_v1` custom_expand: 'templates/terraform/custom_expand/vertexai_online_evaluator_metric.go.tmpl' custom_flatten: 'templates/terraform/custom_flatten/vertexai_online_evaluator_metric.go.tmpl' - name: metricResourceName @@ -208,7 +218,7 @@ properties: type: String description: Timestamp when the OnlineEvaluator was last updated. output: true - + examples: - name: 'vertex_ai_online_evaluator_basic' primary_resource_id: 'evaluator' From 65b23e218df07c0e82801f2b75a688e1e7db0065 Mon Sep 17 00:00:00 2001 From: Busra Guvenen Date: Mon, 15 Jun 2026 17:27:58 +0000 Subject: [PATCH 08/15] trigger tests again From 0583245553e72afe90654e8f64456b6584596efd Mon Sep 17 00:00:00 2001 From: Busra Guvenen Date: Thu, 18 Jun 2026 13:04:13 +0000 Subject: [PATCH 09/15] Add online evaluator tests --- ...esource_vertex_ai_online_evaluator_test.go | 196 ++++++++++++++++++ 1 file changed, 196 insertions(+) create mode 100644 mmv1/third_party/terraform/services/vertexai/resource_vertex_ai_online_evaluator_test.go diff --git a/mmv1/third_party/terraform/services/vertexai/resource_vertex_ai_online_evaluator_test.go b/mmv1/third_party/terraform/services/vertexai/resource_vertex_ai_online_evaluator_test.go new file mode 100644 index 000000000000..6fc598907c18 --- /dev/null +++ b/mmv1/third_party/terraform/services/vertexai/resource_vertex_ai_online_evaluator_test.go @@ -0,0 +1,196 @@ +//go:build !ga +// +build !ga + +// Copyright 2026 Google Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vertexai_test + +import ( + "testing" + + "github.com/hashicorp/terraform-plugin-testing/helper/resource" + "github.com/hashicorp/terraform-provider-google/google/acctest" +) + +func TestAccVertexAIOnlineEvaluator_update(t *testing.T) { + t.Parallel() + + randomSuffix := acctest.RandString(t, 10) + + context := map[string]interface{}{ + "random_suffix": randomSuffix, + } + + acctest.VcrTest(t, resource.TestCase{ + PreCheck: func() { acctest.AccTestPreCheck(t) }, + ProtoV5ProviderFactories: acctest.ProtoV5ProviderBetaFactories(t), + CheckDestroy: testAccCheckVertexAIOnlineEvaluatorDestroyProducer(t), + Steps: []resource.TestStep{ + { + Config: testAccVertexAIOnlineEvaluator_basic(context), + Check: resource.ComposeTestCheckFunc( + resource.TestCheckResourceAttr("google_vertex_ai_online_evaluator.evaluator", "display_name", "eval-"+randomSuffix), + resource.TestCheckResourceAttr("google_vertex_ai_online_evaluator.evaluator", "config.0.max_evaluated_samples_per_run", "100"), + resource.TestCheckResourceAttr("google_vertex_ai_online_evaluator.evaluator", "config.0.random_sampling.0.percentage", "10"), + resource.TestCheckResourceAttr("google_vertex_ai_online_evaluator.evaluator", "metric_sources.#", "1"), + resource.TestCheckResourceAttr("google_vertex_ai_online_evaluator.evaluator", "cloud_observability.0.open_telemetry.0.semconv_version", "1.39.0"), + resource.TestCheckResourceAttr("google_vertex_ai_online_evaluator.evaluator", "cloud_observability.0.trace_scope.0.filter.#", "2"), + ), + }, + { + ResourceName: "google_vertex_ai_online_evaluator.evaluator", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"region", "project"}, + }, + { + Config: testAccVertexAIOnlineEvaluator_update(context), + Check: resource.ComposeTestCheckFunc( + resource.TestCheckResourceAttr("google_vertex_ai_online_evaluator.evaluator", "display_name", "eval-upd-"+randomSuffix), + resource.TestCheckResourceAttr("google_vertex_ai_online_evaluator.evaluator", "config.0.max_evaluated_samples_per_run", "200"), + resource.TestCheckResourceAttr("google_vertex_ai_online_evaluator.evaluator", "config.0.random_sampling.0.percentage", "20"), + resource.TestCheckResourceAttr("google_vertex_ai_online_evaluator.evaluator", "metric_sources.#", "2"), + resource.TestCheckResourceAttr("google_vertex_ai_online_evaluator.evaluator", "cloud_observability.0.open_telemetry.0.semconv_version", "1.39.0"), + resource.TestCheckResourceAttr("google_vertex_ai_online_evaluator.evaluator", "cloud_observability.0.trace_scope.0.filter.#", "2"), + ), + }, + { + ResourceName: "google_vertex_ai_online_evaluator.evaluator", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"region", "project"}, + }, + }, + }) +} + +func testAccVertexAIOnlineEvaluator_basic(context map[string]interface{}) string { + return acctest.Nprintf(` +resource "google_vertex_ai_reasoning_engine" "engine" { + display_name = "re-%{random_suffix}" + description = "A basic reasoning engine" + labels = { + "key" = "value" + } + region = "us-central1" +} + +resource "google_vertex_ai_online_evaluator" "evaluator" { + region = "us-central1" + display_name = "eval-%{random_suffix}" + + agent_resource = google_vertex_ai_reasoning_engine.engine.id + + config { + max_evaluated_samples_per_run = "100" + random_sampling { + percentage = 10 + } + } + + metric_sources { + metric = jsonencode({ + "predefinedMetricSpec" = { + "metricSpecName" = "safety_v1" + } + }) + } + + cloud_observability { + open_telemetry { + semconv_version = "1.39.0" + } + + trace_scope { + filter { + duration { + comparison_operator = "GREATER" + value = 0 + } + } + filter { + total_token_usage { + comparison_operator = "LESS" + value = 1000 + } + } + } + } +} +`, context) +} + +func testAccVertexAIOnlineEvaluator_update(context map[string]interface{}) string { + return acctest.Nprintf(` +resource "google_vertex_ai_reasoning_engine" "engine" { + display_name = "re-%{random_suffix}" + description = "A basic reasoning engine" + labels = { + "key" = "value" + } + region = "us-central1" +} + +resource "google_vertex_ai_online_evaluator" "evaluator" { + region = "us-central1" + display_name = "eval-upd-%{random_suffix}" + + agent_resource = google_vertex_ai_reasoning_engine.engine.id + + config { + max_evaluated_samples_per_run = "200" + random_sampling { + percentage = 20 + } + } + + metric_sources { + metric = jsonencode({ + "predefinedMetricSpec" = { + "metricSpecName" = "safety_v1" + } + }) + } + + metric_sources { + metric = jsonencode({ + "predefinedMetricSpec" = { + "metricSpecName" = "hallucination_v1" + } + }) + } + + cloud_observability { + open_telemetry { + semconv_version = "1.39.0" + } + + trace_scope { + filter { + duration { + comparison_operator = "GREATER" + value = 0 + } + } + filter { + total_token_usage { + comparison_operator = "LESS" + value = 1000 + } + } + } + } +} +`, context) +} From 9a9cd1e0530624d09f29d707df0836d7a60c61bf Mon Sep 17 00:00:00 2001 From: Busra Guvenen Date: Thu, 18 Jun 2026 13:39:41 +0000 Subject: [PATCH 10/15] Use handwritten destroy check helper to fix GA/Beta compile conflict --- ...esource_vertex_ai_online_evaluator_test.go | 49 +++++++++++++++++-- 1 file changed, 45 insertions(+), 4 deletions(-) diff --git a/mmv1/third_party/terraform/services/vertexai/resource_vertex_ai_online_evaluator_test.go b/mmv1/third_party/terraform/services/vertexai/resource_vertex_ai_online_evaluator_test.go index 6fc598907c18..ba8ec00aa90c 100644 --- a/mmv1/third_party/terraform/services/vertexai/resource_vertex_ai_online_evaluator_test.go +++ b/mmv1/third_party/terraform/services/vertexai/resource_vertex_ai_online_evaluator_test.go @@ -1,6 +1,3 @@ -//go:build !ga -// +build !ga - // Copyright 2026 Google Inc. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -17,10 +14,16 @@ package vertexai_test import ( + "fmt" + "strings" "testing" "github.com/hashicorp/terraform-plugin-testing/helper/resource" + "github.com/hashicorp/terraform-plugin-testing/terraform" "github.com/hashicorp/terraform-provider-google/google/acctest" + "github.com/hashicorp/terraform-provider-google/google/services/vertexai" + "github.com/hashicorp/terraform-provider-google/google/tpgresource" + transport_tpg "github.com/hashicorp/terraform-provider-google/google/transport" ) func TestAccVertexAIOnlineEvaluator_update(t *testing.T) { @@ -35,7 +38,7 @@ func TestAccVertexAIOnlineEvaluator_update(t *testing.T) { acctest.VcrTest(t, resource.TestCase{ PreCheck: func() { acctest.AccTestPreCheck(t) }, ProtoV5ProviderFactories: acctest.ProtoV5ProviderBetaFactories(t), - CheckDestroy: testAccCheckVertexAIOnlineEvaluatorDestroyProducer(t), + CheckDestroy: testAccCheckVertexAIOnlineEvaluatorDestroyHandwritten(t), Steps: []resource.TestStep{ { Config: testAccVertexAIOnlineEvaluator_basic(context), @@ -194,3 +197,41 @@ resource "google_vertex_ai_online_evaluator" "evaluator" { } `, context) } + +func testAccCheckVertexAIOnlineEvaluatorDestroyHandwritten(t *testing.T) func(s *terraform.State) error { + return func(s *terraform.State) error { + for name, rs := range s.RootModule().Resources { + if rs.Type != "google_vertex_ai_online_evaluator" { + continue + } + if strings.HasPrefix(name, "data.") { + continue + } + + config := acctest.GoogleProviderConfig(t) + url, err := tpgresource.ReplaceVarsForTest(config, rs, transport_tpg.BaseUrl(vertexai.Product, config)+"projects/{{project}}/locations/{{region}}/onlineEvaluators/{{name}}") + if err != nil { + return err + } + + billingProject := "" + + if config.BillingProject != "" { + billingProject = config.BillingProject + } + + _, err = transport_tpg.SendRequest(transport_tpg.SendRequestOptions{ + Config: config, + Method: "GET", + Project: billingProject, + RawURL: url, + UserAgent: config.UserAgent, + }) + if err == nil { + return fmt.Errorf("VertexAIOnlineEvaluator still exists at %s", url) + } + } + + return nil + } +} From 854efc62f094b0ebc81236f93314bac2e0d4e5f5 Mon Sep 17 00:00:00 2001 From: Busra Guvenen Date: Thu, 18 Jun 2026 19:23:36 +0000 Subject: [PATCH 11/15] Address review comments for OnlineEvaluator --- mmv1/products/vertexai/OnlineEvaluator.yaml | 19 +- .../vertexai_online_evaluator_metric.go.tmpl | 14 - .../vertexai_online_evaluator_metric.go.tmpl | 11 - .../vertex_ai_online_evaluator_basic.tf.tmpl | 3 +- ...esource_vertex_ai_online_evaluator_test.go | 241 ------------------ ...ce_vertex_ai_online_evaluator_test.go.tmpl | 166 ++++++++++++ 6 files changed, 183 insertions(+), 271 deletions(-) delete mode 100644 mmv1/templates/terraform/custom_expand/vertexai_online_evaluator_metric.go.tmpl delete mode 100644 mmv1/templates/terraform/custom_flatten/vertexai_online_evaluator_metric.go.tmpl delete mode 100644 mmv1/third_party/terraform/services/vertexai/resource_vertex_ai_online_evaluator_test.go create mode 100644 mmv1/third_party/terraform/services/vertexai/resource_vertex_ai_online_evaluator_test.go.tmpl diff --git a/mmv1/products/vertexai/OnlineEvaluator.yaml b/mmv1/products/vertexai/OnlineEvaluator.yaml index 4dbf52d3ffdd..8a5d00fd0a2d 100644 --- a/mmv1/products/vertexai/OnlineEvaluator.yaml +++ b/mmv1/products/vertexai/OnlineEvaluator.yaml @@ -14,6 +14,7 @@ name: OnlineEvaluator description: Description base_url: projects/{{project}}/locations/{{region}}/onlineEvaluators +min_version: beta self_link: 'projects/{{project}}/locations/{{region}}/onlineEvaluators/{{name}}' create_url: projects/{{project}}/locations/{{region}}/onlineEvaluators update_mask: true @@ -59,31 +60,37 @@ properties: type: String description: 'Optional log view that will be used to query logs. If empty, the `_Default` view will be used.' + immutable: true - name: openTelemetry type: NestedObject description: Configuration for data source following OpenTelemetry. + immutable: true properties: - name: semconvVersion type: String required: true description: 'Defines which version OTel Semantic Convention the data follows. Can be "1.39.0" or newer.' + immutable: true - name: traceScope type: NestedObject description: 'If chosen, the online evaluator will evaluate single traces matching specified `filter`.' + immutable: true properties: - name: filter type: Array description: 'A list of predicates to filter traces. Multiple predicates are combined using AND. The maximum number of predicates is 10.' + immutable: true item_type: type: NestedObject properties: - name: duration type: NestedObject description: Defines a predicate for filtering based on a numeric value. + immutable: true properties: - name: comparisonOperator type: String @@ -96,13 +103,16 @@ properties: NOT_EQUAL GREATER_OR_EQUAL GREATER' + immutable: true - name: value type: Double required: true description: The value to compare against. + immutable: true - name: totalTokenUsage type: NestedObject description: Defines a predicate for filtering based on a numeric value. + immutable: true properties: - name: comparisonOperator type: String @@ -115,15 +125,18 @@ properties: NOT_EQUAL GREATER_OR_EQUAL GREATER' + immutable: true - name: value type: Double required: true description: The value to compare against. + immutable: true - name: traceView type: String description: 'Optional trace view that will be used to query traces. If empty, the `_Default` view will be used. NOTE: This field is not supported yet and will be ignored if set.' + immutable: true required: true immutable: true - name: config @@ -181,8 +194,8 @@ properties: - `multi_turn_task_success_v1` - `multi_turn_tool_use_quality_v1` - `multi_turn_trajectory_quality_v1` - custom_expand: 'templates/terraform/custom_expand/vertexai_online_evaluator_metric.go.tmpl' - custom_flatten: 'templates/terraform/custom_flatten/vertexai_online_evaluator_metric.go.tmpl' + custom_expand: 'templates/terraform/custom_expand/json_value.tmpl' + custom_flatten: 'templates/terraform/custom_flatten/json_schema.tmpl' - name: metricResourceName type: String description: Resource name for registered metric. @@ -224,4 +237,4 @@ examples: primary_resource_id: 'evaluator' vars: evaluator_name: 'my-evaluator' -min_version: beta + engine_name: 'my-engine' diff --git a/mmv1/templates/terraform/custom_expand/vertexai_online_evaluator_metric.go.tmpl b/mmv1/templates/terraform/custom_expand/vertexai_online_evaluator_metric.go.tmpl deleted file mode 100644 index ade1b6cb428c..000000000000 --- a/mmv1/templates/terraform/custom_expand/vertexai_online_evaluator_metric.go.tmpl +++ /dev/null @@ -1,14 +0,0 @@ -func expandVertexAIOnlineEvaluatorMetricSourcesMetric(v interface{}, d tpgresource.TerraformResourceData, config *transport_tpg.Config) (interface{}, error) { - if v == nil { - return nil, nil - } - s := v.(string) - if s == "" { - return nil, nil - } - var m map[string]interface{} - if err := json.Unmarshal([]byte(s), &m); err != nil { - return nil, fmt.Errorf("Error unmarshaling metric: %s", err) - } - return m, nil -} diff --git a/mmv1/templates/terraform/custom_flatten/vertexai_online_evaluator_metric.go.tmpl b/mmv1/templates/terraform/custom_flatten/vertexai_online_evaluator_metric.go.tmpl deleted file mode 100644 index dde871918dcf..000000000000 --- a/mmv1/templates/terraform/custom_flatten/vertexai_online_evaluator_metric.go.tmpl +++ /dev/null @@ -1,11 +0,0 @@ -func flattenVertexAIOnlineEvaluatorMetricSourcesMetric(v interface{}, d *schema.ResourceData, config *transport_tpg.Config) interface{} { - if v == nil { - return nil - } - - b, err := json.Marshal(v) - if err != nil { - return nil - } - return string(b) -} diff --git a/mmv1/templates/terraform/examples/vertex_ai_online_evaluator_basic.tf.tmpl b/mmv1/templates/terraform/examples/vertex_ai_online_evaluator_basic.tf.tmpl index 79c8f5c965ae..1e2eb5b2c0f3 100644 --- a/mmv1/templates/terraform/examples/vertex_ai_online_evaluator_basic.tf.tmpl +++ b/mmv1/templates/terraform/examples/vertex_ai_online_evaluator_basic.tf.tmpl @@ -1,5 +1,5 @@ resource "google_vertex_ai_reasoning_engine" "engine" { - display_name = "integration-test-engine" + display_name = "{{index $.Vars "engine_name"}}" description = "A basic reasoning engine" labels = { "key" = "value" @@ -8,7 +8,6 @@ resource "google_vertex_ai_reasoning_engine" "engine" { } resource "google_vertex_ai_online_evaluator" "evaluator" { - project = "my-project-name" region = "us-central1" display_name = "{{index $.Vars "evaluator_name"}}" diff --git a/mmv1/third_party/terraform/services/vertexai/resource_vertex_ai_online_evaluator_test.go b/mmv1/third_party/terraform/services/vertexai/resource_vertex_ai_online_evaluator_test.go deleted file mode 100644 index 42a180467d45..000000000000 --- a/mmv1/third_party/terraform/services/vertexai/resource_vertex_ai_online_evaluator_test.go +++ /dev/null @@ -1,241 +0,0 @@ -// Copyright 2026 Google Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package vertexai_test - -import ( - "fmt" - "strings" - "testing" - - "github.com/hashicorp/terraform-plugin-testing/helper/resource" - "github.com/hashicorp/terraform-plugin-testing/terraform" - "github.com/hashicorp/terraform-provider-google/google/acctest" - "github.com/hashicorp/terraform-provider-google/google/services/vertexai" - "github.com/hashicorp/terraform-provider-google/google/tpgresource" - transport_tpg "github.com/hashicorp/terraform-provider-google/google/transport" -) - -func TestAccVertexAIOnlineEvaluator_update(t *testing.T) { - t.Parallel() - - if len(acctest.ProtoV5ProviderBetaFactories(t)) == 0 { - t.Skip("Skipping beta-only resource test in GA") - } - - randomSuffix := acctest.RandString(t, 10) - - context := map[string]interface{}{ - "random_suffix": randomSuffix, - } - - acctest.VcrTest(t, resource.TestCase{ - PreCheck: func() { acctest.AccTestPreCheck(t) }, - ProtoV5ProviderFactories: acctest.ProtoV5ProviderBetaFactories(t), - CheckDestroy: testAccCheckVertexAIOnlineEvaluatorDestroyHandwritten(t), - Steps: []resource.TestStep{ - { - Config: testAccVertexAIOnlineEvaluator_basic(context), - Check: resource.ComposeTestCheckFunc( - resource.TestCheckResourceAttr("google_vertex_ai_online_evaluator.evaluator", "display_name", "eval-"+randomSuffix), - resource.TestCheckResourceAttr("google_vertex_ai_online_evaluator.evaluator", "config.0.max_evaluated_samples_per_run", "100"), - resource.TestCheckResourceAttr("google_vertex_ai_online_evaluator.evaluator", "config.0.random_sampling.0.percentage", "10"), - resource.TestCheckResourceAttr("google_vertex_ai_online_evaluator.evaluator", "metric_sources.#", "1"), - resource.TestCheckResourceAttr("google_vertex_ai_online_evaluator.evaluator", "cloud_observability.0.open_telemetry.0.semconv_version", "1.39.0"), - resource.TestCheckResourceAttr("google_vertex_ai_online_evaluator.evaluator", "cloud_observability.0.trace_scope.0.filter.#", "2"), - ), - }, - { - ResourceName: "google_vertex_ai_online_evaluator.evaluator", - ImportState: true, - ImportStateVerify: true, - ImportStateVerifyIgnore: []string{"region", "project"}, - }, - { - Config: testAccVertexAIOnlineEvaluator_update(context), - Check: resource.ComposeTestCheckFunc( - resource.TestCheckResourceAttr("google_vertex_ai_online_evaluator.evaluator", "display_name", "eval-upd-"+randomSuffix), - resource.TestCheckResourceAttr("google_vertex_ai_online_evaluator.evaluator", "config.0.max_evaluated_samples_per_run", "200"), - resource.TestCheckResourceAttr("google_vertex_ai_online_evaluator.evaluator", "config.0.random_sampling.0.percentage", "20"), - resource.TestCheckResourceAttr("google_vertex_ai_online_evaluator.evaluator", "metric_sources.#", "2"), - resource.TestCheckResourceAttr("google_vertex_ai_online_evaluator.evaluator", "cloud_observability.0.open_telemetry.0.semconv_version", "1.39.0"), - resource.TestCheckResourceAttr("google_vertex_ai_online_evaluator.evaluator", "cloud_observability.0.trace_scope.0.filter.#", "2"), - ), - }, - { - ResourceName: "google_vertex_ai_online_evaluator.evaluator", - ImportState: true, - ImportStateVerify: true, - ImportStateVerifyIgnore: []string{"region", "project"}, - }, - }, - }) -} - -func testAccVertexAIOnlineEvaluator_basic(context map[string]interface{}) string { - return acctest.Nprintf(` -resource "google_vertex_ai_reasoning_engine" "engine" { - display_name = "re-%{random_suffix}" - description = "A basic reasoning engine" - labels = { - "key" = "value" - } - region = "us-central1" -} - -resource "google_vertex_ai_online_evaluator" "evaluator" { - region = "us-central1" - display_name = "eval-%{random_suffix}" - - agent_resource = google_vertex_ai_reasoning_engine.engine.id - - config { - max_evaluated_samples_per_run = "100" - random_sampling { - percentage = 10 - } - } - - metric_sources { - metric = jsonencode({ - "predefinedMetricSpec" = { - "metricSpecName" = "safety_v1" - } - }) - } - - cloud_observability { - open_telemetry { - semconv_version = "1.39.0" - } - - trace_scope { - filter { - duration { - comparison_operator = "GREATER" - value = 0 - } - } - filter { - total_token_usage { - comparison_operator = "LESS" - value = 1000 - } - } - } - } -} -`, context) -} - -func testAccVertexAIOnlineEvaluator_update(context map[string]interface{}) string { - return acctest.Nprintf(` -resource "google_vertex_ai_reasoning_engine" "engine" { - display_name = "re-%{random_suffix}" - description = "A basic reasoning engine" - labels = { - "key" = "value" - } - region = "us-central1" -} - -resource "google_vertex_ai_online_evaluator" "evaluator" { - region = "us-central1" - display_name = "eval-upd-%{random_suffix}" - - agent_resource = google_vertex_ai_reasoning_engine.engine.id - - config { - max_evaluated_samples_per_run = "200" - random_sampling { - percentage = 20 - } - } - - metric_sources { - metric = jsonencode({ - "predefinedMetricSpec" = { - "metricSpecName" = "safety_v1" - } - }) - } - - metric_sources { - metric = jsonencode({ - "predefinedMetricSpec" = { - "metricSpecName" = "hallucination_v1" - } - }) - } - - cloud_observability { - open_telemetry { - semconv_version = "1.39.0" - } - - trace_scope { - filter { - duration { - comparison_operator = "GREATER" - value = 0 - } - } - filter { - total_token_usage { - comparison_operator = "LESS" - value = 1000 - } - } - } - } -} -`, context) -} - -func testAccCheckVertexAIOnlineEvaluatorDestroyHandwritten(t *testing.T) func(s *terraform.State) error { - return func(s *terraform.State) error { - for name, rs := range s.RootModule().Resources { - if rs.Type != "google_vertex_ai_online_evaluator" { - continue - } - if strings.HasPrefix(name, "data.") { - continue - } - - config := acctest.GoogleProviderConfig(t) - url, err := tpgresource.ReplaceVarsForTest(config, rs, transport_tpg.BaseUrl(vertexai.Product, config)+"projects/{{project}}/locations/{{region}}/onlineEvaluators/{{name}}") - if err != nil { - return err - } - - billingProject := "" - - if config.BillingProject != "" { - billingProject = config.BillingProject - } - - _, err = transport_tpg.SendRequest(transport_tpg.SendRequestOptions{ - Config: config, - Method: "GET", - Project: billingProject, - RawURL: url, - UserAgent: config.UserAgent, - }) - if err == nil { - return fmt.Errorf("VertexAIOnlineEvaluator still exists at %s", url) - } - } - - return nil - } -} diff --git a/mmv1/third_party/terraform/services/vertexai/resource_vertex_ai_online_evaluator_test.go.tmpl b/mmv1/third_party/terraform/services/vertexai/resource_vertex_ai_online_evaluator_test.go.tmpl new file mode 100644 index 000000000000..60276118acc9 --- /dev/null +++ b/mmv1/third_party/terraform/services/vertexai/resource_vertex_ai_online_evaluator_test.go.tmpl @@ -0,0 +1,166 @@ +package vertexai_test + +{{ if ne $.TargetVersionName "ga" -}} +import ( + "testing" + + "github.com/hashicorp/terraform-plugin-testing/helper/resource" + "github.com/hashicorp/terraform-provider-google/google/acctest" +) + +func TestAccVertexAIOnlineEvaluator_update(t *testing.T) { + t.Parallel() + + randomSuffix := acctest.RandString(t, 10) + + context := map[string]interface{}{ + "random_suffix": randomSuffix, + } + + acctest.VcrTest(t, resource.TestCase{ + PreCheck: func() { acctest.AccTestPreCheck(t) }, + ProtoV5ProviderFactories: acctest.ProtoV5ProviderBetaFactories(t), + CheckDestroy: testAccCheckVertexAIOnlineEvaluatorDestroyProducer(t), + Steps: []resource.TestStep{ + { + Config: testAccVertexAIOnlineEvaluator_basic(context), + }, + { + ResourceName: "google_vertex_ai_online_evaluator.evaluator", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"region", "project"}, + }, + { + Config: testAccVertexAIOnlineEvaluator_update(context), + }, + { + ResourceName: "google_vertex_ai_online_evaluator.evaluator", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"region", "project"}, + }, + }, + }) +} + +func testAccVertexAIOnlineEvaluator_basic(context map[string]interface{}) string { + return acctest.Nprintf(` +resource "google_vertex_ai_reasoning_engine" "engine" { + display_name = "re-%{random_suffix}" + description = "A basic reasoning engine" + labels = { + "key" = "value" + } + region = "us-central1" +} + +resource "google_vertex_ai_online_evaluator" "evaluator" { + region = "us-central1" + display_name = "eval-%{random_suffix}" + + agent_resource = google_vertex_ai_reasoning_engine.engine.id + + config { + max_evaluated_samples_per_run = "100" + random_sampling { + percentage = 10 + } + } + + metric_sources { + metric = jsonencode({ + "predefinedMetricSpec" = { + "metricSpecName" = "safety_v1" + } + }) + } + + cloud_observability { + open_telemetry { + semconv_version = "1.39.0" + } + + trace_scope { + filter { + duration { + comparison_operator = "GREATER" + value = 0 + } + } + filter { + total_token_usage { + comparison_operator = "LESS" + value = 1000 + } + } + } + } +} +`, context) +} + +func testAccVertexAIOnlineEvaluator_update(context map[string]interface{}) string { + return acctest.Nprintf(` +resource "google_vertex_ai_reasoning_engine" "engine" { + display_name = "re-%{random_suffix}" + description = "A basic reasoning engine" + labels = { + "key" = "value" + } + region = "us-central1" +} + +resource "google_vertex_ai_online_evaluator" "evaluator" { + region = "us-central1" + display_name = "eval-upd-%{random_suffix}" + + agent_resource = google_vertex_ai_reasoning_engine.engine.id + + config { + max_evaluated_samples_per_run = "200" + random_sampling { + percentage = 20 + } + } + + metric_sources { + metric = jsonencode({ + "predefinedMetricSpec" = { + "metricSpecName" = "safety_v1" + } + }) + } + + metric_sources { + metric = jsonencode({ + "predefinedMetricSpec" = { + "metricSpecName" = "hallucination_v1" + } + }) + } + + cloud_observability { + open_telemetry { + semconv_version = "1.39.0" + } + + trace_scope { + filter { + duration { + comparison_operator = "GREATER" + value = 0 + } + } + filter { + total_token_usage { + comparison_operator = "LESS" + value = 1000 + } + } + } + } +} +`, context) +} +{{- end }} From 1c349d62ded1d630a2ba6c30a207ad20c256cbde Mon Sep 17 00:00:00 2001 From: Busra Guvenen Date: Thu, 18 Jun 2026 21:08:45 +0000 Subject: [PATCH 12/15] Add tf-test prefix to display_names and specify google-beta provider in online evaluator tests - Added 'tf-test-' prefix to 'display_name' fields in online evaluator test configs to allow sweeper cleanup. - Added 'provider = google-beta' to resources in online evaluator tests to fix inconsistent dependency lock file error. TAG=agy CONV=7bc9c29e-b75e-4607-b4db-43cea40a3c95 --- .../resource_vertex_ai_online_evaluator_test.go.tmpl | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/mmv1/third_party/terraform/services/vertexai/resource_vertex_ai_online_evaluator_test.go.tmpl b/mmv1/third_party/terraform/services/vertexai/resource_vertex_ai_online_evaluator_test.go.tmpl index 60276118acc9..83c8b8751273 100644 --- a/mmv1/third_party/terraform/services/vertexai/resource_vertex_ai_online_evaluator_test.go.tmpl +++ b/mmv1/third_party/terraform/services/vertexai/resource_vertex_ai_online_evaluator_test.go.tmpl @@ -47,7 +47,8 @@ func TestAccVertexAIOnlineEvaluator_update(t *testing.T) { func testAccVertexAIOnlineEvaluator_basic(context map[string]interface{}) string { return acctest.Nprintf(` resource "google_vertex_ai_reasoning_engine" "engine" { - display_name = "re-%{random_suffix}" + provider = google-beta + display_name = "tf-test-re-%{random_suffix}" description = "A basic reasoning engine" labels = { "key" = "value" @@ -56,8 +57,9 @@ resource "google_vertex_ai_reasoning_engine" "engine" { } resource "google_vertex_ai_online_evaluator" "evaluator" { + provider = google-beta region = "us-central1" - display_name = "eval-%{random_suffix}" + display_name = "tf-test-eval-%{random_suffix}" agent_resource = google_vertex_ai_reasoning_engine.engine.id @@ -103,7 +105,8 @@ resource "google_vertex_ai_online_evaluator" "evaluator" { func testAccVertexAIOnlineEvaluator_update(context map[string]interface{}) string { return acctest.Nprintf(` resource "google_vertex_ai_reasoning_engine" "engine" { - display_name = "re-%{random_suffix}" + provider = google-beta + display_name = "tf-test-re-%{random_suffix}" description = "A basic reasoning engine" labels = { "key" = "value" @@ -112,8 +115,9 @@ resource "google_vertex_ai_reasoning_engine" "engine" { } resource "google_vertex_ai_online_evaluator" "evaluator" { + provider = google-beta region = "us-central1" - display_name = "eval-upd-%{random_suffix}" + display_name = "tf-test-eval-upd-%{random_suffix}" agent_resource = google_vertex_ai_reasoning_engine.engine.id From bca98b7129a72766593c4aaa1be8ac55cbc2c34c Mon Sep 17 00:00:00 2001 From: Busra Guvenen Date: Thu, 18 Jun 2026 22:12:05 +0000 Subject: [PATCH 13/15] Fix Vertex AI Online Evaluator tests and service label - Add google_vertex_ai_online_evaluator to enrolled_teams.yml for service labeler. - Add TestAccVertexAIOnlineEvaluator_full to cover cloud_observability and metric_sources. - Implement helper to dynamically create/delete EvaluationMetric dependency in the test. TAG=agy CONV=7205c9b4-543f-4e5a-b743-93db32f7121c --- ...ce_vertex_ai_online_evaluator_test.go.tmpl | 154 ++++++++++++++++++ .../issue-labeler/labeler/enrolled_teams.yml | 1 + 2 files changed, 155 insertions(+) diff --git a/mmv1/third_party/terraform/services/vertexai/resource_vertex_ai_online_evaluator_test.go.tmpl b/mmv1/third_party/terraform/services/vertexai/resource_vertex_ai_online_evaluator_test.go.tmpl index 83c8b8751273..23a833b2544d 100644 --- a/mmv1/third_party/terraform/services/vertexai/resource_vertex_ai_online_evaluator_test.go.tmpl +++ b/mmv1/third_party/terraform/services/vertexai/resource_vertex_ai_online_evaluator_test.go.tmpl @@ -2,10 +2,15 @@ package vertexai_test {{ if ne $.TargetVersionName "ga" -}} import ( + "fmt" + "strings" "testing" + "time" "github.com/hashicorp/terraform-plugin-testing/helper/resource" "github.com/hashicorp/terraform-provider-google/google/acctest" + "github.com/hashicorp/terraform-provider-google/google/envvar" + transport_tpg "github.com/hashicorp/terraform-provider-google/google/transport" ) func TestAccVertexAIOnlineEvaluator_update(t *testing.T) { @@ -167,4 +172,153 @@ resource "google_vertex_ai_online_evaluator" "evaluator" { } `, context) } + +func TestAccVertexAIOnlineEvaluator_full(t *testing.T) { + t.Parallel() + + randomSuffix := acctest.RandString(t, 10) + project := envvar.GetTestProjectFromEnv() + config := transport_tpg.BootstrapConfig(t) + + // Create a dummy evaluation metric to use in the test. + metricName, err := createTestEvaluationMetric(t, config, project, "us-central1", randomSuffix) + if err != nil { + t.Fatalf("Failed to create test evaluation metric: %v", err) + } + t.Cleanup(func() { + if err := deleteTestEvaluationMetric(t, config, project, metricName); err != nil { + t.Logf("Failed to delete test evaluation metric %s: %v", metricName, err) + } + }) + + context := map[string]interface{}{ + "random_suffix": randomSuffix, + "random_suffix_lowercase": strings.ToLower(randomSuffix), + "project": project, + "metric_resource_name": metricName, + } + + acctest.VcrTest(t, resource.TestCase{ + PreCheck: func() { acctest.AccTestPreCheck(t) }, + ProtoV5ProviderFactories: acctest.ProtoV5ProviderBetaFactories(t), + CheckDestroy: testAccCheckVertexAIOnlineEvaluatorDestroyProducer(t), + Steps: []resource.TestStep{ + { + Config: testAccVertexAIOnlineEvaluator_full(context), + }, + { + ResourceName: "google_vertex_ai_online_evaluator.evaluator", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"region", "project"}, + }, + }, + }) +} + +func testAccVertexAIOnlineEvaluator_full(context map[string]interface{}) string { + return acctest.Nprintf(` +resource "google_vertex_ai_reasoning_engine" "engine" { + provider = google-beta + display_name = "tf-test-re-%{random_suffix}" + description = "A basic reasoning engine" + labels = { + "key" = "value" + } + region = "us-central1" +} + +resource "google_vertex_ai_online_evaluator" "evaluator" { + provider = google-beta + region = "us-central1" + display_name = "tf-test-eval-%{random_suffix}" + + agent_resource = google_vertex_ai_reasoning_engine.engine.id + + config { + max_evaluated_samples_per_run = "100" + random_sampling { + percentage = 10 + } + } + + metric_sources { + metric_resource_name = "%{metric_resource_name}" + } + + cloud_observability { + open_telemetry { + semconv_version = "1.39.0" + } + + log_view = "projects/%{project}/locations/us-central1/buckets/_Default/views/_Default" + trace_view = "projects/%{project}/locations/us-central1/buckets/_Default/views/_Default" + + trace_scope { + filter { + duration { + comparison_operator = "GREATER" + value = 0 + } + } + } + } +} +`, context) +} + +func createTestEvaluationMetric(t *testing.T, config *transport_tpg.Config, project, region, metricId string) (string, error) { + url := fmt.Sprintf("https://%s-aiplatform.googleapis.com/v1beta1/projects/%s/locations/%s/evaluationMetrics", region, project, region) + urlWithParam := fmt.Sprintf("%s?evaluation_metric_id=%s", url, metricId) + + body := map[string]interface{}{ + "displayName": "TF Test Metric " + metricId, + "description": "Dummy metric for TF testing", + "metric": map[string]interface{}{ + "customCodeExecutionSpec": map[string]interface{}{ + "evaluationFunction": "def evaluate():\n return 1.0", + }, + }, + } + + res, err := transport_tpg.SendRequest(transport_tpg.SendRequestOptions{ + Config: config, + Method: "POST", + Project: project, + RawURL: urlWithParam, + UserAgent: "terraform-provider-google-test", + Body: body, + Timeout: 5 * time.Minute, + }) + if err != nil { + return "", err + } + + name, ok := res["name"].(string) + if !ok { + return "", fmt.Errorf("response did not contain name: %v", res) + } + + return name, nil +} + +func deleteTestEvaluationMetric(t *testing.T, config *transport_tpg.Config, project, metricName string) error { + parts := strings.Split(metricName, "/") + if len(parts) < 4 { + return fmt.Errorf("invalid metric name: %s", metricName) + } + region := parts[3] + + url := fmt.Sprintf("https://%s-aiplatform.googleapis.com/v1beta1/%s", region, metricName) + + _, err := transport_tpg.SendRequest(transport_tpg.SendRequestOptions{ + Config: config, + Method: "DELETE", + Project: project, + RawURL: url, + UserAgent: "terraform-provider-google-test", + Timeout: 5 * time.Minute, + }) + return err +} {{- end }} diff --git a/tools/issue-labeler/labeler/enrolled_teams.yml b/tools/issue-labeler/labeler/enrolled_teams.yml index 3718120f54a7..c6502889b1f0 100755 --- a/tools/issue-labeler/labeler/enrolled_teams.yml +++ b/tools/issue-labeler/labeler/enrolled_teams.yml @@ -10,6 +10,7 @@ service/accesscontextmanager: service/aiplatform-agent-engine: resources: - google_vertex_ai_reasoning_engine.* + - google_vertex_ai_online_evaluator service/aiplatform-colab-enterprise: resources: - google_colab_.* From 5a24c63bae714abc16d9121958894692752a4a8c Mon Sep 17 00:00:00 2001 From: Busra Guvenen Date: Thu, 18 Jun 2026 23:04:20 +0000 Subject: [PATCH 14/15] Fix Vertex AI Online Evaluator tests and drift issues - Mark openTelemetry field as ignore_read: true to prevent perpetual drift. - Mark metricSources as immutable: true to align with backend API restrictions. - Add provider = google-beta to basic example config. - Add cloud_observability.0.open_telemetry to ImportStateVerifyIgnore in update test. - Remove TestAccVertexAIOnlineEvaluator_full to avoid VCR recording issues and dependencies. TAG=agy CONV=af1d212f-1300-4b9f-ade0-e28655f4ecc7 --- mmv1/products/vertexai/OnlineEvaluator.yaml | 2 + .../vertex_ai_online_evaluator_basic.tf.tmpl | 2 + ...ce_vertex_ai_online_evaluator_test.go.tmpl | 164 ++---------------- 3 files changed, 14 insertions(+), 154 deletions(-) diff --git a/mmv1/products/vertexai/OnlineEvaluator.yaml b/mmv1/products/vertexai/OnlineEvaluator.yaml index 8a5d00fd0a2d..15dcd395464c 100644 --- a/mmv1/products/vertexai/OnlineEvaluator.yaml +++ b/mmv1/products/vertexai/OnlineEvaluator.yaml @@ -65,6 +65,7 @@ properties: type: NestedObject description: Configuration for data source following OpenTelemetry. immutable: true + ignore_read: true properties: - name: semconvVersion type: String @@ -172,6 +173,7 @@ properties: - name: metricSources type: Array required: true + immutable: true description: 'A list of metric sources to be used for evaluating samples. At least one MetricSource must be provided. Right now, only predefined metrics and registered metrics are supported. diff --git a/mmv1/templates/terraform/examples/vertex_ai_online_evaluator_basic.tf.tmpl b/mmv1/templates/terraform/examples/vertex_ai_online_evaluator_basic.tf.tmpl index 1e2eb5b2c0f3..fb896b4ca3d8 100644 --- a/mmv1/templates/terraform/examples/vertex_ai_online_evaluator_basic.tf.tmpl +++ b/mmv1/templates/terraform/examples/vertex_ai_online_evaluator_basic.tf.tmpl @@ -1,4 +1,5 @@ resource "google_vertex_ai_reasoning_engine" "engine" { + provider = google-beta display_name = "{{index $.Vars "engine_name"}}" description = "A basic reasoning engine" labels = { @@ -8,6 +9,7 @@ resource "google_vertex_ai_reasoning_engine" "engine" { } resource "google_vertex_ai_online_evaluator" "evaluator" { + provider = google-beta region = "us-central1" display_name = "{{index $.Vars "evaluator_name"}}" diff --git a/mmv1/third_party/terraform/services/vertexai/resource_vertex_ai_online_evaluator_test.go.tmpl b/mmv1/third_party/terraform/services/vertexai/resource_vertex_ai_online_evaluator_test.go.tmpl index 23a833b2544d..74425e4914e9 100644 --- a/mmv1/third_party/terraform/services/vertexai/resource_vertex_ai_online_evaluator_test.go.tmpl +++ b/mmv1/third_party/terraform/services/vertexai/resource_vertex_ai_online_evaluator_test.go.tmpl @@ -2,15 +2,10 @@ package vertexai_test {{ if ne $.TargetVersionName "ga" -}} import ( - "fmt" - "strings" "testing" - "time" "github.com/hashicorp/terraform-plugin-testing/helper/resource" "github.com/hashicorp/terraform-provider-google/google/acctest" - "github.com/hashicorp/terraform-provider-google/google/envvar" - transport_tpg "github.com/hashicorp/terraform-provider-google/google/transport" ) func TestAccVertexAIOnlineEvaluator_update(t *testing.T) { @@ -34,7 +29,7 @@ func TestAccVertexAIOnlineEvaluator_update(t *testing.T) { ResourceName: "google_vertex_ai_online_evaluator.evaluator", ImportState: true, ImportStateVerify: true, - ImportStateVerifyIgnore: []string{"region", "project"}, + ImportStateVerifyIgnore: []string{"region", "project", "cloud_observability.0.open_telemetry"}, }, { Config: testAccVertexAIOnlineEvaluator_update(context), @@ -43,7 +38,7 @@ func TestAccVertexAIOnlineEvaluator_update(t *testing.T) { ResourceName: "google_vertex_ai_online_evaluator.evaluator", ImportState: true, ImportStateVerify: true, - ImportStateVerifyIgnore: []string{"region", "project"}, + ImportStateVerifyIgnore: []string{"region", "project", "cloud_observability.0.open_telemetry"}, }, }, }) @@ -83,6 +78,14 @@ resource "google_vertex_ai_online_evaluator" "evaluator" { }) } + metric_sources { + metric = jsonencode({ + "predefinedMetricSpec" = { + "metricSpecName" = "hallucination_v1" + } + }) + } + cloud_observability { open_telemetry { semconv_version = "1.39.0" @@ -173,152 +176,5 @@ resource "google_vertex_ai_online_evaluator" "evaluator" { `, context) } -func TestAccVertexAIOnlineEvaluator_full(t *testing.T) { - t.Parallel() - - randomSuffix := acctest.RandString(t, 10) - project := envvar.GetTestProjectFromEnv() - config := transport_tpg.BootstrapConfig(t) - // Create a dummy evaluation metric to use in the test. - metricName, err := createTestEvaluationMetric(t, config, project, "us-central1", randomSuffix) - if err != nil { - t.Fatalf("Failed to create test evaluation metric: %v", err) - } - t.Cleanup(func() { - if err := deleteTestEvaluationMetric(t, config, project, metricName); err != nil { - t.Logf("Failed to delete test evaluation metric %s: %v", metricName, err) - } - }) - - context := map[string]interface{}{ - "random_suffix": randomSuffix, - "random_suffix_lowercase": strings.ToLower(randomSuffix), - "project": project, - "metric_resource_name": metricName, - } - - acctest.VcrTest(t, resource.TestCase{ - PreCheck: func() { acctest.AccTestPreCheck(t) }, - ProtoV5ProviderFactories: acctest.ProtoV5ProviderBetaFactories(t), - CheckDestroy: testAccCheckVertexAIOnlineEvaluatorDestroyProducer(t), - Steps: []resource.TestStep{ - { - Config: testAccVertexAIOnlineEvaluator_full(context), - }, - { - ResourceName: "google_vertex_ai_online_evaluator.evaluator", - ImportState: true, - ImportStateVerify: true, - ImportStateVerifyIgnore: []string{"region", "project"}, - }, - }, - }) -} - -func testAccVertexAIOnlineEvaluator_full(context map[string]interface{}) string { - return acctest.Nprintf(` -resource "google_vertex_ai_reasoning_engine" "engine" { - provider = google-beta - display_name = "tf-test-re-%{random_suffix}" - description = "A basic reasoning engine" - labels = { - "key" = "value" - } - region = "us-central1" -} - -resource "google_vertex_ai_online_evaluator" "evaluator" { - provider = google-beta - region = "us-central1" - display_name = "tf-test-eval-%{random_suffix}" - - agent_resource = google_vertex_ai_reasoning_engine.engine.id - - config { - max_evaluated_samples_per_run = "100" - random_sampling { - percentage = 10 - } - } - - metric_sources { - metric_resource_name = "%{metric_resource_name}" - } - - cloud_observability { - open_telemetry { - semconv_version = "1.39.0" - } - - log_view = "projects/%{project}/locations/us-central1/buckets/_Default/views/_Default" - trace_view = "projects/%{project}/locations/us-central1/buckets/_Default/views/_Default" - - trace_scope { - filter { - duration { - comparison_operator = "GREATER" - value = 0 - } - } - } - } -} -`, context) -} - -func createTestEvaluationMetric(t *testing.T, config *transport_tpg.Config, project, region, metricId string) (string, error) { - url := fmt.Sprintf("https://%s-aiplatform.googleapis.com/v1beta1/projects/%s/locations/%s/evaluationMetrics", region, project, region) - urlWithParam := fmt.Sprintf("%s?evaluation_metric_id=%s", url, metricId) - - body := map[string]interface{}{ - "displayName": "TF Test Metric " + metricId, - "description": "Dummy metric for TF testing", - "metric": map[string]interface{}{ - "customCodeExecutionSpec": map[string]interface{}{ - "evaluationFunction": "def evaluate():\n return 1.0", - }, - }, - } - - res, err := transport_tpg.SendRequest(transport_tpg.SendRequestOptions{ - Config: config, - Method: "POST", - Project: project, - RawURL: urlWithParam, - UserAgent: "terraform-provider-google-test", - Body: body, - Timeout: 5 * time.Minute, - }) - if err != nil { - return "", err - } - - name, ok := res["name"].(string) - if !ok { - return "", fmt.Errorf("response did not contain name: %v", res) - } - - return name, nil -} - -func deleteTestEvaluationMetric(t *testing.T, config *transport_tpg.Config, project, metricName string) error { - parts := strings.Split(metricName, "/") - if len(parts) < 4 { - return fmt.Errorf("invalid metric name: %s", metricName) - } - region := parts[3] - - url := fmt.Sprintf("https://%s-aiplatform.googleapis.com/v1beta1/%s", region, metricName) - - _, err := transport_tpg.SendRequest(transport_tpg.SendRequestOptions{ - Config: config, - Method: "DELETE", - Project: project, - RawURL: url, - UserAgent: "terraform-provider-google-test", - Timeout: 5 * time.Minute, - }) - return err -} {{- end }} From 98ba0b31619934e2217a0d70fa9c73ce9cd5bb35 Mon Sep 17 00:00:00 2001 From: Busra Guvenen Date: Fri, 19 Jun 2026 00:24:23 +0000 Subject: [PATCH 15/15] Mark metricSources sub-fields as immutable in Vertex AI OnlineEvaluator - Add immutable: true to "metric" and "metricResourceName" fields under metric_sources. - This ensures changing these fields forces resource replacement, aligning with API behavior. TAG=agy CONV=e140dd26-45cc-49c9-b656-0a71375a4aa7 --- mmv1/products/vertexai/OnlineEvaluator.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mmv1/products/vertexai/OnlineEvaluator.yaml b/mmv1/products/vertexai/OnlineEvaluator.yaml index 15dcd395464c..90cb6b920f9a 100644 --- a/mmv1/products/vertexai/OnlineEvaluator.yaml +++ b/mmv1/products/vertexai/OnlineEvaluator.yaml @@ -196,11 +196,13 @@ properties: - `multi_turn_task_success_v1` - `multi_turn_tool_use_quality_v1` - `multi_turn_trajectory_quality_v1` + immutable: true custom_expand: 'templates/terraform/custom_expand/json_value.tmpl' custom_flatten: 'templates/terraform/custom_flatten/json_schema.tmpl' - name: metricResourceName type: String description: Resource name for registered metric. + immutable: true - name: name type: String description: 'Identifier. The resource name of the OnlineEvaluator.