Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -203,7 +203,7 @@ const SecretProviderTable = ({type}: {type: "standard" | "custom"}) => {
size="small"
onClick={() => setIsConfigProviderOpen(true)}
>
Custom Provider
OpenAI-compatible endpoint
</Button>
) : (
<Button
Expand Down
30 changes: 15 additions & 15 deletions web/oss/tests/playwright/acceptance/settings/model-hub.ts
Original file line number Diff line number Diff line change
@@ -1,9 +1,3 @@
import {test} from "@agenta/web-tests/tests/fixtures/base.fixture"

import {expect} from "@agenta/web-tests/utils"
import {expectAuthenticatedSession} from "../utils/auth"
import {createScenarios} from "../utils/scenarios"
import {buildAcceptanceTags} from "../utils/tags"
import {
TestCoverage,
TestcaseType,
Expand All @@ -15,6 +9,12 @@ import {
TestRoleType,
TestSpeedType,
} from "@agenta/web-tests/playwright/config/testTags"
import {test} from "@agenta/web-tests/tests/fixtures/base.fixture"
import {expect} from "@agenta/web-tests/utils"

import {expectAuthenticatedSession} from "../utils/auth"
import {createScenarios} from "../utils/scenarios"
import {buildAcceptanceTags} from "../utils/tags"

/**
* E2E: Model Hub & API Keys Management
Expand Down Expand Up @@ -67,7 +67,7 @@ const modelHubTests = () => {

await scenarios.then('the "Custom providers" table lists the "mock" provider', async () => {
const customProvidersSection = page
.getByText("Custom Provider", {exact: true})
.getByText("OpenAI-compatible endpoint", {exact: true})
.locator("xpath=ancestor::section[1]")
.first()
const providersTable = customProvidersSection.getByRole("table").first()
Expand Down Expand Up @@ -206,14 +206,14 @@ const modelHubTests = () => {
"the user creates a new custom provider via the drawer",
async () => {
const customProvidersSection = page
.getByText("Custom Provider", {exact: true})
.getByText("OpenAI-compatible endpoint", {exact: true})
.locator("xpath=ancestor::section[1]")
.first()

// The section's own label IS the trigger button now — "Create" was
// renamed to "Custom Provider".
// renamed to "OpenAI-compatible endpoint".
const createButton = customProvidersSection.getByRole("button", {
name: "Custom Provider",
name: "OpenAI-compatible endpoint",
exact: true,
})
await expect(createButton).toBeVisible({timeout: 15000})
Expand All @@ -225,7 +225,7 @@ const modelHubTests = () => {
timeout: 15000,
})

// Select "Custom Provider" from the provider type dropdown
// Select "OpenAI-compatible endpoint" from the provider type dropdown
const providerSelect = drawer.locator(".ant-select").first()
await expect(providerSelect).toBeVisible({timeout: 15000})
await providerSelect.click()
Expand All @@ -235,7 +235,7 @@ const modelHubTests = () => {

const optionTexts = (await options.allTextContents()).map((t) => t.trim())
const customProviderIndex = optionTexts.findIndex(
(t) => t === "Custom Provider",
(t) => t === "OpenAI-compatible endpoint",
)

// Click the target option directly — keyboard ArrowDown navigation is unreliable with AntD v5 selects
Expand Down Expand Up @@ -266,7 +266,7 @@ const modelHubTests = () => {
"the new custom provider row appears in the Custom providers table",
async () => {
const customProvidersSection = page
.getByText("Custom Provider", {exact: true})
.getByText("OpenAI-compatible endpoint", {exact: true})
.locator("xpath=ancestor::section[1]")
.first()

Expand All @@ -285,7 +285,7 @@ const modelHubTests = () => {

await scenarios.when("the user deletes the newly created custom provider", async () => {
const customProvidersSection = page
.getByText("Custom Provider", {exact: true})
.getByText("OpenAI-compatible endpoint", {exact: true})
.locator("xpath=ancestor::section[1]")
.first()

Expand All @@ -306,7 +306,7 @@ const modelHubTests = () => {

await scenarios.then("the deleted provider row is no longer visible", async () => {
const customProvidersSection = page
.getByText("Custom Provider", {exact: true})
.getByText("OpenAI-compatible endpoint", {exact: true})
.locator("xpath=ancestor::section[1]")
.first()

Expand Down
4 changes: 3 additions & 1 deletion web/packages/agenta-entities/src/secret/core/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,9 @@ export const PROVIDER_LABELS: Record<string, string> = {
bedrock: "AWS Bedrock",
azure: "Azure OpenAI",
minimax: "MiniMax",
custom: "Custom Provider",
// Stored value stays "custom"; only the user-visible label changes. The v1 custom deployment
// speaks the OpenAI Chat Completions dialect, so name it for what it connects to.
custom: "OpenAI-compatible endpoint",
}

export const PROVIDER_KINDS: Record<string, string> = {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ const CUSTOM_PROVIDER_ROWS: {kind: string; label: string}[] = [
{kind: CustomProviderKind.Azure, label: "Azure OpenAI"},
{kind: CustomProviderKind.Bedrock, label: "AWS Bedrock"},
{kind: CustomProviderKind.VertexAi, label: "Vertex AI"},
{kind: CustomProviderKind.Custom, label: "Custom provider"},
{kind: CustomProviderKind.Custom, label: "OpenAI-compatible endpoint"},
]

/** Family spellings that must compare equal across catalog names, vault env names, and titles. */
Expand Down Expand Up @@ -413,7 +413,7 @@ export function ProviderCredentialsSection({
label={secret.name ?? "?"}
/>
}
label={secret.name ?? "Custom provider"}
label={secret.name ?? "OpenAI-compatible endpoint"}
trailing={
<span className="h-1.5 w-1.5 shrink-0 rounded-full bg-[var(--ag-colorSuccess)]" />
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -432,6 +432,10 @@ export function vaultPickedProviderFamily(
const family = familyFromModelId(modelId, capabilities)
if (family) return family
if (metadataProvider && !isDeploymentProviderKind(metadataProvider)) return metadataProvider
// The OpenAI-compatible (`custom`) deployment defaults to openai rather than deferring to the
// caller's prior (likely unrelated) provider fallback.
if (metadataProvider?.toLowerCase() === OPENAI_COMPATIBLE_KIND)
return OPENAI_COMPATIBLE_DEFAULT_FAMILY
return null
}

Expand All @@ -444,6 +448,13 @@ export function vaultPickedProviderFamily(
// provider), so both must be recognized here or one flavor's connections silently vanish.
const DEPLOYMENT_KINDS = new Set(["direct", "custom", "azure", "bedrock", "vertex_ai", "sagemaker"])

// The `custom` deployment kind is the "OpenAI-compatible endpoint": its models are bare ids that
// encode no vendor, and the v1 runner speaks the OpenAI Chat Completions dialect. So it resolves to
// the `openai` provider family — used both to default a provider-less pick and to gate visibility
// (a harness must reach openai, not just consume the `custom` deployment).
const OPENAI_COMPATIBLE_KIND = "custom"
const OPENAI_COMPATIBLE_DEFAULT_FAMILY = "openai"

/**
* Whether a custom_provider `kind` names a DEPLOYMENT surface (a hosting mechanism, not itself a
* model family — e.g. "bedrock" hosts many families) rather than a plain provider family (e.g.
Expand All @@ -466,7 +477,15 @@ function harnessReachesCustomProviderKind(
): boolean {
if (isDeploymentProviderKind(kind)) {
const consumable = allowedDeployments(capabilities, harness)
return consumable.includes("*") || consumable.some((d) => d.toLowerCase() === kind)
const deploymentOk =
consumable.includes("*") || consumable.some((d) => d.toLowerCase() === kind)
if (!deploymentOk) return false
// An OpenAI-compatible (`custom`) deployment resolves to the openai provider family, so the
// harness must ALSO reach openai — otherwise a Claude-only-Anthropic harness that consumes
// `custom` would still be offered an OpenAI-compatible connection it cannot run.
if (kind === OPENAI_COMPATIBLE_KIND)
return harnessAllowsProvider(capabilities, harness, OPENAI_COMPATIBLE_DEFAULT_FAMILY)
return true
}
const providers = allowedProviders(capabilities, harness)
return providers.includes("*") || providers.some((p) => p.toLowerCase() === kind)
Expand Down
137 changes: 132 additions & 5 deletions web/packages/agenta-entity-ui/tests/unit/connectionUtils.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,15 @@ const CAPABILITIES: HarnessCapabilitiesMap = {
},
},
},
// A Pi-family harness that consumes the OpenAI-compatible `custom` deployment AND reaches the
// openai provider family — the one combination an OpenAI-compatible connection may surface for.
pi_openai_compat: {
providers: ["openai"],
deployments: ["direct", "custom"],
connection_modes: ["agenta", "self_managed"],
model_selection: "provider/id",
models: {openai: ["gpt-5.5"]},
},
}

describe("connectionUtils: modelIdFromConfig", () => {
Expand Down Expand Up @@ -359,17 +368,57 @@ describe("connectionUtils: vaultModelGroups (custom_provider connections)", () =
).toEqual([])
})

it("gates a deployment-kind connection (custom/bedrock/vertex_ai) against consumable deployments, not providers", () => {
// claude's capability entry declares "custom" as a consumable deployment.
it("gates a NON-custom deployment kind (bedrock) against consumable deployments only", () => {
// claude declares "bedrock" as a consumable deployment; its models encode their own family,
// so deployment consumption alone is the gate (no single-family provider check).
expect(
vaultModelGroups(
[{name: "my-gateway", provider: "custom", models: ["gpt-oss"]}],
[
{
name: "my-bedrock",
provider: "bedrock",
models: ["eu.anthropic.claude-haiku-4-5"],
},
],
CAPABILITIES,
"claude",
),
).toHaveLength(1)
// pi_core only consumes "direct" — a "custom" deployment connection stays hidden there
// (matches the runner: Pi ignores a resolved custom endpoint/base_url in v1).
// pi_core only consumes "direct" — a "bedrock" deployment connection stays hidden there.
expect(
vaultModelGroups(
[
{
name: "my-bedrock",
provider: "bedrock",
models: ["eu.anthropic.claude-haiku-4-5"],
},
],
CAPABILITIES,
"pi_core",
),
).toEqual([])
})

it("gates an OpenAI-compatible (custom) connection by BOTH consumable deployment and openai reach", () => {
// A harness that consumes `custom` AND reaches openai is offered it.
expect(
vaultModelGroups(
[{name: "my-gateway", provider: "custom", models: ["gpt-oss"]}],
CAPABILITIES,
"pi_openai_compat",
),
).toHaveLength(1)
// claude consumes `custom` but only reaches anthropic — the OpenAI-compatible connection is
// NOT offered (it resolves to the openai family Claude cannot run).
expect(
vaultModelGroups(
[{name: "my-gateway", provider: "custom", models: ["gpt-oss"]}],
CAPABILITIES,
"claude",
),
).toEqual([])
// pi_core does not consume `custom` at all — hidden regardless of provider reach.
expect(
vaultModelGroups(
[{name: "my-gateway", provider: "custom", models: ["gpt-oss"]}],
Expand All @@ -387,6 +436,14 @@ describe("connectionUtils: vaultModelGroups (custom_provider connections)", () =
"pi_core",
),
).toHaveLength(1)
// A custom connection is also offered under a missing map (both gates read permissive).
expect(
vaultModelGroups(
[{name: "my-gateway", provider: "custom", models: ["gpt-oss"]}],
null,
"pi_core",
),
).toHaveLength(1)
})

it("skips connections with no slug or no models", () => {
Expand All @@ -401,6 +458,41 @@ describe("connectionUtils: vaultModelGroups (custom_provider connections)", () =
),
).toEqual([])
})

// Regression pins: the OpenAI-compatible feature adds a `custom`-only branch. Every flow that
// does NOT involve a custom connection must stay byte-identical to pre-feature behavior.
it("regression: an empty vault yields no groups on any harness (default picker state)", () => {
expect(vaultModelGroups([], CAPABILITIES, "pi_core")).toEqual([])
expect(vaultModelGroups([], CAPABILITIES, "claude")).toEqual([])
expect(vaultModelGroups(null, CAPABILITIES, "claude")).toEqual([])
})

it("regression: a Claude harness still sees its reachable (anthropic) vault connection unchanged", () => {
expect(
vaultModelGroups(
[{name: "my-anthropic", provider: "anthropic", models: ["a1"]}],
CAPABILITIES,
"claude",
),
).toHaveLength(1)
})

it("regression: a non-custom deployment connection to Claude is unchanged by the custom gate", () => {
// Claude consumes bedrock; the new openai-family check must NOT touch non-custom kinds.
expect(
vaultModelGroups(
[
{
name: "my-bedrock",
provider: "bedrock",
models: ["eu.anthropic.claude-haiku-4-5"],
},
],
CAPABILITIES,
"claude",
),
).toHaveLength(1)
})
})

describe("connectionUtils: isDeploymentProviderKind", () => {
Expand Down Expand Up @@ -451,4 +543,39 @@ describe("connectionUtils: vaultPickedProviderFamily (F1 — vault pick must per
it("still resolves the family from metadata alone when the id is absent", () => {
expect(vaultPickedProviderFamily(null, "openai", CAPABILITIES)).toBe("openai")
})

it("defaults an OpenAI-compatible (custom) connection with a bare model id to openai", () => {
// The `custom` kind is a deployment surface (not itself a family), but the OpenAI-compatible
// endpoint speaks the OpenAI dialect — so a provider-less pick resolves to openai instead of
// deferring to the caller's prior-provider fallback (design Decision 8).
expect(vaultPickedProviderFamily("gpt-oss", "custom", CAPABILITIES)).toBe("openai")
expect(vaultPickedProviderFamily("qwen2.5-coder:7b", "custom", CAPABILITIES)).toBe("openai")
})

it("still prefers an explicit id-encoded family over the custom openai default", () => {
// If the id itself encodes a known family, that wins even for a custom connection.
expect(
vaultPickedProviderFamily("eu.anthropic.claude-haiku-4-5", "custom", CAPABILITIES),
).toBe("anthropic")
})
})

describe("connectionUtils: custom pick persists openai family AND keeps the connection slug", () => {
// Mirrors what `useModelHarness.writeModel` composes for a picked OpenAI-compatible option: the
// resolved family (openai, from vaultPickedProviderFamily) plus the option's own connection slug
// (threaded through `metadata.connectionSlug`). Neither may be dropped.
it("composes a ModelRef with provider openai and the preserved agenta slug", () => {
const provider = vaultPickedProviderFamily("gpt-oss", "custom", CAPABILITIES)
const ref = composeModelValue({
modelId: "gpt-oss",
provider,
mode: "agenta",
slug: "my-gateway",
})
expect(ref).toEqual({
model: "gpt-oss",
provider: "openai",
connection: {mode: "agenta", slug: "my-gateway"},
})
})
})
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,7 @@ async function waitForModelsPageReady(page: Page): Promise<void> {
.isVisible()
.catch(() => false)
const createButtonEnabled = await customProvidersSection
.getByRole("button", {name: "Custom Provider"})
.getByRole("button", {name: "OpenAI-compatible endpoint"})
.isEnabled()
.catch(() => false)

Expand Down Expand Up @@ -190,9 +190,9 @@ async function navigateToModels(page: Page, uiHelpers: UIHelpers): Promise<void>

function getCustomProvidersSection(page: Page): Locator {
// The custom-providers section no longer has a dedicated header label — its
// section-identifying text IS the "Custom Provider" trigger button now.
// section-identifying text IS the "OpenAI-compatible endpoint" trigger button now.
return page
.getByText("Custom Provider", {exact: true})
.getByText("OpenAI-compatible endpoint", {exact: true})
.locator("xpath=ancestor::section[1]")
.first()
}
Expand Down
Loading