Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 8 additions & 4 deletions apps/web/src/app/api/fim/completions/route.ts
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import { readDb } from '@/lib/drizzle';
import { debugSaveProxyRequest } from '@/lib/debugUtils';
import { sentryLogger } from '@/lib/utils.server';
import { getBYOKforOrganization, getBYOKforUser } from '@/lib/ai-gateway/byok';
import type { UserByokProviderId } from '@/lib/ai-gateway/providers/openrouter/inference-provider-id';

// Mistral exposes FIM on two separate, key-incompatible endpoints:
// - https://api.mistral.ai (La Plateforme, paid tier keys)
Expand Down Expand Up @@ -153,11 +154,12 @@ export async function POST(request: NextRequest) {
// Extract properties for usage context
const promptInfo = extractFimPromptInfo(requestBody);

const byokProviderKey = fimProvider === 'mistral' ? 'codestral' : 'inception';
const byokProviderKeys: UserByokProviderId[] =
fimProvider === 'mistral' ? ['codestral', 'mistral'] : ['inception'];
Comment thread
chrarnoldus marked this conversation as resolved.

const userByok = organizationId
? await getBYOKforOrganization(readDb, organizationId, [byokProviderKey])
: await getBYOKforUser(readDb, user.id, [byokProviderKey]);
? await getBYOKforOrganization(readDb, organizationId, byokProviderKeys)
: await getBYOKforUser(readDb, user.id, byokProviderKeys);

const usageContext: MicrodollarUsageContext = {
api_kind: 'fim_completions',
Expand Down Expand Up @@ -232,7 +234,9 @@ export async function POST(request: NextRequest) {
}

const systemKey = getSystemApiKey(fimProvider);
const userByokEntry = userByok?.at(0);
const userByokEntry = byokProviderKeys
.map(providerId => userByok?.find(entry => entry.providerId === providerId))
.find(entry => entry !== undefined);
const apiKey = userByokEntry?.decryptedAPIKey ?? systemKey;
const upstreamUrl = resolveFimUpstreamUrl(fimProvider, userByokEntry?.providerId === 'codestral');

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ const VERCEL_BYOK_PROVIDER_NAMES = {
fireworks: 'Fireworks',
google: 'Google AI Studio',
minimax: 'MiniMax',
mistral: 'Mistral AI (other models)',
mistral: 'Mistral AI',
moonshotai: 'Moonshot AI',
novita: 'Novita',
perplexity: 'Perplexity',
Expand All @@ -81,7 +81,7 @@ const VERCEL_BYOK_PROVIDER_NAMES = {

const VERCEL_BYOK_PROVIDERS = [
...Object.entries(VERCEL_BYOK_PROVIDER_NAMES).map(([id, name]) => ({ id, name })),
{ id: DirectUserByokInferenceProviderIdSchema.enum.codestral, name: 'Mistral AI (Codestral)' },
{ id: DirectUserByokInferenceProviderIdSchema.enum.codestral, name: 'Legacy Codestral-only key' },
];

const DIRECT_BYOK_PROVIDERS_LIST = Object.entries(DIRECT_BYOK_PROVIDERS_META).map(([id, name]) => ({
Expand All @@ -92,6 +92,9 @@ const DIRECT_BYOK_PROVIDERS_LIST = Object.entries(DIRECT_BYOK_PROVIDERS_META).ma
const BYOK_PROVIDERS = [...DIRECT_BYOK_PROVIDERS_LIST, ...VERCEL_BYOK_PROVIDERS].toSorted((a, b) =>
a.name.localeCompare(b.name)
);
const ADD_BYOK_PROVIDERS = BYOK_PROVIDERS.filter(
provider => provider.id !== DirectUserByokInferenceProviderIdSchema.enum.codestral
);

function BYOKDescription({ showsCodingPlanKey = false }: { showsCodingPlanKey?: boolean }) {
return (
Expand Down Expand Up @@ -604,7 +607,7 @@ export function BYOKKeysManager({ organizationId }: BYOKKeysManagerProps) {
<SelectValue placeholder="Select provider" />
</SelectTrigger>
<SelectContent>
{BYOK_PROVIDERS.map(provider => {
{(editingKeyId ? BYOK_PROVIDERS : ADD_BYOK_PROVIDERS).map(provider => {
const isDisabled = !editingKeyId && hasExistingKey(provider.id);
return (
<SelectItem
Expand Down
8 changes: 4 additions & 4 deletions apps/web/src/lib/ai-gateway/byok/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,22 +15,22 @@ import type { BYOKResult } from '@/lib/ai-gateway/providers/types';
import { getVercelModelsMetadata } from '@/lib/ai-gateway/providers/gateway-models-cache';

export async function getModelUserByokProviders(modelId: string): Promise<UserByokProviderId[]> {
if (isCodestralModel(modelId)) {
return ['codestral'];
}
const vercelModelMetadata = await getVercelModelsMetadata();
if (Object.keys(vercelModelMetadata).length === 0) {
console.error('[getModelUserByokProviders] no Vercel model metadata in the database');
return [];
}
const providers =
const providers: UserByokProviderId[] =
vercelModelMetadata[mapModelIdToVercel(modelId, false)]?.endpoints
.map(ep => VercelUserByokInferenceProviderIdSchema.safeParse(ep.provider_name ?? ep.tag).data)
.filter(providerId => providerId !== undefined) ?? [];
if (providers.length === 0) {
console.debug(`[getModelUserByokProviders] no user byok providers for ${modelId}`);
return [];
}
if (isCodestralModel(modelId)) {
providers.unshift('codestral');
}
console.debug('[getModelUserByokProviders] found user byok providers for %s', modelId, providers);
return providers;
}
Expand Down
2 changes: 0 additions & 2 deletions apps/web/src/routers/byok-router.ts
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,6 @@ import { getVercelInferenceProviderConfigForUserByok } from '@/lib/ai-gateway/pr
import { decryptByokRow } from '@/lib/ai-gateway/byok';
import type { GatewayProviderOptions } from '@ai-sdk/gateway';
import { mapModelIdToVercel } from '@/lib/ai-gateway/providers/vercel/mapModelIdToVercel';
import { isCodestralModel } from '@/lib/ai-gateway/providers/mistral';
import { isKiloExclusiveModel } from '@/lib/ai-gateway/models';
import DIRECT_BYOK_PROVIDERS from '@/lib/ai-gateway/providers/direct-byok/direct-byok-definitions';
import {
Expand Down Expand Up @@ -114,7 +113,6 @@ async function fetchSupportedModels(): Promise<Record<string, string[]>> {
if (isKiloExclusiveModel(openRouterModel.id)) continue;
const vercelModel = vercelModelMetadata[mapModelIdToVercel(openRouterModel.id, false)];
if (!vercelModel) continue;
if (isCodestralModel(vercelModel.id)) continue;
if (vercelModel.type !== 'language') continue;
for (const endpoint of vercelModel.endpoints) {
const providerParsed = VercelUserByokInferenceProviderIdSchema.safeParse(
Expand Down