feat: auto-fill custom provider prices from models.dev

.changeset/modelsdev-custom-provider-prices.md

@@ -0,0 +1,5 @@
+---
+'manifest': patch
+---
+
+Auto-fill custom provider model prices from exact models.dev provider/model matches, and mark exact model-only matches as estimated when no provider match is available.

packages/backend/src/database/models-dev-sync.service.spec.ts

@@ -169,6 +169,19 @@ const MOCK_API_RESPONSE = {
       },
     },
   },
+  kilo: {
+    id: 'kilo',
+    name: 'Kilo Gateway',
+    models: {
+      'openai/gpt-4o-mini': {
+        id: 'openai/gpt-4o-mini',
+        name: 'GPT-4o mini',
+        cost: { input: 0.15, output: 0.6, cache_read: 0.075 },
+        limit: { context: 128000, output: 16384 },
+        modalities: { input: ['text'], output: ['text'] },
+      },
+    },
+  },
   'unknown-provider': {
     id: 'unknown-provider',
     name: 'Unknown',
@@ -967,4 +980,67 @@ describe('ModelsDevSyncService', () => {
       expect(lower.length).toBeGreaterThan(0);
     });
   });
+
+  describe('lookupCustomProviderModel', () => {
+    beforeEach(async () => {
+      fetchSpy.mockResolvedValue({
+        ok: true,
+        json: async () => MOCK_API_RESPONSE,
+      });
+      await service.refreshCache();
+    });
+
+    it('should find arbitrary models.dev providers by display name', () => {
+      const model = service.lookupCustomProviderModel('Kilo Gateway', 'openai/gpt-4o-mini');
+      expect(model).not.toBeNull();
+      expect(model!.name).toBe('GPT-4o mini');
+      expect(model!.inputPricePerToken).toBe(0.15 / 1_000_000);
+      expect(model!.outputPricePerToken).toBe(0.6 / 1_000_000);
+      expect(model!.contextWindow).toBe(128000);
+    });
+
+    it('should normalize custom provider names and IDs', () => {
+      expect(service.lookupCustomProviderModel('kilo', 'openai/gpt-4o-mini')).not.toBeNull();
+      expect(
+        service.lookupCustomProviderModel('kilo-gateway', 'openai/gpt-4o-mini'),
+      ).not.toBeNull();
+    });
+
+    it('should keep native provider support scoped to PROVIDER_ID_MAP', () => {
+      expect(service.getModelsForProvider('kilo')).toEqual([]);
+    });
+
+    it('should return null when provider or model is missing', () => {
+      expect(service.lookupCustomProviderModel('Mammouth', 'openai/gpt-4o-mini')).toBeNull();
+      expect(service.lookupCustomProviderModel('Kilo Gateway', 'missing-model')).toBeNull();
+    });
+  });
+
+  describe('lookupModelAcrossProviders', () => {
+    beforeEach(async () => {
+      fetchSpy.mockResolvedValue({
+        ok: true,
+        json: async () => MOCK_API_RESPONSE,
+      });
+      await service.refreshCache();
+    });
+
+    it('should match provider-prefixed model IDs against official provider catalogs first', () => {
+      const model = service.lookupModelAcrossProviders('openai/gpt-4o');
+      expect(model).not.toBeNull();
+      expect(model!.name).toBe('GPT-4o');
+      expect(model!.inputPricePerToken).toBe(2.5 / 1_000_000);
+    });
+
+    it('should fall back to exact model IDs from non-native provider catalogs', () => {
+      const model = service.lookupModelAcrossProviders('openai/gpt-4o-mini');
+      expect(model).not.toBeNull();
+      expect(model!.name).toBe('GPT-4o mini');
+      expect(model!.inputPricePerToken).toBe(0.15 / 1_000_000);
+    });
+
+    it('should return null when no provider contains the model ID', () => {
+      expect(service.lookupModelAcrossProviders('missing-model')).toBeNull();
+    });
+  });
 });

packages/backend/src/database/models-dev-sync.service.ts

@@ -1,6 +1,6 @@
 import { Injectable, Logger, OnModuleInit } from '@nestjs/common';
 import { Cron, CronExpression } from '@nestjs/schedule';
-import type { ModelCapability } from 'manifest-shared';
+import { normalizeProviderName, type ModelCapability } from 'manifest-shared';
 import { PROVIDER_BY_ID_OR_ALIAS } from '../common/constants/providers';
 import { capabilitiesFromModelsDev } from '../model-discovery/model-capabilities';
 
@@ -98,6 +98,10 @@ export class ModelsDevSyncService implements OnModuleInit {
   private readonly logger = new Logger(ModelsDevSyncService.name);
   /** Map: our provider ID → Map<model ID (native), entry> */
   private cache = new Map<string, Map<string, ModelsDevModelEntry>>();
+  /** Map: models.dev provider ID → Map<model ID, entry>. Includes providers Manifest does not natively know. */
+  private customProviderCache = new Map<string, Map<string, ModelsDevModelEntry>>();
+  /** Map: provider id/display-name aliases → models.dev provider ID. */
+  private customProviderIndex = new Map<string, string>();
   private lastFetchedAt: Date | null = null;
   private initialLoad: Promise<void> | null = null;
 
@@ -124,8 +128,25 @@ export class ModelsDevSyncService implements OnModuleInit {
     if (!raw) return 0;
 
     const newCache = new Map<string, Map<string, ModelsDevModelEntry>>();
+    const newCustomProviderCache = new Map<string, Map<string, ModelsDevModelEntry>>();
+    const newCustomProviderIndex = new Map<string, string>();
     let totalModels = 0;
 
+    for (const [modelsDevId, provider] of Object.entries(raw)) {
+      if (!provider?.models) continue;
+
+      const modelMap = new Map<string, ModelsDevModelEntry>();
+      for (const [modelId, model] of Object.entries(provider.models)) {
+        if (!this.isChatCompatible(model)) continue;
+        modelMap.set(modelId, this.parseModel(modelsDevId, modelId, model));
+      }
+
+      if (modelMap.size > 0) {
+        newCustomProviderCache.set(modelsDevId, modelMap);
+        this.indexCustomProvider(newCustomProviderIndex, modelsDevId, provider);
+      }
+    }
+
     for (const [ourId, modelsDevId] of Object.entries(PROVIDER_ID_MAP)) {
       const provider = raw[modelsDevId];
       if (!provider?.models) continue;
@@ -144,6 +165,8 @@ export class ModelsDevSyncService implements OnModuleInit {
     }
 
     this.cache = newCache;
+    this.customProviderCache = newCustomProviderCache;
+    this.customProviderIndex = newCustomProviderIndex;
     this.lastFetchedAt = new Date();
     this.logger.log(`models.dev cache loaded: ${newCache.size} providers, ${totalModels} models`);
 
@@ -165,7 +188,55 @@ export class ModelsDevSyncService implements OnModuleInit {
   lookupModel(providerId: string, modelId: string): ModelsDevModelEntry | null {
     const providerModels = this.cache.get(resolveProviderId(providerId));
     if (!providerModels) return null;
+    return this.lookupModelInProvider(providerModels, modelId);
+  }
 
+  /**
+   * Look up a model for a user-defined custom provider by matching the custom
+   * provider display name against models.dev provider IDs and names.
+   */
+  lookupCustomProviderModel(providerName: string, modelId: string): ModelsDevModelEntry | null {
+    const providerKey = this.resolveCustomProviderKey(providerName);
+    if (!providerKey) return null;
+    const providerModels = this.customProviderCache.get(providerKey);
+    if (!providerModels) return null;
+    return this.lookupModelInProvider(providerModels, modelId);
+  }
+
+  /**
+   * Conservative model-only fallback for custom providers that are not listed
+   * on models.dev. Prefer official provider catalogs, then exact IDs from
+   * aggregator catalogs. This is intentionally not fuzzy.
+   */
+  lookupModelAcrossProviders(modelId: string): ModelsDevModelEntry | null {
+    const providerScoped = this.lookupProviderScopedModel(modelId);
+    if (providerScoped) return providerScoped;
+
+    for (const providerModels of this.cache.values()) {
+      const found = this.lookupModelInProvider(providerModels, modelId);
+      if (found) return found;
+    }
+
+    for (const providerModels of this.customProviderCache.values()) {
+      const exact = providerModels.get(modelId);
+      if (exact) return exact;
+    }
+
+    return null;
+  }
+
+  private lookupProviderScopedModel(modelId: string): ModelsDevModelEntry | null {
+    const slash = modelId.indexOf('/');
+    if (slash <= 0 || slash === modelId.length - 1) return null;
+    const providerPart = modelId.slice(0, slash);
+    const nativeModelId = modelId.slice(slash + 1);
+    return this.lookupModel(providerPart, nativeModelId);
+  }
+
+  private lookupModelInProvider(
+    providerModels: Map<string, ModelsDevModelEntry>,
+    modelId: string,
+  ): ModelsDevModelEntry | null {
     // 1. Exact match
     const exact = providerModels.get(modelId);
     if (exact) return exact;
@@ -255,6 +326,16 @@ export class ModelsDevSyncService implements OnModuleInit {
     return null;
   }
 
+  private resolveCustomProviderKey(providerName: string): string | null {
+    const trimmed = providerName.trim();
+    if (!trimmed) return null;
+    return (
+      this.customProviderIndex.get(trimmed.toLowerCase()) ??
+      this.customProviderIndex.get(normalizeProviderName(trimmed)) ??
+      null
+    );
+  }
+
   /**
    * Get all models for a provider (by our internal provider ID).
    * Returns an empty array if the provider is not found.
@@ -274,6 +355,29 @@ export class ModelsDevSyncService implements OnModuleInit {
     return this.lastFetchedAt;
   }
 
+  private indexCustomProvider(
+    index: Map<string, string>,
+    modelsDevId: string,
+    provider: RawModelsDevProvider,
+  ): void {
+    this.addCustomProviderIndexEntry(index, modelsDevId, modelsDevId);
+    if (provider.id) this.addCustomProviderIndexEntry(index, provider.id, modelsDevId);
+    if (provider.name) this.addCustomProviderIndexEntry(index, provider.name, modelsDevId);
+  }
+
+  private addCustomProviderIndexEntry(
+    index: Map<string, string>,
+    value: string,
+    modelsDevId: string,
+  ): void {
+    const trimmed = value.trim();
+    if (!trimmed) return;
+    const lower = trimmed.toLowerCase();
+    if (!index.has(lower)) index.set(lower, modelsDevId);
+    const normalized = normalizeProviderName(trimmed);
+    if (!index.has(normalized)) index.set(normalized, modelsDevId);
+  }
+
   private parseModel(
     providerId: string,
     modelId: string,

packages/backend/src/entities/custom-provider.entity.ts

@@ -7,6 +7,7 @@ export interface CustomProviderModel {
   input_price_per_million_tokens?: number;
   output_price_per_million_tokens?: number;
   context_window?: number;
+  price_estimated?: boolean;
 }
 
 export type CustomProviderApiKind = 'openai' | 'anthropic';

packages/backend/src/routing/custom-provider.controller.spec.ts

@@ -261,6 +261,7 @@ describe('CustomProviderController', () => {
         'http://host.docker.internal:8000/v1',
         'sk-x',
         undefined,
+        undefined,
       );
       expect(result).toEqual({ models: [{ model_name: 'm1' }, { model_name: 'm2' }] });
     });
@@ -275,6 +276,21 @@ describe('CustomProviderController', () => {
         'https://api.anthropic.com',
         'sk-ant-x',
         'anthropic',
+        undefined,
+      );
+    });
+
+    it('forwards provider_name so probe results can be price-enriched', async () => {
+      await controller.probe(mockUser, 'test-agent', {
+        base_url: 'https://api.kilo.ai/api/gateway',
+        apiKey: 'kilo-x',
+        provider_name: 'Kilo Gateway',
+      } as never);
+      expect(mockCustomProviderService.probeModels).toHaveBeenCalledWith(
+        'https://api.kilo.ai/api/gateway',
+        'kilo-x',
+        undefined,
+        'Kilo Gateway',
       );
     });
 

packages/backend/src/routing/custom-provider/custom-provider.controller.ts

@@ -56,6 +56,7 @@ export class CustomProviderController {
       body.base_url,
       body.apiKey,
       body.api_kind,
+      body.provider_name,
     );
     return { models };
   }