Performance enhancements galore! New themining system

This is once again a huge commit, but its mostly performance improvements along with some bug fixes and refactoring. It also includes changes to the theming systems. I'm still not 100% happy with the theming system, but its better than before. Model fetching has been dramatically improved! Nearly all the important computation and pre-processing has been moved to the server. This has also somehow fixed the way model details are loaded, which was causing many models to be missing their details despite models.dev having them. The markdown renderer has once again been changed, but I'm mostly certain that this is the last time major changes will be made to it. The renderer is not spamming components, bloating memory usage, and its not using a bug prone custom written chunking system. There's also a lot more that I haven't mentioned and honestly forgot. I need to get better commit hygiene tbh.
2026-02-19 23:44:23 -06:00
parent 32a4f7f95d
commit 59bb7fbc12
85 changed files with 3523 additions and 2039 deletions
@@ -8,20 +8,19 @@ export default defineEventHandler(async (event) => {

    const success = cancelPendingGeneration(generationId!);

-    if (!success) {
-        const generation = await httpClient.fetchOne(httpClient.query('generations').Where('id', '=', generationId!));
-        if (generation !== null && generation.status === 'pending') {
-            await httpClient.update('generations', generationId!, {
-                status: 'cancelled',
-            });
-            return;
-        }
+    const generation = await httpClient.fetchOne(httpClient.query('generations').Where('id', '=', generationId!));
+    if (generation !== null && generation.status === 'pending') {
+        await httpClient.update('generations', generationId!, {
+            status: 'cancelled',
+        });
+    }

+    if (!success) {
        throw createError({
            statusCode: 400,
            message: 'Generation not found or already completed',
        });
    }

-    return;
+    return 'ok';
 });
@@ -9,7 +9,8 @@ import { httpClient } from '~~/server/lib/triplit';
 import { addPendingGeneration, completeGeneration } from '~~/server/utils/generations';
 import type { schema } from '~~/triplit/schema';
 import { spawn } from 'child_process';
-import { getGateway, type ModelGateway } from '~~/server/utils/ai-provider';
+import { GatewayFetchError, getProviderDetails, type ModelGateway } from '~~/server/utils/ai-provider';
+import { assert } from '~~/utils/assert';

 export const messagesSchema = z.array(modelMessageSchema);

@@ -73,7 +74,26 @@ export default defineEventHandler(async (event) => {
        });
    }

-    const { gateway, streamTransformer: transformer } = await getGateway(provider, model, providerApiKey);
+    const providerDetails = await getProviderDetails(provider, providerApiKey, model);
+    if (!providerDetails.ok) {
+        switch (providerDetails.error) {
+            case GatewayFetchError.NoProviderApiKey: {
+                throw createError({
+                    statusCode: 400,
+                    message: `${provider.type} provider requires an API key`,
+                });
+            }
+            case GatewayFetchError.NoProviderBaseUrl: {
+                throw createError({
+                    statusCode: 400,
+                    message: 'Invalid provider URL',
+                });
+            }
+        }
+    }
+
+    const { gateway } = providerDetails.data;
+    assert(gateway !== null, 'Invalid gateway');

    const generationId = nanoid();
    const message = await httpClient.insert('messages', {
@@ -98,14 +118,20 @@ export default defineEventHandler(async (event) => {
    let logMessage: ((message: string) => void) | undefined;

    if (process.env.GENERATION_DEBUG) {
-        logFile = await fs.open(path.join(process.env.LOG_DIR!, `${Date.now()}-${generationId}.log`), 'w');
-        logMessage = (message: string) => {
-            logFile!.write(message + '\n');
-        };
+        if (process.env.LOG_DIR) {
+            await fs.mkdir(process.env.LOG_DIR!, { recursive: true });
+
+            logFile = await fs.open(path.join(process.env.LOG_DIR!, `${Date.now()}-${generationId}.log`), 'w');
+            logMessage = (message: string) => {
+                logFile!.write(message + '\n');
+            };
+        } else {
+            console.warn('Generation debug logging is enabled but GENERATION_DEBUG is not set');
+        }
    }

    event.waitUntil(
-        generateResponse(message, { gateway, model, parameters: args }, generationId, userId, topicId, messages, transformer, logMessage, logFile),
+        generateResponse(message, { gateway: gateway.gateway, model, parameters: args }, generationId, userId, topicId, messages, gateway.streamTransformer, logMessage, logFile),
    );

    return {
@@ -755,8 +781,6 @@ async function generateResponse(
                        status: 'failed',
                        error,
                    });
-
-                    throw new Error(error);
                } break;
                case 'finish': {
                    let tps;
@@ -1,7 +1,13 @@
+import { type Entity } from '@triplit/client';
+import Big from 'big.js';
 import * as z from 'zod';
-import { providerBaseUrls, SupportedModalities } from '~/types/model';
+import { SupportedModalities } from '~/types/model';
 import { Providers } from '~/types/model';
 import { httpClient } from '~~/server/lib/triplit';
+import { GatewayFetchError, getProviderDetails } from '~~/server/utils/ai-provider';
+import { getModelsDevData } from '~~/server/utils/models-dev';
+import { schema } from '~~/triplit/schema';
+import { Err, Ok, type Result } from '~~/types/result';

 export default defineEventHandler(async (event) => {
    await protectRoute(event);
@@ -32,109 +38,130 @@ export default defineEventHandler(async (event) => {
        });
    }

-    let baseUrl;
-    let fetchUrl;
-    let headers;
-    switch (provider.type) {
-        case 'cohere':
-        case 'cerebras':
-            if (!result.data.providerApiKey) {
+    console.log("apiKey", result.data.providerApiKey);
+
+    const [providerModelsRes, modelsDevRes] = await Promise.all([
+        fetchProviderModels(provider, result.data.providerApiKey),
+        getModelsDevData(),
+    ]);
+    if (!providerModelsRes.ok) {
+        switch (providerModelsRes.error) {
+            case ProviderFetchError.NoProviderApiKey: {
                throw createError({
                    statusCode: 400,
                    message: `${provider.type} provider requires an API key`,
                });
            }
-        case 'google':
-        case 'openrouter':
-            baseUrl = !!provider.config.apiProxyUrl ? provider.config.apiProxyUrl : providerBaseUrls[provider.type];
-            baseUrl = baseUrl.replace(/\/$/, '');
-
-            if (baseUrl === '') {
+            case ProviderFetchError.NoProviderBaseUrl: {
                throw createError({
                    statusCode: 400,
                    message: 'Invalid provider URL',
                });
            }
+        }
+    }

-            fetchUrl = `${baseUrl}/models`;
-            break;
-        case 'ollama':
-            if (!provider.config.apiProxyUrl) {
-                throw createError({
-                    statusCode: 400,
-                    message: 'Ollama provider requires an API proxy URL',
-                });
+    console.log("providerModelsRes.data", providerModelsRes.data);
+
+    const normalizedModels = await normalizeResponse(
+        providerModelsRes.data.data,
+        provider.type,
+        providerModelsRes.data.baseURL,
+        modelsDevRes
+    );
+
+    return {
+        models: normalizedModels,
+    };
+});
+
+enum ProviderFetchError {
+    NoProviderApiKey = 0,
+    NoProviderBaseUrl,
+}
+
+const fetchProviderModels = async (provider: Entity<typeof schema, 'providers'>, providerApiKey: string | undefined): Promise<Result<{ data: Record<string, any>, baseURL: string }, ProviderFetchError>> => {
+    const providerDetails = await getProviderDetails(provider, providerApiKey);
+    if (!providerDetails.ok) {
+        switch (providerDetails.error) {
+            case GatewayFetchError.NoProviderApiKey: {
+                // throw createError({
+                //     statusCode: 400,
+                //     message: `${provider.type} provider requires an API key`,
+                // });
+                return Err(ProviderFetchError.NoProviderApiKey);
            }
-            baseUrl = provider.config.apiProxyUrl;
-            baseUrl = baseUrl.replace(/\/$/, '');
+            case GatewayFetchError.NoProviderBaseUrl: {
+                // throw createError({
+                //     statusCode: 400,
+                //     message: 'Invalid provider URL',
+                // });
+                return Err(ProviderFetchError.NoProviderBaseUrl);
+            }
+        }
+    }

-            fetchUrl = `${baseUrl}/api/tags`;
-            break;
+    const { endpoint: { baseURL, modelsEndpoint, headers } } = providerDetails.data;
+
+    switch (provider.type) {
        case 'longcat':
            // longcat doesn't have a model list endpoint so we just hardcode them here,
            // sry. I talked to Meituan and this is what they said:
            // 后续如果我们新增了这样的接口会及时同步您。
            // en (approx): If we add an interface like this in the future, we will promptly update you
-            return {
-                models: [
-                    {
-                        id: "LongCat-Flash-Chat",
-                        name: "LongCat Flash Chat",
-                        attributes: {
-                            inputModalities: ['text'],
-                            outputModalities: ['text'],
-                            capabilities: ['tools'],
-                            contextWindow: 256_000,
+            return Ok({
+                data: {
+                    models: [
+                        {
+                            id: "LongCat-Flash-Chat",
+                            name: "LongCat Flash Chat",
+                            attributes: {
+                                inputModalities: ['text'],
+                                outputModalities: ['text'],
+                                capabilities: ['tools'],
+                                contextWindow: 256_000,
+                            }
+                        },
+                        {
+                            id: "LongCat-Flash-Thinking",
+                            name: "LongCat Flash Thinking",
+                            attributes: {
+                                inputModalities: ['text'],
+                                outputModalities: ['text'],
+                                capabilities: ['reasoning', 'tools'],
+                                contextWindow: 256_000,
+                            }
+                        },
+                        {
+                            id: "LongCat-Flash-Thinking-2601",
+                            name: "LongCat Flash Thinking (2601)",
+                            attributes: {
+                                inputModalities: ['text'],
+                                outputModalities: ['text'],
+                                capabilities: ['reasoning', 'tools'],
+                                contextWindow: 256_000,
+                            }
+                        },
+                        {
+                            id: "LongCat-Flash-Lite",
+                            name: "LongCat Flash Lite",
+                            attributes: {
+                                inputModalities: ['text'],
+                                outputModalities: ['text'],
+                                capabilities: ['tools'],
+                                contextWindow: 320_000,
+                            }
                        }
-                    },
-                    {
-                        id: "LongCat-Flash-Thinking",
-                        name: "LongCat Flash Thinking",
-                        attributes: {
-                            inputModalities: ['text'],
-                            outputModalities: ['text'],
-                            capabilities: ['reasoning', 'tools'],
-                            contextWindow: 256_000,
-                        }
-                    },
-                    {
-                        id: "LongCat-Flash-Thinking-2601",
-                        name: "LongCat Flash Thinking (2601)",
-                        attributes: {
-                            inputModalities: ['text'],
-                            outputModalities: ['text'],
-                            capabilities: ['reasoning', 'tools'],
-                            contextWindow: 256_000,
-                        }
-                    },
-                    {
-                        id: "LongCat-Flash-Lite",
-                        name: "LongCat Flash Lite",
-                        attributes: {
-                            inputModalities: ['text'],
-                            outputModalities: ['text'],
-                            capabilities: ['tools'],
-                            contextWindow: 320_000,
-                        }
-                    }
-                ]
-            };
-    }
-
-    if (provider.type === 'google') {
-        headers = {
-            'x-goog-api-key': `${result.data.providerApiKey}`
-        }
-    } else {
-        headers = {
-            'Authorization': `Bearer ${result.data.providerApiKey}`
-        }
+                    ],
+                },
+                baseURL
+            });
    }

    let res;
    let data;
    try {
-        res = await fetch(fetchUrl, {
+        res = await fetch(`${baseURL}${modelsEndpoint}`, {
            method: 'GET',
            headers
        });
@@ -154,16 +181,89 @@ export default defineEventHandler(async (event) => {
        });
    }

-    return {
-        models: await normalizeResponse(data, provider.type, baseUrl)
-    };
-});
+    return Ok({ data, baseURL });
+}

-const normalizeResponse = async (response: Record<string, any>, provider: typeof Providers[number], baseUrl: string) => {
+function mergeSets(setA: Set<string>, setB: Set<string>): string[] {
+    return Array.from(new Set([...setA, ...setB]));
+}
+
+const getModelData = (modelId: string, providerId: string, modelsDevData: any) => {
+    const modelData = modelsDevData[providerId]?.models[modelId];
+    console.log("modelData", modelData);
+
+    if (modelData === undefined) return {
+        cost: {},
+        attributes: {
+            inputModalities: ['text'],
+            outputModalities: ['text'],
+            capabilities: [],
+        }
+    };
+
+    const capabilities = new Set<string>();
+    if (modelData.reasoning) {
+        capabilities.add('reasoning');
+    }
+
+    if (modelData.tool_call) {
+        capabilities.add('tools');
+    }
+
+    let inputModalities = modelData.modalities.input.filter((m: string) => (SupportedModalities as Readonly<string[]>).includes(m));
+    let outputModalities = modelData.modalities.output.filter((m: string) => (SupportedModalities as Readonly<string[]>).includes(m));
+    let supportedParameters = [];
+    if (modelData.temperature) {
+        supportedParameters.push('temperature');
+    }
+
+    let contextWindow = modelData.limit?.context || null;
+    let cost: Record<string, string> = {};
+    for (const key in modelData.cost) {
+        switch (key) {
+            case 'input': {
+                cost.prompt = modelData.cost[key].toString();
+            } break;
+            case 'output': {
+                cost.completion = modelData.cost[key].toString();
+            } break;
+        }
+    }
+
+    let releasedAt = (new Date(modelData.release_date)).getTime();
+
+    return {
+        name: modelData.name,
+        attributes: {
+            inputModalities,
+            outputModalities,
+            capabilities: modelData.capabilities || [],
+            contextWindow,
+            supported_parameters: modelData.supportedParameters,
+        },
+        cost,
+        releasedAt,
+    }
+}
+
+const formatBig = (bigValue: Big) => {
+    let str = bigValue.toString();
+
+    if (!str.includes('.')) return str + '.00';
+    if (str.split('.')[1]!.length === 1) return str + '0';
+
+    return str;
+};
+
+const normalizeResponse = async (response: Record<string, any>, provider: typeof Providers[number], baseUrl: string, modelsDevData: any) => {
    switch (provider) {
        case 'cerebras': {
-            console.log(response);
-            return response.data.map((model: any) => ({ id: model.id, releasedAt: model.created }));
+            console.log("response.data", response.data);
+            for (const model of response.data) {
+                console.log(model.created);
+            }
+
+            return response.data.map((model: any) => ({ ...getModelData(model.id, provider, modelsDevData), id: model.id }));
        }
        case 'openrouter': {
            const models = [];
@@ -182,18 +282,69 @@ const normalizeResponse = async (response: Record<string, any>, provider: typeof
                    }
                }

+                const pricing = {} as Record<string, string>;
+                for (const key of Object.keys(model.pricing)) {
+                    // normalize to /1M tokens since models.dev is already in /1M tokens
+
+                    switch (key) {
+                        case 'prompt':
+                        case 'completion':
+                        case 'request':
+                        case 'image':
+                        case 'audio':
+                        case 'discount':
+                            pricing[key] = formatBig(Big(model.pricing[key]).mul(1_000_000));
+                            break;
+                        case 'image_tokens':
+                            pricing['imageTokens'] = formatBig(Big(model.pricing[key]).mul(1_000_000));
+                            break;
+                        case 'image_output':
+                            pricing['imageOutput'] = formatBig(Big(model.pricing[key]).mul(1_000_000));
+                            break;
+                        case 'audio_output':
+                            pricing['audioOutput'] = formatBig(Big(model.pricing[key]).mul(1_000_000));
+                            break;
+                        case 'input_audio_cache':
+                            pricing['inputAudioCache'] = formatBig(Big(model.pricing[key]).mul(1_000_000));
+                            break;
+                        case 'web_search':
+                            pricing['webSearch'] = formatBig(Big(model.pricing[key]).mul(1_000_000));
+                            break;
+                        case 'internal_reasoning':
+                            pricing['internalReasoning'] = formatBig(Big(model.pricing[key]).mul(1_000_000));
+                            break;
+                        case 'input_cache_read':
+                            pricing['inputCacheRead'] = formatBig(Big(model.pricing[key]).mul(1_000_000));
+                            break;
+                        case 'input_cache_write':
+                            pricing['inputCacheWrite'] = formatBig(Big(model.pricing[key]).mul(1_000_000));
+                            break;
+                    }
+                }
+
+                // TODO: do I really need to pull in data from models.dev for OR models?
+                // const modelData = getModelData(model.id, provider, modelsDevData);
+
                models.push({
                    id: model.id as string,
                    name: model.name as string,
-                    pricing: model.pricing,
+                    cost: pricing,
                    attributes: {
+                        // inputModalities: mergeSets(
+                        //     new Set(modelData.inputModalities || []),
+                        //     new Set(model.architecture.input_modalities.filter((modality: string) => (SupportedModalities as Readonly<string[]>).includes(modality))),
+                        // ),
+                        // outputModalities: mergeSets(
+                        //     new Set(modelData.outputModalities || []),
+                        //     new Set(model.architecture.output_modalities.filter((modality: string) => (SupportedModalities as Readonly<string[]>).includes(modality))),
+                        // ),
                        inputModalities: model.architecture.input_modalities.filter((modality: string) => (SupportedModalities as Readonly<string[]>).includes(modality)),
                        outputModalities: model.architecture.output_modalities.filter((modality: string) => (SupportedModalities as Readonly<string[]>).includes(modality)),
                        capabilities: Array.from(capabilities),
                        contextWindow: model.context_length,
                        supported_parameters: model.supported_parameters,
                    },
-                    created: model.created,
+                    releasedAt: model.created * 1000,
                });
            }

@@ -255,12 +406,26 @@ const normalizeResponse = async (response: Record<string, any>, provider: typeof
                        console.error(e);
                    }

+                    let normalizedId = model.name;
+                    normalizedId = normalizedId.replace(/:cloud$/, '');
+                    normalizedId = normalizedId.replace(/-cloud$/, '');
+                    normalizedId = normalizedId.replace(/:latest$/, '');
+                    const modelData = getModelData(normalizedId, 'ollama-cloud', modelsDevData);
+
                    models.set(model.name, {
+                        ...modelData,
                        id: model.name,
                        name: model.name,
                        attributes: {
-                            inputModalities: Array.from(inputModalities),
-                            capabilities: Array.from(capabilities),
+                            ...modelData.attributes,
+                            inputModalities: mergeSets(
+                                inputModalities,
+                                new Set(modelData.attributes?.inputModalities || [])
+                            ),
+                            capabilities: mergeSets(
+                                capabilities,
+                                new Set(modelData.attributes?.capabilities || [])
+                            ),
                            contextWindow,
                        }
                    });
@@ -272,12 +437,10 @@ const normalizeResponse = async (response: Record<string, any>, provider: typeof
            return Array.from(models.values());
        }
        case 'google': {
-            console.log(response);
-            return response.models.map((model: any) => ({ id: model.name.replace('models/', ''), name: model.displayName }));
+            return response.models.map((model: any) => ({ ...getModelData(model.name.replace('models/', ''), provider, modelsDevData), id: model.name.replace('models/', ''), name: model.displayName }));
        }
        case 'longcat': {
-            console.log(response);
-            return response.models.map((model: any) => ({ id: model.name, name: model.name }));
+            return response.models;
        }
        case 'cohere': {
            const models = [];
@@ -299,13 +462,22 @@ const normalizeResponse = async (response: Record<string, any>, provider: typeof
                    }
                }

+                const modelData = getModelData(model.name, provider, modelsDevData);
+
                models.push({
+                    ...modelData,
                    id: model.name,
-                    name: model.name,
                    attributes: {
-                        contextWindow: model.context_length,
-                        inputModalities: Array.from(inputModalities),
-                        capabilities: Array.from(capabilities),
+                        ...modelData.attributes,
+                        inputModalities: mergeSets(
+                            inputModalities,
+                            new Set(modelData.attributes?.inputModalities || [])
+                        ),
+                        capabilities: mergeSets(
+                            capabilities,
+                            new Set(modelData.attributes?.capabilities || [])
+                        ),
+                        contextWindow: model.context_length || modelData.attributes?.contextWindow,
                    }
                });
            }
@@ -1,3 +1,4 @@
+import { httpClient } from '~~/server/lib/triplit';
 import { cancelPendingRename } from '~~/server/utils/renames';
 import { assert } from '~~/utils/assert';

@@ -7,7 +8,15 @@ export default defineEventHandler(async (event) => {
    const { renameId } = event.context.params!;
    assert(renameId);

-    if (cancelPendingRename(renameId)) {
+    const [success, pendingRename] = cancelPendingRename(renameId);
+    if (success) {
+        const topic = await httpClient.fetchOne(httpClient.query('topics').Where('id', '=', pendingRename!.topicId));
+        if (topic !== null && topic.renaming) {
+            await httpClient.update('topics', topic.id, {
+                renaming: false,
+            });
+        }
+
        return {
            success: true,
        };
@@ -4,8 +4,9 @@ import { renamePrompt } from '~~/prompts';
 import { schema } from '~~/triplit/schema';
 import { type Entity } from '@triplit/client';
 import { generateText } from 'ai';
-import { getGateway, type ModelGateway } from '~~/server/utils/ai-provider';
+import { GatewayFetchError, getProviderDetails, type ModelGateway } from '~~/server/utils/ai-provider';
 import { addPendingRename } from '~~/server/utils/renames';
+import { assert } from '~~/utils/assert';

 export default defineEventHandler(async (event) => {
    await protectRoute(event);
@@ -38,10 +39,31 @@ export default defineEventHandler(async (event) => {
            message: 'Invalid model',
        });
    }
+    assert(model.provider !== null, 'Invalid model provider');

-    const { gateway, textTransformer } = await getGateway(model.provider!, model, providerApiKey);
-    const [renameId, abortController] = addPendingRename();
-    event.waitUntil(autoRename(topicId, abortController, { gateway, model }, textTransformer, prompt));
+    const providerDetails = await getProviderDetails(model.provider, providerApiKey, model);
+    if (!providerDetails.ok) {
+        switch (providerDetails.error) {
+            case GatewayFetchError.NoProviderApiKey: {
+                throw createError({
+                    statusCode: 400,
+                    message: `${model.provider.type} provider requires an API key`,
+                });
+            }
+            case GatewayFetchError.NoProviderBaseUrl: {
+                throw createError({
+                    statusCode: 400,
+                    message: 'Invalid provider URL',
+                });
+            }
+        }
+    }
+
+    const { gateway } = providerDetails.data;
+    assert(gateway !== null, 'Invalid gateway');
+
+    const [renameId, pendingRename] = addPendingRename(topicId);
+    event.waitUntil(autoRename(topicId, renameId, pendingRename.abortController, { gateway: gateway.gateway, model }, gateway.textTransformer, prompt));

    return {
        success: true,
@@ -51,6 +73,7 @@ export default defineEventHandler(async (event) => {

 const autoRename = async (
    topicId: string,
+    renameId: string,
    abortController: AbortController,
    model: {
        gateway: ModelGateway,
@@ -86,6 +109,7 @@ const autoRename = async (
    } catch (error) {
        console.error('Failed to auto-rename:', error);
    } finally {
+        completeRename(renameId);
        await httpClient.update('topics', topicId, {
            renaming: false
        });
@@ -3,6 +3,6 @@ import { schema } from '#triplit/schema';

 export const httpClient = new HttpClient({
    schema,
-    serverUrl: process.env.NUXT_PUBLIC_TRIPLIT_URL,
+    serverUrl: process.env.NUXT_LOCAL_TRIPLIT_URL || process.env.NUXT_PUBLIC_TRIPLIT_URL,
    token: process.env.TRIPLIT_SERVICE_TOKEN,
 });
@@ -0,0 +1,10 @@
+export default defineEventHandler((event) => {
+    const start = Date.now(); //
+
+    // Hook to run when the response is about to be sent
+    event.node.res.on('finish', () => {
+        const end = Date.now();
+        const duration = end - start;
+        console.log(`Request to ${event.req.url} took ${duration}ms to render.`); //
+    });
+});
@@ -9,6 +9,7 @@ import { schema } from "~~/triplit/schema";
 import { type StreamTextTransform } from "ai";
 import { transformCerebrasReasoningStream } from "./cerebras";
 import { providerBaseUrls } from "~/types/model";
+import { type Result, Err, Ok } from "~~/types/result";
 // import { createLongcatTransformer } from "./longcat";

 export type ModelGateway = OpenRouterProvider | OllamaProvider | CerebrasProvider | GoogleGenerativeAIProvider | OpenAICompatibleProvider | CohereProvider;
@@ -19,90 +20,133 @@ export interface Gateway {
    textTransformer: ((text: string) => string) | ((text: string) => string)[] | undefined;
 }

-export async function getGateway(provider: Entity<typeof schema, 'providers'>, model: Entity<typeof schema, 'models'>, providerApiKey?: string): Promise<Gateway> {
-    let gateway: ModelGateway;
-    let streamTransformer = undefined;
-    let textTransformer = undefined;
+export interface Provider {
+    gateway: Gateway | null;
+    endpoint: {
+        baseURL: string;
+        modelsEndpoint: string | null;
+        headers: Record<string, string>;
+    }
+}
+
+export enum GatewayFetchError {
+    NoProviderApiKey = 0,
+    NoProviderBaseUrl,
+}
+
+export async function getProviderDetails(provider: Entity<typeof schema, 'providers'>, providerApiKey?: string, model?: Entity<typeof schema, 'models'>): Promise<Result<Provider, GatewayFetchError>> {
+    let gateway: Gateway = {} as Gateway;

    let baseURL = undefined;
+    let modelsEndpoint = undefined;
+    let headers: Record<string, string> = {};

    if (provider.config.apiProxyUrl && provider.config.apiProxyUrl.trim() !== '') {
        baseURL = provider.config.apiProxyUrl;
+    } else {
+        baseURL = providerBaseUrls[provider.type];
    }
+    baseURL = baseURL.replace(/\/$/, '');

    switch (provider.type) {
        case 'openrouter': {
            if (providerApiKey === undefined) {
-                throw createError({
-                    statusCode: 400,
-                    message: 'OpenRouter provider requires an API key',
-                });
+                return Err(GatewayFetchError.NoProviderApiKey);
            }

-            gateway = createOpenRouter({
+            gateway.gateway = createOpenRouter({
                apiKey: providerApiKey,
                headers: {
                    'HTTP-Referer': 'https://localhost:3000',
                    'X-Title': 'Veridian',
                },
            });
-            break;
-        }
+            headers['Authorization'] = `Bearer ${providerApiKey}`
+            modelsEndpoint = `/models`;
+        } break;
        case 'ollama': {
            if (baseURL === undefined) {
-                throw createError({
-                    statusCode: 400,
-                    message: 'Ollama provider requires an API proxy URL',
-                });
+                return Err(GatewayFetchError.NoProviderBaseUrl);
            }

-            const innerGateway = createOllama({
-                apiKey: providerApiKey,
-                baseURL,
-            })
+            if (providerApiKey !== undefined) {
+                headers['Authorization'] = `Bearer ${providerApiKey}`
+            }
+            modelsEndpoint = `/api/tags`;

-            gateway = ((modelId: string) => innerGateway(modelId, { think: [...model.attributes.capabilities].includes('reasoning') })) as OllamaProvider;
-            break;
-        }
+            if (model !== undefined) {
+                const innerGateway = createOllama({
+                    apiKey: providerApiKey,
+                    baseURL,
+                })
+
+                gateway.gateway = ((modelId: string) => innerGateway(modelId, { think: [...model.attributes.capabilities].includes('reasoning') })) as OllamaProvider;
+            }
+        } break;
        case 'cerebras': {
-            gateway = createCerebras({
+            if (providerApiKey === undefined) {
+                return Err(GatewayFetchError.NoProviderApiKey);
+            }
+
+            gateway.gateway = createCerebras({
                apiKey: providerApiKey,
                baseURL,
            })

-            streamTransformer = transformCerebrasReasoningStream() as StreamTextTransform<{}>;
-            textTransformer = (text: string) => {
+            gateway.streamTransformer = transformCerebrasReasoningStream() as StreamTextTransform<{}>;
+            gateway.textTransformer = (text: string) => {
                return text.split('</think>').at(-1)!.trim()
            };
-            break;
-        }
+            headers['Authorization'] = `Bearer ${providerApiKey}`
+            modelsEndpoint = `/models`;
+        } break;
        case 'google': {
-            gateway = createGoogleGenerativeAI({
+            if (providerApiKey === undefined) {
+                return Err(GatewayFetchError.NoProviderApiKey);
+            }
+
+            gateway.gateway = createGoogleGenerativeAI({
                apiKey: providerApiKey,
                baseURL,
-            })
-            break;
-        }
+            });
+            headers['x-goog-api-key'] = `${providerApiKey}`
+            modelsEndpoint = `/models`;
+        } break;
        case 'longcat': {
-            gateway = createOpenAICompatible({
+            if (providerApiKey === undefined) {
+                return Err(GatewayFetchError.NoProviderApiKey);
+            }
+
+            gateway.gateway = createOpenAICompatible({
                name: 'LongCat',
                apiKey: providerApiKey,
                baseURL: baseURL ?? providerBaseUrls[provider.type],
                includeUsage: true,
            })
+            headers['Authorization'] = `Bearer ${providerApiKey}`
+            modelsEndpoint = null;
            // streamTransformer = createLongcatTransformer() as StreamTextTransform<{}>;
        } break;
        case 'cohere': {
-            gateway = createCohere({
+            if (providerApiKey === undefined) {
+                return Err(GatewayFetchError.NoProviderApiKey);
+            }
+
+            gateway.gateway = createCohere({
                apiKey: providerApiKey,
                baseURL,
-            })
-        }
+            });
+            headers['Authorization'] = `Bearer ${providerApiKey}`
+            modelsEndpoint = `/models`;
+        } break;
    }

-    return {
+    return Ok({
        gateway,
-        streamTransformer,
-        textTransformer,
-    };
+        endpoint: {
+            baseURL,
+            modelsEndpoint,
+            headers,
+        }
+    });
 }
@@ -1,4 +1,4 @@
-import { ToolSet, type TextStreamPart, type ToolCallPart } from 'ai';
+import { type ToolSet, type TextStreamPart, type ToolCallPart } from 'ai';

 export function createLongcatTransformer<TOOLS extends ToolSet>(): (options: {
    tools: TOOLS;
@@ -0,0 +1,51 @@
+interface ModelsDevCache {
+    data: any;
+    etag: string | null;
+    lastFetched: number;
+}
+
+let modelsDevCache: ModelsDevCache = {
+    data: null,
+    etag: null,
+    lastFetched: 0,
+};
+
+const MODELS_DEV_URL = 'https://models.dev/api.json';
+const CACHE_TTL = 60 * 1000; // 60 seconds
+
+export async function getModelsDevData(): Promise<any> {
+    const now = Date.now();
+
+    if (modelsDevCache.data &&
+        (now - modelsDevCache.lastFetched < CACHE_TTL)) {
+        return modelsDevCache.data;
+    }
+
+    try {
+        const headers: HeadersInit = {};
+        if (modelsDevCache.etag) {
+            headers['If-None-Match'] = modelsDevCache.etag;
+        }
+
+        const response = await fetch(MODELS_DEV_URL, { headers });
+
+        if (response.status === 304) {
+            modelsDevCache.lastFetched = now;
+            return modelsDevCache.data;
+        }
+
+        const newData = await response.json();
+        const newEtag = response.headers.get('ETag') || null;
+
+        modelsDevCache = {
+            data: newData,
+            etag: newEtag,
+            lastFetched: now,
+        };
+
+        return newData;
+    } catch (error) {
+        console.error('Error fetching models.dev:', error);
+        return modelsDevCache.data || {};
+    }
+}
@@ -1,13 +1,19 @@
-const pendingRenames: Map<string, AbortController> = new Map();
+interface PendingRename {
+    topicId: string;
+    abortController: AbortController;
+}

-export const cancelPendingRename = (renameId: string): boolean => {
-    const controller = pendingRenames.get(renameId);
-    if (controller) {
-        controller.abort();
+const pendingRenames: Map<string, PendingRename> = new Map();
+
+export const cancelPendingRename = (renameId: string): [boolean, PendingRename?] => {
+    const pendingRename = pendingRenames.get(renameId);
+    if (pendingRename?.abortController) {
+        pendingRename.abortController.abort();
        pendingRenames.delete(renameId);
-        return true;
+        return [true, pendingRename];
    }
-    return false;
+
+    return [false, undefined];
 };

 export const completeRename = (renameId: string) => {
@@ -17,10 +23,11 @@ export const completeRename = (renameId: string) => {
    }
 };

-export const addPendingRename = (): [string, AbortController] => {
+export const addPendingRename = (topicId: string): [string, PendingRename] => {
    const id = crypto.randomUUID();
-    const controller = new AbortController();
+    const abortController = new AbortController();
+    const pendingRename = { topicId, abortController };

-    pendingRenames.set(id, controller);
-    return [id, controller];
+    pendingRenames.set(id, pendingRename);
+    return [id, pendingRename];
 };