Performance enhancements galore! New themining system

This is once again a huge commit, but its mostly performance
improvements along with some bug fixes and refactoring. It also includes
changes to the theming systems. I'm still not 100% happy with the
theming system, but its better than before.

Model fetching has been dramatically improved! Nearly all the important
computation and pre-processing has been moved to the server. This has
also somehow fixed the way model details are loaded, which was causing
many models to be missing their details despite models.dev having them.

The markdown renderer has once again been changed, but I'm mostly
certain that this is the last time major changes will be made to it. The
renderer is not spamming components, bloating memory usage, and its not
using a bug prone custom written chunking system.

There's also a lot more that I haven't mentioned and honestly forgot. I
need to get better commit hygiene tbh.

server/utils/ai-provider.ts

@@ -9,6 +9,7 @@ import { schema } from "~~/triplit/schema";
 import { type StreamTextTransform } from "ai";
 import { transformCerebrasReasoningStream } from "./cerebras";
 import { providerBaseUrls } from "~/types/model";
+import { type Result, Err, Ok } from "~~/types/result";
 // import { createLongcatTransformer } from "./longcat";
 
 export type ModelGateway = OpenRouterProvider | OllamaProvider | CerebrasProvider | GoogleGenerativeAIProvider | OpenAICompatibleProvider | CohereProvider;
@@ -19,90 +20,133 @@ export interface Gateway {
     textTransformer: ((text: string) => string) | ((text: string) => string)[] | undefined;
 }
 
-export async function getGateway(provider: Entity<typeof schema, 'providers'>, model: Entity<typeof schema, 'models'>, providerApiKey?: string): Promise<Gateway> {
-    let gateway: ModelGateway;
-    let streamTransformer = undefined;
-    let textTransformer = undefined;
+export interface Provider {
+    gateway: Gateway | null;
+    endpoint: {
+        baseURL: string;
+        modelsEndpoint: string | null;
+        headers: Record<string, string>;
+    }
+}
+
+export enum GatewayFetchError {
+    NoProviderApiKey = 0,
+    NoProviderBaseUrl,
+}
+
+export async function getProviderDetails(provider: Entity<typeof schema, 'providers'>, providerApiKey?: string, model?: Entity<typeof schema, 'models'>): Promise<Result<Provider, GatewayFetchError>> {
+    let gateway: Gateway = {} as Gateway;
 
     let baseURL = undefined;
+    let modelsEndpoint = undefined;
+    let headers: Record<string, string> = {};
 
     if (provider.config.apiProxyUrl && provider.config.apiProxyUrl.trim() !== '') {
         baseURL = provider.config.apiProxyUrl;
+    } else {
+        baseURL = providerBaseUrls[provider.type];
     }
+    baseURL = baseURL.replace(/\/$/, '');
 
     switch (provider.type) {
         case 'openrouter': {
             if (providerApiKey === undefined) {
-                throw createError({
-                    statusCode: 400,
-                    message: 'OpenRouter provider requires an API key',
-                });
+                return Err(GatewayFetchError.NoProviderApiKey);
             }
 
-            gateway = createOpenRouter({
+            gateway.gateway = createOpenRouter({
                 apiKey: providerApiKey,
                 headers: {
                     'HTTP-Referer': 'https://localhost:3000',
                     'X-Title': 'Veridian',
                 },
             });
-            break;
-        }
+            headers['Authorization'] = `Bearer ${providerApiKey}`
+            modelsEndpoint = `/models`;
+        } break;
         case 'ollama': {
             if (baseURL === undefined) {
-                throw createError({
-                    statusCode: 400,
-                    message: 'Ollama provider requires an API proxy URL',
-                });
+                return Err(GatewayFetchError.NoProviderBaseUrl);
             }
 
-            const innerGateway = createOllama({
-                apiKey: providerApiKey,
-                baseURL,
-            })
+            if (providerApiKey !== undefined) {
+                headers['Authorization'] = `Bearer ${providerApiKey}`
+            }
+            modelsEndpoint = `/api/tags`;
 
-            gateway = ((modelId: string) => innerGateway(modelId, { think: [...model.attributes.capabilities].includes('reasoning') })) as OllamaProvider;
-            break;
-        }
+            if (model !== undefined) {
+                const innerGateway = createOllama({
+                    apiKey: providerApiKey,
+                    baseURL,
+                })
+
+                gateway.gateway = ((modelId: string) => innerGateway(modelId, { think: [...model.attributes.capabilities].includes('reasoning') })) as OllamaProvider;
+            }
+        } break;
         case 'cerebras': {
-            gateway = createCerebras({
+            if (providerApiKey === undefined) {
+                return Err(GatewayFetchError.NoProviderApiKey);
+            }
+
+            gateway.gateway = createCerebras({
                 apiKey: providerApiKey,
                 baseURL,
             })
 
-            streamTransformer = transformCerebrasReasoningStream() as StreamTextTransform<{}>;
-            textTransformer = (text: string) => {
+            gateway.streamTransformer = transformCerebrasReasoningStream() as StreamTextTransform<{}>;
+            gateway.textTransformer = (text: string) => {
                 return text.split('</think>').at(-1)!.trim()
             };
-            break;
-        }
+            headers['Authorization'] = `Bearer ${providerApiKey}`
+            modelsEndpoint = `/models`;
+        } break;
         case 'google': {
-            gateway = createGoogleGenerativeAI({
+            if (providerApiKey === undefined) {
+                return Err(GatewayFetchError.NoProviderApiKey);
+            }
+
+            gateway.gateway = createGoogleGenerativeAI({
                 apiKey: providerApiKey,
                 baseURL,
-            })
-            break;
-        }
+            });
+            headers['x-goog-api-key'] = `${providerApiKey}`
+            modelsEndpoint = `/models`;
+        } break;
         case 'longcat': {
-            gateway = createOpenAICompatible({
+            if (providerApiKey === undefined) {
+                return Err(GatewayFetchError.NoProviderApiKey);
+            }
+
+            gateway.gateway = createOpenAICompatible({
                 name: 'LongCat',
                 apiKey: providerApiKey,
                 baseURL: baseURL ?? providerBaseUrls[provider.type],
                 includeUsage: true,
             })
+            headers['Authorization'] = `Bearer ${providerApiKey}`
+            modelsEndpoint = null;
             // streamTransformer = createLongcatTransformer() as StreamTextTransform<{}>;
         } break;
         case 'cohere': {
-            gateway = createCohere({
+            if (providerApiKey === undefined) {
+                return Err(GatewayFetchError.NoProviderApiKey);
+            }
+
+            gateway.gateway = createCohere({
                 apiKey: providerApiKey,
                 baseURL,
-            })
-        }
+            });
+            headers['Authorization'] = `Bearer ${providerApiKey}`
+            modelsEndpoint = `/models`;
+        } break;
     }
 
-    return {
+    return Ok({
         gateway,
-        streamTransformer,
-        textTransformer,
-    };
+        endpoint: {
+            baseURL,
+            modelsEndpoint,
+            headers,
+        }
+    });
 }

server/utils/longcat.ts

@@ -1,4 +1,4 @@
-import { ToolSet, type TextStreamPart, type ToolCallPart } from 'ai';
+import { type ToolSet, type TextStreamPart, type ToolCallPart } from 'ai';
 
 export function createLongcatTransformer<TOOLS extends ToolSet>(): (options: {
     tools: TOOLS;

server/utils/models-dev.ts

@@ -0,0 +1,51 @@
+interface ModelsDevCache {
+    data: any;
+    etag: string | null;
+    lastFetched: number;
+}
+
+let modelsDevCache: ModelsDevCache = {
+    data: null,
+    etag: null,
+    lastFetched: 0,
+};
+
+const MODELS_DEV_URL = 'https://models.dev/api.json';
+const CACHE_TTL = 60 * 1000; // 60 seconds
+
+export async function getModelsDevData(): Promise<any> {
+    const now = Date.now();
+
+    if (modelsDevCache.data &&
+        (now - modelsDevCache.lastFetched < CACHE_TTL)) {
+        return modelsDevCache.data;
+    }
+
+    try {
+        const headers: HeadersInit = {};
+        if (modelsDevCache.etag) {
+            headers['If-None-Match'] = modelsDevCache.etag;
+        }
+
+        const response = await fetch(MODELS_DEV_URL, { headers });
+
+        if (response.status === 304) {
+            modelsDevCache.lastFetched = now;
+            return modelsDevCache.data;
+        }
+
+        const newData = await response.json();
+        const newEtag = response.headers.get('ETag') || null;
+
+        modelsDevCache = {
+            data: newData,
+            etag: newEtag,
+            lastFetched: now,
+        };
+
+        return newData;
+    } catch (error) {
+        console.error('Error fetching models.dev:', error);
+        return modelsDevCache.data || {};
+    }
+}

server/utils/renames.ts

@@ -1,13 +1,19 @@
-const pendingRenames: Map<string, AbortController> = new Map();
+interface PendingRename {
+    topicId: string;
+    abortController: AbortController;
+}
 
-export const cancelPendingRename = (renameId: string): boolean => {
-    const controller = pendingRenames.get(renameId);
-    if (controller) {
-        controller.abort();
+const pendingRenames: Map<string, PendingRename> = new Map();
+
+export const cancelPendingRename = (renameId: string): [boolean, PendingRename?] => {
+    const pendingRename = pendingRenames.get(renameId);
+    if (pendingRename?.abortController) {
+        pendingRename.abortController.abort();
         pendingRenames.delete(renameId);
-        return true;
+        return [true, pendingRename];
     }
-    return false;
+
+    return [false, undefined];
 };
 
 export const completeRename = (renameId: string) => {
@@ -17,10 +23,11 @@ export const completeRename = (renameId: string) => {
     }
 };
 
-export const addPendingRename = (): [string, AbortController] => {
+export const addPendingRename = (topicId: string): [string, PendingRename] => {
     const id = crypto.randomUUID();
-    const controller = new AbortController();
+    const abortController = new AbortController();
+    const pendingRename = { topicId, abortController };
 
-    pendingRenames.set(id, controller);
-    return [id, controller];
+    pendingRenames.set(id, pendingRename);
+    return [id, pendingRename];
 };