feat: add better provider support, icons, regen, and a lot more
This commit is contained in:
+467
-246
@@ -1,6 +1,5 @@
|
||||
import { createOpenRouter, type OpenRouterProvider } from '@openrouter/ai-sdk-provider';
|
||||
import type { Entity } from '@triplit/client';
|
||||
import { type ModelMessage, modelMessageSchema, streamText, tool } from 'ai';
|
||||
import { type ModelMessage, modelMessageSchema, streamText, type StreamTextTransform, tool } from 'ai';
|
||||
import { promises as fs } from 'fs';
|
||||
import { glob } from 'glob';
|
||||
import { nanoid } from 'nanoid';
|
||||
@@ -9,17 +8,11 @@ import * as z from 'zod';
|
||||
import { httpClient } from '~~/server/lib/triplit';
|
||||
import { addPendingGeneration, completeGeneration } from '~~/server/utils/generations';
|
||||
import type { schema } from '~~/triplit/schema';
|
||||
import { spawn } from 'child_process';
|
||||
import { getGateway, type ModelGateway } from '~~/server/utils/ai-provider';
|
||||
|
||||
export const messagesSchema = z.array(modelMessageSchema);
|
||||
|
||||
// quick access
|
||||
// const ACTIVE_MODEL_ID = 'openrouter/free';
|
||||
// const ACTIVE_MODEL_ID = 'x-ai/grok-4.1-fast';
|
||||
const ACTIVE_MODEL_ID = 'google/gemini-3-flash-preview';
|
||||
// const ACTIVE_MODEL_ID = 'arcee-ai/trinity-mini:free';
|
||||
|
||||
type ModelGateway = OpenRouterProvider;
|
||||
|
||||
export default defineEventHandler(async (event) => {
|
||||
await protectRoute(event);
|
||||
|
||||
@@ -28,10 +21,11 @@ export default defineEventHandler(async (event) => {
|
||||
.object({
|
||||
messages: messagesSchema.min(1),
|
||||
topicId: z.string(),
|
||||
parentMessageId: z.string().nullable(),
|
||||
model: z.object({
|
||||
providerId: z.string(),
|
||||
modelId: z.string(),
|
||||
args: z.any(),
|
||||
args: z.record(z.string(), z.any()),
|
||||
}),
|
||||
providerApiKey: z.string().optional(),
|
||||
})
|
||||
@@ -47,9 +41,17 @@ export default defineEventHandler(async (event) => {
|
||||
|
||||
const userId = event.context.user!.id;
|
||||
|
||||
const { messages, topicId, model: { modelId, providerId }, providerApiKey } = result.data;
|
||||
const { messages, topicId, parentMessageId, model: { modelId, providerId, args }, providerApiKey } = result.data;
|
||||
|
||||
const provider = await httpClient.fetchOne(httpClient.query('providers').Where('id', '=', providerId));
|
||||
const fetchPromises = [];
|
||||
|
||||
fetchPromises.push(httpClient.fetchOne(httpClient.query('providers').Where('id', '=', providerId)));
|
||||
fetchPromises.push(httpClient.fetchOne(httpClient.query('models').Where('id', '=', modelId)));
|
||||
fetchPromises.push(httpClient.fetchOne(
|
||||
httpClient.query('generations').Where('topicId', '=', topicId).Where('status', '=', 'pending'),
|
||||
));
|
||||
|
||||
const [provider, model, existingPendingGenerations] = await Promise.all(fetchPromises) as [Entity<typeof schema, 'providers'> | null, Entity<typeof schema, 'models'> | null, Entity<typeof schema, 'generations'> | null];
|
||||
if (provider === null || provider.userId !== userId) {
|
||||
throw createError({
|
||||
statusCode: 400,
|
||||
@@ -57,7 +59,6 @@ export default defineEventHandler(async (event) => {
|
||||
});
|
||||
}
|
||||
|
||||
const model = await httpClient.fetchOne(httpClient.query('models').Where('id', '=', modelId));
|
||||
if (model === null || model.providerId !== model.providerId || model.userId !== userId) {
|
||||
throw createError({
|
||||
statusCode: 400,
|
||||
@@ -65,9 +66,6 @@ export default defineEventHandler(async (event) => {
|
||||
});
|
||||
}
|
||||
|
||||
const existingPendingGenerations = await httpClient.fetchOne(
|
||||
httpClient.query('generations').Where('topicId', '=', topicId).Where('status', '=', 'pending'),
|
||||
);
|
||||
if (existingPendingGenerations !== null) {
|
||||
throw createError({
|
||||
statusCode: 400,
|
||||
@@ -75,39 +73,21 @@ export default defineEventHandler(async (event) => {
|
||||
});
|
||||
}
|
||||
|
||||
let gateway: ModelGateway;
|
||||
|
||||
switch (provider.type) {
|
||||
case 'openrouter': {
|
||||
if (providerApiKey === undefined) {
|
||||
throw createError({
|
||||
statusCode: 400,
|
||||
message: 'OpenRouter provider requires an API key',
|
||||
});
|
||||
}
|
||||
|
||||
gateway = createOpenRouter({
|
||||
apiKey: providerApiKey,
|
||||
headers: {
|
||||
'HTTP-Referer': 'https://localhost:3000',
|
||||
'X-Title': 'Veridian',
|
||||
},
|
||||
});
|
||||
break;
|
||||
}
|
||||
default:
|
||||
throw new Error(`Unknown provider type: ${provider.type}`);
|
||||
}
|
||||
const { gateway, streamTransformer: transformer } = await getGateway(provider, model, providerApiKey);
|
||||
|
||||
const generationId = nanoid();
|
||||
const message = await httpClient.insert('messages', {
|
||||
topicId,
|
||||
userId,
|
||||
focusedIndex: parentMessageId ? null : 0,
|
||||
generationId,
|
||||
parentMessageId,
|
||||
content: '',
|
||||
role: 'assistant',
|
||||
});
|
||||
await httpClient.insert('generations', {
|
||||
id: generationId,
|
||||
userId,
|
||||
topicId,
|
||||
modelId: model.externalId,
|
||||
status: 'pending',
|
||||
@@ -125,7 +105,7 @@ export default defineEventHandler(async (event) => {
|
||||
}
|
||||
|
||||
event.waitUntil(
|
||||
generateResponse(message, { gateway, model: model.externalId }, generationId, userId, messages, logMessage, logFile),
|
||||
generateResponse(message, { gateway, model, parameters: args }, generationId, userId, topicId, messages, transformer, logMessage, logFile),
|
||||
);
|
||||
|
||||
return {
|
||||
@@ -142,196 +122,265 @@ const todo = (...args: any[]) => {
|
||||
throw new Error('TODO');
|
||||
};
|
||||
|
||||
const evalPython = async (code: string) => {
|
||||
// This is the wrapper logic from above, minified or stored as a string
|
||||
// Or you can save the wrapper script to a file and call that.
|
||||
const wrapper = `
|
||||
import ast
|
||||
import sys
|
||||
code = sys.stdin.read()
|
||||
tree = ast.parse(code)
|
||||
last_node = tree.body[-1] if tree.body else None
|
||||
namespace = {}
|
||||
if len(tree.body) > 1:
|
||||
exec(compile(ast.Module(tree.body[:-1], []), "<ast>", "exec"), namespace)
|
||||
if isinstance(last_node, ast.Expr):
|
||||
res = eval(compile(ast.Expression(last_node.value), "<ast>", "eval"), namespace)
|
||||
if res is not None: print(res)
|
||||
elif last_node:
|
||||
exec(compile(ast.Module([last_node], []), "<ast>", "exec"), namespace)
|
||||
`.trim();
|
||||
|
||||
return new Promise<string>((resolve, reject) => {
|
||||
const child = spawn('python3', ['-c', wrapper]);
|
||||
|
||||
let output = '';
|
||||
let errorOutput = '';
|
||||
|
||||
child.stdout.on('data', (data) => {
|
||||
output += data.toString();
|
||||
});
|
||||
|
||||
child.stderr.on('data', (data) => {
|
||||
errorOutput += data.toString();
|
||||
});
|
||||
|
||||
child.on('close', (exitCode) => {
|
||||
if (exitCode !== 0) {
|
||||
reject(errorOutput || `Exit code ${exitCode}`);
|
||||
} else {
|
||||
resolve(output.trim());
|
||||
}
|
||||
});
|
||||
|
||||
// Send the agent's code to the wrapper via stdin
|
||||
child.stdin.write(code);
|
||||
child.stdin.end();
|
||||
});
|
||||
};
|
||||
|
||||
// TODO: obviously come up with a better way for the user to define their own tools
|
||||
const { listDirectoryTool, globTool, readFileTool, readFilesTool, fetchUrlTool, pythonTool, bashTool } = {
|
||||
listDirectoryTool: tool({
|
||||
description: 'Lists the contents of a directory',
|
||||
inputSchema: z.object({
|
||||
path: z.string(),
|
||||
}),
|
||||
outputSchema: z.object({
|
||||
files: z.array(z.object({ name: z.string(), type: z.string() })),
|
||||
}),
|
||||
execute: async ({ path }) => {
|
||||
const rawFiles = await fs.readdir(path, { withFileTypes: true });
|
||||
const files = rawFiles.map((file) => ({
|
||||
name: file.name,
|
||||
type: file.isFile() ? 'file' : 'directory',
|
||||
}));
|
||||
|
||||
return {
|
||||
files,
|
||||
};
|
||||
},
|
||||
}),
|
||||
globTool: tool({
|
||||
description: 'Lists files matching a glob pattern',
|
||||
inputSchema: z.object({
|
||||
pattern: z.string(),
|
||||
}),
|
||||
outputSchema: z.object({
|
||||
files: z.array(z.object({ path: z.string(), type: z.string() })),
|
||||
}),
|
||||
execute: async ({ pattern }) => {
|
||||
const rawFiles = await glob(pattern, { withFileTypes: true });
|
||||
const files = rawFiles.map((file) => ({
|
||||
path: file.parentPath + '/' + file.name,
|
||||
type: file.isFile() ? 'file' : 'directory',
|
||||
}));
|
||||
|
||||
return {
|
||||
files,
|
||||
};
|
||||
},
|
||||
}),
|
||||
readFileTool: tool({
|
||||
description: 'Reads the contents of a file',
|
||||
inputSchema: z.object({
|
||||
path: z.string(),
|
||||
}),
|
||||
outputSchema: z.object({
|
||||
path: z.string(),
|
||||
content: z.string(),
|
||||
}),
|
||||
execute: async ({ path }) => {
|
||||
const file = await fs.readFile(path);
|
||||
return {
|
||||
path,
|
||||
content: file.toString(),
|
||||
};
|
||||
},
|
||||
}),
|
||||
readFilesTool: tool({
|
||||
description: 'Reads the contents of multiple files',
|
||||
inputSchema: z.object({
|
||||
paths: z.array(z.string()).describe('The file paths to read'),
|
||||
}),
|
||||
outputSchema: z.object({
|
||||
files: z.array(
|
||||
z.object({
|
||||
path: z.string(),
|
||||
content: z.string(),
|
||||
}),
|
||||
),
|
||||
}),
|
||||
execute: async ({ paths }) => {
|
||||
const files = await Promise.all(
|
||||
paths.map(async (path) => {
|
||||
const file = await fs.readFile(path);
|
||||
return {
|
||||
path: path,
|
||||
content: file.toString(),
|
||||
};
|
||||
}),
|
||||
);
|
||||
|
||||
return {
|
||||
files,
|
||||
};
|
||||
},
|
||||
}),
|
||||
fetchUrlTool: tool({
|
||||
description: 'Fetches the content of a URL',
|
||||
inputSchema: z.object({
|
||||
url: z.string(),
|
||||
}),
|
||||
outputSchema: z.object({
|
||||
content: z.string(),
|
||||
}),
|
||||
execute: async ({ url }) => {
|
||||
const response = await fetch(url);
|
||||
const content = await response.text();
|
||||
return {
|
||||
content,
|
||||
};
|
||||
},
|
||||
}),
|
||||
pythonTool: tool({
|
||||
description: 'Executes a Python code snippet',
|
||||
inputSchema: z.object({
|
||||
code: z.string(),
|
||||
}),
|
||||
outputSchema: z.object({
|
||||
output: z.string(),
|
||||
}),
|
||||
execute: async ({ code }) => {
|
||||
const output = await evalPython(code);
|
||||
return {
|
||||
output,
|
||||
};
|
||||
},
|
||||
}),
|
||||
bashTool: tool({
|
||||
description: 'Executes a Bash command',
|
||||
inputSchema: z.object({
|
||||
code: z.string(),
|
||||
}),
|
||||
outputSchema: z.object({
|
||||
output: z.string(),
|
||||
}),
|
||||
execute: async ({ code }) => {
|
||||
const { exec } = await import('child_process');
|
||||
const { promisify } = await import('util');
|
||||
const execAsync = promisify(exec);
|
||||
|
||||
async function runCommand(command: string) {
|
||||
const { stdout, stderr } = await execAsync(command);
|
||||
if (stderr) {
|
||||
console.error(`Error: ${stderr}`);
|
||||
return stderr;
|
||||
} else {
|
||||
return stdout;
|
||||
}
|
||||
}
|
||||
|
||||
const output = await runCommand(code);
|
||||
return {
|
||||
output,
|
||||
};
|
||||
},
|
||||
}),
|
||||
}
|
||||
|
||||
async function generateResponse(
|
||||
message: Entity<typeof schema, 'messages'>,
|
||||
model: {
|
||||
gateway: ModelGateway,
|
||||
model: string,
|
||||
model: Entity<typeof schema, 'models'>,
|
||||
parameters: Record<string, any>,
|
||||
},
|
||||
generationId: string,
|
||||
userId: string,
|
||||
topicId: string,
|
||||
messages: ModelMessage[],
|
||||
streamTransoforms: StreamTextTransform<{}> | StreamTextTransform<{}>[] | undefined,
|
||||
log?: (message: string) => void,
|
||||
logFile?: fs.FileHandle,
|
||||
) {
|
||||
const controller = new AbortController();
|
||||
addPendingGeneration(generationId, controller);
|
||||
|
||||
let requestStart = undefined;
|
||||
let ttft = undefined;
|
||||
const activeParts = new Map<string, { id: string; accumulatedContent: string; providerOptions?: any }>();
|
||||
const activeToolCalls = new Map<string, { id: string }>();
|
||||
const activeToolCalls = new Map<string, void>();
|
||||
|
||||
const tools = {
|
||||
// writeFile: tool({
|
||||
// inputSchema: z.object({
|
||||
// path: z.string(),
|
||||
// content: z.string(),
|
||||
// }),
|
||||
// outputSchema: z.object({
|
||||
// success: z.boolean(),
|
||||
// }),
|
||||
// execute: async ({ path, content }) => {
|
||||
// await fs.writeFile(path, content);
|
||||
// return {
|
||||
// success: true,
|
||||
// };
|
||||
// }
|
||||
// }),
|
||||
listDirectory: listDirectoryTool,
|
||||
glob: globTool,
|
||||
readFile: readFileTool,
|
||||
readFiles: readFilesTool,
|
||||
// fetchUrl: fetchUrlTool,
|
||||
python: pythonTool,
|
||||
bash: bashTool,
|
||||
};
|
||||
|
||||
const response = streamText({
|
||||
model: model.gateway(model.model),
|
||||
model: model.gateway(model.model.externalId),
|
||||
messages,
|
||||
// a little trick that makes it so that the stream doesnt stop because of tool calls, and will continue an unbounded amount of time and steps
|
||||
stopWhen: [],
|
||||
// tools: {
|
||||
// // writeFile: tool({
|
||||
// // inputSchema: z.object({
|
||||
// // path: z.string(),
|
||||
// // content: z.string(),
|
||||
// // }),
|
||||
// // outputSchema: z.object({
|
||||
// // success: z.boolean(),
|
||||
// // }),
|
||||
// // execute: async ({ path, content }) => {
|
||||
// // await fs.writeFile(path, content);
|
||||
// // return {
|
||||
// // success: true,
|
||||
// // };
|
||||
// // }
|
||||
// // }),
|
||||
// listDirectory: tool({
|
||||
// inputSchema: z.object({
|
||||
// path: z.string(),
|
||||
// }),
|
||||
// outputSchema: z.object({
|
||||
// files: z.array(z.object({ name: z.string(), type: z.string() })),
|
||||
// }),
|
||||
// execute: async ({ path }) => {
|
||||
// const rawFiles = await fs.readdir(path, { withFileTypes: true });
|
||||
// const files = rawFiles.map((file) => ({
|
||||
// name: file.name,
|
||||
// type: file.isFile() ? 'file' : 'directory',
|
||||
// }));
|
||||
|
||||
// return {
|
||||
// files,
|
||||
// };
|
||||
// },
|
||||
// }),
|
||||
// glob: tool({
|
||||
// inputSchema: z.object({
|
||||
// pattern: z.string(),
|
||||
// }),
|
||||
// outputSchema: z.object({
|
||||
// files: z.array(z.object({ name: z.string(), type: z.string() })),
|
||||
// }),
|
||||
// execute: async ({ pattern }) => {
|
||||
// const rawFiles = await glob(pattern, { withFileTypes: true });
|
||||
// const files = rawFiles.map((file) => ({
|
||||
// name: file.name,
|
||||
// type: file.isFile() ? 'file' : 'directory',
|
||||
// }));
|
||||
|
||||
// return {
|
||||
// files,
|
||||
// };
|
||||
// },
|
||||
// }),
|
||||
// readFile: tool({
|
||||
// inputSchema: z.object({
|
||||
// path: z.string(),
|
||||
// }),
|
||||
// outputSchema: z.object({
|
||||
// path: z.string(),
|
||||
// content: z.string(),
|
||||
// }),
|
||||
// execute: async ({ path }) => {
|
||||
// const file = await fs.readFile(path);
|
||||
// return {
|
||||
// path,
|
||||
// content: file.toString(),
|
||||
// };
|
||||
// },
|
||||
// }),
|
||||
// readFiles: tool({
|
||||
// inputSchema: z.object({
|
||||
// paths: z.array(z.string()).describe('The file paths to read'),
|
||||
// }),
|
||||
// outputSchema: z.object({
|
||||
// files: z.array(
|
||||
// z.object({
|
||||
// path: z.string(),
|
||||
// content: z.string(),
|
||||
// }),
|
||||
// ),
|
||||
// }),
|
||||
// execute: async ({ paths }) => {
|
||||
// const files = await Promise.all(
|
||||
// paths.map(async (path) => {
|
||||
// const file = await fs.readFile(path);
|
||||
// return {
|
||||
// path: path,
|
||||
// content: file.toString(),
|
||||
// };
|
||||
// }),
|
||||
// );
|
||||
|
||||
// return {
|
||||
// files,
|
||||
// };
|
||||
// },
|
||||
// }),
|
||||
// fetchUrl: tool({
|
||||
// inputSchema: z.object({
|
||||
// url: z.string(),
|
||||
// }),
|
||||
// outputSchema: z.object({
|
||||
// content: z.string(),
|
||||
// }),
|
||||
// execute: async ({ url }) => {
|
||||
// const response = await fetch(url);
|
||||
// const content = await response.text();
|
||||
// return {
|
||||
// content,
|
||||
// };
|
||||
// },
|
||||
// }),
|
||||
// },
|
||||
onStepFinish: async (result) => {
|
||||
if (result.toolResults.length > 0) {
|
||||
for (const toolResult of result.toolResults) {
|
||||
let outputType: 'text' | 'json' = 'text';
|
||||
let outputValue: string = '';
|
||||
|
||||
switch (typeof toolResult.output) {
|
||||
case 'string':
|
||||
outputType = 'text';
|
||||
outputValue = toolResult.output;
|
||||
break;
|
||||
case 'object':
|
||||
outputType = 'json';
|
||||
outputValue = JSON.stringify(toolResult.output, null, 2);
|
||||
break;
|
||||
default:
|
||||
console.error('Unknown output type', toolResult.output);
|
||||
await httpClient.update('tool_calls', toolResult.toolCallId, {
|
||||
status: 'failed',
|
||||
error: {
|
||||
type: 'text',
|
||||
value: 'Tool returned invalid output',
|
||||
},
|
||||
});
|
||||
break;
|
||||
}
|
||||
|
||||
await httpClient.update('tool_calls', toolResult.toolCallId, {
|
||||
status: 'completed',
|
||||
output: {
|
||||
type: outputType,
|
||||
value: outputValue,
|
||||
},
|
||||
});
|
||||
|
||||
activeToolCalls.delete(toolResult.toolCallId);
|
||||
}
|
||||
providerOptions: {
|
||||
openrouter: {
|
||||
debug: {
|
||||
echo_upstream_body: true,
|
||||
},
|
||||
}
|
||||
},
|
||||
onFinish: async (result) => {
|
||||
await httpClient.update('generations', generationId, {
|
||||
status: 'completed',
|
||||
tokens: {
|
||||
input: result.totalUsage.inputTokens,
|
||||
cache: {
|
||||
read: result.totalUsage.inputTokenDetails.cacheReadTokens,
|
||||
write: result.totalUsage.inputTokenDetails.cacheWriteTokens,
|
||||
},
|
||||
output: result.totalUsage.outputTokens,
|
||||
thinking: result.totalUsage.outputTokenDetails.reasoningTokens,
|
||||
},
|
||||
});
|
||||
},
|
||||
experimental_transform: streamTransoforms,
|
||||
// a little trick that makes it so that the stream doesnt stop because of tool calls, and will continue an unbounded amount of time and steps
|
||||
stopWhen: [],
|
||||
tools: [...model.model.attributes.capabilities].includes('tools') ? tools : undefined,
|
||||
onError: async (error: any) => {
|
||||
console.error('generation error', error);
|
||||
log?.(error);
|
||||
@@ -343,19 +392,27 @@ async function generateResponse(
|
||||
});
|
||||
}
|
||||
|
||||
for (const activeToolCall of activeToolCalls.values()) {
|
||||
await httpClient.update('tool_calls', activeToolCall.id, {
|
||||
for (const toolCallId of activeToolCalls.keys()) {
|
||||
await httpClient.update('tool_calls', toolCallId, {
|
||||
status: 'failed',
|
||||
error: {
|
||||
type: 'text',
|
||||
value: 'An unknown error occurred',
|
||||
},
|
||||
});
|
||||
|
||||
activeToolCalls.delete(toolCallId);
|
||||
}
|
||||
|
||||
let errValue = error.message || error;
|
||||
|
||||
if (typeof errValue === 'object') {
|
||||
errValue = JSON.stringify(errValue, null, 2);
|
||||
}
|
||||
|
||||
await httpClient.update('generations', generationId, {
|
||||
status: 'failed',
|
||||
error: error.message,
|
||||
error: errValue,
|
||||
});
|
||||
},
|
||||
abortSignal: controller.signal,
|
||||
@@ -396,6 +453,9 @@ async function generateResponse(
|
||||
log?.(JSON.stringify(token, null, 2));
|
||||
|
||||
switch (token.type) {
|
||||
case 'start': {
|
||||
requestStart = Date.now();
|
||||
} break;
|
||||
case 'start-step': {
|
||||
curStepIdx++;
|
||||
} break;
|
||||
@@ -405,7 +465,9 @@ async function generateResponse(
|
||||
const toolCallId = token.id;
|
||||
|
||||
part = await httpClient.insert('message_parts', {
|
||||
topicId,
|
||||
messageId: message.id,
|
||||
userId,
|
||||
toolCallId,
|
||||
type: 'tool-call',
|
||||
content: '',
|
||||
@@ -425,7 +487,7 @@ async function generateResponse(
|
||||
createdAt: new Date(),
|
||||
});
|
||||
|
||||
activeToolCalls.set(key, { id: toolCallId });
|
||||
activeToolCalls.set(toolCallId);
|
||||
|
||||
activeParts.set(key, { id: part.id, accumulatedContent: '' });
|
||||
} break;
|
||||
@@ -435,7 +497,9 @@ async function generateResponse(
|
||||
key = `${type}-${curStepIdx}`;
|
||||
|
||||
part = await httpClient.insert('message_parts', {
|
||||
topicId,
|
||||
messageId: message.id,
|
||||
userId,
|
||||
type: type as 'text' | 'reasoning',
|
||||
content: '',
|
||||
finished: false,
|
||||
@@ -447,6 +511,10 @@ async function generateResponse(
|
||||
} break;
|
||||
case 'text-delta':
|
||||
case 'reasoning-delta': {
|
||||
if (ttft === undefined) {
|
||||
ttft = Date.now() - requestStart!;
|
||||
}
|
||||
|
||||
type = token.type.split('-')[0] as 'text' | 'reasoning';
|
||||
key = `${type}-${curStepIdx}`;
|
||||
part = activeParts.get(key);
|
||||
@@ -512,28 +580,95 @@ async function generateResponse(
|
||||
break;
|
||||
case 'object':
|
||||
inputType = 'json';
|
||||
inputValue = JSON.stringify(token.input, null, 2);
|
||||
inputValue = JSON.stringify(token.input);
|
||||
break;
|
||||
default:
|
||||
console.error('Unknown input type', token.input);
|
||||
break;
|
||||
}
|
||||
|
||||
await httpClient.update('tool_calls', token.toolCallId, {
|
||||
status: 'pending',
|
||||
input: {
|
||||
type: inputType,
|
||||
value: inputValue,
|
||||
},
|
||||
});
|
||||
if (activeToolCalls.has(token.toolCallId)) {
|
||||
await httpClient.update('tool_calls', token.toolCallId, {
|
||||
status: 'pending',
|
||||
input: {
|
||||
type: inputType,
|
||||
value: inputValue,
|
||||
},
|
||||
});
|
||||
} else {
|
||||
key = `tool-call-${curStepIdx}`;
|
||||
|
||||
const toolCallId = token.toolCallId;
|
||||
|
||||
part = await httpClient.insert('message_parts', {
|
||||
topicId,
|
||||
messageId: message.id,
|
||||
userId,
|
||||
toolCallId,
|
||||
type: 'tool-call',
|
||||
content: '',
|
||||
finished: false,
|
||||
createdAt: new Date(),
|
||||
lastUpdatedAt: new Date(),
|
||||
});
|
||||
|
||||
await httpClient.insert('tool_calls', {
|
||||
id: toolCallId,
|
||||
userId: userId,
|
||||
toolName: token.toolName,
|
||||
status: 'pending',
|
||||
input: {
|
||||
type: inputType,
|
||||
value: inputValue,
|
||||
},
|
||||
output: null,
|
||||
error: null,
|
||||
createdAt: new Date(),
|
||||
});
|
||||
|
||||
activeToolCalls.set(toolCallId);
|
||||
|
||||
activeParts.set(key, { id: part.id, accumulatedContent: '' });
|
||||
}
|
||||
} break;
|
||||
case 'tool-error': {
|
||||
const toolCall = activeToolCalls.get(token.toolCallId);
|
||||
if (toolCall === undefined) {
|
||||
console.error('Received tool-error without a start');
|
||||
break;
|
||||
case 'tool-result': {
|
||||
let outputType: 'text' | 'json' = 'text';
|
||||
let outputValue: string = '';
|
||||
|
||||
switch (typeof token.output) {
|
||||
case 'string':
|
||||
outputType = 'text';
|
||||
outputValue = token.output;
|
||||
break;
|
||||
case 'object':
|
||||
outputType = 'json';
|
||||
outputValue = JSON.stringify(token.output);
|
||||
break;
|
||||
default:
|
||||
console.error('Unknown output type', token.output);
|
||||
await httpClient.update('tool_calls', token.toolCallId, {
|
||||
status: 'failed',
|
||||
error: {
|
||||
type: 'text',
|
||||
value: 'Tool returned invalid output',
|
||||
},
|
||||
});
|
||||
|
||||
activeToolCalls.delete(token.toolCallId);
|
||||
break;
|
||||
}
|
||||
|
||||
await httpClient.update('tool_calls', token.toolCallId, {
|
||||
status: 'completed',
|
||||
output: {
|
||||
type: outputType,
|
||||
value: outputValue,
|
||||
},
|
||||
});
|
||||
|
||||
activeToolCalls.delete(token.toolCallId);
|
||||
} break;
|
||||
case 'tool-error': {
|
||||
let outputType: 'text' | 'json';
|
||||
let outputValue: string;
|
||||
|
||||
@@ -544,7 +679,7 @@ async function generateResponse(
|
||||
break;
|
||||
case 'object':
|
||||
outputType = 'json';
|
||||
outputValue = JSON.stringify(token.error, null, 2);
|
||||
outputValue = JSON.stringify(token.error);
|
||||
break;
|
||||
default:
|
||||
console.error('Unknown error type', token.error);
|
||||
@@ -553,20 +688,63 @@ async function generateResponse(
|
||||
break;
|
||||
}
|
||||
|
||||
await httpClient.update('tool_calls', toolCall.id, {
|
||||
status: 'failed',
|
||||
error: {
|
||||
type: outputType,
|
||||
value: outputValue,
|
||||
},
|
||||
});
|
||||
if (activeToolCalls.has(token.toolCallId)) {
|
||||
await httpClient.update('tool_calls', token.toolCallId, {
|
||||
status: 'failed',
|
||||
error: {
|
||||
type: outputType,
|
||||
value: outputValue,
|
||||
},
|
||||
});
|
||||
} else {
|
||||
key = `tool-call-${curStepIdx}`;
|
||||
|
||||
const toolCallId = token.toolCallId;
|
||||
|
||||
part = await httpClient.insert('message_parts', {
|
||||
topicId,
|
||||
messageId: message.id,
|
||||
userId,
|
||||
toolCallId: token.toolCallId,
|
||||
type: 'tool-call',
|
||||
content: '',
|
||||
finished: false,
|
||||
createdAt: new Date(),
|
||||
lastUpdatedAt: new Date(),
|
||||
});
|
||||
|
||||
await httpClient.insert('tool_calls', {
|
||||
id: toolCallId,
|
||||
userId: userId,
|
||||
toolName: token.toolName,
|
||||
status: 'failed',
|
||||
input: null,
|
||||
output: null,
|
||||
error: {
|
||||
type: outputType,
|
||||
value: outputValue,
|
||||
},
|
||||
createdAt: new Date(),
|
||||
});
|
||||
|
||||
activeParts.set(key, { id: part.id, accumulatedContent: '' });
|
||||
}
|
||||
|
||||
activeToolCalls.delete(token.toolCallId);
|
||||
} break;
|
||||
case 'error': {
|
||||
for (const activePart of activeParts.values()) {
|
||||
await httpClient.update('message_parts', activePart.id, {
|
||||
finished: true,
|
||||
lastUpdatedAt: new Date(),
|
||||
});
|
||||
}
|
||||
|
||||
let error = INTERNAL_ERROR;
|
||||
if (typeof token.error === 'string') {
|
||||
error = token.error;
|
||||
} else if (typeof token.error === 'object') {
|
||||
error = JSON.stringify(token.error, null, 2);
|
||||
error = JSON.stringify(token.error);
|
||||
}
|
||||
|
||||
await httpClient.update('generations', generationId, {
|
||||
@@ -575,6 +753,23 @@ async function generateResponse(
|
||||
});
|
||||
} break;
|
||||
case 'finish': {
|
||||
let tps;
|
||||
if (ttft !== undefined && token.totalUsage.outputTokens !== undefined) {
|
||||
const tokenStreamStart = requestStart! + ttft;
|
||||
// this is the *real* request duration, excluding the
|
||||
// TTFT
|
||||
const requestDuration = Date.now() - tokenStreamStart;
|
||||
|
||||
tps = token.totalUsage.outputTokens / (requestDuration / 1000);
|
||||
}
|
||||
|
||||
for (const activePart of activeParts.values()) {
|
||||
await httpClient.update('message_parts', activePart.id, {
|
||||
finished: true,
|
||||
lastUpdatedAt: new Date(),
|
||||
});
|
||||
}
|
||||
|
||||
switch (token.finishReason) {
|
||||
case 'error':
|
||||
await httpClient.update('generations', generationId, {
|
||||
@@ -590,11 +785,36 @@ async function generateResponse(
|
||||
break;
|
||||
}
|
||||
|
||||
const generation = await httpClient.fetchOne(httpClient.query('generations').Where('id', '=', generationId));
|
||||
if (generation === null || generation.status === 'failed') return;
|
||||
|
||||
await httpClient.update('generations', generationId, {
|
||||
status: 'completed',
|
||||
tokens: {
|
||||
input: token.totalUsage.inputTokens,
|
||||
cache: {
|
||||
read: token.totalUsage.inputTokenDetails.cacheReadTokens,
|
||||
write: token.totalUsage.inputTokenDetails.cacheWriteTokens,
|
||||
},
|
||||
output: token.totalUsage.outputTokens,
|
||||
thinking: token.totalUsage.outputTokenDetails.reasoningTokens,
|
||||
ttft,
|
||||
tps,
|
||||
},
|
||||
});
|
||||
|
||||
// I hate you switch fallthroughs
|
||||
} break;
|
||||
case 'abort': {
|
||||
for (const activeToolCall of activeToolCalls.values()) {
|
||||
await httpClient.update('tool_calls', activeToolCall.id, {
|
||||
for (const activePart of activeParts.values()) {
|
||||
await httpClient.update('message_parts', activePart.id, {
|
||||
finished: true,
|
||||
lastUpdatedAt: new Date(),
|
||||
});
|
||||
}
|
||||
|
||||
for (const toolCallId of activeToolCalls.keys()) {
|
||||
await httpClient.update('tool_calls', toolCallId, {
|
||||
status: 'cancelled',
|
||||
});
|
||||
}
|
||||
@@ -615,14 +835,13 @@ async function generateResponse(
|
||||
case 'tool-approval-request':
|
||||
todo('tool-approval-request token type', token);
|
||||
break;
|
||||
case 'tool-output-denied':
|
||||
todo('tool-output-denied token type', token);
|
||||
break;
|
||||
case 'start':
|
||||
// typescript thinks this is not a real token type?
|
||||
// case 'tool-output-denied':
|
||||
// todo('tool-output-denied token type', token);
|
||||
// break;
|
||||
case 'finish-step':
|
||||
case 'tool-input-delta':
|
||||
case 'tool-input-end':
|
||||
case 'tool-result':
|
||||
// handled or irrelevant
|
||||
break;
|
||||
}
|
||||
@@ -637,14 +856,16 @@ async function generateResponse(
|
||||
});
|
||||
}
|
||||
|
||||
for (const activeToolCall of activeToolCalls.values()) {
|
||||
await httpClient.update('tool_calls', activeToolCall.id, {
|
||||
for (const toolCallId of activeToolCalls.keys()) {
|
||||
await httpClient.update('tool_calls', toolCallId, {
|
||||
status: 'failed',
|
||||
error: {
|
||||
type: 'text',
|
||||
value: 'An unknown error occurred',
|
||||
},
|
||||
});
|
||||
|
||||
activeToolCalls.delete(toolCallId);
|
||||
}
|
||||
|
||||
await httpClient.update('generations', generationId, {
|
||||
|
||||
@@ -0,0 +1,316 @@
|
||||
import * as z from 'zod';
|
||||
import { providerBaseUrls, SupportedModalities } from '~/types/model';
|
||||
import { Providers } from '~/types/model';
|
||||
import { httpClient } from '~~/server/lib/triplit';
|
||||
|
||||
export default defineEventHandler(async (event) => {
|
||||
await protectRoute(event);
|
||||
const userId = event.context.user!.id;
|
||||
|
||||
const result = await readValidatedBody(event, (body) =>
|
||||
z
|
||||
.object({
|
||||
providerApiKey: z.string().optional(),
|
||||
})
|
||||
.safeParse(body),
|
||||
);
|
||||
|
||||
if (!result.success) {
|
||||
throw createError({
|
||||
statusCode: 400,
|
||||
message: result.error.issues[0]!.message,
|
||||
});
|
||||
}
|
||||
|
||||
const providerId = getRouterParam(event, 'providerId')
|
||||
|
||||
const provider = await httpClient.fetchOne(httpClient.query('providers').Where('id', '=', providerId));
|
||||
if (provider === null || provider.userId !== userId) {
|
||||
throw createError({
|
||||
statusCode: 400,
|
||||
message: 'Invalid provider',
|
||||
});
|
||||
}
|
||||
|
||||
let baseUrl;
|
||||
let fetchUrl;
|
||||
let headers;
|
||||
switch (provider.type) {
|
||||
case 'cohere':
|
||||
case 'cerebras':
|
||||
if (!result.data.providerApiKey) {
|
||||
throw createError({
|
||||
statusCode: 400,
|
||||
message: `${provider.type} provider requires an API key`,
|
||||
});
|
||||
}
|
||||
case 'google':
|
||||
case 'openrouter':
|
||||
baseUrl = !!provider.config.apiProxyUrl ? provider.config.apiProxyUrl : providerBaseUrls[provider.type];
|
||||
baseUrl = baseUrl.replace(/\/$/, '');
|
||||
|
||||
if (baseUrl === '') {
|
||||
throw createError({
|
||||
statusCode: 400,
|
||||
message: 'Invalid provider URL',
|
||||
});
|
||||
}
|
||||
|
||||
fetchUrl = `${baseUrl}/models`;
|
||||
break;
|
||||
case 'ollama':
|
||||
if (!provider.config.apiProxyUrl) {
|
||||
throw createError({
|
||||
statusCode: 400,
|
||||
message: 'Ollama provider requires an API proxy URL',
|
||||
});
|
||||
}
|
||||
baseUrl = provider.config.apiProxyUrl;
|
||||
baseUrl = baseUrl.replace(/\/$/, '');
|
||||
|
||||
fetchUrl = `${baseUrl}/api/tags`;
|
||||
break;
|
||||
case 'longcat':
|
||||
// longcat doesn't have a model list endpoint so we just hardcode them here,
|
||||
// sry. I talked to Meituan and this is what they said:
|
||||
// 后续如果我们新增了这样的接口会及时同步您。
|
||||
// en (approx): If we add an interface like this in the future, we will promptly update you
|
||||
return {
|
||||
models: [
|
||||
{
|
||||
id: "LongCat-Flash-Chat",
|
||||
name: "LongCat Flash Chat",
|
||||
attributes: {
|
||||
inputModalities: ['text'],
|
||||
outputModalities: ['text'],
|
||||
capabilities: ['tools'],
|
||||
contextWindow: 256_000,
|
||||
}
|
||||
},
|
||||
{
|
||||
id: "LongCat-Flash-Thinking",
|
||||
name: "LongCat Flash Thinking",
|
||||
attributes: {
|
||||
inputModalities: ['text'],
|
||||
outputModalities: ['text'],
|
||||
capabilities: ['reasoning', 'tools'],
|
||||
contextWindow: 256_000,
|
||||
}
|
||||
},
|
||||
{
|
||||
id: "LongCat-Flash-Thinking-2601",
|
||||
name: "LongCat Flash Thinking (2601)",
|
||||
attributes: {
|
||||
inputModalities: ['text'],
|
||||
outputModalities: ['text'],
|
||||
capabilities: ['reasoning', 'tools'],
|
||||
contextWindow: 256_000,
|
||||
}
|
||||
},
|
||||
{
|
||||
id: "LongCat-Flash-Lite",
|
||||
name: "LongCat Flash Lite",
|
||||
attributes: {
|
||||
inputModalities: ['text'],
|
||||
outputModalities: ['text'],
|
||||
capabilities: ['tools'],
|
||||
contextWindow: 320_000,
|
||||
}
|
||||
}
|
||||
]
|
||||
};
|
||||
}
|
||||
|
||||
if (provider.type === 'google') {
|
||||
headers = {
|
||||
'x-goog-api-key': `${result.data.providerApiKey}`
|
||||
}
|
||||
} else {
|
||||
headers = {
|
||||
'Authorization': `Bearer ${result.data.providerApiKey}`
|
||||
}
|
||||
}
|
||||
|
||||
let res;
|
||||
let data;
|
||||
try {
|
||||
res = await fetch(fetchUrl, {
|
||||
method: 'GET',
|
||||
headers
|
||||
});
|
||||
data = await res.json();
|
||||
} catch (e) {
|
||||
console.error('Failed to fetch models:', e);
|
||||
throw createError({
|
||||
statusCode: 500,
|
||||
message: 'Failed to fetch models ' + e,
|
||||
});
|
||||
}
|
||||
|
||||
if (!res.ok) {
|
||||
throw createError({
|
||||
statusCode: res.status,
|
||||
message: JSON.stringify(data),
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
models: await normalizeResponse(data, provider.type, baseUrl)
|
||||
};
|
||||
});
|
||||
|
||||
const normalizeResponse = async (response: Record<string, any>, provider: typeof Providers[number], baseUrl: string) => {
|
||||
switch (provider) {
|
||||
case 'cerebras': {
|
||||
console.log(response);
|
||||
return response.data.map((model: any) => ({ id: model.id, releasedAt: model.created }));
|
||||
}
|
||||
case 'openrouter': {
|
||||
const models = [];
|
||||
|
||||
for (const model of response.data) {
|
||||
const capabilities = new Set<string>();
|
||||
|
||||
for (const capability of model.supported_parameters) {
|
||||
switch (capability) {
|
||||
case 'reasoning': {
|
||||
capabilities.add('reasoning');
|
||||
} break;
|
||||
case 'tools': {
|
||||
capabilities.add('tools');
|
||||
} break;
|
||||
}
|
||||
}
|
||||
|
||||
models.push({
|
||||
id: model.id as string,
|
||||
name: model.name as string,
|
||||
pricing: model.pricing,
|
||||
attributes: {
|
||||
inputModalities: model.architecture.input_modalities.filter((modality: string) => (SupportedModalities as Readonly<string[]>).includes(modality)),
|
||||
outputModalities: model.architecture.output_modalities.filter((modality: string) => (SupportedModalities as Readonly<string[]>).includes(modality)),
|
||||
capabilities: Array.from(capabilities),
|
||||
contextWindow: model.context_length,
|
||||
supported_parameters: model.supported_parameters,
|
||||
},
|
||||
created: model.created,
|
||||
});
|
||||
}
|
||||
|
||||
return models;
|
||||
}
|
||||
case 'ollama': {
|
||||
const headers: Record<string, string> = {
|
||||
'Content-Type': 'application/json',
|
||||
'User-Agent': 'Mozilla/5.0'
|
||||
}
|
||||
|
||||
if (response.token) {
|
||||
headers['Authorization'] = `Bearer ${response.token}`;
|
||||
}
|
||||
|
||||
const models = new Map<string, any>();
|
||||
const infoPromises = [];
|
||||
|
||||
for (const model of response.models) {
|
||||
infoPromises.push(fetch(`${baseUrl}/api/show`, {
|
||||
method: 'POST',
|
||||
headers,
|
||||
body: JSON.stringify({
|
||||
model: model.name,
|
||||
}),
|
||||
}).then((res) => {
|
||||
return res.json()
|
||||
}).then((json) => {
|
||||
const inputModalities = new Set<string>();
|
||||
const capabilities = new Set<string>();
|
||||
|
||||
for (const capability of json.capabilities) {
|
||||
switch (capability) {
|
||||
case 'thinking':
|
||||
capabilities.add('reasoning');
|
||||
break;
|
||||
case 'tools':
|
||||
capabilities.add('tools');
|
||||
break;
|
||||
case 'completion':
|
||||
inputModalities.add('text');
|
||||
break;
|
||||
case 'vision':
|
||||
inputModalities.add('image');
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
let contextWindow: number | undefined;
|
||||
|
||||
try {
|
||||
for (const key of Object.keys(json.model_info)) {
|
||||
if (key.endsWith('context_length')) {
|
||||
contextWindow = json.model_info[key];
|
||||
break;
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
console.error(e);
|
||||
}
|
||||
|
||||
models.set(model.name, {
|
||||
id: model.name,
|
||||
name: model.name,
|
||||
attributes: {
|
||||
inputModalities: Array.from(inputModalities),
|
||||
capabilities: Array.from(capabilities),
|
||||
contextWindow,
|
||||
}
|
||||
});
|
||||
}));
|
||||
}
|
||||
|
||||
await Promise.all(infoPromises);
|
||||
|
||||
return Array.from(models.values());
|
||||
}
|
||||
case 'google': {
|
||||
console.log(response);
|
||||
return response.models.map((model: any) => ({ id: model.name.replace('models/', ''), name: model.displayName }));
|
||||
}
|
||||
case 'longcat': {
|
||||
console.log(response);
|
||||
return response.models.map((model: any) => ({ id: model.name, name: model.name }));
|
||||
}
|
||||
case 'cohere': {
|
||||
const models = [];
|
||||
for (const model of response.models) {
|
||||
const inputModalities = new Set<string>();
|
||||
const capabilities = new Set<string>();
|
||||
|
||||
for (const feature of model.features || []) {
|
||||
switch (feature) {
|
||||
case 'tools': {
|
||||
capabilities.add('tools');
|
||||
} break;
|
||||
case 'vision': {
|
||||
inputModalities.add('image');
|
||||
} break;
|
||||
case 'reasoning': {
|
||||
capabilities.add('reasoning');
|
||||
} break;
|
||||
}
|
||||
}
|
||||
|
||||
models.push({
|
||||
id: model.name,
|
||||
name: model.name,
|
||||
attributes: {
|
||||
contextWindow: model.context_length,
|
||||
inputModalities: Array.from(inputModalities),
|
||||
capabilities: Array.from(capabilities),
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
return models;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
import { cancelPendingRename } from '~~/server/utils/renames';
|
||||
import { assert } from '~~/utils/assert';
|
||||
|
||||
export default defineEventHandler(async (event) => {
|
||||
await protectRoute(event);
|
||||
|
||||
const { renameId } = event.context.params!;
|
||||
assert(renameId);
|
||||
|
||||
if (cancelPendingRename(renameId)) {
|
||||
return {
|
||||
success: true,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
success: false,
|
||||
};
|
||||
});
|
||||
@@ -0,0 +1,93 @@
|
||||
import * as z from 'zod';
|
||||
import { httpClient } from '~~/server/lib/triplit';
|
||||
import { renamePrompt } from '~~/prompts';
|
||||
import { schema } from '~~/triplit/schema';
|
||||
import { type Entity } from '@triplit/client';
|
||||
import { generateText } from 'ai';
|
||||
import { getGateway, type ModelGateway } from '~~/server/utils/ai-provider';
|
||||
import { addPendingRename } from '~~/server/utils/renames';
|
||||
|
||||
export default defineEventHandler(async (event) => {
|
||||
await protectRoute(event);
|
||||
|
||||
const userId = event.context.user!.id;
|
||||
|
||||
const result = await readValidatedBody(event, (body) =>
|
||||
z
|
||||
.object({
|
||||
modelId: z.string(),
|
||||
topicId: z.string(),
|
||||
prompt: z.string(),
|
||||
providerApiKey: z.string().optional(),
|
||||
})
|
||||
.safeParse(body),
|
||||
);
|
||||
if (!result.success) {
|
||||
throw createError({
|
||||
statusCode: 400,
|
||||
message: result.error.issues[0]!.message,
|
||||
});
|
||||
}
|
||||
|
||||
const { modelId, topicId, prompt, providerApiKey } = result.data;
|
||||
|
||||
const model = await httpClient.fetchOne(httpClient.query('models').Where('id', '=', modelId).Include('provider'));
|
||||
if (model === null || model.providerId !== model.providerId || model.userId !== userId) {
|
||||
throw createError({
|
||||
statusCode: 400,
|
||||
message: 'Invalid model',
|
||||
});
|
||||
}
|
||||
|
||||
const { gateway, textTransformer } = await getGateway(model.provider!, model, providerApiKey);
|
||||
const [renameId, abortController] = addPendingRename();
|
||||
event.waitUntil(autoRename(topicId, abortController, { gateway, model }, textTransformer, prompt));
|
||||
|
||||
return {
|
||||
success: true,
|
||||
renameId,
|
||||
};
|
||||
});
|
||||
|
||||
const autoRename = async (
|
||||
topicId: string,
|
||||
abortController: AbortController,
|
||||
model: {
|
||||
gateway: ModelGateway,
|
||||
model: Entity<typeof schema, 'models'>,
|
||||
},
|
||||
textTransformer: ((text: string) => string) | ((text: string) => string)[] | undefined,
|
||||
prompt: string,
|
||||
) => {
|
||||
try {
|
||||
const response = await generateText({
|
||||
model: model.gateway(model.model.externalId),
|
||||
system: renamePrompt,
|
||||
prompt,
|
||||
timeout: 90 * 1000,
|
||||
abortSignal: abortController.signal,
|
||||
})
|
||||
|
||||
let text = response.text;
|
||||
|
||||
if (textTransformer !== undefined) {
|
||||
if (Array.isArray(textTransformer)) {
|
||||
for (const transformer of textTransformer) {
|
||||
text = transformer(text);
|
||||
}
|
||||
} else {
|
||||
text = textTransformer(text);
|
||||
}
|
||||
}
|
||||
|
||||
await httpClient.update('topics', topicId, {
|
||||
name: text,
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Failed to auto-rename:', error);
|
||||
} finally {
|
||||
await httpClient.update('topics', topicId, {
|
||||
renaming: false
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
import { createCerebras, type CerebrasProvider } from "@ai-sdk/cerebras";
|
||||
import { createGoogleGenerativeAI, type GoogleGenerativeAIProvider } from "@ai-sdk/google";
|
||||
import { createOpenRouter, type OpenRouterProvider } from "@openrouter/ai-sdk-provider";
|
||||
import { createOllama, type OllamaProvider } from "ai-sdk-ollama";
|
||||
import { createOpenAICompatible, type OpenAICompatibleProvider } from '@ai-sdk/openai-compatible';
|
||||
import { createCohere, type CohereProvider } from '@ai-sdk/cohere';
|
||||
import { type Entity } from "@triplit/client";
|
||||
import { schema } from "~~/triplit/schema";
|
||||
import { type StreamTextTransform } from "ai";
|
||||
import { transformCerebrasReasoningStream } from "./cerebras";
|
||||
import { providerBaseUrls } from "~/types/model";
|
||||
// import { createLongcatTransformer } from "./longcat";
|
||||
|
||||
export type ModelGateway = OpenRouterProvider | OllamaProvider | CerebrasProvider | GoogleGenerativeAIProvider | OpenAICompatibleProvider | CohereProvider;
|
||||
|
||||
export interface Gateway {
|
||||
gateway: ModelGateway;
|
||||
streamTransformer: StreamTextTransform<{}> | StreamTextTransform<{}>[] | undefined;
|
||||
textTransformer: ((text: string) => string) | ((text: string) => string)[] | undefined;
|
||||
}
|
||||
|
||||
export async function getGateway(provider: Entity<typeof schema, 'providers'>, model: Entity<typeof schema, 'models'>, providerApiKey?: string): Promise<Gateway> {
|
||||
let gateway: ModelGateway;
|
||||
let streamTransformer = undefined;
|
||||
let textTransformer = undefined;
|
||||
|
||||
let baseURL = undefined;
|
||||
|
||||
if (provider.config.apiProxyUrl && provider.config.apiProxyUrl.trim() !== '') {
|
||||
baseURL = provider.config.apiProxyUrl;
|
||||
}
|
||||
|
||||
switch (provider.type) {
|
||||
case 'openrouter': {
|
||||
if (providerApiKey === undefined) {
|
||||
throw createError({
|
||||
statusCode: 400,
|
||||
message: 'OpenRouter provider requires an API key',
|
||||
});
|
||||
}
|
||||
|
||||
gateway = createOpenRouter({
|
||||
apiKey: providerApiKey,
|
||||
headers: {
|
||||
'HTTP-Referer': 'https://localhost:3000',
|
||||
'X-Title': 'Veridian',
|
||||
},
|
||||
});
|
||||
break;
|
||||
}
|
||||
case 'ollama': {
|
||||
if (baseURL === undefined) {
|
||||
throw createError({
|
||||
statusCode: 400,
|
||||
message: 'Ollama provider requires an API proxy URL',
|
||||
});
|
||||
}
|
||||
|
||||
const innerGateway = createOllama({
|
||||
apiKey: providerApiKey,
|
||||
baseURL,
|
||||
})
|
||||
|
||||
gateway = ((modelId: string) => innerGateway(modelId, { think: [...model.attributes.capabilities].includes('reasoning') })) as OllamaProvider;
|
||||
break;
|
||||
}
|
||||
case 'cerebras': {
|
||||
gateway = createCerebras({
|
||||
apiKey: providerApiKey,
|
||||
baseURL,
|
||||
})
|
||||
|
||||
streamTransformer = transformCerebrasReasoningStream() as StreamTextTransform<{}>;
|
||||
textTransformer = (text: string) => {
|
||||
return text.split('</think>').at(-1)!.trim()
|
||||
};
|
||||
break;
|
||||
}
|
||||
case 'google': {
|
||||
gateway = createGoogleGenerativeAI({
|
||||
apiKey: providerApiKey,
|
||||
baseURL,
|
||||
})
|
||||
break;
|
||||
}
|
||||
case 'longcat': {
|
||||
gateway = createOpenAICompatible({
|
||||
name: 'LongCat',
|
||||
apiKey: providerApiKey,
|
||||
baseURL: baseURL ?? providerBaseUrls[provider.type],
|
||||
includeUsage: true,
|
||||
})
|
||||
// streamTransformer = createLongcatTransformer() as StreamTextTransform<{}>;
|
||||
} break;
|
||||
case 'cohere': {
|
||||
gateway = createCohere({
|
||||
apiKey: providerApiKey,
|
||||
baseURL,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
gateway,
|
||||
streamTransformer,
|
||||
textTransformer,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
import type { TextStreamPart, ToolSet } from 'ai';
|
||||
|
||||
export function transformCerebrasReasoningStream<TOOLS extends ToolSet>(): (options: {
|
||||
tools: TOOLS;
|
||||
stopStream: () => void;
|
||||
}) => TransformStream<TextStreamPart<TOOLS>, TextStreamPart<TOOLS>> {
|
||||
|
||||
return (_opts) => {
|
||||
let isThinking = false;
|
||||
let currentReasoningId: string | null = null;
|
||||
|
||||
let bufferedTextStart: TextStreamPart<TOOLS> | null = null;
|
||||
let hasEmittedTextStart = false;
|
||||
|
||||
return new TransformStream<TextStreamPart<TOOLS>, TextStreamPart<TOOLS>>({
|
||||
transform(chunk, controller) {
|
||||
if (chunk.type === 'text-start') {
|
||||
bufferedTextStart = chunk;
|
||||
return;
|
||||
}
|
||||
|
||||
if (chunk.type === 'text-delta') {
|
||||
let text = chunk.text;
|
||||
|
||||
if (text.includes('<think>')) {
|
||||
isThinking = true;
|
||||
currentReasoningId = crypto.randomUUID();
|
||||
|
||||
const [before, after] = text.split('<think>');
|
||||
|
||||
if (before && before.trim().length > 0) {
|
||||
if (bufferedTextStart && !hasEmittedTextStart) {
|
||||
controller.enqueue(bufferedTextStart);
|
||||
hasEmittedTextStart = true;
|
||||
}
|
||||
controller.enqueue({ type: 'text-delta', text: before, id: chunk.id });
|
||||
}
|
||||
|
||||
controller.enqueue({ type: 'reasoning-start', id: currentReasoningId });
|
||||
|
||||
if (after) {
|
||||
controller.enqueue({ type: 'reasoning-delta', text: after, id: currentReasoningId });
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (text.includes('</think>')) {
|
||||
isThinking = false;
|
||||
const [before, after] = text.split('</think>');
|
||||
|
||||
if (before && currentReasoningId !== null) {
|
||||
controller.enqueue({ type: 'reasoning-delta', text: before, id: currentReasoningId });
|
||||
}
|
||||
|
||||
if (currentReasoningId !== null) {
|
||||
controller.enqueue({ type: 'reasoning-end', id: currentReasoningId });
|
||||
}
|
||||
|
||||
if (after && after.length > 0) {
|
||||
if (bufferedTextStart && !hasEmittedTextStart) {
|
||||
controller.enqueue(bufferedTextStart);
|
||||
hasEmittedTextStart = true;
|
||||
}
|
||||
controller.enqueue({ type: 'text-delta', text: after, id: chunk.id });
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (isThinking && currentReasoningId !== null) {
|
||||
controller.enqueue({ type: 'reasoning-delta', text: text, id: currentReasoningId });
|
||||
} else {
|
||||
if (bufferedTextStart && !hasEmittedTextStart) {
|
||||
controller.enqueue(bufferedTextStart);
|
||||
hasEmittedTextStart = true;
|
||||
}
|
||||
controller.enqueue(chunk);
|
||||
}
|
||||
} else {
|
||||
controller.enqueue(chunk);
|
||||
}
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -1,22 +1,22 @@
|
||||
const pendingGenerations: Record<string, AbortController> = {};
|
||||
const pendingGenerations: Map<string, AbortController> = new Map();
|
||||
|
||||
export const cancelPendingGeneration = (generationId: string): boolean => {
|
||||
const controller = pendingGenerations[generationId];
|
||||
const controller = pendingGenerations.get(generationId);
|
||||
if (controller) {
|
||||
controller.abort();
|
||||
delete pendingGenerations[generationId];
|
||||
pendingGenerations.delete(generationId);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
};
|
||||
|
||||
export const completeGeneration = (generationId: string) => {
|
||||
const controller = pendingGenerations[generationId];
|
||||
const controller = pendingGenerations.get(generationId);
|
||||
if (controller) {
|
||||
delete pendingGenerations[generationId];
|
||||
pendingGenerations.delete(generationId);
|
||||
}
|
||||
};
|
||||
|
||||
export const addPendingGeneration = (generationId: string, controller: AbortController) => {
|
||||
pendingGenerations[generationId] = controller;
|
||||
pendingGenerations.set(generationId, controller);
|
||||
};
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
import { ToolSet, type TextStreamPart, type ToolCallPart } from 'ai';
|
||||
|
||||
export function createLongcatTransformer<TOOLS extends ToolSet>(): (options: {
|
||||
tools: TOOLS;
|
||||
stopStream: () => void;
|
||||
}) => TransformStream<TextStreamPart<TOOLS>, TextStreamPart<TOOLS>> {
|
||||
let buffer = '';
|
||||
let hasToolCallInStep = false;
|
||||
let lastChunkId: string | undefined;
|
||||
let lastChunkType: 'text' | 'reasoning' | undefined;
|
||||
let step = 0;
|
||||
|
||||
return (_opts) => {
|
||||
return new TransformStream<TextStreamPart<TOOLS>, TextStreamPart<TOOLS>>({
|
||||
transform(chunk, controller) {
|
||||
if (chunk.type === 'finish-step' || chunk.type === 'finish') {
|
||||
step++;
|
||||
|
||||
if (hasToolCallInStep) {
|
||||
// We clone the chunk and overwrite the finishReason.
|
||||
// This tricks the SDK into thinking the model requested a tool natively.
|
||||
const modifiedChunk = {
|
||||
...chunk,
|
||||
finishReason: 'tool-calls' as const,
|
||||
};
|
||||
|
||||
// Reset for the next potential step
|
||||
if (chunk.type === 'finish-step') {
|
||||
hasToolCallInStep = false;
|
||||
}
|
||||
|
||||
controller.enqueue(modifiedChunk);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if (chunk.type === 'text-start' || chunk.type === 'reasoning-start') {
|
||||
lastChunkId = chunk.id;
|
||||
lastChunkType = chunk.type.split('-')[1] as 'text' | 'reasoning';
|
||||
}
|
||||
|
||||
// We only care about text chunks
|
||||
if (chunk.type !== 'text-delta' && chunk.type !== 'reasoning-delta') {
|
||||
controller.enqueue(chunk);
|
||||
return;
|
||||
}
|
||||
|
||||
buffer += chunk.text;
|
||||
|
||||
// Check if we have a full tool call in the buffer
|
||||
const pattern = /<longcat_tool_call>([\s\S]*?)<\/longcat_tool_call>/g;
|
||||
let lastIndex = 0;
|
||||
let match;
|
||||
|
||||
while ((match = pattern.exec(buffer)) !== null) {
|
||||
console.log("longcat tool call found at index", match.index);
|
||||
|
||||
// 1. Enqueue any text that appeared BEFORE the tool call
|
||||
const textBefore = buffer.substring(lastIndex, match.index);
|
||||
if (textBefore) {
|
||||
controller.enqueue({ type: chunk.type, text: textBefore, id: lastChunkId ?? chunk.type.includes('reasoning') ? `reasoning-${step}` : `text-${step}` });
|
||||
}
|
||||
|
||||
// 2. Parse the XML content
|
||||
const content = match[1]!.trim();
|
||||
const toolNameMatch = content.match(/^([^\s<]+)/);
|
||||
|
||||
if (toolNameMatch) {
|
||||
hasToolCallInStep = true;
|
||||
|
||||
const toolName = toolNameMatch[1];
|
||||
const args: Record<string, any> = {};
|
||||
const argRegex = /<longcat_arg_key>(.*?)<\/longcat_arg_key>\s*<longcat_arg_value>(.*?)<\/longcat_arg_value>/gs;
|
||||
|
||||
let argMatch;
|
||||
while ((argMatch = argRegex.exec(content)) !== null) {
|
||||
args[argMatch[1]!.trim()] = argMatch[2]!.trim();
|
||||
}
|
||||
|
||||
// 3. EMIT A TOOL CALL PART
|
||||
// This is the "magic" - the SDK will see this and act as if the LLM
|
||||
// called a native tool.
|
||||
const toolCallId = `lc-${Date.now()}-${Math.random().toString(36).substr(2, 5)}`;
|
||||
controller.enqueue({
|
||||
type: 'tool-call',
|
||||
// @ts-ignore
|
||||
id: toolCallId,
|
||||
toolCallId,
|
||||
toolName,
|
||||
input: args,
|
||||
dynamic: true,
|
||||
});
|
||||
}
|
||||
|
||||
lastIndex = pattern.lastIndex;
|
||||
}
|
||||
|
||||
// Keep the remaining buffer (unclosed tags) for the next chunk
|
||||
buffer = buffer.substring(lastIndex);
|
||||
|
||||
// If there's no open tag starting, we can flush the buffer as text
|
||||
if (!buffer.includes('<longcat_tool_call>')) {
|
||||
if (buffer) {
|
||||
controller.enqueue({ type: chunk.type, text: buffer, id: lastChunkId ?? chunk.type.includes('reasoning') ? `reasoning-${step}` : `text-${step}` });
|
||||
buffer = '';
|
||||
}
|
||||
}
|
||||
},
|
||||
flush(controller) {
|
||||
if (buffer && lastChunkId && lastChunkType) {
|
||||
controller.enqueue({ type: `${lastChunkType}-delta`, text: buffer, id: lastChunkId });
|
||||
}
|
||||
}
|
||||
});
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
const pendingRenames: Map<string, AbortController> = new Map();
|
||||
|
||||
export const cancelPendingRename = (renameId: string): boolean => {
|
||||
const controller = pendingRenames.get(renameId);
|
||||
if (controller) {
|
||||
controller.abort();
|
||||
pendingRenames.delete(renameId);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
};
|
||||
|
||||
export const completeRename = (renameId: string) => {
|
||||
const controller = pendingRenames.get(renameId);
|
||||
if (controller) {
|
||||
pendingRenames.delete(renameId);
|
||||
}
|
||||
};
|
||||
|
||||
export const addPendingRename = (): [string, AbortController] => {
|
||||
const id = crypto.randomUUID();
|
||||
const controller = new AbortController();
|
||||
|
||||
pendingRenames.set(id, controller);
|
||||
return [id, controller];
|
||||
};
|
||||
Reference in New Issue
Block a user