diff --git a/server/api/chat/generate.post.ts b/server/api/chat/generate.post.ts index d6ced58..fbca3b6 100644 --- a/server/api/chat/generate.post.ts +++ b/server/api/chat/generate.post.ts @@ -44,15 +44,13 @@ export default defineEventHandler(async (event) => { const { messages, topicId, parentMessageId, model: { modelId, providerId, args }, providerApiKey } = result.data; - const fetchPromises = []; - - fetchPromises.push(httpClient.fetchOne(httpClient.query('providers').Where('id', '=', providerId))); - fetchPromises.push(httpClient.fetchOne(httpClient.query('models').Where('id', '=', modelId))); - fetchPromises.push(httpClient.fetchOne( - httpClient.query('generations').Where('topicId', '=', topicId).Where('status', '=', 'pending'), - )); - - const [provider, model, existingPendingGenerations] = await Promise.all(fetchPromises) as [Entity | null, Entity | null, Entity | null]; + const [provider, model, existingPendingGenerations] = await Promise.all([ + httpClient.fetchOne(httpClient.query('providers').Where('id', '=', providerId)), + httpClient.fetchOne(httpClient.query('models').Where('id', '=', modelId)), + httpClient.fetchOne( + httpClient.query('generations').Where('topicId', '=', topicId).Where('status', '=', 'pending'), + ), + ]); if (provider === null || provider.userId !== userId) { throw createError({ statusCode: 400, @@ -93,9 +91,19 @@ export default defineEventHandler(async (event) => { } const { gateway } = providerDetails.data; - assert(gateway !== null, 'Invalid gateway'); + if (gateway === null) { + throw createError({ + statusCode: 500, + statusMessage: 'Invalid gateway', + data: { + code: 'INVALID_GATEWAY', + ok: false, + } + }); + } const generationId = nanoid(); + // TODO: do these inserts on the client so that the feedback is instant const message = await httpClient.insert('messages', { topicId, userId, @@ -368,6 +376,7 @@ async function generateResponse( const activeParts = new Map(); const activeToolCalls = new Map(); + // TODO: somehow let the user turn on and off tools const tools = { // writeFile: tool({ // inputSchema: z.object({ @@ -480,7 +489,7 @@ async function generateResponse( switch (token.type) { case 'start': { - requestStart = Date.now(); + requestStart = performance.now(); } break; case 'start-step': { curStepIdx++; @@ -538,7 +547,7 @@ async function generateResponse( case 'text-delta': case 'reasoning-delta': { if (ttft === undefined) { - ttft = Date.now() - requestStart!; + ttft = performance.now() - requestStart!; } type = token.type.split('-')[0] as 'text' | 'reasoning'; @@ -788,7 +797,7 @@ async function generateResponse( const tokenStreamStart = requestStart! + ttft; // this is the *real* request duration, excluding the // TTFT - const requestDuration = Date.now() - tokenStreamStart; + const requestDuration = performance.now() - tokenStreamStart; tps = token.totalUsage.outputTokens / (requestDuration / 1000); }