zoeissleeping/veridian
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: use performance.now for more precise tps

Browse Source
This commit is contained in:
Zoe
2026-02-25 19:47:05 -06:00
parent 5f3f81da72
commit 50cb195486
1 changed files with 22 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/api/chat/generate.post.ts
+21 -12
View File
@@ -44,15 +44,13 @@ export default defineEventHandler(async (event) => {
const { messages, topicId, parentMessageId, model: { modelId, providerId, args }, providerApiKey } = result.data; const { messages, topicId, parentMessageId, model: { modelId, providerId, args }, providerApiKey } = result.data;
const fetchPromises = []; const [provider, model, existingPendingGenerations] = await Promise.all([
httpClient.fetchOne(httpClient.query('providers').Where('id', '=', providerId)),
fetchPromises.push(httpClient.fetchOne(httpClient.query('providers').Where('id', '=', providerId))); httpClient.fetchOne(httpClient.query('models').Where('id', '=', modelId)),
fetchPromises.push(httpClient.fetchOne(httpClient.query('models').Where('id', '=', modelId))); httpClient.fetchOne(
fetchPromises.push(httpClient.fetchOne(
httpClient.query('generations').Where('topicId', '=', topicId).Where('status', '=', 'pending'), httpClient.query('generations').Where('topicId', '=', topicId).Where('status', '=', 'pending'),
)); ),
]);
const [provider, model, existingPendingGenerations] = await Promise.all(fetchPromises) as [Entity<typeof schema, 'providers'> | null, Entity<typeof schema, 'models'> | null, Entity<typeof schema, 'generations'> | null];
if (provider === null || provider.userId !== userId) { if (provider === null || provider.userId !== userId) {
throw createError({ throw createError({
statusCode: 400, statusCode: 400,
@@ -93,9 +91,19 @@ export default defineEventHandler(async (event) => {
} }
const { gateway } = providerDetails.data; const { gateway } = providerDetails.data;
assert(gateway !== null, 'Invalid gateway'); if (gateway === null) {
throw createError({
statusCode: 500,
statusMessage: 'Invalid gateway',
data: {
code: 'INVALID_GATEWAY',
ok: false,
}
});
}
const generationId = nanoid(); const generationId = nanoid();
// TODO: do these inserts on the client so that the feedback is instant
const message = await httpClient.insert('messages', { const message = await httpClient.insert('messages', {
topicId, topicId,
userId, userId,
@@ -368,6 +376,7 @@ async function generateResponse(
const activeParts = new Map<string, { id: string; accumulatedContent: string; providerOptions?: any }>(); const activeParts = new Map<string, { id: string; accumulatedContent: string; providerOptions?: any }>();
const activeToolCalls = new Map<string, void>(); const activeToolCalls = new Map<string, void>();
// TODO: somehow let the user turn on and off tools
const tools = { const tools = {
// writeFile: tool({ // writeFile: tool({
// inputSchema: z.object({ // inputSchema: z.object({
@@ -480,7 +489,7 @@ async function generateResponse(
switch (token.type) { switch (token.type) {
case 'start': { case 'start': {
requestStart = Date.now(); requestStart = performance.now();
} break; } break;
case 'start-step': { case 'start-step': {
curStepIdx++; curStepIdx++;
@@ -538,7 +547,7 @@ async function generateResponse(
case 'text-delta': case 'text-delta':
case 'reasoning-delta': { case 'reasoning-delta': {
if (ttft === undefined) { if (ttft === undefined) {
ttft = Date.now() - requestStart!; ttft = performance.now() - requestStart!;
} }
type = token.type.split('-')[0] as 'text' | 'reasoning'; type = token.type.split('-')[0] as 'text' | 'reasoning';
@@ -788,7 +797,7 @@ async function generateResponse(
const tokenStreamStart = requestStart! + ttft; const tokenStreamStart = requestStart! + ttft;
// this is the *real* request duration, excluding the // this is the *real* request duration, excluding the
// TTFT // TTFT
const requestDuration = Date.now() - tokenStreamStart; const requestDuration = performance.now() - tokenStreamStart;
tps = token.totalUsage.outputTokens / (requestDuration / 1000); tps = token.totalUsage.outputTokens / (requestDuration / 1000);
} }