zoeissleeping/veridian
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: use performance.now for more precise tps

Browse Source
This commit is contained in:
Zoe
2026-02-25 19:47:05 -06:00
parent 5f3f81da72
commit 50cb195486
1 changed files with 22 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/api/chat/generate.post.ts
+22 -13
View File
@@ -44,15 +44,13 @@ export default defineEventHandler(async (event) => {
const { messages, topicId, parentMessageId, model: { modelId, providerId, args }, providerApiKey } = result.data;
const fetchPromises = [];
fetchPromises.push(httpClient.fetchOne(httpClient.query('providers').Where('id', '=', providerId)));
fetchPromises.push(httpClient.fetchOne(httpClient.query('models').Where('id', '=', modelId)));
fetchPromises.push(httpClient.fetchOne(
httpClient.query('generations').Where('topicId', '=', topicId).Where('status', '=', 'pending'),
));
const [provider, model, existingPendingGenerations] = await Promise.all(fetchPromises) as [Entity<typeof schema, 'providers'> | null, Entity<typeof schema, 'models'> | null, Entity<typeof schema, 'generations'> | null];
const [provider, model, existingPendingGenerations] = await Promise.all([
httpClient.fetchOne(httpClient.query('providers').Where('id', '=', providerId)),
httpClient.fetchOne(httpClient.query('models').Where('id', '=', modelId)),
httpClient.fetchOne(
httpClient.query('generations').Where('topicId', '=', topicId).Where('status', '=', 'pending'),
),
]);
if (provider === null || provider.userId !== userId) {
throw createError({
statusCode: 400,
@@ -93,9 +91,19 @@ export default defineEventHandler(async (event) => {
}
const { gateway } = providerDetails.data;
assert(gateway !== null, 'Invalid gateway');
if (gateway === null) {
throw createError({
statusCode: 500,
statusMessage: 'Invalid gateway',
data: {
code: 'INVALID_GATEWAY',
ok: false,
}
});
}
const generationId = nanoid();
// TODO: do these inserts on the client so that the feedback is instant
const message = await httpClient.insert('messages', {
topicId,
userId,
@@ -368,6 +376,7 @@ async function generateResponse(
const activeParts = new Map<string, { id: string; accumulatedContent: string; providerOptions?: any }>();
const activeToolCalls = new Map<string, void>();
// TODO: somehow let the user turn on and off tools
const tools = {
// writeFile: tool({
// inputSchema: z.object({
@@ -480,7 +489,7 @@ async function generateResponse(
switch (token.type) {
case 'start': {
requestStart = Date.now();
requestStart = performance.now();
} break;
case 'start-step': {
curStepIdx++;
@@ -538,7 +547,7 @@ async function generateResponse(
case 'text-delta':
case 'reasoning-delta': {
if (ttft === undefined) {
ttft = Date.now() - requestStart!;
ttft = performance.now() - requestStart!;
}
type = token.type.split('-')[0] as 'text' | 'reasoning';
@@ -788,7 +797,7 @@ async function generateResponse(
const tokenStreamStart = requestStart! + ttft;
// this is the *real* request duration, excluding the
// TTFT
const requestDuration = Date.now() - tokenStreamStart;
const requestDuration = performance.now() - tokenStreamStart;
tps = token.totalUsage.outputTokens / (requestDuration / 1000);
}