import { db } from '../server/lib/db'; import * as drizzleSchema from '../drizzle/schema'; import { HttpClient } from '@triplit/client'; import { schema } from '../triplit/schema'; import fs from 'node:fs/promises'; export const httpClient = new HttpClient({ schema, serverUrl: process.env.NUXT_LOCAL_TRIPLIT_URL || process.env.NUXT_PUBLIC_TRIPLIT_URL, token: process.env.TRIPLIT_SERVICE_TOKEN, }); /** * Simple Argument Parser * Use: --skip=users,accounts --topics-file=ids.txt */ const args = process.argv.slice(2); const skipList = args.find(a => a.startsWith('--skip='))?.split('=')[1].split(',') || []; const topicsFilePath = args.find(a => a.startsWith('--topics-file='))?.split('=')[1]; async function migrate() { console.log('🚀 Starting Complex Relational Migration...'); // Load allowed topic IDs if a file was provided let allowedTopicIds: Set | null = null; if (topicsFilePath) { try { const fileContent = await fs.readFile(topicsFilePath, 'utf-8'); allowedTopicIds = new Set(fileContent.split('\n').map(id => id.trim()).filter(Boolean)); console.log(`📂 Loaded ${allowedTopicIds.size} topic IDs from filter file.`); } catch (e) { console.error(`❌ Failed to read topics file: ${topicsFilePath}`); process.exit(1); } } const shouldSkip = (name: string) => skipList.includes(name); if (!shouldSkip('users')) { console.log('📦 Migrating Users...'); const users = await httpClient.fetch(httpClient.query('users')); if (users.length) { await db.insert(drizzleSchema.users).values(users).onConflictDoNothing(); console.log(` - Migrated ${users.length} users`); } } if (!shouldSkip('accounts')) { console.log('📦 Migrating Accounts...'); const accounts = await httpClient.fetch(httpClient.query('accounts')); if (accounts.length) { await db.insert(drizzleSchema.accounts).values(accounts).onConflictDoNothing(); console.log(` - Migrated ${accounts.length} accounts`); } } if (!shouldSkip('sessions')) { console.log('📦 Migrating Sessions...'); const sessions = await httpClient.fetch(httpClient.query('sessions')); if (sessions.length) { await db.insert(drizzleSchema.sessions).values(sessions).onConflictDoNothing(); console.log(` - Migrated ${sessions.length} sessions`); } } if (!shouldSkip('verifications')) { console.log('📦 Migrating Verifications...'); const verifications = await httpClient.fetch(httpClient.query('verifications')); if (verifications.length) { await db.insert(drizzleSchema.verifications).values(verifications).onConflictDoNothing(); console.log(` - Migrated ${verifications.length} verifications`); } } if (!shouldSkip('settings')) { console.log('📦 Migrating Settings...'); const settings = await httpClient.fetch(httpClient.query('settings')); if (settings.length) { await db.insert(drizzleSchema.settings).values(settings).onConflictDoNothing(); console.log(` - Migrated ${settings.length} settings`); } } // Providers and Models are usually needed for Topics/Agents, so skip with caution let providers = await httpClient.fetch(httpClient.query('providers')); if (!shouldSkip('providers') && providers.length) { console.log('📦 Migrating Providers...'); const cleanedProviders = providers.map(p => ({ ...p, config: typeof p.config === 'string' ? JSON.parse(p.config) : p.config })); await db.insert(drizzleSchema.providers).values(cleanedProviders).onConflictDoNothing(); console.log(` - Migrated ${providers.length} providers`); } const rawModels = await httpClient.fetch(httpClient.query('models')); const models = rawModels.map(m => ({ ...m, cost: typeof m.cost === 'string' ? JSON.parse(m.cost) : m.cost, attributes: typeof m.attributes === 'string' ? JSON.parse(m.attributes) : m.attributes })); if (!shouldSkip('models') && models.length) { console.log('📦 Migrating Models...'); for (const model of models) { const { attributes, ...rest } = model; let drizzleModel = { inputModalities: Array.from(attributes.inputModalities || ['text']), outputModalities: Array.from(attributes.outputModalities || ['text']), capabilities: Array.from(attributes.capabilities || []), contextWindow: attributes.contextWindow, supportedParameters: Array.from(attributes.supported_parameters || []), ...rest, }; await db.insert(drizzleSchema.models).values(drizzleModel).onConflictDoNothing(); } console.log(` - Migrated ${models.length} models`); } let agents = await httpClient.fetch(httpClient.query('agents')); if (!shouldSkip('agents') && agents.length) { console.log('📦 Migrating Agents...'); await db.insert(drizzleSchema.agents).values(agents).onConflictDoNothing(); console.log(` - Migrated ${agents.length} agents`); } console.log('📦 Migrating Topics...'); let topics = await httpClient.fetch(httpClient.query('topics')); // Filter by Topic ID file if provided if (allowedTopicIds) { topics = topics.filter(t => allowedTopicIds!.has(t.id)); } if (topics.length) { await db.insert(drizzleSchema.topics).values(topics).onConflictDoNothing(); console.log(` - Migrated ${topics.length} topics`); } // Relational Filtering Helper const isTopicAllowed = (topicId: string) => topics.some(t => t.id === topicId); let generations = await httpClient.fetch(httpClient.query('generations')); generations = generations.filter(g => isTopicAllowed(g.topicId)); if (generations.length) { console.log('📦 Migrating Generations...'); await db.insert(drizzleSchema.generations).values(generations).onConflictDoNothing(); console.log(` - Migrated ${generations.length} generations`); } let messages = await httpClient.fetch(httpClient.query('messages').Order('createdAt', 'ASC')); messages = messages.filter(m => isTopicAllowed(m.topicId)); if (messages.length) { console.log('📦 Migrating Messages...'); for (const msg of messages) { await db.insert(drizzleSchema.messages).values(msg).onConflictDoNothing(); } console.log(` - Migrated ${messages.length} messages`); } if (!shouldSkip('tool_calls')) { console.log('📦 Migrating Tool Calls...'); let toolCalls = await httpClient.fetch(httpClient.query('tool_calls')); if (toolCalls.length) { const cleaned = toolCalls.map(tc => ({ ...tc, input: typeof tc.input === 'string' ? JSON.parse(tc.input) : tc.input, output: typeof tc.output === 'string' ? JSON.parse(tc.output) : tc.output, error: typeof tc.error === 'string' ? JSON.parse(tc.error) : tc.error, })); await db.insert(drizzleSchema.toolCalls).values(cleaned).onConflictDoNothing(); console.log(` - Migrated ${toolCalls.length} tool calls`); } } let parts = await httpClient.fetch(httpClient.query('message_parts').Order('createdAt', 'ASC')); parts = parts.filter(p => isTopicAllowed(p.topicId)); if (parts.length) { console.log('📦 Migrating Message Parts...'); for (const part of parts) { await db.insert(drizzleSchema.messageParts).values(part).onConflictDoNothing(); } console.log(` - Migrated ${parts.length} parts`); } if (!shouldSkip('files')) { console.log('📦 Migrating Files...'); let files = await httpClient.fetch(httpClient.query('files')); if (files.length) { await db.insert(drizzleSchema.files).values(files).onConflictDoNothing(); console.log(` - Migrated ${files.length} files`); } } console.log('📦 Migrating Attachments...'); let attachments = await httpClient.fetch(httpClient.query('attachments')); attachments = attachments.filter(a => isTopicAllowed(a.topicId)); if (attachments.length) { await db.insert(drizzleSchema.attachments).values(attachments).onConflictDoNothing(); console.log(` - Migrated ${attachments.length} attachments`); } console.log('✅ Relational migration finished.'); process.exit(0); } migrate();