206 lines
8.6 KiB
TypeScript
206 lines
8.6 KiB
TypeScript
import { db } from '../server/lib/db';
|
|
import * as drizzleSchema from '../drizzle/schema';
|
|
import { HttpClient } from '@triplit/client';
|
|
import { schema } from '../triplit/schema';
|
|
import fs from 'node:fs/promises';
|
|
|
|
export const httpClient = new HttpClient({
|
|
schema,
|
|
serverUrl: process.env.NUXT_LOCAL_TRIPLIT_URL || process.env.NUXT_PUBLIC_TRIPLIT_URL,
|
|
token: process.env.TRIPLIT_SERVICE_TOKEN,
|
|
});
|
|
|
|
/**
|
|
* Simple Argument Parser
|
|
* Use: --skip=users,accounts --topics-file=ids.txt
|
|
*/
|
|
const args = process.argv.slice(2);
|
|
const skipList = args.find(a => a.startsWith('--skip='))?.split('=')[1].split(',') || [];
|
|
const topicsFilePath = args.find(a => a.startsWith('--topics-file='))?.split('=')[1];
|
|
|
|
async function migrate() {
|
|
console.log('🚀 Starting Complex Relational Migration...');
|
|
|
|
// Load allowed topic IDs if a file was provided
|
|
let allowedTopicIds: Set<string> | null = null;
|
|
if (topicsFilePath) {
|
|
try {
|
|
const fileContent = await fs.readFile(topicsFilePath, 'utf-8');
|
|
allowedTopicIds = new Set(fileContent.split('\n').map(id => id.trim()).filter(Boolean));
|
|
console.log(`📂 Loaded ${allowedTopicIds.size} topic IDs from filter file.`);
|
|
} catch (e) {
|
|
console.error(`❌ Failed to read topics file: ${topicsFilePath}`);
|
|
process.exit(1);
|
|
}
|
|
}
|
|
|
|
const shouldSkip = (name: string) => skipList.includes(name);
|
|
|
|
if (!shouldSkip('users')) {
|
|
console.log('📦 Migrating Users...');
|
|
const users = await httpClient.fetch(httpClient.query('users'));
|
|
if (users.length) {
|
|
await db.insert(drizzleSchema.users).values(users).onConflictDoNothing();
|
|
console.log(` - Migrated ${users.length} users`);
|
|
}
|
|
}
|
|
|
|
if (!shouldSkip('accounts')) {
|
|
console.log('📦 Migrating Accounts...');
|
|
const accounts = await httpClient.fetch(httpClient.query('accounts'));
|
|
if (accounts.length) {
|
|
await db.insert(drizzleSchema.accounts).values(accounts).onConflictDoNothing();
|
|
console.log(` - Migrated ${accounts.length} accounts`);
|
|
}
|
|
}
|
|
|
|
if (!shouldSkip('sessions')) {
|
|
console.log('📦 Migrating Sessions...');
|
|
const sessions = await httpClient.fetch(httpClient.query('sessions'));
|
|
if (sessions.length) {
|
|
await db.insert(drizzleSchema.sessions).values(sessions).onConflictDoNothing();
|
|
console.log(` - Migrated ${sessions.length} sessions`);
|
|
}
|
|
}
|
|
|
|
if (!shouldSkip('verifications')) {
|
|
console.log('📦 Migrating Verifications...');
|
|
const verifications = await httpClient.fetch(httpClient.query('verifications'));
|
|
if (verifications.length) {
|
|
await db.insert(drizzleSchema.verifications).values(verifications).onConflictDoNothing();
|
|
console.log(` - Migrated ${verifications.length} verifications`);
|
|
}
|
|
}
|
|
|
|
if (!shouldSkip('settings')) {
|
|
console.log('📦 Migrating Settings...');
|
|
const settings = await httpClient.fetch(httpClient.query('settings'));
|
|
if (settings.length) {
|
|
await db.insert(drizzleSchema.settings).values(settings).onConflictDoNothing();
|
|
console.log(` - Migrated ${settings.length} settings`);
|
|
}
|
|
}
|
|
|
|
// Providers and Models are usually needed for Topics/Agents, so skip with caution
|
|
let providers = await httpClient.fetch(httpClient.query('providers'));
|
|
if (!shouldSkip('providers') && providers.length) {
|
|
console.log('📦 Migrating Providers...');
|
|
const cleanedProviders = providers.map(p => ({
|
|
...p,
|
|
config: typeof p.config === 'string' ? JSON.parse(p.config) : p.config
|
|
}));
|
|
await db.insert(drizzleSchema.providers).values(cleanedProviders).onConflictDoNothing();
|
|
console.log(` - Migrated ${providers.length} providers`);
|
|
}
|
|
|
|
const rawModels = await httpClient.fetch(httpClient.query('models'));
|
|
const models = rawModels.map(m => ({
|
|
...m,
|
|
cost: typeof m.cost === 'string' ? JSON.parse(m.cost) : m.cost,
|
|
attributes: typeof m.attributes === 'string' ? JSON.parse(m.attributes) : m.attributes
|
|
}));
|
|
|
|
if (!shouldSkip('models') && models.length) {
|
|
console.log('📦 Migrating Models...');
|
|
for (const model of models) {
|
|
const { attributes, ...rest } = model;
|
|
let drizzleModel = {
|
|
inputModalities: Array.from(attributes.inputModalities || ['text']),
|
|
outputModalities: Array.from(attributes.outputModalities || ['text']),
|
|
capabilities: Array.from(attributes.capabilities || []),
|
|
contextWindow: attributes.contextWindow,
|
|
supportedParameters: Array.from(attributes.supported_parameters || []),
|
|
...rest,
|
|
};
|
|
await db.insert(drizzleSchema.models).values(drizzleModel).onConflictDoNothing();
|
|
}
|
|
console.log(` - Migrated ${models.length} models`);
|
|
}
|
|
|
|
let agents = await httpClient.fetch(httpClient.query('agents'));
|
|
if (!shouldSkip('agents') && agents.length) {
|
|
console.log('📦 Migrating Agents...');
|
|
await db.insert(drizzleSchema.agents).values(agents).onConflictDoNothing();
|
|
console.log(` - Migrated ${agents.length} agents`);
|
|
}
|
|
|
|
console.log('📦 Migrating Topics...');
|
|
let topics = await httpClient.fetch(httpClient.query('topics'));
|
|
// Filter by Topic ID file if provided
|
|
if (allowedTopicIds) {
|
|
topics = topics.filter(t => allowedTopicIds!.has(t.id));
|
|
}
|
|
if (topics.length) {
|
|
await db.insert(drizzleSchema.topics).values(topics).onConflictDoNothing();
|
|
console.log(` - Migrated ${topics.length} topics`);
|
|
}
|
|
|
|
// Relational Filtering Helper
|
|
const isTopicAllowed = (topicId: string) => topics.some(t => t.id === topicId);
|
|
|
|
let generations = await httpClient.fetch(httpClient.query('generations'));
|
|
generations = generations.filter(g => isTopicAllowed(g.topicId));
|
|
if (generations.length) {
|
|
console.log('📦 Migrating Generations...');
|
|
await db.insert(drizzleSchema.generations).values(generations).onConflictDoNothing();
|
|
console.log(` - Migrated ${generations.length} generations`);
|
|
}
|
|
|
|
let messages = await httpClient.fetch(httpClient.query('messages').Order('createdAt', 'ASC'));
|
|
messages = messages.filter(m => isTopicAllowed(m.topicId));
|
|
if (messages.length) {
|
|
console.log('📦 Migrating Messages...');
|
|
for (const msg of messages) {
|
|
await db.insert(drizzleSchema.messages).values(msg).onConflictDoNothing();
|
|
}
|
|
console.log(` - Migrated ${messages.length} messages`);
|
|
}
|
|
|
|
if (!shouldSkip('tool_calls')) {
|
|
console.log('📦 Migrating Tool Calls...');
|
|
let toolCalls = await httpClient.fetch(httpClient.query('tool_calls'));
|
|
if (toolCalls.length) {
|
|
const cleaned = toolCalls.map(tc => ({
|
|
...tc,
|
|
input: typeof tc.input === 'string' ? JSON.parse(tc.input) : tc.input,
|
|
output: typeof tc.output === 'string' ? JSON.parse(tc.output) : tc.output,
|
|
error: typeof tc.error === 'string' ? JSON.parse(tc.error) : tc.error,
|
|
}));
|
|
await db.insert(drizzleSchema.toolCalls).values(cleaned).onConflictDoNothing();
|
|
console.log(` - Migrated ${toolCalls.length} tool calls`);
|
|
}
|
|
}
|
|
|
|
let parts = await httpClient.fetch(httpClient.query('message_parts').Order('createdAt', 'ASC'));
|
|
parts = parts.filter(p => isTopicAllowed(p.topicId));
|
|
if (parts.length) {
|
|
console.log('📦 Migrating Message Parts...');
|
|
for (const part of parts) {
|
|
await db.insert(drizzleSchema.messageParts).values(part).onConflictDoNothing();
|
|
}
|
|
console.log(` - Migrated ${parts.length} parts`);
|
|
}
|
|
|
|
if (!shouldSkip('files')) {
|
|
console.log('📦 Migrating Files...');
|
|
let files = await httpClient.fetch(httpClient.query('files'));
|
|
if (files.length) {
|
|
await db.insert(drizzleSchema.files).values(files).onConflictDoNothing();
|
|
console.log(` - Migrated ${files.length} files`);
|
|
}
|
|
}
|
|
|
|
console.log('📦 Migrating Attachments...');
|
|
let attachments = await httpClient.fetch(httpClient.query('attachments'));
|
|
attachments = attachments.filter(a => isTopicAllowed(a.topicId));
|
|
if (attachments.length) {
|
|
await db.insert(drizzleSchema.attachments).values(attachments).onConflictDoNothing();
|
|
console.log(` - Migrated ${attachments.length} attachments`);
|
|
}
|
|
|
|
console.log('✅ Relational migration finished.');
|
|
process.exit(0);
|
|
}
|
|
|
|
migrate();
|