feat: ditch triplit, move to postgresql + drizzle orm
This commit is contained in:
@@ -0,0 +1,205 @@
|
||||
import { db } from '../server/lib/db';
|
||||
import * as drizzleSchema from '../drizzle/schema';
|
||||
import { HttpClient } from '@triplit/client';
|
||||
import { schema } from '../triplit/schema';
|
||||
import fs from 'node:fs/promises';
|
||||
|
||||
export const httpClient = new HttpClient({
|
||||
schema,
|
||||
serverUrl: process.env.NUXT_LOCAL_TRIPLIT_URL || process.env.NUXT_PUBLIC_TRIPLIT_URL,
|
||||
token: process.env.TRIPLIT_SERVICE_TOKEN,
|
||||
});
|
||||
|
||||
/**
|
||||
* Simple Argument Parser
|
||||
* Use: --skip=users,accounts --topics-file=ids.txt
|
||||
*/
|
||||
const args = process.argv.slice(2);
|
||||
const skipList = args.find(a => a.startsWith('--skip='))?.split('=')[1].split(',') || [];
|
||||
const topicsFilePath = args.find(a => a.startsWith('--topics-file='))?.split('=')[1];
|
||||
|
||||
async function migrate() {
|
||||
console.log('🚀 Starting Complex Relational Migration...');
|
||||
|
||||
// Load allowed topic IDs if a file was provided
|
||||
let allowedTopicIds: Set<string> | null = null;
|
||||
if (topicsFilePath) {
|
||||
try {
|
||||
const fileContent = await fs.readFile(topicsFilePath, 'utf-8');
|
||||
allowedTopicIds = new Set(fileContent.split('\n').map(id => id.trim()).filter(Boolean));
|
||||
console.log(`📂 Loaded ${allowedTopicIds.size} topic IDs from filter file.`);
|
||||
} catch (e) {
|
||||
console.error(`❌ Failed to read topics file: ${topicsFilePath}`);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
const shouldSkip = (name: string) => skipList.includes(name);
|
||||
|
||||
if (!shouldSkip('users')) {
|
||||
console.log('📦 Migrating Users...');
|
||||
const users = await httpClient.fetch(httpClient.query('users'));
|
||||
if (users.length) {
|
||||
await db.insert(drizzleSchema.users).values(users).onConflictDoNothing();
|
||||
console.log(` - Migrated ${users.length} users`);
|
||||
}
|
||||
}
|
||||
|
||||
if (!shouldSkip('accounts')) {
|
||||
console.log('📦 Migrating Accounts...');
|
||||
const accounts = await httpClient.fetch(httpClient.query('accounts'));
|
||||
if (accounts.length) {
|
||||
await db.insert(drizzleSchema.accounts).values(accounts).onConflictDoNothing();
|
||||
console.log(` - Migrated ${accounts.length} accounts`);
|
||||
}
|
||||
}
|
||||
|
||||
if (!shouldSkip('sessions')) {
|
||||
console.log('📦 Migrating Sessions...');
|
||||
const sessions = await httpClient.fetch(httpClient.query('sessions'));
|
||||
if (sessions.length) {
|
||||
await db.insert(drizzleSchema.sessions).values(sessions).onConflictDoNothing();
|
||||
console.log(` - Migrated ${sessions.length} sessions`);
|
||||
}
|
||||
}
|
||||
|
||||
if (!shouldSkip('verifications')) {
|
||||
console.log('📦 Migrating Verifications...');
|
||||
const verifications = await httpClient.fetch(httpClient.query('verifications'));
|
||||
if (verifications.length) {
|
||||
await db.insert(drizzleSchema.verifications).values(verifications).onConflictDoNothing();
|
||||
console.log(` - Migrated ${verifications.length} verifications`);
|
||||
}
|
||||
}
|
||||
|
||||
if (!shouldSkip('settings')) {
|
||||
console.log('📦 Migrating Settings...');
|
||||
const settings = await httpClient.fetch(httpClient.query('settings'));
|
||||
if (settings.length) {
|
||||
await db.insert(drizzleSchema.settings).values(settings).onConflictDoNothing();
|
||||
console.log(` - Migrated ${settings.length} settings`);
|
||||
}
|
||||
}
|
||||
|
||||
// Providers and Models are usually needed for Topics/Agents, so skip with caution
|
||||
let providers = await httpClient.fetch(httpClient.query('providers'));
|
||||
if (!shouldSkip('providers') && providers.length) {
|
||||
console.log('📦 Migrating Providers...');
|
||||
const cleanedProviders = providers.map(p => ({
|
||||
...p,
|
||||
config: typeof p.config === 'string' ? JSON.parse(p.config) : p.config
|
||||
}));
|
||||
await db.insert(drizzleSchema.providers).values(cleanedProviders).onConflictDoNothing();
|
||||
console.log(` - Migrated ${providers.length} providers`);
|
||||
}
|
||||
|
||||
const rawModels = await httpClient.fetch(httpClient.query('models'));
|
||||
const models = rawModels.map(m => ({
|
||||
...m,
|
||||
cost: typeof m.cost === 'string' ? JSON.parse(m.cost) : m.cost,
|
||||
attributes: typeof m.attributes === 'string' ? JSON.parse(m.attributes) : m.attributes
|
||||
}));
|
||||
|
||||
if (!shouldSkip('models') && models.length) {
|
||||
console.log('📦 Migrating Models...');
|
||||
for (const model of models) {
|
||||
const { attributes, ...rest } = model;
|
||||
let drizzleModel = {
|
||||
inputModalities: Array.from(attributes.inputModalities || ['text']),
|
||||
outputModalities: Array.from(attributes.outputModalities || ['text']),
|
||||
capabilities: Array.from(attributes.capabilities || []),
|
||||
contextWindow: attributes.contextWindow,
|
||||
supportedParameters: Array.from(attributes.supported_parameters || []),
|
||||
...rest,
|
||||
};
|
||||
await db.insert(drizzleSchema.models).values(drizzleModel).onConflictDoNothing();
|
||||
}
|
||||
console.log(` - Migrated ${models.length} models`);
|
||||
}
|
||||
|
||||
let agents = await httpClient.fetch(httpClient.query('agents'));
|
||||
if (!shouldSkip('agents') && agents.length) {
|
||||
console.log('📦 Migrating Agents...');
|
||||
await db.insert(drizzleSchema.agents).values(agents).onConflictDoNothing();
|
||||
console.log(` - Migrated ${agents.length} agents`);
|
||||
}
|
||||
|
||||
console.log('📦 Migrating Topics...');
|
||||
let topics = await httpClient.fetch(httpClient.query('topics'));
|
||||
// Filter by Topic ID file if provided
|
||||
if (allowedTopicIds) {
|
||||
topics = topics.filter(t => allowedTopicIds!.has(t.id));
|
||||
}
|
||||
if (topics.length) {
|
||||
await db.insert(drizzleSchema.topics).values(topics).onConflictDoNothing();
|
||||
console.log(` - Migrated ${topics.length} topics`);
|
||||
}
|
||||
|
||||
// Relational Filtering Helper
|
||||
const isTopicAllowed = (topicId: string) => topics.some(t => t.id === topicId);
|
||||
|
||||
let generations = await httpClient.fetch(httpClient.query('generations'));
|
||||
generations = generations.filter(g => isTopicAllowed(g.topicId));
|
||||
if (generations.length) {
|
||||
console.log('📦 Migrating Generations...');
|
||||
await db.insert(drizzleSchema.generations).values(generations).onConflictDoNothing();
|
||||
console.log(` - Migrated ${generations.length} generations`);
|
||||
}
|
||||
|
||||
let messages = await httpClient.fetch(httpClient.query('messages').Order('createdAt', 'ASC'));
|
||||
messages = messages.filter(m => isTopicAllowed(m.topicId));
|
||||
if (messages.length) {
|
||||
console.log('📦 Migrating Messages...');
|
||||
for (const msg of messages) {
|
||||
await db.insert(drizzleSchema.messages).values(msg).onConflictDoNothing();
|
||||
}
|
||||
console.log(` - Migrated ${messages.length} messages`);
|
||||
}
|
||||
|
||||
if (!shouldSkip('tool_calls')) {
|
||||
console.log('📦 Migrating Tool Calls...');
|
||||
let toolCalls = await httpClient.fetch(httpClient.query('tool_calls'));
|
||||
if (toolCalls.length) {
|
||||
const cleaned = toolCalls.map(tc => ({
|
||||
...tc,
|
||||
input: typeof tc.input === 'string' ? JSON.parse(tc.input) : tc.input,
|
||||
output: typeof tc.output === 'string' ? JSON.parse(tc.output) : tc.output,
|
||||
error: typeof tc.error === 'string' ? JSON.parse(tc.error) : tc.error,
|
||||
}));
|
||||
await db.insert(drizzleSchema.toolCalls).values(cleaned).onConflictDoNothing();
|
||||
console.log(` - Migrated ${toolCalls.length} tool calls`);
|
||||
}
|
||||
}
|
||||
|
||||
let parts = await httpClient.fetch(httpClient.query('message_parts').Order('createdAt', 'ASC'));
|
||||
parts = parts.filter(p => isTopicAllowed(p.topicId));
|
||||
if (parts.length) {
|
||||
console.log('📦 Migrating Message Parts...');
|
||||
for (const part of parts) {
|
||||
await db.insert(drizzleSchema.messageParts).values(part).onConflictDoNothing();
|
||||
}
|
||||
console.log(` - Migrated ${parts.length} parts`);
|
||||
}
|
||||
|
||||
if (!shouldSkip('files')) {
|
||||
console.log('📦 Migrating Files...');
|
||||
let files = await httpClient.fetch(httpClient.query('files'));
|
||||
if (files.length) {
|
||||
await db.insert(drizzleSchema.files).values(files).onConflictDoNothing();
|
||||
console.log(` - Migrated ${files.length} files`);
|
||||
}
|
||||
}
|
||||
|
||||
console.log('📦 Migrating Attachments...');
|
||||
let attachments = await httpClient.fetch(httpClient.query('attachments'));
|
||||
attachments = attachments.filter(a => isTopicAllowed(a.topicId));
|
||||
if (attachments.length) {
|
||||
await db.insert(drizzleSchema.attachments).values(attachments).onConflictDoNothing();
|
||||
console.log(` - Migrated ${attachments.length} attachments`);
|
||||
}
|
||||
|
||||
console.log('✅ Relational migration finished.');
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
migrate();
|
||||
Reference in New Issue
Block a user