Performance enhancements galore! New themining system
This is once again a huge commit, but its mostly performance improvements along with some bug fixes and refactoring. It also includes changes to the theming systems. I'm still not 100% happy with the theming system, but its better than before. Model fetching has been dramatically improved! Nearly all the important computation and pre-processing has been moved to the server. This has also somehow fixed the way model details are loaded, which was causing many models to be missing their details despite models.dev having them. The markdown renderer has once again been changed, but I'm mostly certain that this is the last time major changes will be made to it. The renderer is not spamming components, bloating memory usage, and its not using a bug prone custom written chunking system. There's also a lot more that I haven't mentioned and honestly forgot. I need to get better commit hygiene tbh.
This commit is contained in:
@@ -1,11 +1,11 @@
|
||||
<script setup lang="ts">
|
||||
import { sortByReleaseDate } from '~/utils/sort';
|
||||
import { encryptData, decrypt, uint8ArrayToBase64, base64ToUint8Array } from '~/utils/crypto';
|
||||
import { providerBaseUrls, SupportedModalities, type Model } from '~/types/model';
|
||||
import { providerBaseUrls, type Model } from '~/types/model';
|
||||
import { useSettings } from '~/composables/useSettings';
|
||||
import ModelItem from './ModelItem.vue';
|
||||
// @ts-ignore
|
||||
import { DynamicScroller, DynamicScrollerItem } from 'vue-virtual-scroller'
|
||||
import 'vue-virtual-scroller/dist/vue-virtual-scroller.css'
|
||||
const triplit = useTriplitClient();
|
||||
|
||||
const { pageParams } = useSettings();
|
||||
@@ -96,29 +96,17 @@ const updateProxyUrl = async (value: string) => {
|
||||
const fetchingModels = ref(false);
|
||||
|
||||
const fetchModels = async () => {
|
||||
const { user } = useAuth();
|
||||
const { user } = useAuth()
|
||||
|
||||
fetchingModels.value = true;
|
||||
|
||||
try {
|
||||
const [providerResponse, devDataResponse] = await Promise.all([
|
||||
$fetch(`/api/provider/${provider.value!.id}/models`, {
|
||||
method: 'POST',
|
||||
body: JSON.stringify({
|
||||
providerApiKey: apiKey.value
|
||||
})
|
||||
}) as any,
|
||||
$fetch('https://models.dev/api.json') as any
|
||||
]);
|
||||
|
||||
// TODO: get model details correctly for ollama-cloud models
|
||||
let providerType = provider.value!.type as string;
|
||||
if (providerType === 'ollama') {
|
||||
providerType = 'ollama-cloud';
|
||||
}
|
||||
|
||||
const modelDetails = devDataResponse[providerType]?.models || {};
|
||||
console.log(modelDetails);
|
||||
const modelsData = await $fetch(`/api/provider/${provider.value!.id}/models`, {
|
||||
method: 'POST',
|
||||
body: JSON.stringify({
|
||||
providerApiKey: apiKey.value
|
||||
})
|
||||
}) as any;
|
||||
|
||||
const existingModelsMap = new Map(
|
||||
(provider.value?.models || []).map((m: any) => [m.externalId, m])
|
||||
@@ -127,103 +115,34 @@ const fetchModels = async () => {
|
||||
const toInsert: any[] = [];
|
||||
const toUpdate: { id: string, data: any }[] = [];
|
||||
|
||||
providerResponse.models.forEach((pModel: any) => {
|
||||
let slug: string = pModel.id.toLowerCase();
|
||||
if (providerType === 'ollama-cloud') {
|
||||
slug = slug.replace(/:cloud$/, '');
|
||||
slug = slug.replace(/-cloud$/, '');
|
||||
slug = slug.replace(/:latest$/, '');
|
||||
}
|
||||
|
||||
let info = modelDetails[slug] || {};
|
||||
|
||||
const capabilities = [];
|
||||
|
||||
if (info.reasoning) {
|
||||
capabilities.push('reasoning');
|
||||
}
|
||||
|
||||
if (info.tool_call) {
|
||||
capabilities.push('tools');
|
||||
}
|
||||
|
||||
let inputModalities = info.modalities?.input.filter((m: string) => (SupportedModalities as Readonly<string[]>).includes(m));
|
||||
if (inputModalities === undefined || inputModalities.length === 0) {
|
||||
inputModalities = ['text'];
|
||||
}
|
||||
|
||||
let outputModalities = info.modalities?.output.filter((m: string) => (SupportedModalities as Readonly<string[]>).includes(m));
|
||||
if (outputModalities === undefined || outputModalities.length === 0) {
|
||||
outputModalities = ['text'];
|
||||
}
|
||||
|
||||
// merge pModel.attributes and info.modalities, with a preference for pModel.attributes
|
||||
const attributes = {
|
||||
inputModalities: new Set(inputModalities),
|
||||
outputModalities: new Set(outputModalities),
|
||||
capabilities,
|
||||
contextWindow: pModel.context_length || info.limit?.context || null,
|
||||
supported_parameters: new Set(pModel.supported_parameters || ["temperature", "max_tokens"]),
|
||||
...(pModel.attributes || {}),
|
||||
};
|
||||
|
||||
let cost;
|
||||
if (pModel.pricing === undefined) {
|
||||
cost = {}
|
||||
} else {
|
||||
cost = {
|
||||
prompt: pModel.pricing.prompt || null,
|
||||
completion: pModel.pricing.completion || null,
|
||||
request: pModel.pricing.request || null,
|
||||
image: pModel.pricing.image || null,
|
||||
imageTokens: pModel.pricing.image_tokens || null,
|
||||
imageOutput: pModel.pricing.image_output || null,
|
||||
audio: pModel.pricing.audio || null,
|
||||
audioOutput: pModel.pricing.audio_output || null,
|
||||
inputAudioCache: pModel.pricing.input_audio_cache || null,
|
||||
webSearch: pModel.pricing.web_search || null,
|
||||
internalReasoning: pModel.pricing.internal_reasoning || null,
|
||||
inputCacheRead: pModel.pricing.input_cache_read || null,
|
||||
inputCacheWrite: pModel.pricing.input_cache_write || null,
|
||||
discount: pModel.pricing.discount || null,
|
||||
}
|
||||
}
|
||||
|
||||
const existing = existingModelsMap.get(pModel.id);
|
||||
for (const model of modelsData.models) {
|
||||
const existing = existingModelsMap.get(model.id);
|
||||
|
||||
if (existing) {
|
||||
// UPDATE logic: Remove 'id' from the payload as per Triplit requirements
|
||||
const { id, ...existingWithoutId } = existing;
|
||||
const { id, ...existingWithoutId } = model;
|
||||
console.log("existingWithoutId", existingWithoutId);
|
||||
|
||||
toUpdate.push({
|
||||
id: existing.id,
|
||||
data: {
|
||||
...existingWithoutId,
|
||||
name: existing.name || pModel.name || info.name || pModel.id,
|
||||
cost,
|
||||
attributes, // Update tech specs
|
||||
releasedAt: pModel.created ? new Date(pModel.created * 1000) : null,
|
||||
}
|
||||
data: existingWithoutId,
|
||||
});
|
||||
} else {
|
||||
// INSERT logic: This is a brand new model
|
||||
toInsert.push({
|
||||
userId: user.value?.id,
|
||||
providerId: provider.value!.id,
|
||||
externalId: pModel.id,
|
||||
name: pModel.name || info.name || pModel.id,
|
||||
userId: user.value?.id!,
|
||||
externalId: model.id,
|
||||
providerId: provider.value!.id!,
|
||||
name: model.name || model.id,
|
||||
cost: model.cost || {},
|
||||
attributes: model.attributes,
|
||||
isCustom: false,
|
||||
enabled: false,
|
||||
cost,
|
||||
attributes,
|
||||
releasedAt: pModel.created ? new Date(pModel.created * 1000) : null,
|
||||
createdAt: new Date(),
|
||||
releasedAt: model.releasedAt,
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// delete models that are not in the API response and are not custom models
|
||||
const apiModelIds = new Set(providerResponse.models.map((p: any) => p.id));
|
||||
const apiModelIds = new Set(existingModelsMap.values().map((m: any) => m.externalId));
|
||||
const toDelete = (provider.value?.models || []).filter((m: any) =>
|
||||
!m.isCustom && !apiModelIds.has(m.externalId)
|
||||
);
|
||||
@@ -250,12 +169,12 @@ const deleteModels = async () => {
|
||||
|
||||
const enabledModels = computed(() =>
|
||||
filterModels(provider.value?.models.filter(m => m.enabled === true) as Model[] || [], modelSearch.value)
|
||||
.sort((a, b) => a.releasedAt && b.releasedAt ? b.releasedAt.getTime() - a.releasedAt.getTime() : 0)
|
||||
.sort(sortByReleaseDate)
|
||||
)
|
||||
|
||||
const disabledModels = computed(() =>
|
||||
filterModels(provider.value?.models.filter(m => m.enabled === false) as Model[] || [], modelSearch.value)
|
||||
.sort((a, b) => a.releasedAt && b.releasedAt ? b.releasedAt.getTime() - a.releasedAt.getTime() : 0)
|
||||
.sort(sortByReleaseDate)
|
||||
)
|
||||
|
||||
onUnmounted(() => {
|
||||
@@ -274,13 +193,13 @@ defineEmits(['navigate']);
|
||||
|
||||
<div class="flex flex-row justify-between gap-16">
|
||||
<label class="whitespace-nowrap" for="provider-api-key">API Key</label>
|
||||
<div
|
||||
class="text-sm font-mono flex flex-row rounded-md bg-[var(--color-highlight)] items-center gap-1 w-7/10">
|
||||
<input class="w-full p-0 pl-2 py-1 bg-transparent" :type="apiKeyVisible ? 'text' : 'password'"
|
||||
id="provider-api-key" :value="apiKey" autocomplete="false" spellcheck="false"
|
||||
<div class="text-sm font-mono flex flex-row rounded-md bg-[var(--bg-container)] items-center gap-1 w-7/10">
|
||||
<input class="placeholder:text-[var(--text-tertiary)] w-full p-0 pl-2 py-1 bg-transparent"
|
||||
:type="apiKeyVisible ? 'text' : 'password'" id="provider-api-key" :value="apiKey"
|
||||
autocomplete="false" spellcheck="false"
|
||||
@input="updateApiKey(($event.target! as HTMLInputElement).value)" />
|
||||
<button @click="apiKeyVisible = !apiKeyVisible"
|
||||
class="text-sm p-2 text-[var(--color-muted)] hover:text-[var(--color-text)]">
|
||||
class="text-sm p-2 text-[var(--text-secondary)] hover:text-[var(--text-primary)]">
|
||||
<Icon :name="apiKeyVisible ? 'mynaui:eye' : 'mynaui:eye-slash'" class="text-4 min-h-4 min-w-4" />
|
||||
</button>
|
||||
</div>
|
||||
@@ -288,16 +207,16 @@ defineEmits(['navigate']);
|
||||
|
||||
<div class="flex flex-row justify-between gap-16">
|
||||
<label class="whitespace-nowrap" for="provider-api-key">API Proxy URL</label>
|
||||
<div
|
||||
class="text-sm font-mono flex flex-row rounded-md bg-[var(--color-highlight)] items-center gap-1 w-7/10">
|
||||
<input :placeholder="providerBaseUrls[provider!.type] ?? ''" class="w-full px-2 py-1 bg-transparent"
|
||||
type="text" id="provider-proxy-url" :value="apiProxyUrl"
|
||||
<div class="text-sm font-mono flex flex-row rounded-md bg-[var(--bg-container)] items-center gap-1 w-7/10">
|
||||
<input :placeholder="providerBaseUrls[provider!.type] ?? ''"
|
||||
class="placeholder:text-[var(--text-tertiary)] w-full px-2 py-1 bg-transparent" type="text"
|
||||
id="provider-proxy-url" :value="apiProxyUrl"
|
||||
@input="updateProxyUrl(($event.target! as HTMLInputElement).value)" />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="flex flex-row justify-center text-xs">
|
||||
<p class="text-[var(--color-muted)]">
|
||||
<p class="text-[var(--text-secondary)]">
|
||||
<Icon name="mynaui:lock" /> Your API key is encrypted using <a
|
||||
href="https://datatracker.ietf.org/doc/html/draft-ietf-avt-srtp-aes-gcm-01">AES-GCM</a> encryption.
|
||||
</p>
|
||||
@@ -307,26 +226,27 @@ defineEmits(['navigate']);
|
||||
<div class="pt-5 justify-between w-full flex flex-wrap gap-y-1 items-center">
|
||||
<h4 class="whitespace-nowrap m-0 flex gap-x-2 items-start">
|
||||
Model List
|
||||
<span class="text-sm text-[var(--color-muted)] font-normal text-xs flex items-center gap-1">
|
||||
{{ provider?.models.length }} models available <button @click="deleteModels">
|
||||
<span class="text-sm text-[var(--text-secondary)] font-normal text-xs flex items-center gap-1">
|
||||
{{ provider?.models.length }} models available <button
|
||||
class="p-0.5 hover:bg-[var(--color-hover)] rounded transition-colors duration-200"
|
||||
@click="deleteModels">
|
||||
<Icon name="mynaui:x-solid" />
|
||||
</button>
|
||||
</span>
|
||||
</h4>
|
||||
|
||||
<div class="flex items-center gap-2">
|
||||
<div
|
||||
class="flex items-center justify-center px-2 py-1 bg-[var(--color-highlight)] text-xs rounded-md">
|
||||
<input class="p-0 bg-transparent" v-model="modelSearch" type="text"
|
||||
placeholder="Search models..." />
|
||||
<div class="flex items-center justify-center px-2 py-1 bg-[var(--bg-container)] text-xs rounded-md">
|
||||
<input class="placeholder:text-[var(--text-tertiary)] p-0 bg-transparent" v-model="modelSearch"
|
||||
type="text" placeholder="Search models..." />
|
||||
<button :class="modelSearch.length > 0 ? 'visible' : 'invisible'" @click="modelSearch = ''"
|
||||
class="right-1 hover:bg-[var(--color-highlight)] rounded p-0.5">
|
||||
<Icon name="mynaui:x" class="text-3.5 block text-[var(--color-subtle)]" />
|
||||
class="right-1 hover:bg-[var(--color-hover)] rounded transition-colors duration-200 p-0.5">
|
||||
<Icon name="mynaui:x" class="text-3.5 block text-[var(--text-secondary)]" />
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<button @click="fetchModels"
|
||||
class="whitespace-nowrap flex bg-[var(--color-highlight)] hover:bg-[var(--color-highlight-high)] text-sm rounded-md items-center px-2 py-0.5 gap-2 transition-colors duration-200 ease-[cubic-bezier(0.5,_1,_0.89,_1)]">
|
||||
class="whitespace-nowrap flex bg-[var(--bg-container)] hover:bg-[var(--color-hover)] text-sm rounded-md items-center px-2 py-0.5 gap-2 transition-colors duration-200 ease-[cubic-bezier(0.5,_1,_0.89,_1)]">
|
||||
<Icon :class="[fetchingModels ? 'animate-rotate' : '']" name="mynaui:refresh" />
|
||||
fetch models
|
||||
</button>
|
||||
@@ -335,14 +255,14 @@ defineEmits(['navigate']);
|
||||
|
||||
<div v-if="provider?.models?.length === 0" class="flex flex-row items-center justify-center gap-2 mt-2">
|
||||
<Icon name="mynaui:info-circle" class="text-4" />
|
||||
<span class="text-sm text-[var(--color-muted)]">
|
||||
<span class="text-sm text-[var(--text-secondary)]">
|
||||
No models found
|
||||
</span>
|
||||
</div>
|
||||
|
||||
<ClientOnly v-else>
|
||||
<div class="flex flex-col gap-1 mt-2">
|
||||
<span v-if="enabledModels.length > 0" class="text-sm text-[var(--color-muted)]">
|
||||
<span v-if="enabledModels.length > 0" class="text-sm text-[var(--text-secondary)]">
|
||||
Enabled
|
||||
</span>
|
||||
<div class="flex flex-col gap-1">
|
||||
@@ -359,7 +279,7 @@ defineEmits(['navigate']);
|
||||
</template>
|
||||
</DynamicScroller>
|
||||
|
||||
<span v-if="disabledModels.length > 0" class="text-sm text-[var(--color-muted)]">
|
||||
<span v-if="disabledModels.length > 0" class="text-sm text-[var(--text-secondary)]">
|
||||
Disabled
|
||||
</span>
|
||||
<div class="flex flex-col gap-1">
|
||||
@@ -383,19 +303,3 @@ defineEmits(['navigate']);
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<style>
|
||||
.animate-rotate {
|
||||
animation: rotate 1s linear infinite;
|
||||
}
|
||||
|
||||
@keyframes rotate {
|
||||
0% {
|
||||
transform: rotate(0deg);
|
||||
}
|
||||
|
||||
100% {
|
||||
transform: rotate(360deg);
|
||||
}
|
||||
}
|
||||
</style>
|
||||
Reference in New Issue
Block a user