llmrouter: leichtgewichtiger Gateway-Ersatz mit Alternativ-Auflösung
OpenAI-kompatibler Reverse-Proxy in Bun: Client-Key -> ACL, Alias -> Modell (alternative: günstigstes Modell derselben Klasse über llmlite/openrouter-auswahl inkl. Effort-Achse; fest: lokale Upstreams, TTS/STT, Embeddings), Usage/Kosten aus der OpenRouter-Antwort (auch im Stream), JSONL + SQLite-Log, Tagesstatistik je Key. Altnamen der LiteLLM-Zeit werden übersetzt. Kein 3456/stdcmpt, kein role/*, kein -auto (Andreas 2026-09-05). Lokal verifiziert: Chat, Stream, ACL, Altname, Kosten im Log. Parallelbetrieb auf :4001 geplant. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Claude Fable 5.1
@@ -0,0 +1,64 @@
|
||||
/**
|
||||
* config.ts — laedt config.json, key-acls.json und die Token-Datei.
|
||||
*
|
||||
* Drei Quellen, bewusst getrennt:
|
||||
* - config.json versioniert: Aliasse, Upstreams, Altnamen
|
||||
* - key-acls.json versioniert: Name -> erlaubte Aliasse
|
||||
* - KEYS_FILE NUR auf dem Host: Token -> Name (Secret)
|
||||
* Upstream-Schluessel kommen aus der Umgebung (keyEnv je Upstream).
|
||||
*/
|
||||
import { readFileSync } from "fs";
|
||||
import { dirname, resolve } from "path";
|
||||
|
||||
export type Upstream = { base: string; keyEnv?: string; maxParallel?: number };
|
||||
export type Alias =
|
||||
| { art: "alternative"; referenz: string }
|
||||
| { art: "fest"; upstream: string; model: string; body?: Record<string, unknown> };
|
||||
export type Config = {
|
||||
port: number;
|
||||
logDir: string;
|
||||
refreshHours: number;
|
||||
aaRefreshDays: number;
|
||||
lasttest: boolean;
|
||||
upstreams: Record<string, Upstream>;
|
||||
aliases: Record<string, Alias>;
|
||||
altnamen: Record<string, string>;
|
||||
};
|
||||
|
||||
export const BASE = resolve(dirname(import.meta.path), "..");
|
||||
export const CONFIG_PATH = process.env.LLMROUTER_CONFIG ?? `${BASE}/config.json`;
|
||||
|
||||
export function configLaden(): Config {
|
||||
const c = JSON.parse(readFileSync(CONFIG_PATH, "utf8")) as Config;
|
||||
for (const [name, a] of Object.entries(c.aliases)) {
|
||||
if (a.art === "fest" && c.upstreams[a.upstream] === undefined) throw new Error(`Alias ${name}: Upstream ${a.upstream} fehlt`);
|
||||
}
|
||||
for (const [alt, ziel] of Object.entries(c.altnamen)) {
|
||||
if (c.aliases[ziel] === undefined) throw new Error(`Altname ${alt} zeigt auf unbekannten Alias ${ziel}`);
|
||||
}
|
||||
c.logDir = resolve(dirname(CONFIG_PATH), c.logDir);
|
||||
if (process.env.LLMROUTER_LASTTEST === "0") c.lasttest = false; // fuer lokale Tests: spart ~210k Token je Wechsel
|
||||
return c;
|
||||
}
|
||||
|
||||
/** Name -> erlaubte Aliasse; leere Liste = alles. */
|
||||
export function aclLaden(): Map<string, string[]> {
|
||||
const pfad = process.env.LLMROUTER_ACLS ?? `${dirname(CONFIG_PATH)}/key-acls.json`;
|
||||
const d = JSON.parse(readFileSync(pfad, "utf8")) as { keys: Record<string, string[]> };
|
||||
return new Map(Object.entries(d.keys));
|
||||
}
|
||||
|
||||
/** Token -> Name. Datei fehlt = kein Client kommt rein (bewusst, nicht "alle"). */
|
||||
export function tokenLaden(): Map<string, string> {
|
||||
const pfad = process.env.KEYS_FILE;
|
||||
if (pfad === undefined) return new Map();
|
||||
try {
|
||||
return new Map(Object.entries(JSON.parse(readFileSync(pfad, "utf8")) as Record<string, string>));
|
||||
} catch {
|
||||
return new Map();
|
||||
}
|
||||
}
|
||||
|
||||
export function upstreamKey(u: Upstream): string | undefined {
|
||||
return u.keyEnv === undefined ? undefined : process.env[u.keyEnv];
|
||||
}
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
/**
|
||||
* keys.ts — Client-Token verwalten. Die Datei (KEYS_FILE) liegt nur auf dem
|
||||
* Host, nie im Repo. `add <name>` erzeugt ein Token und gibt es EINMAL aus;
|
||||
* `add <name> --token <t>` uebernimmt ein bestehendes (Migration von LiteLLM,
|
||||
* damit kein Container angefasst werden muss). `list` zeigt Namen.
|
||||
*/
|
||||
import { readFileSync, writeFileSync } from "fs";
|
||||
import { randomBytes } from "crypto";
|
||||
|
||||
const pfad = process.env.KEYS_FILE;
|
||||
if (pfad === undefined) { console.error("KEYS_FILE nicht gesetzt"); process.exit(1); }
|
||||
let daten: Record<string, string> = {};
|
||||
try { daten = JSON.parse(readFileSync(pfad, "utf8")); } catch { /* neu */ }
|
||||
|
||||
const [befehl, name] = process.argv.slice(2);
|
||||
if (befehl === "list") {
|
||||
for (const n of [...new Set(Object.values(daten))].sort()) console.log(n);
|
||||
} else if (befehl === "add" && name) {
|
||||
const i = process.argv.indexOf("--token");
|
||||
const token = i > 0 ? process.argv[i + 1] : `sk-42i-${randomBytes(24).toString("base64url")}`;
|
||||
daten[token] = name;
|
||||
writeFileSync(pfad, JSON.stringify(daten, null, 1), { mode: 0o600 });
|
||||
console.log(i > 0 ? `uebernommen fuer ${name}` : token);
|
||||
} else if (befehl === "remove" && name) {
|
||||
for (const [t, n] of Object.entries(daten)) if (n === name) delete daten[t];
|
||||
writeFileSync(pfad, JSON.stringify(daten, null, 1), { mode: 0o600 });
|
||||
} else {
|
||||
console.log("keys.ts list | add <name> [--token <t>] | remove <name>");
|
||||
}
|
||||
+73
@@ -0,0 +1,73 @@
|
||||
/**
|
||||
* log.ts — ein Eintrag je Anfrage, zweimal: als JSONL (grep-bar, ueberlebt
|
||||
* alles) und in SQLite (fuer die Tagesstatistik je Key). Die Kosten kommen
|
||||
* von OpenRouter aus der Antwort (usage.cost), fuer lokale Upstreams bleibt
|
||||
* das Feld null -- wir erfinden keine Preise.
|
||||
*/
|
||||
import { appendFileSync, mkdirSync } from "fs";
|
||||
import { Database } from "bun:sqlite";
|
||||
|
||||
export type Eintrag = {
|
||||
ts: string;
|
||||
key: string;
|
||||
alias: string;
|
||||
angefragt: string;
|
||||
model: string;
|
||||
upstream: string;
|
||||
effort: string | null;
|
||||
pfad: string;
|
||||
status: number;
|
||||
ms: number;
|
||||
stream: boolean;
|
||||
prompt_tokens: number | null;
|
||||
completion_tokens: number | null;
|
||||
cached_tokens: number | null;
|
||||
cost: number | null;
|
||||
fehler?: string;
|
||||
};
|
||||
|
||||
export class Log {
|
||||
private db: Database;
|
||||
private jsonl: string;
|
||||
constructor(dir: string) {
|
||||
mkdirSync(dir, { recursive: true });
|
||||
this.jsonl = `${dir}/requests.jsonl`;
|
||||
this.db = new Database(`${dir}/requests.sqlite`);
|
||||
this.db.run(`CREATE TABLE IF NOT EXISTS requests (
|
||||
ts TEXT, key TEXT, alias TEXT, angefragt TEXT, model TEXT, upstream TEXT, effort TEXT, pfad TEXT,
|
||||
status INTEGER, ms INTEGER, stream INTEGER, prompt_tokens INTEGER, completion_tokens INTEGER,
|
||||
cached_tokens INTEGER, cost REAL, fehler TEXT)`);
|
||||
this.db.run(`CREATE INDEX IF NOT EXISTS requests_ts_key ON requests (ts, key)`);
|
||||
}
|
||||
schreiben(e: Eintrag): void {
|
||||
appendFileSync(this.jsonl, JSON.stringify(e) + "\n");
|
||||
this.db.run(
|
||||
`INSERT INTO requests VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,
|
||||
[e.ts, e.key, e.alias, e.angefragt, e.model, e.upstream, e.effort, e.pfad, e.status, e.ms, e.stream ? 1 : 0,
|
||||
e.prompt_tokens, e.completion_tokens, e.cached_tokens, e.cost, e.fehler ?? null],
|
||||
);
|
||||
}
|
||||
/** Je Tag und Key: Anfragen, Tokens, Kosten. */
|
||||
statistik(tage: number): { tag: string; key: string; anfragen: number; prompt: number; completion: number; cost: number; modelle: string }[] {
|
||||
return this.db.query(`
|
||||
SELECT substr(ts,1,10) AS tag, key,
|
||||
count(*) AS anfragen,
|
||||
coalesce(sum(prompt_tokens),0) AS prompt,
|
||||
coalesce(sum(completion_tokens),0) AS completion,
|
||||
coalesce(sum(cost),0) AS cost,
|
||||
group_concat(DISTINCT model) AS modelle
|
||||
FROM requests WHERE ts >= datetime('now', ?)
|
||||
GROUP BY tag, key ORDER BY tag DESC, cost DESC`).all(`-${tage} days`) as never;
|
||||
}
|
||||
}
|
||||
|
||||
/** Zieht Tokens und Kosten aus einer OpenAI-/OpenRouter-Usage. */
|
||||
export function usageLesen(u: unknown): Pick<Eintrag, "prompt_tokens" | "completion_tokens" | "cached_tokens" | "cost"> {
|
||||
const usage = (u ?? {}) as { prompt_tokens?: number; completion_tokens?: number; cost?: number; prompt_tokens_details?: { cached_tokens?: number } };
|
||||
return {
|
||||
prompt_tokens: usage.prompt_tokens ?? null,
|
||||
completion_tokens: usage.completion_tokens ?? null,
|
||||
cached_tokens: usage.prompt_tokens_details?.cached_tokens ?? null,
|
||||
cost: typeof usage.cost === "number" ? usage.cost : null,
|
||||
};
|
||||
}
|
||||
+115
@@ -0,0 +1,115 @@
|
||||
/**
|
||||
* resolver.ts — loest die "alternative"-Aliasse auf das billigste Modell
|
||||
* derselben Klasse auf (openrouter-auswahl.ts aus llmlite, gleiche Regel wie
|
||||
* das Anzeige-Werkzeug und der alte Tier-Updater).
|
||||
*
|
||||
* Takt: OpenRouter-Preise aendern sich oft -> alle refreshHours neu holen.
|
||||
* AA-Indexe aendern sich nie, es kommen nur Modelle hinzu -> aa-cache.json,
|
||||
* alle aaRefreshDays erneuert. Ergebnis in state.json, damit ein Neustart
|
||||
* nicht mit leerer Zuordnung startet; Wechsel landen in changes.log.
|
||||
*
|
||||
* Aufruf als Skript: `bun run src/resolver.ts [--force] [--dry-run]`
|
||||
*/
|
||||
import { readFileSync, writeFileSync, appendFileSync, mkdirSync, existsSync, statSync } from "fs";
|
||||
import {
|
||||
apiKeyHolen, aaKeyHolen, listenHolen, kandidatenBauen, sperrliste, guenstigereAlternative,
|
||||
variantenHolen, referenzParsen, referenzFormat, type Variante, type Effort,
|
||||
} from "../../llmlite/openrouter-auswahl.ts";
|
||||
import { configLaden, type Config } from "./config.ts";
|
||||
|
||||
export type Zuordnung = {
|
||||
model: string;
|
||||
effort: Effort | null;
|
||||
index: number;
|
||||
weighted: number;
|
||||
referenz: string;
|
||||
seit: string;
|
||||
/** true = nichts Billigeres gefunden, die Referenz selbst wird gerufen */
|
||||
istReferenz: boolean;
|
||||
};
|
||||
export type State = { aktualisiert: string; zuordnung: Record<string, Zuordnung> };
|
||||
|
||||
const statePfad = (c: Config) => `${c.logDir}/state.json`;
|
||||
const aaPfad = (c: Config) => `${c.logDir}/aa-cache.json`;
|
||||
|
||||
export function stateLaden(c: Config): State {
|
||||
try {
|
||||
return JSON.parse(readFileSync(statePfad(c), "utf8")) as State;
|
||||
} catch {
|
||||
return { aktualisiert: "1970-01-01T00:00:00.000Z", zuordnung: {} };
|
||||
}
|
||||
}
|
||||
|
||||
export function stateVeraltet(c: Config, s: State): boolean {
|
||||
return Date.now() - Date.parse(s.aktualisiert) > c.refreshHours * 3_600_000;
|
||||
}
|
||||
|
||||
async function variantenMitCache(c: Config, modelle: Parameters<typeof variantenHolen>[1]): Promise<Variante[]> {
|
||||
const pfad = aaPfad(c);
|
||||
const frisch = existsSync(pfad) && Date.now() - statSync(pfad).mtimeMs < c.aaRefreshDays * 86_400_000;
|
||||
if (frisch) return JSON.parse(readFileSync(pfad, "utf8")) as Variante[];
|
||||
const aaKey = aaKeyHolen();
|
||||
if (aaKey === undefined) {
|
||||
if (existsSync(pfad)) return JSON.parse(readFileSync(pfad, "utf8")) as Variante[];
|
||||
throw new Error("kein Artificial-Analysis-Schluessel und kein aa-cache.json");
|
||||
}
|
||||
const { varianten } = await variantenHolen(aaKey, modelle);
|
||||
mkdirSync(c.logDir, { recursive: true });
|
||||
writeFileSync(pfad, JSON.stringify(varianten));
|
||||
return varianten;
|
||||
}
|
||||
|
||||
/**
|
||||
* Loest alle alternative-Aliasse auf. Die Last-Probe (78k Kontext, dreimal,
|
||||
* kostet ~210k Token) laeuft nur fuer einen Kandidaten, der die bisherige
|
||||
* Zuordnung abloesen wuerde -- wie im alten Tier-Updater.
|
||||
*/
|
||||
export async function aufloesen(c: Config, bisher: State, opts?: { dryRun?: boolean; log?: (z: string) => void }): Promise<State> {
|
||||
const log = opts?.log ?? ((z: string) => console.log(z));
|
||||
const key = apiKeyHolen();
|
||||
const listen = await listenHolen(key);
|
||||
const gesperrt = sperrliste();
|
||||
const { alle } = kandidatenBauen(listen, gesperrt);
|
||||
const varianten = await variantenMitCache(c, listen.voll);
|
||||
const neu: State = { aktualisiert: new Date().toISOString(), zuordnung: { ...bisher.zuordnung } };
|
||||
|
||||
for (const [alias, def] of Object.entries(c.aliases)) {
|
||||
if (def.art !== "alternative") continue;
|
||||
const alt = bisher.zuordnung[alias];
|
||||
try {
|
||||
const abgelehnt: string[] = [];
|
||||
const { referenz, auswahl } = await guenstigereAlternative(def.referenz, alle, key,
|
||||
(id, grund) => abgelehnt.push(`${id}: ${grund}`), gesperrt,
|
||||
{ varianten, lasttest: c.lasttest, bestand: alt?.model });
|
||||
const ziel: Zuordnung = auswahl === null
|
||||
? { model: referenz.id, effort: referenz.effort ?? null, index: referenz.index, weighted: referenz.weighted, referenz: def.referenz, seit: alt?.seit ?? neu.aktualisiert, istReferenz: true }
|
||||
: { model: auswahl.candidate.id, effort: auswahl.candidate.effort ?? null, index: auswahl.candidate.index, weighted: auswahl.candidate.weighted, referenz: def.referenz, seit: alt?.seit ?? neu.aktualisiert, istReferenz: false };
|
||||
const gleich = alt !== undefined && alt.model === ziel.model && alt.effort === ziel.effort;
|
||||
if (!gleich) {
|
||||
ziel.seit = neu.aktualisiert;
|
||||
const zeile = `${neu.aktualisiert} ${alias}: ${alt ? referenzFormat(alt.model, alt.effort) : "(neu)"} -> ${referenzFormat(ziel.model, ziel.effort)} (Index ${ziel.index.toFixed(1)}, ${ziel.weighted.toFixed(2)} $/1M gewichtet, Referenz ${def.referenz}${ziel.istReferenz ? ", = Referenz" : ""})`;
|
||||
log(zeile);
|
||||
if (!opts?.dryRun) { mkdirSync(c.logDir, { recursive: true }); appendFileSync(`${c.logDir}/changes.log`, zeile + "\n"); }
|
||||
}
|
||||
for (const a of abgelehnt) log(` abgelehnt ${alias}: ${a}`);
|
||||
neu.zuordnung[alias] = ziel;
|
||||
} catch (e) {
|
||||
log(`FEHLER ${alias}: ${(e as Error).message}${alt ? " -- behalte " + referenzFormat(alt.model, alt.effort) : " -- KEINE Zuordnung"}`);
|
||||
}
|
||||
}
|
||||
if (!opts?.dryRun) { mkdirSync(c.logDir, { recursive: true }); writeFileSync(statePfad(c), JSON.stringify(neu, null, 1)); }
|
||||
return neu;
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
const c = configLaden();
|
||||
const s = stateLaden(c);
|
||||
const force = process.argv.includes("--force");
|
||||
const dryRun = process.argv.includes("--dry-run");
|
||||
if (!force && !stateVeraltet(c, s)) {
|
||||
console.log(`state.json ist frisch (${s.aktualisiert}); --force erzwingt.`);
|
||||
} else {
|
||||
const neu = await aufloesen(c, s, { dryRun });
|
||||
console.table(Object.entries(neu.zuordnung).map(([alias, z]) => ({ alias, ziel: referenzFormat(z.model, z.effort), index: z.index.toFixed(1), "$/1M": z.weighted.toFixed(2), referenz: z.referenz, seit: z.seit.slice(0, 16) })));
|
||||
}
|
||||
}
|
||||
+203
@@ -0,0 +1,203 @@
|
||||
/**
|
||||
* server.ts — der llmrouter: OpenAI-kompatibler Reverse-Proxy mit vier
|
||||
* Eingriffen je Anfrage: Client-Key pruefen, Alias aufloesen und Modell
|
||||
* umschreiben, Usage/Kosten aus der Antwort mitlesen, eine Zeile loggen.
|
||||
*
|
||||
* Keine Format-Uebersetzung: alle Upstreams sprechen OpenAI-Format
|
||||
* (OpenRouter, llama-server, Ollama, Mistral). Genau das macht ihn klein.
|
||||
*
|
||||
* Umgebung: PORT (optional), KEYS_FILE (Token -> Name), ADMIN_KEY (fuer
|
||||
* /stats und /zuordnung), OPENROUTER_API_KEY, MISTRAL_API_KEY.
|
||||
*/
|
||||
import { configLaden, aclLaden, tokenLaden, upstreamKey, type Config, type Upstream } from "./config.ts";
|
||||
import { aufloesen, stateLaden, stateVeraltet, type State } from "./resolver.ts";
|
||||
import { Log, usageLesen, type Eintrag } from "./log.ts";
|
||||
import { referenzFormat } from "../../llmlite/openrouter-auswahl.ts";
|
||||
|
||||
const config: Config = configLaden();
|
||||
const acl = aclLaden();
|
||||
let tokens = tokenLaden();
|
||||
let state: State = stateLaden(config);
|
||||
const log = new Log(config.logDir);
|
||||
const jetzt = () => new Date().toISOString();
|
||||
const sagen = (z: string) => console.log(`${jetzt()} ${z}`);
|
||||
|
||||
// --- Zuordnung frisch halten ------------------------------------------------
|
||||
let laeuft = false;
|
||||
async function auffrischen(grund: string): Promise<void> {
|
||||
if (laeuft) return;
|
||||
laeuft = true;
|
||||
try {
|
||||
sagen(`Zuordnung auffrischen (${grund})`);
|
||||
state = await aufloesen(config, state, { log: sagen });
|
||||
} catch (e) {
|
||||
sagen(`Auffrischen fehlgeschlagen: ${(e as Error).message} -- behalte Stand ${state.aktualisiert}`);
|
||||
} finally {
|
||||
laeuft = false;
|
||||
}
|
||||
}
|
||||
if (stateVeraltet(config, state)) void auffrischen("Start, state veraltet");
|
||||
setInterval(() => { if (stateVeraltet(config, state)) void auffrischen("Takt"); tokens = tokenLaden(); }, 15 * 60_000);
|
||||
|
||||
// --- Nebenlaeufigkeit je Upstream (lokales Qwen: ein Slot) ------------------
|
||||
const semaphoren = new Map<string, { frei: number; warteschlange: (() => void)[] }>();
|
||||
async function belegen(name: string, u: Upstream): Promise<() => void> {
|
||||
if (u.maxParallel === undefined) return () => {};
|
||||
let s = semaphoren.get(name);
|
||||
if (s === undefined) { s = { frei: u.maxParallel, warteschlange: [] }; semaphoren.set(name, s); }
|
||||
if (s.frei > 0) s.frei -= 1;
|
||||
else await new Promise<void>((res) => s!.warteschlange.push(res));
|
||||
return () => { const n = s!.warteschlange.shift(); if (n) n(); else s!.frei += 1; };
|
||||
}
|
||||
|
||||
// --- Aufloesung Alias -> Ziel ------------------------------------------------
|
||||
type Ziel = { alias: string; upstreamName: string; upstream: Upstream; model: string; effort: string | null; body?: Record<string, unknown>; alternative: boolean };
|
||||
function zielBestimmen(angefragt: string, keyName: string): Ziel | { fehler: string; status: number } {
|
||||
const alias = config.altnamen[angefragt] ?? angefragt;
|
||||
const def = config.aliases[alias];
|
||||
if (def === undefined) return { fehler: `Unbekanntes Modell ${angefragt}`, status: 404 };
|
||||
const erlaubt = acl.get(keyName) ?? [];
|
||||
if (erlaubt.length > 0 && !erlaubt.includes(alias)) return { fehler: `${keyName} darf ${alias} nicht rufen`, status: 403 };
|
||||
if (def.art === "fest") {
|
||||
return { alias, upstreamName: def.upstream, upstream: config.upstreams[def.upstream], model: def.model, effort: null, body: def.body, alternative: false };
|
||||
}
|
||||
const z = state.zuordnung[alias];
|
||||
if (z === undefined) return { fehler: `${alias} ist noch nicht aufgeloest (Zuordnung fehlt) -- spaeter erneut`, status: 503 };
|
||||
return { alias, upstreamName: "openrouter", upstream: config.upstreams.openrouter, model: z.model, effort: z.effort, alternative: true };
|
||||
}
|
||||
|
||||
// --- Anfrage bauen -------------------------------------------------------------
|
||||
const JSON_PFADE = new Set(["/chat/completions", "/completions", "/embeddings", "/audio/speech", "/responses"]);
|
||||
const FORM_PFADE = new Set(["/audio/transcriptions", "/audio/translations"]);
|
||||
|
||||
function jsonBody(orig: Record<string, unknown>, ziel: Ziel): Record<string, unknown> {
|
||||
const b: Record<string, unknown> = { ...orig, model: ziel.model, ...(ziel.body ?? {}) };
|
||||
if (ziel.upstreamName === "openrouter") {
|
||||
b.usage = { include: true }; // Kosten kommen in der Antwort mit
|
||||
if (b.stream === true) b.stream_options = { ...(b.stream_options as object ?? {}), include_usage: true };
|
||||
}
|
||||
if (ziel.alternative) {
|
||||
// Die Stufe gehoert zum Alias, nicht zum Client: wer marvin:low ruft,
|
||||
// bekommt die Klasse von marvin:low -- auch wenn er reasoning_effort mitschickt.
|
||||
delete b.reasoning_effort; delete b.reasoning;
|
||||
if (ziel.effort !== null) b.reasoning = { effort: ziel.effort };
|
||||
}
|
||||
return b;
|
||||
}
|
||||
|
||||
// --- Server -------------------------------------------------------------------
|
||||
Bun.serve({
|
||||
port: Number(process.env.PORT ?? config.port),
|
||||
idleTimeout: 255,
|
||||
async fetch(req) {
|
||||
const url = new URL(req.url);
|
||||
const pfad = url.pathname.replace(/^\/v1/u, "");
|
||||
if (pfad === "/health") return Response.json({ ok: true, zuordnung: state.aktualisiert });
|
||||
|
||||
// Admin
|
||||
const auth = req.headers.get("authorization")?.replace(/^Bearer\s+/iu, "") ?? req.headers.get("x-api-key") ?? "";
|
||||
const admin = process.env.ADMIN_KEY !== undefined && auth === process.env.ADMIN_KEY;
|
||||
if (pfad === "/stats") return admin ? Response.json(log.statistik(Number(url.searchParams.get("days") ?? 7))) : new Response("admin", { status: 401 });
|
||||
if (pfad === "/zuordnung") return admin ? Response.json(state) : new Response("admin", { status: 401 });
|
||||
if (pfad === "/auffrischen" && req.method === "POST") { if (!admin) return new Response("admin", { status: 401 }); void auffrischen("manuell"); return Response.json({ gestartet: !laeuft }); }
|
||||
|
||||
// Client
|
||||
const keyName = tokens.get(auth);
|
||||
if (keyName === undefined) return Response.json({ error: { message: "Ungueltiger API-Key" } }, { status: 401 });
|
||||
if (pfad === "/models") {
|
||||
const erlaubt = acl.get(keyName) ?? [];
|
||||
const namen = Object.keys(config.aliases).filter((a) => erlaubt.length === 0 || erlaubt.includes(a));
|
||||
return Response.json({ object: "list", data: namen.map((id) => ({ id, object: "model", owned_by: "42i" })) });
|
||||
}
|
||||
if (req.method !== "POST" || !(JSON_PFADE.has(pfad) || FORM_PFADE.has(pfad))) return Response.json({ error: { message: `Pfad ${pfad} wird nicht bedient` } }, { status: 404 });
|
||||
|
||||
const start = Date.now();
|
||||
let angefragt = "";
|
||||
let upstreamBody: BodyInit;
|
||||
let stream = false;
|
||||
let ziel: Ziel;
|
||||
const headers = new Headers();
|
||||
|
||||
if (JSON_PFADE.has(pfad)) {
|
||||
let orig: Record<string, unknown>;
|
||||
try { orig = (await req.json()) as Record<string, unknown>; } catch { return Response.json({ error: { message: "Body ist kein JSON" } }, { status: 400 }); }
|
||||
angefragt = String(orig.model ?? "");
|
||||
const z = zielBestimmen(angefragt, keyName);
|
||||
if ("fehler" in z) return Response.json({ error: { message: z.fehler } }, { status: z.status });
|
||||
ziel = z;
|
||||
stream = orig.stream === true;
|
||||
upstreamBody = JSON.stringify(jsonBody(orig, ziel));
|
||||
headers.set("content-type", "application/json");
|
||||
} else {
|
||||
const form = await req.formData();
|
||||
angefragt = String(form.get("model") ?? "");
|
||||
const z = zielBestimmen(angefragt, keyName);
|
||||
if ("fehler" in z) return Response.json({ error: { message: z.fehler } }, { status: z.status });
|
||||
ziel = z;
|
||||
form.set("model", ziel.model);
|
||||
upstreamBody = form;
|
||||
}
|
||||
const k = upstreamKey(ziel.upstream);
|
||||
if (k !== undefined) headers.set("authorization", `Bearer ${k}`);
|
||||
if (ziel.upstreamName === "openrouter") { headers.set("http-referer", "https://llm.lan"); headers.set("x-title", `llmrouter/${keyName}`); }
|
||||
const accept = req.headers.get("accept"); if (accept) headers.set("accept", accept);
|
||||
|
||||
const eintrag: Eintrag = {
|
||||
ts: jetzt(), key: keyName, alias: ziel.alias, angefragt, model: ziel.model, upstream: ziel.upstreamName, effort: ziel.effort,
|
||||
pfad, status: 0, ms: 0, stream, prompt_tokens: null, completion_tokens: null, cached_tokens: null, cost: null,
|
||||
};
|
||||
const abschliessen = (usage?: unknown) => {
|
||||
eintrag.ms = Date.now() - start;
|
||||
if (usage !== undefined) Object.assign(eintrag, usageLesen(usage));
|
||||
try { log.schreiben(eintrag); } catch (e) { sagen(`Log-Fehler: ${(e as Error).message}`); }
|
||||
};
|
||||
|
||||
const freigeben = await belegen(ziel.upstreamName, ziel.upstream);
|
||||
let antwort: Response;
|
||||
try {
|
||||
antwort = await fetch(`${ziel.upstream.base}${pfad}`, { method: "POST", headers, body: upstreamBody, signal: AbortSignal.timeout(600_000) });
|
||||
} catch (e) {
|
||||
freigeben();
|
||||
eintrag.status = 502; eintrag.fehler = (e as Error).message; abschliessen();
|
||||
return Response.json({ error: { message: `Upstream ${ziel.upstreamName} nicht erreichbar: ${eintrag.fehler}` } }, { status: 502 });
|
||||
}
|
||||
eintrag.status = antwort.status;
|
||||
const ct = antwort.headers.get("content-type") ?? "";
|
||||
const weiter = new Headers(antwort.headers);
|
||||
weiter.delete("content-length"); weiter.delete("content-encoding"); weiter.delete("transfer-encoding");
|
||||
weiter.set("x-llmrouter-model", referenzFormat(ziel.model, ziel.effort as never));
|
||||
|
||||
if (ct.startsWith("text/event-stream") && antwort.body !== null) {
|
||||
// Durchreichen und nebenbei den letzten usage-Chunk lesen.
|
||||
let rest = ""; let usage: unknown;
|
||||
const dec = new TextDecoder();
|
||||
const tee = new TransformStream<Uint8Array, Uint8Array>({
|
||||
transform(chunk, ctrl) {
|
||||
ctrl.enqueue(chunk);
|
||||
rest += dec.decode(chunk, { stream: true });
|
||||
let i: number;
|
||||
while ((i = rest.indexOf("\n")) >= 0) {
|
||||
const zeile = rest.slice(0, i).trim(); rest = rest.slice(i + 1);
|
||||
if (zeile.startsWith("data:") && zeile.includes('"usage"')) {
|
||||
try { const j = JSON.parse(zeile.slice(5)); if (j.usage) usage = j.usage; } catch { /* Teilzeile */ }
|
||||
}
|
||||
}
|
||||
},
|
||||
flush() { freigeben(); abschliessen(usage); },
|
||||
});
|
||||
return new Response(antwort.body.pipeThrough(tee), { status: antwort.status, headers: weiter });
|
||||
}
|
||||
freigeben();
|
||||
if (ct.includes("application/json")) {
|
||||
const text = await antwort.text();
|
||||
let usage: unknown; let fehler: string | undefined;
|
||||
try { const j = JSON.parse(text); usage = j.usage; if (j.error) fehler = JSON.stringify(j.error).slice(0, 300); } catch { /* egal */ }
|
||||
if (fehler) eintrag.fehler = fehler;
|
||||
abschliessen(usage);
|
||||
return new Response(text, { status: antwort.status, headers: weiter });
|
||||
}
|
||||
abschliessen();
|
||||
return new Response(antwort.body, { status: antwort.status, headers: weiter });
|
||||
},
|
||||
});
|
||||
sagen(`llmrouter lauscht auf :${process.env.PORT ?? config.port}, ${Object.keys(config.aliases).length} Aliasse, ${tokens.size} Token, Zuordnung von ${state.aktualisiert}`);
|
||||
@@ -0,0 +1,9 @@
|
||||
/** stats.ts — Tagesstatistik je Key aus requests.sqlite. `bun run src/stats.ts [tage]` */
|
||||
import { configLaden } from "./config.ts";
|
||||
import { Log } from "./log.ts";
|
||||
|
||||
const tage = Number(process.argv[2] ?? 7);
|
||||
const log = new Log(configLaden().logDir);
|
||||
const zeilen = log.statistik(tage);
|
||||
if (zeilen.length === 0) console.log(`Keine Anfragen in den letzten ${tage} Tagen.`);
|
||||
else console.table(zeilen.map((z) => ({ ...z, cost: `$${z.cost.toFixed(4)}` })));
|
||||
Reference in New Issue
Block a user