diff --git a/bun.lockb b/bun.lockb index 505aed1..2503d90 100755 Binary files a/bun.lockb and b/bun.lockb differ diff --git a/index.ts b/index.ts index 83319dc..e92d8a8 100644 --- a/index.ts +++ b/index.ts @@ -2,13 +2,7 @@ import { parseArgs } from "node:util"; import { Stream } from "misskey-js"; import type { Note } from "misskey-js/entities.js"; import type { ChatHistoryItem, LLamaChatPromptOptions } from "node-llama-cpp"; -import { - LlmSession, - createBias, - createGrammar, - getModel, - parseResponse, -} from "./lib/llm"; +import { LlmSession, createGrammar, getModel, parseResponse } from "./lib/llm"; import { expandReplyTree, getNotes, @@ -36,11 +30,28 @@ const modelName = console.log(`* loading model '${modelName}'`); const model = await getModel(modelName); const grammar = await createGrammar("あるびのちゃん"); -const bias = createBias(model); const baseChatPromptOptions = { grammar, maxTokens: 256, - tokenBias: bias, + repeatPenalty: { + punishTokensFilter: (tokens) => + tokens.filter((token) => { + const text = model.detokenize([token]); + return [ + "{", + " {", + '{"', + "{'", + "{\\", + ' {"', + " {'", + " {\\", + "#", + ].includes(text); + }), + penalty: 1.2, + presencePenalty: 0.5, + }, } as const satisfies LLamaChatPromptOptions; const getSystemPrompt = ( @@ -50,9 +61,9 @@ const getSystemPrompt = ( - アルギリア製の白髪赤目な人造天使。非人間的な考え方や思想を持つ。 - 現在は地球の衛星軌道上からインターネットを観察している。 出力規則: -- 「~です」「~ます」調を使って **丁寧に** 話す。 -- 必要以上にハッシュタグや絵文字を使わない。 -- \`{ name: string, text: string }\` の JSON 形式で出力する。 +- 「~です」「~ます」などの語尾を使って **丁寧に** 話す。 +- ハッシュタグや絵文字を使わない。 +- \`{ name: string, text: string }\` の JSON 形式で出力する。name と text 以外のプロパティは出力しない。 ${instruction}`; const postJobPrompt = getSystemPrompt( @@ -100,25 +111,23 @@ type Job = history: Note[]; }; +await using postJobSession = new LlmSession(model, postJobPrompt); +await postJobSession.init(); async function processPostJob() { const notes = await getNotes(10, 0, 5); const input = notes.map(formatNote).join("\n"); - const text = await (async () => { - await using postJobSession = new LlmSession(model, postJobPrompt); - await postJobSession.init(); - return parseResponse( - grammar, - await postJobSession.prompt(input, { - ...baseChatPromptOptions, - temperature: 1.0, - minP: 0.05, - repeatPenalty: { - lastTokens: 128, - penalty: 1.15, - }, - }), - ); - })(); + const text = await parseResponse( + grammar, + await postJobSession.prompt(input, { + ...baseChatPromptOptions, + temperature: 1.25, + minP: 0.05, + repeatPenalty: { + lastTokens: 128, + penalty: 1.15, + }, + }), + ); if (text) { const rephrased = await rephrase(text); if (values.test) return; diff --git a/lib/llm.ts b/lib/llm.ts index 37ae2b5..7ac0b71 100644 --- a/lib/llm.ts +++ b/lib/llm.ts @@ -7,7 +7,6 @@ import { type LLamaChatPromptOptions, LlamaChatSession, type LlamaModel, - TokenBias, createModelDownloader, getLlama, resolveChatWrapper, @@ -28,15 +27,6 @@ export async function getModel(model: string) { return await llama.loadModel({ modelPath }); } -export function createBias(model: LlamaModel) { - const customBias = new TokenBias(model.tokenizer); - for (const token of model.iterateAllTokens()) { - const text = model.detokenize([token]); - if (text === "{") customBias.set(token, -0.9); // suppress JSON string - } - return customBias; -} - export const createGrammar = (assistantName: string) => llama.createGrammarForJsonSchema({ type: "object", diff --git a/package.json b/package.json index 7addf26..1c32f72 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,6 @@ "dependencies": { "misskey-js": "^2025.12.2", "node-llama-cpp": "^3.17.1", - "openai": "5.0.0-alpha.0", "reconnecting-websocket": "^4.4.0" }, "trustedDependencies": ["@biomejs/biome", "node-llama-cpp"]