From 84e9ff8bd2cd9705dfde57c1a948e25c8c14161e Mon Sep 17 00:00:00 2001 From: Avraham Sakal Date: Thu, 31 Jul 2025 09:15:32 -0400 Subject: [PATCH] factored-out fact generation into `facts` module --- pages/chat/facts.ts | 96 +++++++++++++++++++++++ pages/chat/provider.ts | 5 ++ pages/chat/trpc.ts | 168 ++++++----------------------------------- 3 files changed, 122 insertions(+), 147 deletions(-) create mode 100644 pages/chat/provider.ts diff --git a/pages/chat/facts.ts b/pages/chat/facts.ts index 1637de1..b2b5769 100644 --- a/pages/chat/facts.ts +++ b/pages/chat/facts.ts @@ -4,6 +4,48 @@ import { createCallerFactory, } from "../../trpc/server.js"; import { db, type Fact } from "../../database/lowdb.js"; +import type { DraftMessage } from "../../types.js"; +import { openrouter } from "./provider.js"; +import { generateObject, generateText, jsonSchema } from "ai"; + +const factsFromNewMessagesSystemPrompt = ({ + previousRunningSummary, + messagesSincePreviousRunningSummary, +}: { + previousRunningSummary: string; + messagesSincePreviousRunningSummary: Array; +}) => `You are an expert at extracting facts from conversations. + +An AI assistant is in the middle of a conversation whose data is given below. The data consists of a summary of a conversation, and optionally some messages exchanged since that summary was produced. The user will provide you with *new* messages. + +Your task is to extract *new* facts that can be gleaned from the *new* messages that the user sends. + +* You should not extract any facts that are already in the summary. +* The user should be referred to as "the user" in the fact text. +* The user's pronouns should be either he or she, NOT "they" or "them", because this summary will be read by an AI assistant to give it context; and excessive use of "they" or "them" will make what they refer to unclear or ambiguous. +* The assistant should be referred to as "I" or "me", because these facts will be read by an AI assistant to give it context. + + + ${previousRunningSummary} + + +${messagesSincePreviousRunningSummary.map( + (message) => + `<${message.role}_message>${message.content}`, +)} +`; + +const factsFromNewMessagesUserPrompt = ({ + newMessages, +}: { + newMessages: Array; +}) => + `${newMessages.map( + (message) => + `<${message.role}_message>${message.content}`, + )} + +Extract new facts from these messages.`; export const facts = router({ fetchByConversationId: publicProcedure @@ -33,6 +75,60 @@ export const facts = router({ db.write(); return { ok: true }; }), + extractFromNewMessages: publicProcedure + .input( + (x) => + x as { + previousRunningSummary: string; + /** will *not* have facts extracted */ + messagesSincePreviousRunningSummary: Array; + /** *will* have facts extracted */ + newMessages: Array; + }, + ) + .query( + async ({ + input: { + previousRunningSummary, + messagesSincePreviousRunningSummary, + newMessages, + }, + }) => { + const factsFromUserMessageResponse = await generateObject<{ + facts: Array; + }>({ + model: openrouter("mistralai/mistral-nemo"), + messages: [ + { + role: "system" as const, + content: factsFromNewMessagesSystemPrompt({ + previousRunningSummary, + messagesSincePreviousRunningSummary, + }), + }, + { + role: "user" as const, + content: factsFromNewMessagesUserPrompt({ + newMessages, + }), + }, + ], + schema: jsonSchema({ + type: "object", + properties: { + facts: { + type: "array", + items: { + type: "string", + }, + }, + }, + }), + temperature: 0.4, + }); + return factsFromUserMessageResponse; + }, + ), }); export const createCaller = createCallerFactory(facts); diff --git a/pages/chat/provider.ts b/pages/chat/provider.ts new file mode 100644 index 0000000..201eaf6 --- /dev/null +++ b/pages/chat/provider.ts @@ -0,0 +1,5 @@ +import { createOpenRouter } from "@openrouter/ai-sdk-provider"; +import { env } from "../../server/env.js"; +export const openrouter = createOpenRouter({ + apiKey: env.OPENROUTER_API_KEY, +}); diff --git a/pages/chat/trpc.ts b/pages/chat/trpc.ts index 0bcf29e..28afc4d 100644 --- a/pages/chat/trpc.ts +++ b/pages/chat/trpc.ts @@ -3,15 +3,12 @@ import { publicProcedure, createCallerFactory, } from "../../trpc/server.js"; -import { createOpenRouter } from "@openrouter/ai-sdk-provider"; import { generateObject, generateText, jsonSchema } from "ai"; -import type { Message as UIMessage } from "ai"; import type { OtherParameters, CommittedMessage, DraftMessage, } from "../../types.js"; -import { env } from "../../server/env.js"; // import { client } from "../../database/milvus"; // import { // ConsistencyLevelEnum, @@ -21,7 +18,10 @@ import { db, type FactTrigger, type Fact } from "../../database/lowdb.js"; import { nanoid } from "nanoid"; import { conversations } from "./conversations.js"; import { messages } from "./messages.js"; -import { facts } from "./facts.js"; +import { facts, createCaller as createCallerFacts } from "./facts.js"; +import { openrouter } from "./provider.js"; + +const factsCaller = createCallerFacts({}); const mainSystemPrompt = ({ systemPrompt, @@ -33,74 +33,6 @@ This is a summary of the conversation so far, from your point-of-view (so "I" an ${previousRunningSummary} `; -const factsFromUserMessageSystemPrompt = ({ - previousRunningSummary, -}: { - previousRunningSummary: string; -}) => `You are an expert at extracting facts from conversations. - -You will be given a summary of a conversation, and the messages exchanged since that summary was produced. - -Your task is to extract *new* facts that can be gleaned from the messages exchanged since the summary was produced. - -* You should not extract any facts that are already in the summary. -* The user should be referred to as "the user" in the fact text. -* The user's pronouns should be either he or she, NOT "they" or "them", because this summary will be read by an AI assistant to give it context; and excessive use of "they" or "them" will make what they refer to unclear or ambiguous. -* The assistant should be referred to as "I" or "me", because these facts will be read by an AI assistant to give it context. - - - ${previousRunningSummary} - -`; - -const factsFromUserMessageUserPrompt = ({ - messagesSincePreviousRunningSummary, -}: { - messagesSincePreviousRunningSummary: Array; -}) => - `${messagesSincePreviousRunningSummary.map( - (message) => - `<${message.role}_message>${message.content}`, - )} - -Extract new facts from these messages.`; - -const factsFromAssistantMessageSystemPrompt = ({ - previousRunningSummary, -}: { - previousRunningSummary: string; -}) => `You are an expert at extracting facts from conversations. - -You will be given a summary of a conversation, and the messages exchanged since that summary was produced. - -Your task is to extract *new* facts that can be gleaned from the *final assistant response*. - -* You should not extract any facts that are already in the summary or in the ensuing conversation; you should only extract new facts from the final assistant response. -* The user should be referred to as "the user" in the fact text. -* The user's pronouns should be either he or she, NOT "they" or "them", because this summary will be read by an AI assistant to give it context; and excessive use of "they" or "them" will make what they refer to unclear or ambiguous. -* The assistant should be referred to as "I" or "me", because these facts will be read by an AI assistant to give it context. - - - ${previousRunningSummary} - -`; - -const factsFromAssistantMessageUserPrompt = ({ - messagesSincePreviousRunningSummary, - mainResponseContent, -}: { - messagesSincePreviousRunningSummary: Array; - mainResponseContent: string; -}) => - `${messagesSincePreviousRunningSummary.map( - (message) => - `<${message.role}_message>${message.content}`, - )} - -${mainResponseContent} - - -Extract facts from the assistant's response.`; const factTriggersSystemPrompt = ({ previousRunningSummary, @@ -185,10 +117,6 @@ ${mainResponseContent} Generate a new running summary of the conversation.`; -const openrouter = createOpenRouter({ - apiKey: env.OPENROUTER_API_KEY, -}); - export const chat = router({ conversations, messages, @@ -269,39 +197,12 @@ export const chat = router({ * injection, because we're sending the user's message unadulterated to * the model; there's no reason to inject the same Facts that the model is * already using to generate its response.) */ - const factsFromUserMessageResponse = await generateObject<{ - facts: Array; - }>({ - model: openrouter("mistralai/mistral-nemo"), - messages: [ - { - role: "system" as const, - content: factsFromUserMessageSystemPrompt({ - previousRunningSummary, - }), - }, - { - role: "user" as const, - content: factsFromUserMessageUserPrompt({ - messagesSincePreviousRunningSummary, - }), - }, - ], - schema: jsonSchema({ - type: "object", - properties: { - facts: { - type: "array", - items: { - type: "string", - }, - }, - }, - }), - maxSteps: 3, - tools: undefined, - ...parameters, - }); + const factsFromUserMessageResponse = + await factsCaller.extractFromNewMessages({ + previousRunningSummary, + messagesSincePreviousRunningSummary: [], + newMessages: messagesSincePreviousRunningSummary, + }); const insertedFactsFromUserMessage: Array = factsFromUserMessageResponse.object.facts.map((fact) => ({ id: nanoid(), @@ -349,45 +250,18 @@ export const chat = router({ db.data.messages.push(insertedAssistantMessage); /** Extract Facts from the model's response, and add them to the database, * linking the Facts with the messages they came from. */ - const factsFromAssistantMessageResponse = await generateObject<{ - facts: Array; - }>({ - model: openrouter("mistralai/mistral-nemo"), - messages: [ - { - role: "system" as const, - content: factsFromAssistantMessageSystemPrompt({ - previousRunningSummary, - }), - }, - /** Yes, the next message is a `user` message, because models are - * trained to respond to `user` messages. So we wrap the assistant - * response in XML tags to show that it's not the user speaking, - * rather it's input for the model to process. The user is only - * saying "Extract facts..." */ - { - role: "user" as const, - content: factsFromAssistantMessageUserPrompt({ - messagesSincePreviousRunningSummary, - mainResponseContent: mainResponse.text, - }), - }, - ], - schema: jsonSchema({ - type: "object", - properties: { - facts: { - type: "array", - items: { - type: "string", - }, + const factsFromAssistantMessageResponse = + await factsCaller.extractFromNewMessages({ + previousRunningSummary, + messagesSincePreviousRunningSummary, + newMessages: [ + { + role: "assistant" as const, + content: mainResponse.text, }, - }, - }), - maxSteps: 3, - tools: undefined, - ...parameters, - }); + ], + }); + const insertedFactsFromAssistantMessage: Array = factsFromAssistantMessageResponse.object.facts.map((factContent) => ({ id: nanoid(),