avraham
/
context-efficient-ai
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

factored-out fact generation into `facts` module

Browse Source
master
Avraham Sakal 2 months ago
parent 083b7a275c
commit 84e9ff8bd2
3 changed files with 122 additions and 147 deletions
Show Stats Download Patch File Download Diff File

96
pages/chat/facts.ts
Unescape Escape View File

@ -4,6 +4,48 @@ import {
createCallerFactory,
} from "../../trpc/server.js";
import { db, type Fact } from "../../database/lowdb.js";
import type { DraftMessage } from "../../types.js";
import { openrouter } from "./provider.js";
import { generateObject, generateText, jsonSchema } from "ai";
const factsFromNewMessagesSystemPrompt = ({
previousRunningSummary,
messagesSincePreviousRunningSummary,
}: {
previousRunningSummary: string;
messagesSincePreviousRunningSummary: Array<DraftMessage>;
}) => `You are an expert at extracting facts from conversations.
An AI assistant is in the middle of a conversation whose data is given below. The data consists of a summary of a conversation, and optionally some messages exchanged since that summary was produced. The user will provide you with *new* messages.
Your task is to extract *new* facts that can be gleaned from the *new* messages that the user sends.
* You should not extract any facts that are already in the summary.
* The user should be referred to as "the user" in the fact text.
* The user's pronouns should be either he or she, NOT "they" or "them", because this summary will be read by an AI assistant to give it context; and excessive use of "they" or "them" will make what they refer to unclear or ambiguous.
* The assistant should be referred to as "I" or "me", because these facts will be read by an AI assistant to give it context.
<running_summary>
${previousRunningSummary}
</running_summary>
${messagesSincePreviousRunningSummary.map(
(message) =>
`<${message.role}_message>${message.content}</${message.role}_message>`,
)}
`;
const factsFromNewMessagesUserPrompt = ({
newMessages,
}: {
newMessages: Array<DraftMessage>;
}) =>
`${newMessages.map(
(message) =>
`<${message.role}_message>${message.content}</${message.role}_message>`,
)}
Extract new facts from these messages.`;
export const facts = router({
fetchByConversationId: publicProcedure
@ -33,6 +75,60 @@ export const facts = router({
db.write();
return { ok: true };
}),
extractFromNewMessages: publicProcedure
.input(
(x) =>
x as {
previousRunningSummary: string;
/** will *not* have facts extracted */
messagesSincePreviousRunningSummary: Array<DraftMessage>;
/** *will* have facts extracted */
newMessages: Array<DraftMessage>;
},
)
.query(
async ({
input: {
previousRunningSummary,
messagesSincePreviousRunningSummary,
newMessages,
},
}) => {
const factsFromUserMessageResponse = await generateObject<{
facts: Array<string>;
}>({
model: openrouter("mistralai/mistral-nemo"),
messages: [
{
role: "system" as const,
content: factsFromNewMessagesSystemPrompt({
previousRunningSummary,
messagesSincePreviousRunningSummary,
}),
},
{
role: "user" as const,
content: factsFromNewMessagesUserPrompt({
newMessages,
}),
},
],
schema: jsonSchema({
type: "object",
properties: {
facts: {
type: "array",
items: {
type: "string",
},
},
},
}),
temperature: 0.4,
});
return factsFromUserMessageResponse;
},
),
});
export const createCaller = createCallerFactory(facts);

5
pages/chat/provider.ts
Unescape Escape View File

@ -0,0 +1,5 @@
import { createOpenRouter } from "@openrouter/ai-sdk-provider";
import { env } from "../../server/env.js";
export const openrouter = createOpenRouter({
apiKey: env.OPENROUTER_API_KEY,
});

156
pages/chat/trpc.ts
Unescape Escape View File

@ -3,15 +3,12 @@ import {
publicProcedure,
createCallerFactory,
} from "../../trpc/server.js";
import { createOpenRouter } from "@openrouter/ai-sdk-provider";
import { generateObject, generateText, jsonSchema } from "ai";
import type { Message as UIMessage } from "ai";
import type {
OtherParameters,
CommittedMessage,
DraftMessage,
} from "../../types.js";
import { env } from "../../server/env.js";
// import { client } from "../../database/milvus";
// import {
// ConsistencyLevelEnum,
@ -21,7 +18,10 @@ import { db, type FactTrigger, type Fact } from "../../database/lowdb.js";
import { nanoid } from "nanoid";
import { conversations } from "./conversations.js";
import { messages } from "./messages.js";
import { facts } from "./facts.js";
import { facts, createCaller as createCallerFacts } from "./facts.js";
import { openrouter } from "./provider.js";
const factsCaller = createCallerFacts({});
const mainSystemPrompt = ({
systemPrompt,
@ -33,74 +33,6 @@ This is a summary of the conversation so far, from your point-of-view (so "I" an
${previousRunningSummary}
</running_summary>
`;
const factsFromUserMessageSystemPrompt = ({
previousRunningSummary,
}: {
previousRunningSummary: string;
}) => `You are an expert at extracting facts from conversations.
You will be given a summary of a conversation, and the messages exchanged since that summary was produced.
Your task is to extract *new* facts that can be gleaned from the messages exchanged since the summary was produced.
* You should not extract any facts that are already in the summary.
* The user should be referred to as "the user" in the fact text.
* The user's pronouns should be either he or she, NOT "they" or "them", because this summary will be read by an AI assistant to give it context; and excessive use of "they" or "them" will make what they refer to unclear or ambiguous.
* The assistant should be referred to as "I" or "me", because these facts will be read by an AI assistant to give it context.
<running_summary>
${previousRunningSummary}
</running_summary>
`;
const factsFromUserMessageUserPrompt = ({
messagesSincePreviousRunningSummary,
}: {
messagesSincePreviousRunningSummary: Array<DraftMessage>;
}) =>
`${messagesSincePreviousRunningSummary.map(
(message) =>
`<${message.role}_message>${message.content}</${message.role}_message>`,
)}
Extract new facts from these messages.`;
const factsFromAssistantMessageSystemPrompt = ({
previousRunningSummary,
}: {
previousRunningSummary: string;
}) => `You are an expert at extracting facts from conversations.
You will be given a summary of a conversation, and the messages exchanged since that summary was produced.
Your task is to extract *new* facts that can be gleaned from the *final assistant response*.
* You should not extract any facts that are already in the summary or in the ensuing conversation; you should only extract new facts from the final assistant response.
* The user should be referred to as "the user" in the fact text.
* The user's pronouns should be either he or she, NOT "they" or "them", because this summary will be read by an AI assistant to give it context; and excessive use of "they" or "them" will make what they refer to unclear or ambiguous.
* The assistant should be referred to as "I" or "me", because these facts will be read by an AI assistant to give it context.
<running_summary>
${previousRunningSummary}
</running_summary>
`;
const factsFromAssistantMessageUserPrompt = ({
messagesSincePreviousRunningSummary,
mainResponseContent,
}: {
messagesSincePreviousRunningSummary: Array<DraftMessage>;
mainResponseContent: string;
}) =>
`${messagesSincePreviousRunningSummary.map(
(message) =>
`<${message.role}_message>${message.content}</${message.role}_message>`,
)}
<assistant_response>
${mainResponseContent}
</assistant_response>
Extract facts from the assistant's response.`;
const factTriggersSystemPrompt = ({
previousRunningSummary,
@ -185,10 +117,6 @@ ${mainResponseContent}
Generate a new running summary of the conversation.`;
const openrouter = createOpenRouter({
apiKey: env.OPENROUTER_API_KEY,
});
export const chat = router({
conversations,
messages,
@ -269,38 +197,11 @@ export const chat = router({
* injection, because we're sending the user's message unadulterated to
* the model; there's no reason to inject the same Facts that the model is
* already using to generate its response.) */
const factsFromUserMessageResponse = await generateObject<{
facts: Array<string>;
}>({
model: openrouter("mistralai/mistral-nemo"),
messages: [
{
role: "system" as const,
content: factsFromUserMessageSystemPrompt({
const factsFromUserMessageResponse =
await factsCaller.extractFromNewMessages({
previousRunningSummary,
}),
},
{
role: "user" as const,
content: factsFromUserMessageUserPrompt({
messagesSincePreviousRunningSummary,
}),
},
],
schema: jsonSchema({
type: "object",
properties: {
facts: {
type: "array",
items: {
type: "string",
},
},
},
}),
maxSteps: 3,
tools: undefined,
...parameters,
messagesSincePreviousRunningSummary: [],
newMessages: messagesSincePreviousRunningSummary,
});
const insertedFactsFromUserMessage: Array<Fact> =
factsFromUserMessageResponse.object.facts.map((fact) => ({
@ -349,45 +250,18 @@ export const chat = router({
db.data.messages.push(insertedAssistantMessage);
/** Extract Facts from the model's response, and add them to the database,
* linking the Facts with the messages they came from. */
const factsFromAssistantMessageResponse = await generateObject<{
facts: Array<string>;
}>({
model: openrouter("mistralai/mistral-nemo"),
messages: [
{
role: "system" as const,
content: factsFromAssistantMessageSystemPrompt({
const factsFromAssistantMessageResponse =
await factsCaller.extractFromNewMessages({
previousRunningSummary,
}),
},
/** Yes, the next message is a `user` message, because models are
* trained to respond to `user` messages. So we wrap the assistant
* response in XML tags to show that it's not the user speaking,
* rather it's input for the model to process. The user is only
* saying "Extract facts..." */
{
role: "user" as const,
content: factsFromAssistantMessageUserPrompt({
messagesSincePreviousRunningSummary,
mainResponseContent: mainResponse.text,
}),
newMessages: [
{
role: "assistant" as const,
content: mainResponse.text,
},
],
schema: jsonSchema({
type: "object",
properties: {
facts: {
type: "array",
items: {
type: "string",
},
},
},
}),
maxSteps: 3,
tools: undefined,
...parameters,
});
const insertedFactsFromAssistantMessage: Array<Fact> =
factsFromAssistantMessageResponse.object.facts.map((factContent) => ({
id: nanoid(),

Write Preview
Loading…
Cancel
Save