avraham
/
context-efficient-ai
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

each generation/completion uses one system prompt and one user prompt for maximum clarity for the model

Browse Source
master
Avraham Sakal 3 months ago
parent ee9e9424ab
commit caf36cf67b
1 changed files with 144 additions and 61 deletions
Show Stats Download Patch File Download Diff File

205
pages/chat/trpc.ts
Unescape Escape View File

@ -34,21 +34,78 @@ const factsFromUserMessageSystemPrompt = ({
previousRunningSummary, previousRunningSummary,
}: { }: {
previousRunningSummary: string; previousRunningSummary: string;
}) => `Given the following summary of a conversation, coupled with the messages exchanged since that summary was produced, extract new facts that can be gleaned from the conversation. }) => `You are an expert at extracting facts from conversations. Given the following summary of a conversation, coupled with the messages exchanged since that summary was produced (which will be provided by the user), extract new facts that can be gleaned from the messages exchanged since the summary was produced.
<running_summary> <running_summary>
${previousRunningSummary} ${previousRunningSummary}
</running_summary> </running_summary>
`; `;
const factsFromUserMessageUserPrompt = ({
messagesSincePreviousRunningSummary,
}: {
messagesSincePreviousRunningSummary: Array<DraftMessage>;
}) =>
`${messagesSincePreviousRunningSummary.map(
(message) =>
`<${message.role}_message>${message.content}</${message.role}_message>`,
)}
Extract new facts from these messages.`;
const factsFromAssistantMessageSystemPrompt = ({
previousRunningSummary,
}: {
previousRunningSummary: string;
}) => `You are an insightful fact extractor. Given the following summary of a conversation, coupled with the messages exchanged since that summary was produced (which will be provided by the user), extract new facts that can be gleaned from the final assistant response.
<running_summary>
${previousRunningSummary}
</running_summary>
`;
const factsFromAssistantMessageUserPrompt = ({
messagesSincePreviousRunningSummary,
mainResponseContent,
}: {
messagesSincePreviousRunningSummary: Array<DraftMessage>;
mainResponseContent: string;
}) =>
`${messagesSincePreviousRunningSummary.map(
(message) =>
`<${message.role}_message>${message.content}</${message.role}_message>`,
)}
<assistant_response>
${mainResponseContent}
</assistant_response>
Extract facts from the assistant's response.`;
const runningSummarySystemPrompt = ({ const runningSummarySystemPrompt = ({
previousRunningSummary, previousRunningSummary,
}: { }: {
previousRunningSummary: string; previousRunningSummary: string;
}) => `Given the following summary of a conversation, coupled with the messages exchanged since that summary was produced, produce a new summary of the conversation. }) => `You are an expert at summarizing conversations. Given the following summary of a conversation, coupled with the messages exchanged since that summary was produced (which will be provided by the user), produce a new summary of the conversation.
<running_summary> <running_summary>
${previousRunningSummary} ${previousRunningSummary}
</running_summary> </running_summary>
`; `;
const runningSummaryUserPrompt = ({
messagesSincePreviousRunningSummary,
mainResponseContent,
}: {
messagesSincePreviousRunningSummary: Array<DraftMessage>;
mainResponseContent: string;
}) =>
`${messagesSincePreviousRunningSummary.map(
(message) =>
`<${message.role}_message>${message.content}</${message.role}_message>`,
)}
<assistant_response>
${mainResponseContent}
</assistant_response>
Generate a new running summary of the conversation.`;
const openrouter = createOpenRouter({ const openrouter = createOpenRouter({
apiKey: env.OPENROUTER_API_KEY, apiKey: env.OPENROUTER_API_KEY,
}); });
@ -145,6 +202,9 @@ export const chat = router({
? ((messages[previousRunningSummaryIndex] as CommittedMessage) ? ((messages[previousRunningSummaryIndex] as CommittedMessage)
.runningSummary as string) .runningSummary as string)
: ""; : "";
const messagesSincePreviousRunningSummary = messages.slice(
previousRunningSummaryIndex + 1,
);
/** Save the incoming message to the database. */ /** Save the incoming message to the database. */
const insertedUserMessage: CommittedMessage = { const insertedUserMessage: CommittedMessage = {
id: nanoid(), id: nanoid(),
@ -176,7 +236,7 @@ export const chat = router({
previousRunningSummary, previousRunningSummary,
}), }),
}, },
...messages.slice(previousRunningSummaryIndex + 1), ...messagesSincePreviousRunningSummary,
], ],
maxSteps: 3, maxSteps: 3,
tools: undefined, tools: undefined,
@ -201,7 +261,12 @@ export const chat = router({
previousRunningSummary, previousRunningSummary,
}), }),
}, },
...messages.slice(previousRunningSummaryIndex + 1), {
role: "user" as const,
content: factsFromUserMessageUserPrompt({
messagesSincePreviousRunningSummary,
}),
},
], ],
schema: jsonSchema({ schema: jsonSchema({
type: "object", type: "object",
@ -218,7 +283,7 @@ export const chat = router({
tools: undefined, tools: undefined,
...parameters, ...parameters,
}); });
const insertedFacts: Array<Fact> = const insertedFactsFromUserMessage: Array<Fact> =
factsFromUserMessageResponse.object.facts.map((fact) => ({ factsFromUserMessageResponse.object.facts.map((fact) => ({
id: nanoid(), id: nanoid(),
userId: "1", userId: "1",
@ -226,9 +291,57 @@ export const chat = router({
content: fact, content: fact,
createdAt: new Date().toISOString(), createdAt: new Date().toISOString(),
})); }));
db.data.facts.push(...insertedFacts); db.data.facts.push(...insertedFactsFromUserMessage);
/** Extract Facts from the model's response, and add them to the database, /** Extract Facts from the model's response, and add them to the database,
* linking the Facts with the messages they came from. */ * linking the Facts with the messages they came from. */
const factsFromAssistantMessageResponse = await generateObject<{
facts: Array<string>;
}>({
model: openrouter("mistralai/mistral-nemo"),
messages: [
{
role: "system" as const,
content: factsFromAssistantMessageSystemPrompt({
previousRunningSummary,
}),
},
/** Yes, the next message is a `user` message, because models are
* trained to respond to `user` messages. So we wrap the assistant
* response in XML tags to show that it's not the user speaking,
* rather it's input for the model to process. The user is only
* saying "Extract facts..." */
{
role: "user" as const,
content: factsFromAssistantMessageUserPrompt({
messagesSincePreviousRunningSummary,
mainResponseContent: mainResponse.text,
}),
},
],
schema: jsonSchema({
type: "object",
properties: {
facts: {
type: "array",
items: {
type: "string",
},
},
},
}),
maxSteps: 3,
tools: undefined,
...parameters,
});
const insertedFactsFromAssistantMessage: Array<Fact> =
factsFromAssistantMessageResponse.object.facts.map((factContent) => ({
id: nanoid(),
userId: "1",
sourceMessageId: insertedAssistantMessage.id,
content: factContent,
createdAt: new Date().toISOString(),
}));
db.data.facts.push(...insertedFactsFromAssistantMessage);
/** For each Fact produced in the two fact-extraction steps, generate /** For each Fact produced in the two fact-extraction steps, generate
* FactTriggers and add them to the database, linking the FactTriggers * FactTriggers and add them to the database, linking the FactTriggers
* with the Facts they came from. A FactTrigger is a natural language * with the Facts they came from. A FactTrigger is a natural language
@ -238,60 +351,27 @@ export const chat = router({
* with the model's response to the database. The new running summary is * with the model's response to the database. The new running summary is
* based on the previous running summary combined with the all messages * based on the previous running summary combined with the all messages
* since that summary was produced. */ * since that summary was produced. */
const runningSummaryResponse = previousRunningSummary const runningSummaryResponse = await generateText({
? await generateText({ model: openrouter("mistralai/mistral-nemo"),
model: openrouter("mistralai/mistral-nemo"), messages: [
messages: [ {
{ role: "system" as const,
role: "system" as const, content: runningSummarySystemPrompt({
content: runningSummarySystemPrompt({ previousRunningSummary,
previousRunningSummary, }),
}), },
}, {
...messages.slice(previousRunningSummaryIndex + 1), role: "user" as const,
{ content: runningSummaryUserPrompt({
role: "assistant" as const, messagesSincePreviousRunningSummary,
content: mainResponse.text, mainResponseContent: mainResponse.text,
} as UIMessage, }),
/** I might need this next message, because models are trained to },
* respond when the final message in `messages` is from the `user`, ],
* but in our case it's an `assistant` message, so I'm artificially maxSteps: 3,
* adding a `user` message to the end of the conversation. */ tools: undefined,
{ ...parameters,
role: "user" as const, });
content: "What is the new summary of the conversation?",
} as UIMessage,
],
maxSteps: 3,
tools: undefined,
...parameters,
})
: await generateText({
model: openrouter("mistralai/mistral-nemo"),
messages: [
{
role: "system" as const,
content:
"Given the following messages of a conversation, produce a summary of the conversation.",
},
...messages,
{
role: "assistant" as const,
content: mainResponse.text,
} as UIMessage,
/** I might need this next message, because models are trained to
* respond when the final message in `messages` is from the `user`,
* but in our case it's an `assistant` message, so I'm artificially
* adding a `user` message to the end of the conversation. */
{
role: "user" as const,
content: "What is the new summary of the conversation?",
} as UIMessage,
],
maxSteps: 3,
tools: undefined,
...parameters,
});
const insertedAssistantMessage: CommittedMessage = { const insertedAssistantMessage: CommittedMessage = {
id: nanoid(), id: nanoid(),
conversationId, conversationId,
@ -308,7 +388,10 @@ export const chat = router({
return { return {
insertedAssistantMessage, insertedAssistantMessage,
insertedUserMessage, insertedUserMessage,
insertedFacts, insertedFacts: [
...insertedFactsFromUserMessage,
...insertedFactsFromAssistantMessage,
],
}; };
}, },
), ),

Write Preview
Loading…
Cancel
Save