@ -2,13 +2,15 @@ import {
router ,
router ,
publicProcedure ,
publicProcedure ,
createCallerFactory ,
createCallerFactory ,
Validator ,
// Validator
} from "../../trpc/server" ; } from "../../trpc/server" ;
import { createOpenRouter } from "@openrouter/ai-sdk-provider" ; import { createOpenRouter } from "@openrouter/ai-sdk-provider" ;
import { generateText } from "ai" ; import { generateText } from "ai" ;
import type { Message as UIMessage } from "ai" ; import type { Message as UIMessage } from "ai" ;
import type { OtherParameters } from "../../types.js" ; import type {
OtherParameters ,
CommittedMessage ,
DraftMessage ,
} from "../../types.js" ;
import { env } from "../../server/env.js" ; import { env } from "../../server/env.js" ;
// import { client } from "../../database/milvus";
// import { client } from "../../database/milvus";
// import {
// import {
@ -55,7 +57,7 @@ export const chat = router({
. mutation ( async ( { input : { id } } ) = > {
. mutation ( async ( { input : { id } } ) = > {
const result = await db
const result = await db
. deleteFrom ( "conversations" )
. deleteFrom ( "conversations" )
. where ( "id" , "=" , id as ConversationsId )
. where ( "id" , "=" , Number ( id ) as ConversationsId )
. execute ( ) ;
. execute ( ) ;
return result ;
return result ;
} ) ,
} ) ,
@ -71,32 +73,200 @@ export const chat = router({
const result = await db
const result = await db
. updateTable ( "conversations" )
. updateTable ( "conversations" )
. set ( { title } )
. set ( { title } )
. where ( "id" , "=" , id as ConversationsId )
. where ( "id" , "=" , Number ( id ) as ConversationsId )
. execute ( ) ;
. execute ( ) ;
return result [ 0 ] ;
return result [ 0 ] ;
} ) ,
} ) ,
fetchMessages : publicProcedure
. input ( ( x ) = > x as { conversationId : number } )
. query ( async ( { input : { conversationId } } ) = > {
const rows = await db
. selectFrom ( "messages" )
. selectAll ( )
. where ( "conversation_id" , "=" , conversationId as ConversationsId )
. execute ( ) ;
return rows . map ( ( row ) = > ( {
. . . row ,
conversationId : conversationId as ConversationsId ,
runningSummary : row.running_summary ,
} ) ) as Array < CommittedMessage > ;
} ) ,
sendMessage : publicProcedure
sendMessage : publicProcedure
. input (
. input (
( x ) = >
( x ) = >
x as {
x as {
messages : Array < UIMessage > ;
conversationId : number ;
messages : Array < DraftMessage | CommittedMessage > ;
systemPrompt : string ;
systemPrompt : string ;
parameters : OtherParameters ;
parameters : OtherParameters ;
} ,
} ,
)
)
. mutation ( async ( { input : { messages , systemPrompt , parameters } } ) = > {
. mutation (
const response = await generateText ( {
async ( {
model : openrouter ( "mistralai/mistral-nemo" ) ,
input : { conversationId , messages , systemPrompt , parameters } ,
messages : [
} ) = > {
{ role : "system" as const , content : systemPrompt } ,
/ * * T O D O : S a v e a l l u n s a v e d m e s s a g e s ( i . e . t h o s e w i t h o u t a n ` i d ` ) t o t h e
. . . messages ,
* database . Is this dangerous ? Can an attacker just send a bunch of
] ,
* messages , omitting the ids , causing me to save a bunch of them to the
maxSteps : 3 ,
* database ? I guess it ' s no worse than starting new converations , which
tools : undefined ,
* anyone can freely do . * /
. . . parameters ,
const previousRunningSummaryIndex = messages . findLastIndex (
} ) ;
( message ) = >
return response ;
typeof ( message as CommittedMessage ) . runningSummary !== "undefined" ,
} ) ,
) ;
const previousRunningSummary =
previousRunningSummaryIndex >= 0
? ( ( messages [ previousRunningSummaryIndex ] as CommittedMessage )
. runningSummary as string )
: "" ;
/** Save the incoming message to the database. */
const insertedUserMessage = await db
. insertInto ( "messages" )
. values ( {
conversation_id : conversationId as ConversationsId ,
content : messages [ messages . length - 1 ] . content ,
role : "user" as const ,
index : messages.length - 1 ,
created_at : new Date ( ) . toISOString ( ) ,
} )
. returning ( [ "id" , "index" ] )
. executeTakeFirst ( ) ;
/ * * G e n e r a t e a n e w m e s s a g e f r o m t h e m o d e l , b u t h o l d - o f f o n a d d i n g i t t o
* the database until we produce the associated running - summary , below .
* The model should be given the conversation summary thus far , and of
* course the user ' s latest message , unmodified . Invite the model to
* create any tools it needs . The tool needs to be implemented in a
* language which this system can execute ; usually an interpretted
* language like Python or JavaScript . * /
const mainResponse = await generateText ( {
model : openrouter ( "mistralai/mistral-nemo" ) ,
messages : [
previousRunningSummary === ""
? { role : "system" as const , content : systemPrompt }
: {
role : "system" as const ,
content : ` ${ systemPrompt }
This is a summary of the conversation so far , from your point - of - view ( so "I" and "me" refer to you ) :
< running_summary >
$ { previousRunningSummary }
< / running_summary >
` ,
} ,
. . . messages . slice ( previousRunningSummaryIndex + 1 ) ,
] ,
maxSteps : 3 ,
tools : undefined ,
. . . parameters ,
} ) ;
console . log ( "sent" , [
previousRunningSummary === ""
? { role : "system" as const , content : systemPrompt }
: {
role : "system" as const ,
content : ` ${ systemPrompt }
This is a summary of the conversation so far , from your point - of - view ( so "I" and "me" refer to you ) :
< running_summary >
$ { previousRunningSummary }
< / running_summary >
` ,
} ,
. . . messages . slice ( previousRunningSummaryIndex + 1 ) ,
] ) ;
/ * * E x t r a c t F a c t s f r o m t h e u s e r ' s m e s s a g e , a n d a d d t h e m t o t h e d a t a b a s e ,
* linking the Facts with the messages they came from . ( Yes , this should
* be done * after * the model response , not before ; because when we run a
* query to find Facts to inject into the context sent to the model , we
* don 't want Facts from the user' s current message to be candidates for
* injection , because we 're sending the user' s message unadulterated to
* the model ; there ' s no reason to inject the same Facts that the model is
* already using to generate its response . ) * /
/ * * E x t r a c t F a c t s f r o m t h e m o d e l ' s r e s p o n s e , a n d a d d t h e m t o t h e d a t a b a s e ,
* linking the Facts with the messages they came from . * /
/ * * F o r e a c h F a c t p r o d u c e d i n t h e t w o f a c t - e x t r a c t i o n s t e p s , g e n e r a t e
* FactTriggers and add them to the database , linking the FactTriggers
* with the Facts they came from . A FactTrigger is a natural language
* phrase that describes a situation in which it would be useful to invoke
* the Fact . ( e . g . , "When food preferences are discussed" ) . * /
/ * * P r o d u c e a r u n n i n g s u m m a r y o f t h e c o n v e r s a t i o n , a n d s a v e t h a t a l o n g
* with the model ' s response to the database . The new running summary is
* based on the previous running summary combined with the all messages
* since that summary was produced . * /
const runningSummaryResponse = previousRunningSummary
? await generateText ( {
model : openrouter ( "mistralai/mistral-nemo" ) ,
messages : [
{
role : "system" as const ,
content : ` Given the following summary of a conversation, coupled with the messages exchanged since that summary was produced, produce a new summary of the conversation.
< running_summary >
$ { previousRunningSummary }
< / running_summary >
` ,
} ,
. . . messages . slice ( previousRunningSummaryIndex + 1 ) ,
{
role : "assistant" as const ,
content : mainResponse.text ,
} as UIMessage ,
/ * * I m i g h t n e e d t h i s n e x t m e s s a g e , b e c a u s e m o d e l s a r e t r a i n e d t o
* respond when the final message in ` messages ` is from the ` user ` ,
* but in our case it 's an `assistant` message, so I' m artificially
* adding a ` user ` message to the end of the conversation . * /
{
role : "user" as const ,
content : "What is the new summary of the conversation?" ,
} as UIMessage ,
] ,
maxSteps : 3 ,
tools : undefined ,
. . . parameters ,
} )
: await generateText ( {
model : openrouter ( "mistralai/mistral-nemo" ) ,
messages : [
{
role : "system" as const ,
content :
"Given the following messages of a conversation, produce a summary of the conversation." ,
} ,
. . . messages ,
{
role : "assistant" as const ,
content : mainResponse.text ,
} as UIMessage ,
/ * * I m i g h t n e e d t h i s n e x t m e s s a g e , b e c a u s e m o d e l s a r e t r a i n e d t o
* respond when the final message in ` messages ` is from the ` user ` ,
* but in our case it 's an `assistant` message, so I' m artificially
* adding a ` user ` message to the end of the conversation . * /
{
role : "user" as const ,
content : "What is the new summary of the conversation?" ,
} as UIMessage ,
] ,
maxSteps : 3 ,
tools : undefined ,
. . . parameters ,
} ) ;
const insertedAssistantMessage = await db
. insertInto ( "messages" )
. values ( {
conversation_id : conversationId as ConversationsId ,
content : mainResponse.text ,
running_summary : runningSummaryResponse.text ,
role : "assistant" as const ,
index : messages.length ,
created_at : new Date ( ) . toISOString ( ) ,
} )
. returningAll ( )
. executeTakeFirst ( ) ;
/ * * T O D O : n o t i f y t h e c a l l e r , s o m e h o w , t h a t s o m e m e s s a g e s w e r e s a v e d t o
* the database and / or were outfitted with runningSummaries , so the
* caller can update its UI state . * /
return { insertedAssistantMessage , insertedUserMessage } ;
} ,
) ,
} ) ; } ) ;
export const createCaller = createCallerFactory ( chat ) ; export const createCaller = createCallerFactory ( chat ) ;
Avraham Sakal
3 months ago