avraham
/
context-efficient-ai
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

abstract application-to-database interface

Browse Source
master
Avraham Sakal 1 month ago
parent 28a0be808a
commit afee7dd8c9
7 changed files with 242 additions and 88 deletions
Show Stats Download Patch File Download Diff File

9
database/common.ts
Unescape Escape View File

@ -0,0 +1,9 @@
export interface Entity<T, ID> {
construct: (data: T) => T;
create: (data: T) => Promise<T>;
createMany: (data: T[]) => Promise<T[]>;
findAll: () => Promise<T[]>;
findById: (id: ID) => Promise<T | undefined>;
update: (id: ID, data: Partial<T>) => Promise<void>;
delete: (id: ID) => Promise<void>;
}

208
database/lowdb.ts
Unescape Escape View File

@ -1,6 +1,8 @@
import { Low } from "lowdb";
import { JSONFile } from "lowdb/node";
import type { CommittedMessage } from "../types";
import type { Entity } from "./common";
import { nanoid } from "nanoid";
export type Conversation = {
id: string;
@ -51,3 +53,209 @@ export const db = new Low<DB>(new JSONFile("db.json"), {
await db.read();
/** Write the database to the file, in case it didn't exist before. */
await db.write();
const conversations: Entity<Conversation, string> & {
fetchMessages: (conversationId: string) => Promise<Array<CommittedMessage>>;
} = {
construct: (conversation: Conversation) => conversation,
create: async (conversation: Conversation) => {
conversation.id = conversation.id ?? nanoid();
await db.data.conversations.push(conversation);
await db.write();
return conversation;
},
createMany: async (conversations: Array<Conversation>) => {
await db.data.conversations.push(...conversations);
await db.write();
return conversations;
},
findAll: async () => {
return db.data.conversations;
},
findById: async (id) => {
return db.data.conversations.find((c) => c.id === id);
},
update: async (id, data: Partial<Conversation>) => {
const conversationIndex = db.data.conversations.findIndex(
(c) => c.id === id
);
if (conversationIndex === -1) throw new Error("Conversation not found");
db.data.conversations[conversationIndex] = {
...db.data.conversations[conversationIndex],
...data,
};
await db.write();
},
delete: async (id) => {
db.data.conversations.splice(
db.data.conversations.findIndex((c) => c.id === id),
1
);
const deletedMessageIds = db.data.messages
.filter((m) => m.conversationId === id)
.map((m) => m.id);
db.data.messages = db.data.messages.filter((m) => m.conversationId !== id);
const deletedFactIds = db.data.facts
.filter((fact) => deletedMessageIds.includes(fact.sourceMessageId))
.map((fact) => fact.id);
db.data.facts = db.data.facts.filter(
(fact) => !deletedFactIds.includes(fact.id)
);
db.data.factTriggers = db.data.factTriggers.filter(
(factTrigger) => !deletedFactIds.includes(factTrigger.sourceFactId)
);
await db.write();
},
fetchMessages: async (conversationId) => {
const rows = await db.data.messages.filter(
(m) => m.conversationId === conversationId
);
return rows as Array<CommittedMessage>;
},
};
const factTriggers: Entity<FactTrigger, string> & {
findByFactId: (factId: string) => Promise<Array<FactTrigger>>;
} = {
construct: (factTrigger: FactTrigger) => factTrigger,
create: async (factTrigger: FactTrigger) => {
factTrigger.id = factTrigger.id ?? nanoid();
await db.data.factTriggers.push(factTrigger);
await db.write();
return factTrigger;
},
createMany: async (factTriggers: Array<FactTrigger>) => {
await db.data.factTriggers.push(...factTriggers);
await db.write();
return factTriggers;
},
findAll: async () => {
return db.data.factTriggers;
},
findById: async (id) => {
return db.data.factTriggers.find((factTrigger) => factTrigger.id === id);
},
update: async (id, data: Partial<FactTrigger>) => {
const factTriggerIndex = db.data.factTriggers.findIndex(
(factTrigger) => factTrigger.id === id
);
if (factTriggerIndex === -1) throw new Error("Fact trigger not found");
db.data.factTriggers[factTriggerIndex] = {
...db.data.factTriggers[factTriggerIndex],
...data,
};
await db.write();
},
delete: async (id) => {
const deletedFactTriggerIndex = db.data.factTriggers.findIndex(
(factTrigger) => factTrigger.id === id
);
if (deletedFactTriggerIndex === -1)
throw new Error("Fact trigger not found");
db.data.factTriggers.splice(deletedFactTriggerIndex, 1);
await db.write();
},
findByFactId: async (factId: string) => {
return db.data.factTriggers.filter(
(factTrigger) => factTrigger.sourceFactId === factId
);
},
};
const facts: Entity<Fact, string> & {
findByConversationId: (conversationId: string) => Promise<Array<Fact>>;
} = {
construct: (fact: Fact) => fact,
create: async (fact: Fact) => {
fact.id = fact.id ?? nanoid();
await db.data.facts.push(fact);
await db.write();
return fact;
},
createMany: async (facts: Array<Fact>) => {
await db.data.facts.push(...facts);
await db.write();
return facts;
},
findAll: async () => {
return db.data.facts;
},
findById: async (id) => {
return db.data.facts.find((fact) => fact.id === id);
},
update: async (id, data: Partial<Fact>) => {
const factIndex = db.data.facts.findIndex((fact) => fact.id === id);
if (factIndex === -1) throw new Error("Fact not found");
db.data.facts[factIndex] = {
...db.data.facts[factIndex],
...data,
};
await db.write();
},
delete: async (id) => {
const deletedFactId = db.data.facts.findIndex((fact) => fact.id === id);
if (deletedFactId === -1) throw new Error("Fact not found");
db.data.facts.splice(deletedFactId, 1);
await db.write();
},
findByConversationId: async (conversationId) => {
const conversationMessageIds = db.data.messages
.filter((m) => m.conversationId === conversationId)
.map((m) => m.id);
const rows = await db.data.facts.filter((f) =>
conversationMessageIds.includes(f.sourceMessageId)
);
return rows as Array<Fact>;
},
};
const messages: Entity<CommittedMessage, string> & {
findByConversationId: (
conversationId: string
) => Promise<Array<CommittedMessage>>;
} = {
construct: (message: CommittedMessage) => message,
create: async (message: CommittedMessage) => {
message.id = message.id ?? nanoid();
await db.data.messages.push(message);
await db.write();
return message;
},
createMany: async (messages: Array<CommittedMessage>) => {
await db.data.messages.push(...messages);
await db.write();
return messages;
},
findAll: async () => {
return db.data.messages;
},
findById: async (id) => {
return db.data.messages.find((m) => m.id === id);
},
update: async (id, data: Partial<CommittedMessage>) => {
const messageIndex = db.data.messages.findIndex((m) => m.id === id);
if (messageIndex === -1) throw new Error("Message not found");
db.data.messages[messageIndex] = {
...db.data.messages[messageIndex],
...data,
};
await db.write();
},
delete: async (id) => {
db.data.messages.splice(
db.data.messages.findIndex((m) => m.id === id),
1
);
await db.write();
},
findByConversationId: async (conversationId) => {
return db.data.messages.filter((m) => m.conversationId === conversationId);
},
};
export const _db = {
conversations,
factTriggers,
facts,
messages,
};

50
pages/chat/conversations.ts
Unescape Escape View File

@ -3,55 +3,29 @@ import {
publicProcedure,
createCallerFactory,
} from "../../trpc/server";
import type { CommittedMessage } from "../../types.js";
import { db } from "../../database/lowdb";
import { nanoid } from "nanoid";
import { _db } from "../../database/lowdb";
export const conversations = router({
fetchAll: publicProcedure.query(async () => {
const rows = await db.data.conversations;
return rows;
return await _db.conversations.findAll();
}),
fetchOne: publicProcedure
.input((x) => x as { id: string })
.query(async ({ input: { id } }) => {
const row = await db.data.conversations.find((c) => c.id === id);
return row;
return await _db.conversations.findById(id);
}),
start: publicProcedure.mutation(async () => {
const title = "New Conversation";
const row = {
id: nanoid(),
title,
id: "",
title: "New Conversation",
userId: "1",
};
await db.data.conversations.push(row);
db.write();
return row;
return await _db.conversations.create(row);
}),
deleteOne: publicProcedure
.input((x) => x as { id: string })
.mutation(async ({ input: { id } }) => {
db.data.conversations.splice(
db.data.conversations.findIndex((c) => c.id === id),
1
);
const deletedMessageIds = db.data.messages
.filter((m) => m.conversationId === id)
.map((m) => m.id);
db.data.messages = db.data.messages.filter(
(m) => m.conversationId !== id
);
const deletedFactIds = db.data.facts
.filter((fact) => deletedMessageIds.includes(fact.sourceMessageId))
.map((fact) => fact.id);
db.data.facts = db.data.facts.filter(
(fact) => !deletedFactIds.includes(fact.id)
);
db.data.factTriggers = db.data.factTriggers.filter(
(factTrigger) => !deletedFactIds.includes(factTrigger.sourceFactId)
);
db.write();
await _db.conversations.delete(id);
return { ok: true };
}),
updateTitle: publicProcedure
@ -63,19 +37,13 @@ export const conversations = router({
}
)
.mutation(async ({ input: { id, title } }) => {
const conversation = await db.data.conversations.find((c) => c.id === id);
if (!conversation) throw new Error("Conversation not found");
conversation.title = title;
db.write();
await _db.conversations.update(id, { title });
return { ok: true };
}),
fetchMessages: publicProcedure
.input((x) => x as { conversationId: string })
.query(async ({ input: { conversationId } }) => {
const rows = await db.data.messages.filter(
(m) => m.conversationId === conversationId
);
return rows as Array<CommittedMessage>;
return await _db.conversations.fetchMessages(conversationId);
}),
});

21
pages/chat/fact-triggers.ts
Unescape Escape View File

@ -3,7 +3,7 @@ import {
publicProcedure,
createCallerFactory,
} from "../../trpc/server.js";
import { db, type Fact } from "../../database/lowdb.js";
import { _db, type Fact } from "../../database/lowdb.js";
import type { DraftMessage } from "../../types.js";
import { openrouter, MODEL_NAME } from "./provider.js";
import { generateObject, generateText, jsonSchema } from "ai";
@ -61,9 +61,7 @@ export const factTriggers = router({
fetchByFactId: publicProcedure
.input((x) => x as { factId: string })
.query(async ({ input: { factId } }) => {
return db.data.factTriggers.filter(
(factTrigger) => factTrigger.sourceFactId === factId
);
return _db.factTriggers.findByFactId(factId);
}),
deleteOne: publicProcedure
.input(
@ -73,13 +71,7 @@ export const factTriggers = router({
}
)
.mutation(async ({ input: { factTriggerId } }) => {
const deletedFactTriggerIndex = db.data.factTriggers.findIndex(
(factTrigger) => factTrigger.id === factTriggerId
);
if (deletedFactTriggerIndex === -1)
throw new Error("Fact trigger not found");
db.data.factTriggers.splice(deletedFactTriggerIndex, 1);
await db.write();
await _db.factTriggers.delete(factTriggerId);
return { ok: true };
}),
update: publicProcedure
@ -91,12 +83,7 @@ export const factTriggers = router({
}
)
.mutation(async ({ input: { factTriggerId, content } }) => {
const factTriggerIndex = db.data.factTriggers.findIndex(
(factTrigger) => factTrigger.id === factTriggerId
);
if (factTriggerIndex === -1) throw new Error("Fact trigger not found");
db.data.factTriggers[factTriggerIndex].content = content;
await db.write();
_db.factTriggers.update(factTriggerId, { content });
return { ok: true };
}),
generateFromFact: publicProcedure

22
pages/chat/facts.ts
Unescape Escape View File

@ -3,7 +3,7 @@ import {
publicProcedure,
createCallerFactory,
} from "../../trpc/server.js";
import { db, type Fact } from "../../database/lowdb.js";
import { _db } from "../../database/lowdb.js";
import type { DraftMessage } from "../../types.js";
import { MODEL_NAME, openrouter } from "./provider.js";
import { generateObject, generateText, jsonSchema } from "ai";
@ -57,13 +57,7 @@ export const facts = router({
fetchByConversationId: publicProcedure
.input((x) => x as { conversationId: string })
.query(async ({ input: { conversationId } }) => {
const conversationMessageIds = db.data.messages
.filter((m) => m.conversationId === conversationId)
.map((m) => m.id);
const rows = await db.data.facts.filter((f) =>
conversationMessageIds.includes(f.sourceMessageId)
);
return rows as Array<Fact>;
return await _db.facts.findByConversationId(conversationId);
}),
deleteOne: publicProcedure
.input(
@ -73,12 +67,7 @@ export const facts = router({
}
)
.mutation(async ({ input: { factId } }) => {
const deletedFactId = db.data.facts.findIndex(
(fact) => fact.id === factId
);
if (deletedFactId === -1) throw new Error("Fact not found");
db.data.facts.splice(deletedFactId, 1);
db.write();
await _db.facts.delete(factId);
return { ok: true };
}),
update: publicProcedure
@ -90,10 +79,7 @@ export const facts = router({
}
)
.mutation(async ({ input: { factId, content } }) => {
const factIndex = db.data.facts.findIndex((fact) => fact.id === factId);
if (factIndex === -1) throw new Error("Fact not found");
db.data.facts[factIndex].content = content;
await db.write();
await _db.facts.update(factId, { content });
return { ok: true };
}),
extractFromNewMessages: publicProcedure

7
pages/chat/messages.ts
Unescape Escape View File

@ -3,10 +3,10 @@ import {
publicProcedure,
createCallerFactory,
} from "../../trpc/server";
import { createCaller as createConversationsCaller } from "./conversations.js";
import { MODEL_NAME, openrouter } from "./provider.js";
import { generateObject, generateText, jsonSchema } from "ai";
import type { DraftMessage } from "../../types.js";
import { _db } from "../../database/lowdb";
const runningSummarySystemPrompt = ({
previousRunningSummary,
@ -52,10 +52,7 @@ export const messages = router({
fetchByConversationId: publicProcedure
.input((x) => x as { conversationId: string })
.query(async ({ input: { conversationId } }) => {
const caller = createConversationsCaller({});
return await caller.fetchMessages({
conversationId,
});
return await _db.messages.findByConversationId(conversationId);
}),
generateRunningSummary: publicProcedure
.input(

13
pages/chat/trpc.ts
Unescape Escape View File

@ -14,7 +14,7 @@ import type {
// ConsistencyLevelEnum,
// type NumberArrayId,
// } from "@zilliz/milvus2-sdk-node";
import { db, type FactTrigger, type Fact } from "../../database/lowdb.js";
import { db, type FactTrigger, type Fact, _db } from "../../database/lowdb.js";
import { nanoid } from "nanoid";
import { conversations } from "./conversations.js";
import { messages } from "./messages.js";
@ -105,8 +105,7 @@ export const chat = router({
index: messages.length - 1,
createdAt: new Date().toISOString(),
};
db.data.messages.push(insertedUserMessage);
// do not db.write() until the end
await _db.messages.create(insertedUserMessage);
/** Generate a new message from the model, but hold-off on adding it to
* the database until we produce the associated running-summary, below.
@ -163,7 +162,7 @@ export const chat = router({
content: fact,
createdAt: new Date().toISOString(),
}));
db.data.facts.push(...insertedFactsFromUserMessage);
_db.facts.createMany(insertedFactsFromUserMessage);
/** Produce a running summary of the conversation, and save that along
* with the model's response to the database. The new running summary is
@ -185,7 +184,7 @@ export const chat = router({
index: messages.length,
createdAt: new Date().toISOString(),
};
db.data.messages.push(insertedAssistantMessage);
await _db.messages.create(insertedAssistantMessage);
/** Extract Facts from the model's response, and add them to the database,
* linking the Facts with the messages they came from. */
const factsFromAssistantMessageResponse =
@ -209,7 +208,7 @@ export const chat = router({
content: factContent,
createdAt: new Date().toISOString(),
}));
db.data.facts.push(...insertedFactsFromAssistantMessage);
_db.facts.createMany(insertedFactsFromAssistantMessage);
const insertedFacts = [
...insertedFactsFromUserMessage,
@ -238,7 +237,7 @@ export const chat = router({
scopeConversationId: conversationId,
createdAt: new Date().toISOString(),
}));
db.data.factTriggers.push(...insertedFactTriggers);
_db.factTriggers.createMany(insertedFactTriggers);
}
await db.write();

Write Preview
Loading…
Cancel
Save