re-organized code; implemented `getAggregate()` where it was missing

server/src/AggregateDatabase/Calendar/clickhouse.ts

@@ -1,13 +1,13 @@
 import type { CalendarDatabase, CalendarKey } from "./interfaces.js";
 import type { Aggregate } from "../interfaces.js";
-import { query } from "../lib/clickhouse.js";
+import { query } from "../../lib/clickhouse.js";
 
 function makeCalendarDatabase(): CalendarDatabase {
-	const calendarDatabase: Omit<CalendarDatabase, "getCalendars"> = {
+  const calendarDatabase: Omit<CalendarDatabase, "getCalendars"> = {
-		getKeys: async ({ key: { symbol }, date }) => {
+    getKeys: async ({ key: { symbol }, date }) => {
-			const calendarsForSymbolOnDate = await query<
+      const calendarsForSymbolOnDate = await query<
-				Omit<CalendarKey, "symbol">
+        Omit<CalendarKey, "symbol">
-			>(`
+      >(`
 				WITH today_option_contracts AS (
 					SELECT expirationDate, strike, type
 					FROM option_contract_existences
@@ -26,17 +26,66 @@ function makeCalendarDatabase(): CalendarDatabase {
 				AND front_option_contract.expirationDate < back_option_contract.expirationDate
 				`);
 
-			return calendarsForSymbolOnDate.map((calendarWithoutSymbol) => ({
+      return calendarsForSymbolOnDate.map((calendarWithoutSymbol) => ({
-				...calendarWithoutSymbol,
+        ...calendarWithoutSymbol,
-				symbol,
+        symbol,
-			}));
+      }));
-		},
+    },
-		getAggregates: async ({
+    getAggregate: async ({
-			key: { symbol, frontExpirationDate, backExpirationDate, strike, type },
+      key: { symbol, frontExpirationDate, backExpirationDate, strike, type },
-			date,
+      tsStart,
-		}) => {
+    }) => {
-			return (
+      const tsStartString = new Date(tsStart).toISOString();
-				await query<Omit<Aggregate<CalendarKey>, "key">>(`
+      return (
+        await query<Omit<Aggregate<CalendarKey>, "key">>(`
+				WITH front_option_contract_candlestick AS (
+					SELECT
+						tsStart,
+						open,
+						close,
+						high,
+						low
+					FROM option_contract_aggregates
+					WHERE symbol = '${symbol}'
+					AND type = '${type}'
+					AND strike = '${strike}'
+					AND expirationDate = '${frontExpirationDate}'
+					AND tsStart = '${tsStartString}'
+				),
+				back_option_contract_candlestick AS (
+					SELECT
+						tsStart,
+						open,
+						close,
+						high,
+						low
+					FROM option_contract_aggregates
+					WHERE symbol = '${symbol}'
+					AND type = '${type}'
+					AND strike = '${strike}'
+					AND expirationDate = '${backExpirationDate}'
+					AND tsStart = '${tsStartString}'
+				)
+				SELECT
+						toUnixTimestamp(front_option_contract_candlestick.tsStart) as tsStart,
+						back_option_contract_candlestick.open - front_option_contract_candlestick.open as open,
+						back_option_contract_candlestick.close - front_option_contract_candlestick.close as close
+				FROM front_option_contract_candlestick
+				INNER JOIN back_option_contract_candlestick
+				ON front_option_contract_candlestick.tsStart = back_option_contract_candlestick.tsStart
+				ORDER BY front_option_contract_candlestick.tsStart ASC
+			`)
+      ).map((aggregate) => ({
+        ...aggregate,
+        tsStart: aggregate.tsStart * 1000, // unfortunately, `toUnixTimestamp` only returns second-precision
+      }))[0];
+    },
+    getAggregates: async ({
+      key: { symbol, frontExpirationDate, backExpirationDate, strike, type },
+      date,
+    }) => {
+      return (
+        await query<Omit<Aggregate<CalendarKey>, "key">>(`
 				WITH front_option_contract_candlestick AS (
 					SELECT
 						tsStart,
@@ -74,19 +123,19 @@ function makeCalendarDatabase(): CalendarDatabase {
 				ON front_option_contract_candlestick.tsStart = back_option_contract_candlestick.tsStart
 				ORDER BY front_option_contract_candlestick.tsStart ASC
 			`)
-			).map((aggregate) => ({
+      ).map((aggregate) => ({
-				...aggregate,
+        ...aggregate,
-				tsStart: aggregate.tsStart * 1000, // unfortunately, `toUnixTimestamp` only returns second-precision
+        tsStart: aggregate.tsStart * 1000, // unfortunately, `toUnixTimestamp` only returns second-precision
-			}));
+      }));
-		},
+    },
-		insertAggregates: async (aggregates) => {
+    insertAggregates: async (aggregates) => {
-			// no-op: we insert individual option contracts, not calendars
+      // no-op: we insert individual option contracts, not calendars
-		},
+    },
-		getClosingPrice: async ({
+    getClosingPrice: async ({
-			key: { symbol, strike, type, frontExpirationDate, backExpirationDate },
+      key: { symbol, strike, type, frontExpirationDate, backExpirationDate },
-		}) => {
+    }) => {
-			return (
+      return (
-				await query<{ calendarClosingPrice: number }>(`
+        await query<{ calendarClosingPrice: number }>(`
 					WITH front_option_contract_candlestick AS (
 						SELECT
 								tsStart,
@@ -121,22 +170,22 @@ function makeCalendarDatabase(): CalendarDatabase {
 					INNER JOIN back_option_contract_candlestick
 					ON front_option_contract_candlestick.tsStart = back_option_contract_candlestick.tsStart
 				`)
-			)[0]?.calendarClosingPrice;
+      )[0]?.calendarClosingPrice;
-		},
+    },
-		getTargetPriceByProbability: async ({
+    getTargetPriceByProbability: async ({
-			symbol,
+      symbol,
-			calendarSpan,
+      calendarSpan,
-			strikePercentageFromTheMoney,
+      strikePercentageFromTheMoney,
-			historicalProbabilityOfSuccess,
+      historicalProbabilityOfSuccess,
-		}) => {
+    }) => {
-			return 0.24;
+      return 0.24;
-		},
+    },
-	};
+  };
 
-	return {
+  return {
-		...calendarDatabase,
+    ...calendarDatabase,
-		getCalendars: calendarDatabase.getKeys,
+    getCalendars: calendarDatabase.getKeys,
-	};
+  };
 }
 
 export const calendarDatabase: CalendarDatabase = makeCalendarDatabase();

server/src/AggregateDatabase/Calendar/interfaces.ts

@@ -0,0 +1,24 @@
+import type { AggregateDatabase } from "../interfaces.js";
+
+export type CalendarKey = {
+  symbol: string;
+  type: "call" | "put";
+  strike: number;
+  frontExpirationDate: string;
+  backExpirationDate: string;
+};
+
+export type CalendarDatabase = AggregateDatabase<CalendarKey> & {
+  getCalendars: AggregateDatabase<CalendarKey>["getKeys"];
+  getTargetPriceByProbability: ({
+    symbol,
+    calendarSpan,
+    strikePercentageFromTheMoney,
+    historicalProbabilityOfSuccess,
+  }: {
+    symbol: string;
+    calendarSpan: number;
+    strikePercentageFromTheMoney: number;
+    historicalProbabilityOfSuccess: number;
+  }) => Promise<number>;
+};

server/src/AggregateDatabase/Calendar/optiondb-lmdbx.ts

@@ -1,4 +1,4 @@
-import { optionContractDatabase } from "../optiondb/lmdbx.js";
+import { optionContractDatabase } from "../OptionContract/lmdbx.js";
 import type { CalendarDatabase } from "./interfaces.js";
 
 /** Largest possible key according to the `ordered-binary` (used by lmdbx) docs. */

server/src/AggregateDatabase/OptionContract/clickhouse.ts

@@ -3,7 +3,7 @@ import type {
   OptionContractKey,
 } from "./interfaces.js";
 import type { Aggregate } from "../interfaces.js";
-import { clickhouse, query } from "../lib/clickhouse.js";
+import { clickhouse, query } from "../../lib/clickhouse.js";
 
 function makeOptionContractDatabase(): OptionContractDatabase {
   const optionContractDatabase: Omit<
@@ -48,6 +48,31 @@ function makeOptionContractDatabase(): OptionContractDatabase {
         tsStart: aggregate.tsStart * 1000, // unfortunately, `toUnixTimestamp` only returns second-precision
       }));
     },
+    getAggregate: async ({
+      key: { symbol, expirationDate, strike, type },
+      tsStart,
+    }) => {
+      const tsStartString = new Date(tsStart).toISOString();
+      return (
+        await query<Omit<Aggregate<OptionContractKey>, "key">>(`
+          SELECT
+						open,
+						close,
+						high,
+						low
+					FROM option_contract_aggregates
+					WHERE symbol = '${symbol}'
+					AND type = '${type}'
+					AND strike = '${strike}'
+					AND expirationDate = '${expirationDate}'
+					AND tsStart = '${tsStartString}'
+          ORDER BY tsStart ASC
+        `)
+      ).map((aggregate) => ({
+        ...aggregate,
+        tsStart,
+      }))[0];
+    },
     insertAggregates: async (aggregates) => {
       // stock existence is taken care of by clickhouse materialized view
       await clickhouse.insert({
@@ -70,7 +95,7 @@ function makeOptionContractDatabase(): OptionContractDatabase {
             close,
             high,
             low,
-          }),
+          })
         ),
       });
     },

server/src/AggregateDatabase/Stock/clickhouse.ts

@@ -1,6 +1,6 @@
 import type { StockDatabase, StockKey } from "./interfaces.js";
 import type { Aggregate } from "../interfaces.js";
-import { clickhouse, query } from "../lib/clickhouse.js";
+import { clickhouse, query } from "../../lib/clickhouse.js";
 
 function makeStockDatabase(): StockDatabase {
   const stockDatabase: Omit<StockDatabase, "getSymbols"> = {
@@ -31,6 +31,23 @@ function makeStockDatabase(): StockDatabase {
         tsStart: aggregate.tsStart * 1000, // unfortunately, `toUnixTimestamp` only returns second-precision
       }));
     },
+    getAggregate: async ({ key: { symbol }, tsStart }) => {
+      return (
+        await query<Omit<Aggregate<StockKey>, "key">>(`
+          SELECT 
+            open,
+            close,
+            high,
+            low
+          FROM stock_aggregates
+          WHERE symbol = '${symbol}'
+          AND tsStart = '${tsStart}'
+        `)
+      ).map((aggregate) => ({
+        ...aggregate,
+        tsStart,
+      }))[0];
+    },
     insertAggregates: async (aggregates) => {
       // stock existence is taken care of by clickhouse materialized view
       await clickhouse.insert({
@@ -43,7 +60,7 @@ function makeStockDatabase(): StockDatabase {
             close,
             high,
             low,
-          }),
+          })
         ),
       });
     },

server/src/AggregateDatabase/Stock/lmdbx.ts

@@ -45,6 +45,9 @@ function makeStockDatabase(): StockDatabase {
           low: value.low,
         })).asArray;
     },
+    getAggregate: async ({ key: { symbol }, tsStart }) => {
+      return stockAggregatesDb.get([symbol, tsStart]);
+    },
     insertAggregates: async (aggregates) => {
       await stockExistenceDb.batch(() => {
         for (const aggregate of aggregates) {
@@ -53,7 +56,7 @@ function makeStockDatabase(): StockDatabase {
               new Date(aggregate.tsStart).toISOString().substring(0, 10),
               aggregate.key.symbol,
             ],
-            null,
+            null
           );
         }
       });

server/src/backtest.ts

@@ -1,6 +1,6 @@
-import { stockDatabase } from "./stockdb/lmdbx.js";
+import { stockDatabase } from "./AggregateDatabase/Stock/lmdbx.js";
-import { calendarDatabase } from "./calendardb/optiondb-lmdbx.js";
+import { calendarDatabase } from "./AggregateDatabase/Calendar/optiondb-lmdbx.js";
-import type { CalendarKey } from "./calendardb/interfaces.js";
+import type { CalendarKey } from "./AggregateDatabase/Calendar/interfaces.js";
 import { nextDate } from "./lib/utils/nextDate.js";
 
 type BacktestInput = {
@@ -71,9 +71,9 @@ export async function backtest({
           });
         const calendarAggregateAtCurrentTime =
           await calendarDatabase.getAggregate({
-          key: {
+            key: {
-            ...calendar,
+              ...calendar,
-          },
+            },
             tsStart: stockAggregate.tsStart,
           });
         // if there exists a matching calendar candlestick for the current minute:
@@ -141,3 +141,13 @@ export async function backtest({
   console.log("Ending Buying Power:", buyingPower);
   console.log("Portfolio:", portfolio.values());
 }
+
+const args = process.argv.slice(2);
+if (args.length > 0) {
+  if (args.length < 3) {
+    console.error("Please provide at least 3 command line arguments.");
+    process.exit(1);
+  } else {
+    await backtest({ symbol: args[0], startDate: args[1], endDate: args[2] });
+  }
+}

server/src/calendardb/interfaces.ts

@@ -1,24 +0,0 @@
-import type { AggregateDatabase } from "../interfaces.js";
-
-export type CalendarKey = {
-	symbol: string;
-	type: "call" | "put";
-	strike: number;
-	frontExpirationDate: string;
-	backExpirationDate: string;
-};
-
-export type CalendarDatabase = AggregateDatabase<CalendarKey> & {
-	getCalendars: AggregateDatabase<CalendarKey>["getKeys"];
-	getTargetPriceByProbability: ({
-		symbol,
-		calendarSpan,
-		strikePercentageFromTheMoney,
-		historicalProbabilityOfSuccess,
-	}: {
-		symbol: string;
-		calendarSpan: number;
-		strikePercentageFromTheMoney: number;
-		historicalProbabilityOfSuccess: number;
-	}) => Promise<number>;
-};

server/src/scripts/clickhouse-to-lmdbx.ts

@@ -1,4 +1,4 @@
-import type { AggregateDatabase } from "../interfaces.js";
+import type { AggregateDatabase } from "../AggregateDatabase/interfaces.js";
 import { stockDatabase as stockDatabaseClickhouse } from "../stockdb/clickhouse.js";
 import { stockDatabase as stockDatabaseLmdbx } from "../stockdb/lmdbx.js";
 // import { optionContractDatabase as optionContractDatabaseClickhouse } from "../optiondb.clickhouse.js";
@@ -20,7 +20,7 @@ async function syncAggregates<T>({
   date: string;
 }) {
   const aggregatesFrom = (await fromDatabase.getAggregates({ key, date })).map(
-    (aggregateWithoutKey) => ({ ...aggregateWithoutKey, key }),
+    (aggregateWithoutKey) => ({ ...aggregateWithoutKey, key })
   );
   await toDatabase.insertAggregates(aggregatesFrom);
 }
@@ -29,7 +29,10 @@ const symbols = ["AMD", "AAPL", "MSFT", "GOOGL", "NFLX", "NVDA"];
 async function run<T extends StockKey | OptionContractKey>({
   fromDatabase,
   toDatabase,
-}: { fromDatabase: AggregateDatabase<T>; toDatabase: AggregateDatabase<T> }) {
+}: {
+  fromDatabase: AggregateDatabase<T>;
+  toDatabase: AggregateDatabase<T>;
+}) {
   const startDate = process.argv[2];
   const endDate = process.argv[3];
 
@@ -48,7 +51,7 @@ async function run<T extends StockKey | OptionContractKey>({
             key: { symbol } as T,
             date,
           }),
-        { shouldRetry: retryOnTimeout },
+        { shouldRetry: retryOnTimeout }
       );
 
       for (const key of keys) {
@@ -61,7 +64,7 @@ async function run<T extends StockKey | OptionContractKey>({
               key,
               date,
             }),
-          { shouldRetry: retryOnTimeout },
+          { shouldRetry: retryOnTimeout }
         );
       }
     }