improve and extract nextDate() function; improve clickhouse-to-lmdbx sync script

server/.gitignore

@@ -24,4 +24,6 @@ dist-ssr
 *.sln
 *.sw?
 
-.env
+.env
+*.db
+*.db-lck

server/src/backtest.ts

@@ -1,13 +1,8 @@
 import { stockDatabase } from "./stockdb.clickhouse.js";
-import { calendarDatabase } from "./calendardb.clickhouse.js";
+import { calendarDatabase } from "./calendardb.optiondb.lmdbx.js";
 import type { CalendarKey } from "./calendardb.interfaces.js";
 import type { Aggregate } from "./interfaces.js";
-
-function nextDate(date: string) {
-	const dateObject = new Date(date);
-	dateObject.setDate(dateObject.getDate() + 1);
-	return dateObject.toISOString().substring(0, 10);
-}
+import { nextDate } from "./lib/util.js";
 
 type BacktestInput = {
 	symbol: string;
@@ -58,7 +53,7 @@ export async function backtest({
 		}
 		// for each minute of that day for which we have a stock candlestick:
 		for (const stockAggregate of stockAggregates) {
-			console.log("Current Time:", new Date(stockAggregate.tsStart));
+			// console.log("Current Time:", new Date(stockAggregate.tsStart));
 			// filter-out calendars that are far-from-the-money (10%)
 			const calendarsNearTheMoney = calendars.filter(
 				({ strike }) =>

server/src/calendardb.lmdbx.ts

@@ -2,13 +2,13 @@ import type { CalendarDatabase } from "./calendardb.interfaces.js";
 import { open } from "lmdbx";
 
 const calendarAggregatesDb = open({
-	path: "/tmp/calendar-aggregates.db",
+	path: "./calendar-aggregates.db",
 	// any options go here, we can turn on compression like this:
 	compression: true,
 });
 
 const calendarExistenceDb = open({
-	path: "/tmp/calendar-existence.db",
+	path: "./calendar-existence.db",
 	// any options go here, we can turn on compression like this:
 	compression: true,
 });

server/src/calendardb.optiondb.lmdbx.ts

@@ -0,0 +1,155 @@
+import { optionContractDatabase } from "./optiondb.lmdbx.js";
+import type { CalendarDatabase } from "./calendardb.interfaces.js";
+
+/** Largest possible key according to the `ordered-binary` (used by lmdbx) docs. */
+const MAXIMUM_KEY = Buffer.from([0xff]);
+
+function makeCalendarDatabase(): CalendarDatabase {
+	const calendarDatabase: Omit<CalendarDatabase, "getCalendars"> = {
+		getKeys: async ({ key: { symbol }, date }) => {
+			const optionContracts = await optionContractDatabase.getOptionContracts({
+				date,
+				key: { symbol },
+			});
+			return optionContracts.flatMap(
+				(frontOptionContract, i, optionContracts) =>
+					optionContracts
+						.filter((_, j) => i !== j)
+						.map((backOptionContract) => ({
+							symbol,
+							frontExpirationDate: frontOptionContract.expirationDate,
+							backExpirationDate: backOptionContract.expirationDate,
+							strike: frontOptionContract.strike,
+							type: frontOptionContract.type,
+						})),
+			);
+		},
+		getAggregates: async ({
+			key: { symbol, frontExpirationDate, backExpirationDate, strike, type },
+			date,
+		}) => {
+			const frontOptionContractAggregates =
+				await optionContractDatabase.getAggregates({
+					date,
+					key: { symbol, expirationDate: frontExpirationDate, strike, type },
+				});
+			const backOptionContractAggregates =
+				await optionContractDatabase.getAggregates({
+					date,
+					key: { symbol, expirationDate: backExpirationDate, strike, type },
+				});
+			const calendarAggregates = [];
+			let i = 0;
+			let j = 0;
+			while (
+				i < frontOptionContractAggregates.length &&
+				j < backOptionContractAggregates.length
+			) {
+				if (
+					frontOptionContractAggregates[i].tsStart ===
+					backOptionContractAggregates[j].tsStart
+				) {
+					calendarAggregates.push({
+						tsStart: frontOptionContractAggregates[i].tsStart,
+						open:
+							backOptionContractAggregates[j].open -
+							frontOptionContractAggregates[i].open,
+						close:
+							backOptionContractAggregates[j].close -
+							frontOptionContractAggregates[i].close,
+						// the high and low are not exactly correct since we don't know if each contract's high and low happened ata the same moment as the other:
+						high:
+							backOptionContractAggregates[j].high -
+							frontOptionContractAggregates[i].high,
+						low:
+							backOptionContractAggregates[j].low -
+							frontOptionContractAggregates[i].low,
+					});
+					i++;
+					j++;
+				} else if (
+					frontOptionContractAggregates[i].tsStart >
+					backOptionContractAggregates[j].tsStart
+				) {
+					j++;
+				} else {
+					i++;
+				}
+			}
+			return calendarAggregates;
+		},
+		insertAggregates: async (aggregates) => {
+			// right now, no-op
+		},
+		getClosingPrice: async ({
+			key: { symbol, strike, type, frontExpirationDate, backExpirationDate },
+		}) => {
+			const startOfLastHourUnix = new Date(
+				`${frontExpirationDate}T00:00:00Z`,
+			).valueOf();
+			const endOfLastHourUnix = startOfLastHourUnix + 3600 * 1000;
+			const frontOptionContractAggregates = (
+				await optionContractDatabase.getAggregates({
+					date: frontExpirationDate,
+					key: { symbol, expirationDate: frontExpirationDate, strike, type },
+				})
+			).filter(
+				({ tsStart }) =>
+					tsStart >= startOfLastHourUnix && tsStart < endOfLastHourUnix,
+			);
+			const backOptionContractAggregates = (
+				await optionContractDatabase.getAggregates({
+					date: frontExpirationDate,
+					key: { symbol, expirationDate: backExpirationDate, strike, type },
+				})
+			).filter(
+				({ tsStart }) =>
+					tsStart >= startOfLastHourUnix && tsStart < endOfLastHourUnix,
+			);
+			let i = 0;
+			let j = 0;
+			let minPrice = 0;
+			while (
+				i < frontOptionContractAggregates.length &&
+				j < backOptionContractAggregates.length
+			) {
+				if (
+					frontOptionContractAggregates[i].tsStart ===
+					backOptionContractAggregates[j].tsStart
+				) {
+					const calendarClosePrice =
+						backOptionContractAggregates[j].close -
+						frontOptionContractAggregates[j].close;
+					if (calendarClosePrice < minPrice || minPrice === 0) {
+						minPrice = calendarClosePrice;
+					}
+					i++;
+					j++;
+				} else if (
+					frontOptionContractAggregates[i].tsStart >
+					backOptionContractAggregates[j].tsStart
+				) {
+					j++;
+				} else {
+					i++;
+				}
+			}
+			return minPrice;
+		},
+		getTargetPriceByProbability: async ({
+			symbol,
+			calendarSpan,
+			strikePercentageFromTheMoney,
+			historicalProbabilityOfSuccess,
+		}) => {
+			return 0.24;
+		},
+	};
+
+	return {
+		...calendarDatabase,
+		getCalendars: calendarDatabase.getKeys,
+	};
+}
+
+export const calendarDatabase: CalendarDatabase = makeCalendarDatabase();

server/src/lib/util.ts

@@ -0,0 +1,6 @@
+export function nextDate(date: string) {
+  const [year, month, day] = date.split('-').map(Number);
+  const nextDay = new Date(Date.UTC(year, month - 1, day + 1));
+  const nextDateString = nextDay.toISOString().substring(0, 10);
+  return nextDateString;
+}

server/src/optiondb.lmdbx.ts

@@ -2,13 +2,13 @@ import type { OptionContractDatabase } from "./optiondb.interfaces.js";
 import { open } from "lmdbx";
 
 const optionContractAggregatesDb = open({
-  path: "/tmp/option-contract-aggregates.db",
+  path: "./option-contract-aggregates.db",
   // any options go here, we can turn on compression like this:
   compression: true,
 });
 
 const optionContractExistenceDb = open({
-  path: "/tmp/option-contract-existence.db",
+  path: "./option-contract-existence.db",
   // any options go here, we can turn on compression like this:
   compression: true,
 });

server/src/scripts/clickhouse-to-lmdbx.ts

@@ -3,47 +3,52 @@ import type { AggregateDatabase } from "../interfaces.js";
 // import { stockDatabase as stockDatabaseLmdbx } from "../stockdb.lmdbx.js";
 import { optionContractDatabase as optionContractDatabaseClickhouse } from "../optiondb.clickhouse.js";
 import { optionContractDatabase as optionContractDatabaseLmdbx } from "../optiondb.lmdbx.js";
-
-function nextDate(date: string) {
-  const dateObject = new Date(date);
-  dateObject.setDate(dateObject.getDate() + 1);
-  return dateObject.toISOString().substring(0, 10);
-}
+import { nextDate } from "../lib/util.js";
 
 async function syncAggregates<T>({
-  from,
-  to,
+  fromDatabase,
+  toDatabase,
   key,
   date,
 }: {
-  from: AggregateDatabase<T>;
-  to: AggregateDatabase<T>;
+  fromDatabase: AggregateDatabase<T>;
+  toDatabase: AggregateDatabase<T>;
   key: T;
   date: string;
 }) {
-  const aggregatesFrom = (await from.getAggregates({ key, date })).map(
+  const aggregatesFrom = (await fromDatabase.getAggregates({ key, date })).map(
     (aggregateWithoutKey) => ({ ...aggregateWithoutKey, key }),
   );
-  await to.insertAggregates(aggregatesFrom);
+  await toDatabase.insertAggregates(aggregatesFrom);
 }
 
 const symbols = ["AMD", "AAPL", "MSFT", "GOOGL", "NFLX", "NVDA"];
 async function run() {
-  const startDate = "2022-02-01";
-  const endDate = "2024-07-15";
+  const startDate = process.argv[2];
+  const endDate = process.argv[3];
+
+  if (!startDate || !endDate) {
+    console.error("Usage: node clickhouse-to-lmdbx.js <startDate> <endDate>");
+    console.error("Dates should be in YYYY-MM-DD format");
+    process.exit(1);
+  }
   for (let date = startDate; date <= endDate; date = nextDate(date)) {
     // const symbols = await stockDatabaseClickhouse.getSymbols({ date });
     for (const symbol of symbols) {
       console.log(date, symbol);
-			const keys = await optionContractDatabaseClickhouse.getKeys({key: {symbol}, date});
-			for(const key of keys){
-				await syncAggregates({
-					from: optionContractDatabaseClickhouse,
-					to: optionContractDatabaseLmdbx,
-					key,
-					date,
-				});
-			}
+      const keys = await optionContractDatabaseClickhouse.getKeys({
+        key: { symbol },
+        date,
+      });
+      for (const key of keys) {
+        // console.log(date, symbol, key.expirationDate, key.strike, key.type);
+        await syncAggregates({
+          fromDatabase: optionContractDatabaseClickhouse,
+          toDatabase: optionContractDatabaseLmdbx,
+          key,
+          date,
+        });
+      }
     }
   }
 }