From d6762fdae5413ef9f73bdf486028d8bef9b936f1 Mon Sep 17 00:00:00 2001 From: avraham Date: Sun, 11 Aug 2024 18:08:54 -0400 Subject: [PATCH] re-organized code; implemented `getAggregate()` where it was missing --- .../Calendar}/clickhouse.ts | 139 ++++++++++++------ .../AggregateDatabase/Calendar/interfaces.ts | 24 +++ .../Calendar}/lmdbx.ts | 0 .../Calendar}/optiondb-lmdbx.ts | 2 +- .../OptionContract}/clickhouse.ts | 29 +++- .../OptionContract}/interfaces.ts | 0 .../OptionContract}/lmdbx.ts | 0 .../Stock}/clickhouse.ts | 21 ++- .../Stock}/interfaces.ts | 0 .../Stock}/lmdbx.ts | 5 +- .../src/{ => AggregateDatabase}/interfaces.ts | 0 server/src/backtest.ts | 22 ++- server/src/calendardb/interfaces.ts | 24 --- server/src/scripts/clickhouse-to-lmdbx.ts | 13 +- 14 files changed, 193 insertions(+), 86 deletions(-) rename server/src/{calendardb => AggregateDatabase/Calendar}/clickhouse.ts (52%) create mode 100644 server/src/AggregateDatabase/Calendar/interfaces.ts rename server/src/{calendardb => AggregateDatabase/Calendar}/lmdbx.ts (100%) rename server/src/{calendardb => AggregateDatabase/Calendar}/optiondb-lmdbx.ts (99%) rename server/src/{optiondb => AggregateDatabase/OptionContract}/clickhouse.ts (76%) rename server/src/{optiondb => AggregateDatabase/OptionContract}/interfaces.ts (100%) rename server/src/{optiondb => AggregateDatabase/OptionContract}/lmdbx.ts (100%) rename server/src/{stockdb => AggregateDatabase/Stock}/clickhouse.ts (78%) rename server/src/{stockdb => AggregateDatabase/Stock}/interfaces.ts (100%) rename server/src/{stockdb => AggregateDatabase/Stock}/lmdbx.ts (94%) rename server/src/{ => AggregateDatabase}/interfaces.ts (100%) delete mode 100644 server/src/calendardb/interfaces.ts diff --git a/server/src/calendardb/clickhouse.ts b/server/src/AggregateDatabase/Calendar/clickhouse.ts similarity index 52% rename from server/src/calendardb/clickhouse.ts rename to server/src/AggregateDatabase/Calendar/clickhouse.ts index 5616f89..c32b344 100644 --- a/server/src/calendardb/clickhouse.ts +++ b/server/src/AggregateDatabase/Calendar/clickhouse.ts @@ -1,13 +1,13 @@ import type { CalendarDatabase, CalendarKey } from "./interfaces.js"; import type { Aggregate } from "../interfaces.js"; -import { query } from "../lib/clickhouse.js"; +import { query } from "../../lib/clickhouse.js"; function makeCalendarDatabase(): CalendarDatabase { - const calendarDatabase: Omit = { - getKeys: async ({ key: { symbol }, date }) => { - const calendarsForSymbolOnDate = await query< - Omit - >(` + const calendarDatabase: Omit = { + getKeys: async ({ key: { symbol }, date }) => { + const calendarsForSymbolOnDate = await query< + Omit + >(` WITH today_option_contracts AS ( SELECT expirationDate, strike, type FROM option_contract_existences @@ -26,17 +26,66 @@ function makeCalendarDatabase(): CalendarDatabase { AND front_option_contract.expirationDate < back_option_contract.expirationDate `); - return calendarsForSymbolOnDate.map((calendarWithoutSymbol) => ({ - ...calendarWithoutSymbol, - symbol, - })); - }, - getAggregates: async ({ - key: { symbol, frontExpirationDate, backExpirationDate, strike, type }, - date, - }) => { - return ( - await query, "key">>(` + return calendarsForSymbolOnDate.map((calendarWithoutSymbol) => ({ + ...calendarWithoutSymbol, + symbol, + })); + }, + getAggregate: async ({ + key: { symbol, frontExpirationDate, backExpirationDate, strike, type }, + tsStart, + }) => { + const tsStartString = new Date(tsStart).toISOString(); + return ( + await query, "key">>(` + WITH front_option_contract_candlestick AS ( + SELECT + tsStart, + open, + close, + high, + low + FROM option_contract_aggregates + WHERE symbol = '${symbol}' + AND type = '${type}' + AND strike = '${strike}' + AND expirationDate = '${frontExpirationDate}' + AND tsStart = '${tsStartString}' + ), + back_option_contract_candlestick AS ( + SELECT + tsStart, + open, + close, + high, + low + FROM option_contract_aggregates + WHERE symbol = '${symbol}' + AND type = '${type}' + AND strike = '${strike}' + AND expirationDate = '${backExpirationDate}' + AND tsStart = '${tsStartString}' + ) + SELECT + toUnixTimestamp(front_option_contract_candlestick.tsStart) as tsStart, + back_option_contract_candlestick.open - front_option_contract_candlestick.open as open, + back_option_contract_candlestick.close - front_option_contract_candlestick.close as close + FROM front_option_contract_candlestick + INNER JOIN back_option_contract_candlestick + ON front_option_contract_candlestick.tsStart = back_option_contract_candlestick.tsStart + ORDER BY front_option_contract_candlestick.tsStart ASC + `) + ).map((aggregate) => ({ + ...aggregate, + tsStart: aggregate.tsStart * 1000, // unfortunately, `toUnixTimestamp` only returns second-precision + }))[0]; + }, + getAggregates: async ({ + key: { symbol, frontExpirationDate, backExpirationDate, strike, type }, + date, + }) => { + return ( + await query, "key">>(` WITH front_option_contract_candlestick AS ( SELECT tsStart, @@ -74,19 +123,19 @@ function makeCalendarDatabase(): CalendarDatabase { ON front_option_contract_candlestick.tsStart = back_option_contract_candlestick.tsStart ORDER BY front_option_contract_candlestick.tsStart ASC `) - ).map((aggregate) => ({ - ...aggregate, - tsStart: aggregate.tsStart * 1000, // unfortunately, `toUnixTimestamp` only returns second-precision - })); - }, - insertAggregates: async (aggregates) => { - // no-op: we insert individual option contracts, not calendars - }, - getClosingPrice: async ({ - key: { symbol, strike, type, frontExpirationDate, backExpirationDate }, - }) => { - return ( - await query<{ calendarClosingPrice: number }>(` + ).map((aggregate) => ({ + ...aggregate, + tsStart: aggregate.tsStart * 1000, // unfortunately, `toUnixTimestamp` only returns second-precision + })); + }, + insertAggregates: async (aggregates) => { + // no-op: we insert individual option contracts, not calendars + }, + getClosingPrice: async ({ + key: { symbol, strike, type, frontExpirationDate, backExpirationDate }, + }) => { + return ( + await query<{ calendarClosingPrice: number }>(` WITH front_option_contract_candlestick AS ( SELECT tsStart, @@ -121,22 +170,22 @@ function makeCalendarDatabase(): CalendarDatabase { INNER JOIN back_option_contract_candlestick ON front_option_contract_candlestick.tsStart = back_option_contract_candlestick.tsStart `) - )[0]?.calendarClosingPrice; - }, - getTargetPriceByProbability: async ({ - symbol, - calendarSpan, - strikePercentageFromTheMoney, - historicalProbabilityOfSuccess, - }) => { - return 0.24; - }, - }; + )[0]?.calendarClosingPrice; + }, + getTargetPriceByProbability: async ({ + symbol, + calendarSpan, + strikePercentageFromTheMoney, + historicalProbabilityOfSuccess, + }) => { + return 0.24; + }, + }; - return { - ...calendarDatabase, - getCalendars: calendarDatabase.getKeys, - }; + return { + ...calendarDatabase, + getCalendars: calendarDatabase.getKeys, + }; } export const calendarDatabase: CalendarDatabase = makeCalendarDatabase(); diff --git a/server/src/AggregateDatabase/Calendar/interfaces.ts b/server/src/AggregateDatabase/Calendar/interfaces.ts new file mode 100644 index 0000000..feee88a --- /dev/null +++ b/server/src/AggregateDatabase/Calendar/interfaces.ts @@ -0,0 +1,24 @@ +import type { AggregateDatabase } from "../interfaces.js"; + +export type CalendarKey = { + symbol: string; + type: "call" | "put"; + strike: number; + frontExpirationDate: string; + backExpirationDate: string; +}; + +export type CalendarDatabase = AggregateDatabase & { + getCalendars: AggregateDatabase["getKeys"]; + getTargetPriceByProbability: ({ + symbol, + calendarSpan, + strikePercentageFromTheMoney, + historicalProbabilityOfSuccess, + }: { + symbol: string; + calendarSpan: number; + strikePercentageFromTheMoney: number; + historicalProbabilityOfSuccess: number; + }) => Promise; +}; diff --git a/server/src/calendardb/lmdbx.ts b/server/src/AggregateDatabase/Calendar/lmdbx.ts similarity index 100% rename from server/src/calendardb/lmdbx.ts rename to server/src/AggregateDatabase/Calendar/lmdbx.ts diff --git a/server/src/calendardb/optiondb-lmdbx.ts b/server/src/AggregateDatabase/Calendar/optiondb-lmdbx.ts similarity index 99% rename from server/src/calendardb/optiondb-lmdbx.ts rename to server/src/AggregateDatabase/Calendar/optiondb-lmdbx.ts index 5fbc747..bcf650e 100644 --- a/server/src/calendardb/optiondb-lmdbx.ts +++ b/server/src/AggregateDatabase/Calendar/optiondb-lmdbx.ts @@ -1,4 +1,4 @@ -import { optionContractDatabase } from "../optiondb/lmdbx.js"; +import { optionContractDatabase } from "../OptionContract/lmdbx.js"; import type { CalendarDatabase } from "./interfaces.js"; /** Largest possible key according to the `ordered-binary` (used by lmdbx) docs. */ diff --git a/server/src/optiondb/clickhouse.ts b/server/src/AggregateDatabase/OptionContract/clickhouse.ts similarity index 76% rename from server/src/optiondb/clickhouse.ts rename to server/src/AggregateDatabase/OptionContract/clickhouse.ts index beacd04..b331f10 100644 --- a/server/src/optiondb/clickhouse.ts +++ b/server/src/AggregateDatabase/OptionContract/clickhouse.ts @@ -3,7 +3,7 @@ import type { OptionContractKey, } from "./interfaces.js"; import type { Aggregate } from "../interfaces.js"; -import { clickhouse, query } from "../lib/clickhouse.js"; +import { clickhouse, query } from "../../lib/clickhouse.js"; function makeOptionContractDatabase(): OptionContractDatabase { const optionContractDatabase: Omit< @@ -48,6 +48,31 @@ function makeOptionContractDatabase(): OptionContractDatabase { tsStart: aggregate.tsStart * 1000, // unfortunately, `toUnixTimestamp` only returns second-precision })); }, + getAggregate: async ({ + key: { symbol, expirationDate, strike, type }, + tsStart, + }) => { + const tsStartString = new Date(tsStart).toISOString(); + return ( + await query, "key">>(` + SELECT + open, + close, + high, + low + FROM option_contract_aggregates + WHERE symbol = '${symbol}' + AND type = '${type}' + AND strike = '${strike}' + AND expirationDate = '${expirationDate}' + AND tsStart = '${tsStartString}' + ORDER BY tsStart ASC + `) + ).map((aggregate) => ({ + ...aggregate, + tsStart, + }))[0]; + }, insertAggregates: async (aggregates) => { // stock existence is taken care of by clickhouse materialized view await clickhouse.insert({ @@ -70,7 +95,7 @@ function makeOptionContractDatabase(): OptionContractDatabase { close, high, low, - }), + }) ), }); }, diff --git a/server/src/optiondb/interfaces.ts b/server/src/AggregateDatabase/OptionContract/interfaces.ts similarity index 100% rename from server/src/optiondb/interfaces.ts rename to server/src/AggregateDatabase/OptionContract/interfaces.ts diff --git a/server/src/optiondb/lmdbx.ts b/server/src/AggregateDatabase/OptionContract/lmdbx.ts similarity index 100% rename from server/src/optiondb/lmdbx.ts rename to server/src/AggregateDatabase/OptionContract/lmdbx.ts diff --git a/server/src/stockdb/clickhouse.ts b/server/src/AggregateDatabase/Stock/clickhouse.ts similarity index 78% rename from server/src/stockdb/clickhouse.ts rename to server/src/AggregateDatabase/Stock/clickhouse.ts index 121c708..10f54c8 100644 --- a/server/src/stockdb/clickhouse.ts +++ b/server/src/AggregateDatabase/Stock/clickhouse.ts @@ -1,6 +1,6 @@ import type { StockDatabase, StockKey } from "./interfaces.js"; import type { Aggregate } from "../interfaces.js"; -import { clickhouse, query } from "../lib/clickhouse.js"; +import { clickhouse, query } from "../../lib/clickhouse.js"; function makeStockDatabase(): StockDatabase { const stockDatabase: Omit = { @@ -31,6 +31,23 @@ function makeStockDatabase(): StockDatabase { tsStart: aggregate.tsStart * 1000, // unfortunately, `toUnixTimestamp` only returns second-precision })); }, + getAggregate: async ({ key: { symbol }, tsStart }) => { + return ( + await query, "key">>(` + SELECT + open, + close, + high, + low + FROM stock_aggregates + WHERE symbol = '${symbol}' + AND tsStart = '${tsStart}' + `) + ).map((aggregate) => ({ + ...aggregate, + tsStart, + }))[0]; + }, insertAggregates: async (aggregates) => { // stock existence is taken care of by clickhouse materialized view await clickhouse.insert({ @@ -43,7 +60,7 @@ function makeStockDatabase(): StockDatabase { close, high, low, - }), + }) ), }); }, diff --git a/server/src/stockdb/interfaces.ts b/server/src/AggregateDatabase/Stock/interfaces.ts similarity index 100% rename from server/src/stockdb/interfaces.ts rename to server/src/AggregateDatabase/Stock/interfaces.ts diff --git a/server/src/stockdb/lmdbx.ts b/server/src/AggregateDatabase/Stock/lmdbx.ts similarity index 94% rename from server/src/stockdb/lmdbx.ts rename to server/src/AggregateDatabase/Stock/lmdbx.ts index 61cfe30..30fe917 100644 --- a/server/src/stockdb/lmdbx.ts +++ b/server/src/AggregateDatabase/Stock/lmdbx.ts @@ -45,6 +45,9 @@ function makeStockDatabase(): StockDatabase { low: value.low, })).asArray; }, + getAggregate: async ({ key: { symbol }, tsStart }) => { + return stockAggregatesDb.get([symbol, tsStart]); + }, insertAggregates: async (aggregates) => { await stockExistenceDb.batch(() => { for (const aggregate of aggregates) { @@ -53,7 +56,7 @@ function makeStockDatabase(): StockDatabase { new Date(aggregate.tsStart).toISOString().substring(0, 10), aggregate.key.symbol, ], - null, + null ); } }); diff --git a/server/src/interfaces.ts b/server/src/AggregateDatabase/interfaces.ts similarity index 100% rename from server/src/interfaces.ts rename to server/src/AggregateDatabase/interfaces.ts diff --git a/server/src/backtest.ts b/server/src/backtest.ts index 98572aa..1f41261 100644 --- a/server/src/backtest.ts +++ b/server/src/backtest.ts @@ -1,6 +1,6 @@ -import { stockDatabase } from "./stockdb/lmdbx.js"; -import { calendarDatabase } from "./calendardb/optiondb-lmdbx.js"; -import type { CalendarKey } from "./calendardb/interfaces.js"; +import { stockDatabase } from "./AggregateDatabase/Stock/lmdbx.js"; +import { calendarDatabase } from "./AggregateDatabase/Calendar/optiondb-lmdbx.js"; +import type { CalendarKey } from "./AggregateDatabase/Calendar/interfaces.js"; import { nextDate } from "./lib/utils/nextDate.js"; type BacktestInput = { @@ -71,9 +71,9 @@ export async function backtest({ }); const calendarAggregateAtCurrentTime = await calendarDatabase.getAggregate({ - key: { - ...calendar, - }, + key: { + ...calendar, + }, tsStart: stockAggregate.tsStart, }); // if there exists a matching calendar candlestick for the current minute: @@ -141,3 +141,13 @@ export async function backtest({ console.log("Ending Buying Power:", buyingPower); console.log("Portfolio:", portfolio.values()); } + +const args = process.argv.slice(2); +if (args.length > 0) { + if (args.length < 3) { + console.error("Please provide at least 3 command line arguments."); + process.exit(1); + } else { + await backtest({ symbol: args[0], startDate: args[1], endDate: args[2] }); + } +} diff --git a/server/src/calendardb/interfaces.ts b/server/src/calendardb/interfaces.ts deleted file mode 100644 index 26110da..0000000 --- a/server/src/calendardb/interfaces.ts +++ /dev/null @@ -1,24 +0,0 @@ -import type { AggregateDatabase } from "../interfaces.js"; - -export type CalendarKey = { - symbol: string; - type: "call" | "put"; - strike: number; - frontExpirationDate: string; - backExpirationDate: string; -}; - -export type CalendarDatabase = AggregateDatabase & { - getCalendars: AggregateDatabase["getKeys"]; - getTargetPriceByProbability: ({ - symbol, - calendarSpan, - strikePercentageFromTheMoney, - historicalProbabilityOfSuccess, - }: { - symbol: string; - calendarSpan: number; - strikePercentageFromTheMoney: number; - historicalProbabilityOfSuccess: number; - }) => Promise; -}; diff --git a/server/src/scripts/clickhouse-to-lmdbx.ts b/server/src/scripts/clickhouse-to-lmdbx.ts index 6d83243..8687d65 100644 --- a/server/src/scripts/clickhouse-to-lmdbx.ts +++ b/server/src/scripts/clickhouse-to-lmdbx.ts @@ -1,4 +1,4 @@ -import type { AggregateDatabase } from "../interfaces.js"; +import type { AggregateDatabase } from "../AggregateDatabase/interfaces.js"; import { stockDatabase as stockDatabaseClickhouse } from "../stockdb/clickhouse.js"; import { stockDatabase as stockDatabaseLmdbx } from "../stockdb/lmdbx.js"; // import { optionContractDatabase as optionContractDatabaseClickhouse } from "../optiondb.clickhouse.js"; @@ -20,7 +20,7 @@ async function syncAggregates({ date: string; }) { const aggregatesFrom = (await fromDatabase.getAggregates({ key, date })).map( - (aggregateWithoutKey) => ({ ...aggregateWithoutKey, key }), + (aggregateWithoutKey) => ({ ...aggregateWithoutKey, key }) ); await toDatabase.insertAggregates(aggregatesFrom); } @@ -29,7 +29,10 @@ const symbols = ["AMD", "AAPL", "MSFT", "GOOGL", "NFLX", "NVDA"]; async function run({ fromDatabase, toDatabase, -}: { fromDatabase: AggregateDatabase; toDatabase: AggregateDatabase }) { +}: { + fromDatabase: AggregateDatabase; + toDatabase: AggregateDatabase; +}) { const startDate = process.argv[2]; const endDate = process.argv[3]; @@ -48,7 +51,7 @@ async function run({ key: { symbol } as T, date, }), - { shouldRetry: retryOnTimeout }, + { shouldRetry: retryOnTimeout } ); for (const key of keys) { @@ -61,7 +64,7 @@ async function run({ key, date, }), - { shouldRetry: retryOnTimeout }, + { shouldRetry: retryOnTimeout } ); } }