From bf094de46105030c73db16c32a2d9a8597278e1b Mon Sep 17 00:00:00 2001 From: avraham Date: Fri, 2 Aug 2024 16:41:42 -0400 Subject: [PATCH] improve and extract `nextDate()` function; improve clickhouse-to-lmdbx sync script --- server/src/backtest.ts | 11 +- server/src/calendardb.optiondb.lmdbx.ts | 155 ++++++++++++++++++++++ server/src/lib/util.ts | 6 + server/src/scripts/clickhouse-to-lmdbx.ts | 51 +++---- 4 files changed, 192 insertions(+), 31 deletions(-) create mode 100644 server/src/calendardb.optiondb.lmdbx.ts create mode 100644 server/src/lib/util.ts diff --git a/server/src/backtest.ts b/server/src/backtest.ts index b6633cb..4337abe 100644 --- a/server/src/backtest.ts +++ b/server/src/backtest.ts @@ -1,13 +1,8 @@ import { stockDatabase } from "./stockdb.clickhouse.js"; -import { calendarDatabase } from "./calendardb.clickhouse.js"; +import { calendarDatabase } from "./calendardb.optiondb.lmdbx.js"; import type { CalendarKey } from "./calendardb.interfaces.js"; import type { Aggregate } from "./interfaces.js"; - -function nextDate(date: string) { - const dateObject = new Date(date); - dateObject.setDate(dateObject.getDate() + 1); - return dateObject.toISOString().substring(0, 10); -} +import { nextDate } from "./lib/util.js"; type BacktestInput = { symbol: string; @@ -58,7 +53,7 @@ export async function backtest({ } // for each minute of that day for which we have a stock candlestick: for (const stockAggregate of stockAggregates) { - console.log("Current Time:", new Date(stockAggregate.tsStart)); + // console.log("Current Time:", new Date(stockAggregate.tsStart)); // filter-out calendars that are far-from-the-money (10%) const calendarsNearTheMoney = calendars.filter( ({ strike }) => diff --git a/server/src/calendardb.optiondb.lmdbx.ts b/server/src/calendardb.optiondb.lmdbx.ts new file mode 100644 index 0000000..ac9ee04 --- /dev/null +++ b/server/src/calendardb.optiondb.lmdbx.ts @@ -0,0 +1,155 @@ +import { optionContractDatabase } from "./optiondb.lmdbx.js"; +import type { CalendarDatabase } from "./calendardb.interfaces.js"; + +/** Largest possible key according to the `ordered-binary` (used by lmdbx) docs. */ +const MAXIMUM_KEY = Buffer.from([0xff]); + +function makeCalendarDatabase(): CalendarDatabase { + const calendarDatabase: Omit = { + getKeys: async ({ key: { symbol }, date }) => { + const optionContracts = await optionContractDatabase.getOptionContracts({ + date, + key: { symbol }, + }); + return optionContracts.flatMap( + (frontOptionContract, i, optionContracts) => + optionContracts + .filter((_, j) => i !== j) + .map((backOptionContract) => ({ + symbol, + frontExpirationDate: frontOptionContract.expirationDate, + backExpirationDate: backOptionContract.expirationDate, + strike: frontOptionContract.strike, + type: frontOptionContract.type, + })), + ); + }, + getAggregates: async ({ + key: { symbol, frontExpirationDate, backExpirationDate, strike, type }, + date, + }) => { + const frontOptionContractAggregates = + await optionContractDatabase.getAggregates({ + date, + key: { symbol, expirationDate: frontExpirationDate, strike, type }, + }); + const backOptionContractAggregates = + await optionContractDatabase.getAggregates({ + date, + key: { symbol, expirationDate: backExpirationDate, strike, type }, + }); + const calendarAggregates = []; + let i = 0; + let j = 0; + while ( + i < frontOptionContractAggregates.length && + j < backOptionContractAggregates.length + ) { + if ( + frontOptionContractAggregates[i].tsStart === + backOptionContractAggregates[j].tsStart + ) { + calendarAggregates.push({ + tsStart: frontOptionContractAggregates[i].tsStart, + open: + backOptionContractAggregates[j].open - + frontOptionContractAggregates[i].open, + close: + backOptionContractAggregates[j].close - + frontOptionContractAggregates[i].close, + // the high and low are not exactly correct since we don't know if each contract's high and low happened ata the same moment as the other: + high: + backOptionContractAggregates[j].high - + frontOptionContractAggregates[i].high, + low: + backOptionContractAggregates[j].low - + frontOptionContractAggregates[i].low, + }); + i++; + j++; + } else if ( + frontOptionContractAggregates[i].tsStart > + backOptionContractAggregates[j].tsStart + ) { + j++; + } else { + i++; + } + } + return calendarAggregates; + }, + insertAggregates: async (aggregates) => { + // right now, no-op + }, + getClosingPrice: async ({ + key: { symbol, strike, type, frontExpirationDate, backExpirationDate }, + }) => { + const startOfLastHourUnix = new Date( + `${frontExpirationDate}T00:00:00Z`, + ).valueOf(); + const endOfLastHourUnix = startOfLastHourUnix + 3600 * 1000; + const frontOptionContractAggregates = ( + await optionContractDatabase.getAggregates({ + date: frontExpirationDate, + key: { symbol, expirationDate: frontExpirationDate, strike, type }, + }) + ).filter( + ({ tsStart }) => + tsStart >= startOfLastHourUnix && tsStart < endOfLastHourUnix, + ); + const backOptionContractAggregates = ( + await optionContractDatabase.getAggregates({ + date: frontExpirationDate, + key: { symbol, expirationDate: backExpirationDate, strike, type }, + }) + ).filter( + ({ tsStart }) => + tsStart >= startOfLastHourUnix && tsStart < endOfLastHourUnix, + ); + let i = 0; + let j = 0; + let minPrice = 0; + while ( + i < frontOptionContractAggregates.length && + j < backOptionContractAggregates.length + ) { + if ( + frontOptionContractAggregates[i].tsStart === + backOptionContractAggregates[j].tsStart + ) { + const calendarClosePrice = + backOptionContractAggregates[j].close - + frontOptionContractAggregates[j].close; + if (calendarClosePrice < minPrice || minPrice === 0) { + minPrice = calendarClosePrice; + } + i++; + j++; + } else if ( + frontOptionContractAggregates[i].tsStart > + backOptionContractAggregates[j].tsStart + ) { + j++; + } else { + i++; + } + } + return minPrice; + }, + getTargetPriceByProbability: async ({ + symbol, + calendarSpan, + strikePercentageFromTheMoney, + historicalProbabilityOfSuccess, + }) => { + return 0.24; + }, + }; + + return { + ...calendarDatabase, + getCalendars: calendarDatabase.getKeys, + }; +} + +export const calendarDatabase: CalendarDatabase = makeCalendarDatabase(); diff --git a/server/src/lib/util.ts b/server/src/lib/util.ts new file mode 100644 index 0000000..b9ff2c0 --- /dev/null +++ b/server/src/lib/util.ts @@ -0,0 +1,6 @@ +export function nextDate(date: string) { + const [year, month, day] = date.split('-').map(Number); + const nextDay = new Date(Date.UTC(year, month - 1, day + 1)); + const nextDateString = nextDay.toISOString().substring(0, 10); + return nextDateString; +} \ No newline at end of file diff --git a/server/src/scripts/clickhouse-to-lmdbx.ts b/server/src/scripts/clickhouse-to-lmdbx.ts index ffed9e6..692913e 100644 --- a/server/src/scripts/clickhouse-to-lmdbx.ts +++ b/server/src/scripts/clickhouse-to-lmdbx.ts @@ -3,47 +3,52 @@ import type { AggregateDatabase } from "../interfaces.js"; // import { stockDatabase as stockDatabaseLmdbx } from "../stockdb.lmdbx.js"; import { optionContractDatabase as optionContractDatabaseClickhouse } from "../optiondb.clickhouse.js"; import { optionContractDatabase as optionContractDatabaseLmdbx } from "../optiondb.lmdbx.js"; - -function nextDate(date: string) { - const dateObject = new Date(date); - dateObject.setDate(dateObject.getDate() + 1); - return dateObject.toISOString().substring(0, 10); -} +import { nextDate } from "../lib/util.js"; async function syncAggregates({ - from, - to, + fromDatabase, + toDatabase, key, date, }: { - from: AggregateDatabase; - to: AggregateDatabase; + fromDatabase: AggregateDatabase; + toDatabase: AggregateDatabase; key: T; date: string; }) { - const aggregatesFrom = (await from.getAggregates({ key, date })).map( + const aggregatesFrom = (await fromDatabase.getAggregates({ key, date })).map( (aggregateWithoutKey) => ({ ...aggregateWithoutKey, key }), ); - await to.insertAggregates(aggregatesFrom); + await toDatabase.insertAggregates(aggregatesFrom); } const symbols = ["AMD", "AAPL", "MSFT", "GOOGL", "NFLX", "NVDA"]; async function run() { - const startDate = "2022-02-01"; - const endDate = "2024-07-15"; + const startDate = process.argv[2]; + const endDate = process.argv[3]; + + if (!startDate || !endDate) { + console.error("Usage: node clickhouse-to-lmdbx.js "); + console.error("Dates should be in YYYY-MM-DD format"); + process.exit(1); + } for (let date = startDate; date <= endDate; date = nextDate(date)) { // const symbols = await stockDatabaseClickhouse.getSymbols({ date }); for (const symbol of symbols) { console.log(date, symbol); - const keys = await optionContractDatabaseClickhouse.getKeys({key: {symbol}, date}); - for(const key of keys){ - await syncAggregates({ - from: optionContractDatabaseClickhouse, - to: optionContractDatabaseLmdbx, - key, - date, - }); - } + const keys = await optionContractDatabaseClickhouse.getKeys({ + key: { symbol }, + date, + }); + for (const key of keys) { + // console.log(date, symbol, key.expirationDate, key.strike, key.type); + await syncAggregates({ + fromDatabase: optionContractDatabaseClickhouse, + toDatabase: optionContractDatabaseLmdbx, + key, + date, + }); + } } } }