diff --git a/server/src/scripts/clickhouse-to-lmdbx.ts b/server/src/scripts/clickhouse-to-lmdbx.ts index c4746aa..180d550 100644 --- a/server/src/scripts/clickhouse-to-lmdbx.ts +++ b/server/src/scripts/clickhouse-to-lmdbx.ts @@ -3,7 +3,8 @@ import type { AggregateDatabase } from "../interfaces.js"; // import { stockDatabase as stockDatabaseLmdbx } from "../stockdb.lmdbx.js"; import { optionContractDatabase as optionContractDatabaseClickhouse } from "../optiondb.clickhouse.js"; import { optionContractDatabase as optionContractDatabaseLmdbx } from "../optiondb.lmdbx.js"; -import { nextDate } from "../lib/util.js"; +import { nextDate } from "../lib/utils/nextDate.js"; +import { retry, retryOnTimeout } from "../lib/utils/retry.js"; async function syncAggregates({ fromDatabase, @@ -17,7 +18,7 @@ async function syncAggregates({ date: string; }) { const aggregatesFrom = (await fromDatabase.getAggregates({ key, date })).map( - (aggregateWithoutKey) => ({ ...aggregateWithoutKey, key }) + (aggregateWithoutKey) => ({ ...aggregateWithoutKey, key }), ); await toDatabase.insertAggregates(aggregatesFrom); } @@ -36,18 +37,27 @@ async function run() { // const symbols = await stockDatabaseClickhouse.getSymbols({ date }); for (const symbol of symbols) { console.log(date, symbol); - const keys = await optionContractDatabaseClickhouse.getKeys({ - key: { symbol }, - date, - }); + const keys = await retry( + () => + optionContractDatabaseClickhouse.getKeys({ + key: { symbol }, + date, + }), + { shouldRetry: retryOnTimeout }, + ); + for (const key of keys) { // console.log(date, symbol, key.expirationDate, key.strike, key.type); - await syncAggregates({ - fromDatabase: optionContractDatabaseClickhouse, - toDatabase: optionContractDatabaseLmdbx, - key, - date, - }); + await retry( + () => + syncAggregates({ + fromDatabase: optionContractDatabaseClickhouse, + toDatabase: optionContractDatabaseLmdbx, + key, + date, + }), + { shouldRetry: retryOnTimeout }, + ); } } }