fix: Update clickhouse-to-lmdbx script to use retry mechanism and updated utility functions.

server/src/scripts/clickhouse-to-lmdbx.ts

@@ -3,7 +3,8 @@ import type { AggregateDatabase } from "../interfaces.js";
 // import { stockDatabase as stockDatabaseLmdbx } from "../stockdb.lmdbx.js";
 import { optionContractDatabase as optionContractDatabaseClickhouse } from "../optiondb.clickhouse.js";
 import { optionContractDatabase as optionContractDatabaseLmdbx } from "../optiondb.lmdbx.js";
-import { nextDate } from "../lib/util.js";
+import { nextDate } from "../lib/utils/nextDate.js";
+import { retry, retryOnTimeout } from "../lib/utils/retry.js";
 
 async function syncAggregates<T>({
   fromDatabase,
@@ -17,7 +18,7 @@ async function syncAggregates<T>({
   date: string;
 }) {
   const aggregatesFrom = (await fromDatabase.getAggregates({ key, date })).map(
-    (aggregateWithoutKey) => ({ ...aggregateWithoutKey, key })
+    (aggregateWithoutKey) => ({ ...aggregateWithoutKey, key }),
   );
   await toDatabase.insertAggregates(aggregatesFrom);
 }
@@ -36,18 +37,27 @@ async function run() {
     // const symbols = await stockDatabaseClickhouse.getSymbols({ date });
     for (const symbol of symbols) {
       console.log(date, symbol);
-      const keys = await optionContractDatabaseClickhouse.getKeys({
-        key: { symbol },
-        date,
-      });
+      const keys = await retry(
+        () =>
+          optionContractDatabaseClickhouse.getKeys({
+            key: { symbol },
+            date,
+          }),
+        { shouldRetry: retryOnTimeout },
+      );
+
       for (const key of keys) {
         // console.log(date, symbol, key.expirationDate, key.strike, key.type);
-        await syncAggregates({
-          fromDatabase: optionContractDatabaseClickhouse,
-          toDatabase: optionContractDatabaseLmdbx,
-          key,
-          date,
-        });
+        await retry(
+          () =>
+            syncAggregates({
+              fromDatabase: optionContractDatabaseClickhouse,
+              toDatabase: optionContractDatabaseLmdbx,
+              key,
+              date,
+            }),
+          { shouldRetry: retryOnTimeout },
+        );
       }
     }
   }