From 1d83cd419a2263a579708c9811ae0dd8969d240e Mon Sep 17 00:00:00 2001
From: avraham <briansakal@gmail.com>
Date: Sun, 4 Aug 2024 19:26:27 -0400
Subject: [PATCH] fix: Update clickhouse-to-lmdbx script to use retry mechanism
 and updated utility functions.

---
 server/src/scripts/clickhouse-to-lmdbx.ts | 34 +++++++++++++++--------
 1 file changed, 22 insertions(+), 12 deletions(-)

diff --git a/server/src/scripts/clickhouse-to-lmdbx.ts b/server/src/scripts/clickhouse-to-lmdbx.ts
index c4746aa..180d550 100644
--- a/server/src/scripts/clickhouse-to-lmdbx.ts
+++ b/server/src/scripts/clickhouse-to-lmdbx.ts
@@ -3,7 +3,8 @@ import type { AggregateDatabase } from "../interfaces.js";
 // import { stockDatabase as stockDatabaseLmdbx } from "../stockdb.lmdbx.js";
 import { optionContractDatabase as optionContractDatabaseClickhouse } from "../optiondb.clickhouse.js";
 import { optionContractDatabase as optionContractDatabaseLmdbx } from "../optiondb.lmdbx.js";
-import { nextDate } from "../lib/util.js";
+import { nextDate } from "../lib/utils/nextDate.js";
+import { retry, retryOnTimeout } from "../lib/utils/retry.js";
 
 async function syncAggregates<T>({
   fromDatabase,
@@ -17,7 +18,7 @@ async function syncAggregates<T>({
   date: string;
 }) {
   const aggregatesFrom = (await fromDatabase.getAggregates({ key, date })).map(
-    (aggregateWithoutKey) => ({ ...aggregateWithoutKey, key })
+    (aggregateWithoutKey) => ({ ...aggregateWithoutKey, key }),
   );
   await toDatabase.insertAggregates(aggregatesFrom);
 }
@@ -36,18 +37,27 @@ async function run() {
     // const symbols = await stockDatabaseClickhouse.getSymbols({ date });
     for (const symbol of symbols) {
       console.log(date, symbol);
-      const keys = await optionContractDatabaseClickhouse.getKeys({
-        key: { symbol },
-        date,
-      });
+      const keys = await retry(
+        () =>
+          optionContractDatabaseClickhouse.getKeys({
+            key: { symbol },
+            date,
+          }),
+        { shouldRetry: retryOnTimeout },
+      );
+
       for (const key of keys) {
         // console.log(date, symbol, key.expirationDate, key.strike, key.type);
-        await syncAggregates({
-          fromDatabase: optionContractDatabaseClickhouse,
-          toDatabase: optionContractDatabaseLmdbx,
-          key,
-          date,
-        });
+        await retry(
+          () =>
+            syncAggregates({
+              fromDatabase: optionContractDatabaseClickhouse,
+              toDatabase: optionContractDatabaseLmdbx,
+              key,
+              date,
+            }),
+          { shouldRetry: retryOnTimeout },
+        );
       }
     }
   }