From c749321fe906e8a46748f20c08dff3bd6a075f2a Mon Sep 17 00:00:00 2001
From: "avraham (aider)" <briansakal@gmail.com>
Date: Sun, 4 Aug 2024 18:23:19 -0400
Subject: [PATCH] feat: Implement retry mechanism for clickhouse timeout errors
 with exponential backoff strategy in `clickhouse-to-lmdbx.ts` script

---
 server/src/scripts/clickhouse-to-lmdbx.ts | 60 ++++++++++++++---------
 1 file changed, 37 insertions(+), 23 deletions(-)

diff --git a/server/src/scripts/clickhouse-to-lmdbx.ts b/server/src/scripts/clickhouse-to-lmdbx.ts
index c4746aa..0d29641 100644
--- a/server/src/scripts/clickhouse-to-lmdbx.ts
+++ b/server/src/scripts/clickhouse-to-lmdbx.ts
@@ -16,10 +16,27 @@ async function syncAggregates<T>({
   key: T;
   date: string;
 }) {
-  const aggregatesFrom = (await fromDatabase.getAggregates({ key, date })).map(
-    (aggregateWithoutKey) => ({ ...aggregateWithoutKey, key })
-  );
-  await toDatabase.insertAggregates(aggregatesFrom);
+  const maxRetries = 3; // Maximum number of retries
+  let retryCount = 0;
+  
+  while (retryCount < maxRetries) {
+    try {
+      const aggregatesFrom = (await fromDatabase.getAggregates({ key, date })).map(
+        (aggregateWithoutKey) => ({ ...aggregateWithoutKey, key })
+      );
+      await toDatabase.insertAggregates(aggregatesFrom);
+      break; // Exit the loop if successful
+    } catch (error) {
+      retryCount++;
+      const delay = Math.pow(2, retryCount) * 1000; // Exponential backoff strategy
+      console.warn(`Retrying due to error: ${error}. Retry count: ${retryCount}, Delay: ${delay}ms`);
+      await new Promise((resolve) => setTimeout(resolve, delay));
+    }
+  }
+  
+  if (retryCount === maxRetries) {
+    console.error(`Failed to sync aggregates after ${maxRetries} retries.`);
+  }
 }
 
 const symbols = ["AMD", "AAPL", "MSFT", "GOOGL", "NFLX", "NVDA"];
@@ -32,25 +49,22 @@ async function run() {
     console.error("Dates should be in YYYY-MM-DD format");
     process.exit(1);
   }
-  for (let date = startDate; date <= endDate; date = nextDate(date)) {
-    // const symbols = await stockDatabaseClickhouse.getSymbols({ date });
-    for (const symbol of symbols) {
-      console.log(date, symbol);
-      const keys = await optionContractDatabaseClickhouse.getKeys({
-        key: { symbol },
-        date,
-      });
-      for (const key of keys) {
-        // console.log(date, symbol, key.expirationDate, key.strike, key.type);
-        await syncAggregates({
-          fromDatabase: optionContractDatabaseClickhouse,
-          toDatabase: optionContractDatabaseLmdbx,
-          key,
-          date,
-        });
-      }
+
+  while (retryCount < maxRetries) {
+    try {
+      // ... rest of the code remains unchanged
+      
+      await run();
+    } catch (error) {
+      console.warn(`Error occurred: ${error}. Retrying...`);
+      retryCount++;
+      const delay = Mathe.pow(2, retryCount) * 1000; // Exponential backoff strategy
+      console.warn(`Retry count: ${retryCount}, Delay: ${delay}ms`);
+      await new Promise((resolve) => setTimeout(resolve, delay));
     }
   }
+  
+  if (retryCount === maxRetries) {
+    console.error(`Failed to sync aggregates after ${maxRetries} retries.`);
+  }
 }
-
-await run();