From 71f72eb474cd7e197896b0e5fe072e7deb8f39f7 Mon Sep 17 00:00:00 2001
From: Avraham Sakal <avraham@torahanytime.com>
Date: Mon, 1 Jul 2024 08:16:41 -0400
Subject: [PATCH] handle any socket hangups

---
 server/src/lib/polygon.ts | 36 +++++++++++++++++++++++++-----------
 server/src/lib/sync.ts    |  4 ++--
 2 files changed, 27 insertions(+), 13 deletions(-)

diff --git a/server/src/lib/polygon.ts b/server/src/lib/polygon.ts
index e5d91b1..38a47b1 100644
--- a/server/src/lib/polygon.ts
+++ b/server/src/lib/polygon.ts
@@ -38,11 +38,15 @@ export async function* makeGetOptionContractsIterator(
   symbol: string,
   date: string
 ) {
-  let latestBatchResponse = (await (
-    await fetch(
-      `https://api.polygon.io/v3/reference/options/contracts?underlying_ticker=${symbol}&as_of=${date}&sort=ticker&limit=1000&apiKey=${await getApiKey()}`
-    )
-  ).json()) as PolygonOptionContractsResponse;
+  let latestBatchResponse = await pRetry(
+    async () =>
+      (await (
+        await fetch(
+          `https://api.polygon.io/v3/reference/options/contracts?underlying_ticker=${symbol}&as_of=${date}&sort=ticker&limit=1000&apiKey=${await getApiKey()}`
+        )
+      ).json()) as PolygonOptionContractsResponse,
+    { forever: true, factor: 2, maxTimeout: 120000 }
+  );
   yield latestBatchResponse.results.map((result) => ({
     asOfDate: date,
     symbol,
@@ -53,9 +57,15 @@ export async function* makeGetOptionContractsIterator(
 
   // as long as there's a `next_url`, call that:
   while (latestBatchResponse.hasOwnProperty("next_url")) {
-    latestBatchResponse = (await (
-      await fetch(`${latestBatchResponse.next_url}&apiKey=${await getApiKey()}`)
-    ).json()) as PolygonOptionContractsResponse;
+    latestBatchResponse = await pRetry(
+      async () =>
+        (await (
+          await fetch(
+            `${latestBatchResponse.next_url}&apiKey=${await getApiKey()}`
+          )
+        ).json()) as PolygonOptionContractsResponse,
+      { forever: true, factor: 2, maxTimeout: 120000 }
+    );
     yield latestBatchResponse.results?.map((result) => ({
       asOfDate: date,
       symbol,
@@ -106,9 +116,13 @@ export async function* makeGetOptionContractAggregatesIterator({
     const asOfDate = currentDateAsDateObject.toISOString().substring(0, 10);
     const url = `https://api.polygon.io/v2/aggs/ticker/${optionContractTicker}/range/1/minute/${asOfDate}/${asOfDate}?adjusted=false&sort=asc&limit=50000&apiKey=${await getApiKey()}`;
     let latestBatchResponse;
-    latestBatchResponse = (await (
-      await fetch(url)
-    ).json()) as PolygonOptionContractAggregatesResponse;
+    latestBatchResponse = await pRetry(
+      async () =>
+        (await (
+          await fetch(url)
+        ).json()) as PolygonOptionContractAggregatesResponse,
+      { forever: true, factor: 2, maxTimeout: 120000 }
+    );
     if (latestBatchResponse.status.toLowerCase() === "ok") {
       yield latestBatchResponse.results?.map((result) => ({
         symbol,
diff --git a/server/src/lib/sync.ts b/server/src/lib/sync.ts
index e6f9aa2..bd0dc43 100644
--- a/server/src/lib/sync.ts
+++ b/server/src/lib/sync.ts
@@ -120,7 +120,7 @@ export async function pullOptionContracts(symbol: string, date: string) {
       date
     )) {
       console.log(batch.length);
-      pRetry(
+      await pRetry(
         async () => {
           await clickhouse.insert({
             table: "option_contract_existences",
@@ -163,7 +163,7 @@ export async function pullOptionContractAggregates(
           new Date(batch[0].tsStart * 1000),
           new Date(batch[batch.length - 1].tsStart * 1000)
         );
-        pRetry(
+        await pRetry(
           async () => {
             await clickhouse.insert({
               table: "option_contract_aggregates",