From fc2526a4aae79c3588ef4bc54860a2676539c3e3 Mon Sep 17 00:00:00 2001
From: Avraham Sakal <avraham@torahanytime.com>
Date: Sun, 30 Jun 2024 21:14:37 -0400
Subject: [PATCH] fix: option contract aggregate sync on empty batches or
 unauthorized fetches

---
 server/src/lib/polygon.ts | 47 ++++++++++++++++++---------------------
 server/src/lib/sync.ts    | 27 +++++++++++++++-------
 2 files changed, 41 insertions(+), 33 deletions(-)

diff --git a/server/src/lib/polygon.ts b/server/src/lib/polygon.ts
index bb058c0..e5d91b1 100644
--- a/server/src/lib/polygon.ts
+++ b/server/src/lib/polygon.ts
@@ -104,33 +104,30 @@ export async function* makeGetOptionContractAggregatesIterator({
   const currentDateAsDateObject = new Date(firstDate);
   while (currentDateAsDateObject <= expirationDateAsDateObject) {
     const asOfDate = currentDateAsDateObject.toISOString().substring(0, 10);
-    let latestBatchResponse = await pRetry(
-      async () =>
-        (await (
-          await fetch(
-            `https://api.polygon.io/v2/aggs/ticker/${optionContractTicker}/range/1/minute/${asOfDate}/${asOfDate}?adjusted=false&sort=asc&limit=50000&apiKey=${await getApiKey()}`
-          )
-        ).json()) as PolygonOptionContractAggregatesResponse,
-      { retries: 5, factor: 2, minTimeout: 1000, maxTimeout: 60 * 1000 }
-    );
-    if (latestBatchResponse.status.toLowerCase() !== "ok") {
+    const url = `https://api.polygon.io/v2/aggs/ticker/${optionContractTicker}/range/1/minute/${asOfDate}/${asOfDate}?adjusted=false&sort=asc&limit=50000&apiKey=${await getApiKey()}`;
+    let latestBatchResponse;
+    latestBatchResponse = (await (
+      await fetch(url)
+    ).json()) as PolygonOptionContractAggregatesResponse;
+    if (latestBatchResponse.status.toLowerCase() === "ok") {
+      yield latestBatchResponse.results?.map((result) => ({
+        symbol,
+        expirationDate,
+        strike,
+        type,
+
+        tsStart: (result.t || 0) / 1000,
+        open: result.o,
+        close: result.c,
+        low: result.l,
+        high: result.h,
+      })) || [];
+    } else if (latestBatchResponse.status === "NOT_AUTHORIZED") {
+      console.error("Skipping due to:", latestBatchResponse);
+    } else {
       console.error(latestBatchResponse);
-      throw new Error(
-        `error fetching option contract aggregate ${optionContractTicker}`
-      );
+      throw new Error(`error fetching option contract aggregate ${url}`);
     }
-    yield latestBatchResponse.results?.map((result) => ({
-      symbol,
-      expirationDate,
-      strike,
-      type,
-
-      tsStart: (result.t || 0) / 1000,
-      open: result.o,
-      close: result.c,
-      low: result.l,
-      high: result.h,
-    })) || [];
     currentDateAsDateObject.setUTCDate(
       currentDateAsDateObject.getUTCDate() + 1
     );
diff --git a/server/src/lib/sync.ts b/server/src/lib/sync.ts
index d9466b4..61b7a56 100644
--- a/server/src/lib/sync.ts
+++ b/server/src/lib/sync.ts
@@ -148,12 +148,21 @@ export async function pullOptionContractAggregates(
       ...optionContract,
       firstDate,
     })) {
-      console.log(batch.length);
-      await clickhouse.insert({
-        table: "option_contract_aggregates",
-        values: batch,
-        format: "JSONEachRow",
-      });
+      if (batch.length > 0) {
+        console.log(
+          optionContract.symbol,
+          optionContract.expirationDate,
+          optionContract.strike,
+          optionContract.type,
+          new Date(batch[0].tsStart * 1000),
+          new Date(batch[batch.length - 1].tsStart * 1000)
+        );
+        await clickhouse.insert({
+          table: "option_contract_aggregates",
+          values: batch,
+          format: "JSONEachRow",
+        });
+      }
     }
     await setPullOptionContractAggregatesState(ticker, {
       status: OptionContractAggregatesSyncStatus.COMPLETED,
@@ -210,13 +219,15 @@ export async function pullOptionContractAggregatesSince(
   yesterdayAsDateObject.setUTCDate(yesterdayAsDateObject.getUTCDate() - 1);
   while (currentDateAsDateObject <= yesterdayAsDateObject) {
     const currentDate = currentDateAsDateObject.toISOString().substring(0, 10);
+    console.log(`Date: ${currentDate}`);
     for await (const optionContracts of makeGetOptionContractsIterator(
       symbol,
       currentDate
     )) {
-      await pullOptionContractAggregates(optionContracts);
+      for (const optionContract of optionContracts) {
+        await pullOptionContractAggregates(optionContract);
+      }
     }
-    console.log(`Date: ${currentDate}`);
     currentDateAsDateObject.setUTCDate(
       currentDateAsDateObject.getUTCDate() + 1
     );