fix: option contract aggregate sync on empty batches or unauthorized fetches

server/src/lib/polygon.ts

@@ -104,21 +104,12 @@ export async function* makeGetOptionContractAggregatesIterator({
   const currentDateAsDateObject = new Date(firstDate);
   while (currentDateAsDateObject <= expirationDateAsDateObject) {
     const asOfDate = currentDateAsDateObject.toISOString().substring(0, 10);
-    let latestBatchResponse = await pRetry(
+    const url = `https://api.polygon.io/v2/aggs/ticker/${optionContractTicker}/range/1/minute/${asOfDate}/${asOfDate}?adjusted=false&sort=asc&limit=50000&apiKey=${await getApiKey()}`;
-      async () =>
+    let latestBatchResponse;
-        (await (
+    latestBatchResponse = (await (
-          await fetch(
+      await fetch(url)
-            `https://api.polygon.io/v2/aggs/ticker/${optionContractTicker}/range/1/minute/${asOfDate}/${asOfDate}?adjusted=false&sort=asc&limit=50000&apiKey=${await getApiKey()}`
+    ).json()) as PolygonOptionContractAggregatesResponse;
-          )
+    if (latestBatchResponse.status.toLowerCase() === "ok") {
-        ).json()) as PolygonOptionContractAggregatesResponse,
-      { retries: 5, factor: 2, minTimeout: 1000, maxTimeout: 60 * 1000 }
-    );
-    if (latestBatchResponse.status.toLowerCase() !== "ok") {
-      console.error(latestBatchResponse);
-      throw new Error(
-        `error fetching option contract aggregate ${optionContractTicker}`
-      );
-    }
       yield latestBatchResponse.results?.map((result) => ({
         symbol,
         expirationDate,
@@ -131,6 +122,12 @@ export async function* makeGetOptionContractAggregatesIterator({
         low: result.l,
         high: result.h,
       })) || [];
+    } else if (latestBatchResponse.status === "NOT_AUTHORIZED") {
+      console.error("Skipping due to:", latestBatchResponse);
+    } else {
+      console.error(latestBatchResponse);
+      throw new Error(`error fetching option contract aggregate ${url}`);
+    }
     currentDateAsDateObject.setUTCDate(
       currentDateAsDateObject.getUTCDate() + 1
     );

server/src/lib/sync.ts

@@ -148,13 +148,22 @@ export async function pullOptionContractAggregates(
       ...optionContract,
       firstDate,
     })) {
-      console.log(batch.length);
+      if (batch.length > 0) {
+        console.log(
+          optionContract.symbol,
+          optionContract.expirationDate,
+          optionContract.strike,
+          optionContract.type,
+          new Date(batch[0].tsStart * 1000),
+          new Date(batch[batch.length - 1].tsStart * 1000)
+        );
         await clickhouse.insert({
           table: "option_contract_aggregates",
           values: batch,
           format: "JSONEachRow",
         });
       }
+    }
     await setPullOptionContractAggregatesState(ticker, {
       status: OptionContractAggregatesSyncStatus.COMPLETED,
     });
@@ -210,13 +219,15 @@ export async function pullOptionContractAggregatesSince(
   yesterdayAsDateObject.setUTCDate(yesterdayAsDateObject.getUTCDate() - 1);
   while (currentDateAsDateObject <= yesterdayAsDateObject) {
     const currentDate = currentDateAsDateObject.toISOString().substring(0, 10);
+    console.log(`Date: ${currentDate}`);
     for await (const optionContracts of makeGetOptionContractsIterator(
       symbol,
       currentDate
     )) {
-      await pullOptionContractAggregates(optionContracts);
+      for (const optionContract of optionContracts) {
+        await pullOptionContractAggregates(optionContract);
+      }
     }
-    console.log(`Date: ${currentDate}`);
     currentDateAsDateObject.setUTCDate(
       currentDateAsDateObject.getUTCDate() + 1
     );