fix: option contract aggregate sync on empty batches or unauthorized fetches

server/src/lib/polygon.ts

@@ -104,33 +104,30 @@ export async function* makeGetOptionContractAggregatesIterator({
   const currentDateAsDateObject = new Date(firstDate);
   while (currentDateAsDateObject <= expirationDateAsDateObject) {
     const asOfDate = currentDateAsDateObject.toISOString().substring(0, 10);
-    let latestBatchResponse = await pRetry(
+    const url = `https://api.polygon.io/v2/aggs/ticker/${optionContractTicker}/range/1/minute/${asOfDate}/${asOfDate}?adjusted=false&sort=asc&limit=50000&apiKey=${await getApiKey()}`;
-      async () =>
+    let latestBatchResponse;
-        (await (
+    latestBatchResponse = (await (
-          await fetch(
+      await fetch(url)
-            `https://api.polygon.io/v2/aggs/ticker/${optionContractTicker}/range/1/minute/${asOfDate}/${asOfDate}?adjusted=false&sort=asc&limit=50000&apiKey=${await getApiKey()}`
+    ).json()) as PolygonOptionContractAggregatesResponse;
-          )
+    if (latestBatchResponse.status.toLowerCase() === "ok") {
-        ).json()) as PolygonOptionContractAggregatesResponse,
+      yield latestBatchResponse.results?.map((result) => ({
-      { retries: 5, factor: 2, minTimeout: 1000, maxTimeout: 60 * 1000 }
+        symbol,
-    );
+        expirationDate,
-    if (latestBatchResponse.status.toLowerCase() !== "ok") {
+        strike,
+        type,
+
+        tsStart: (result.t || 0) / 1000,
+        open: result.o,
+        close: result.c,
+        low: result.l,
+        high: result.h,
+      })) || [];
+    } else if (latestBatchResponse.status === "NOT_AUTHORIZED") {
+      console.error("Skipping due to:", latestBatchResponse);
+    } else {
       console.error(latestBatchResponse);
-      throw new Error(
+      throw new Error(`error fetching option contract aggregate ${url}`);
-        `error fetching option contract aggregate ${optionContractTicker}`
-      );
     }
-    yield latestBatchResponse.results?.map((result) => ({
-      symbol,
-      expirationDate,
-      strike,
-      type,
-
-      tsStart: (result.t || 0) / 1000,
-      open: result.o,
-      close: result.c,
-      low: result.l,
-      high: result.h,
-    })) || [];
     currentDateAsDateObject.setUTCDate(
       currentDateAsDateObject.getUTCDate() + 1
     );

server/src/lib/sync.ts

@@ -148,12 +148,21 @@ export async function pullOptionContractAggregates(
       ...optionContract,
       firstDate,
     })) {
-      console.log(batch.length);
+      if (batch.length > 0) {
-      await clickhouse.insert({
+        console.log(
-        table: "option_contract_aggregates",
+          optionContract.symbol,
-        values: batch,
+          optionContract.expirationDate,
-        format: "JSONEachRow",
+          optionContract.strike,
-      });
+          optionContract.type,
+          new Date(batch[0].tsStart * 1000),
+          new Date(batch[batch.length - 1].tsStart * 1000)
+        );
+        await clickhouse.insert({
+          table: "option_contract_aggregates",
+          values: batch,
+          format: "JSONEachRow",
+        });
+      }
     }
     await setPullOptionContractAggregatesState(ticker, {
       status: OptionContractAggregatesSyncStatus.COMPLETED,
@@ -210,13 +219,15 @@ export async function pullOptionContractAggregatesSince(
   yesterdayAsDateObject.setUTCDate(yesterdayAsDateObject.getUTCDate() - 1);
   while (currentDateAsDateObject <= yesterdayAsDateObject) {
     const currentDate = currentDateAsDateObject.toISOString().substring(0, 10);
+    console.log(`Date: ${currentDate}`);
     for await (const optionContracts of makeGetOptionContractsIterator(
       symbol,
       currentDate
     )) {
-      await pullOptionContractAggregates(optionContracts);
+      for (const optionContract of optionContracts) {
+        await pullOptionContractAggregates(optionContract);
+      }
     }
-    console.log(`Date: ${currentDate}`);
     currentDateAsDateObject.setUTCDate(
       currentDateAsDateObject.getUTCDate() + 1
     );