add "fill stock quoites" script, reorder scripts

01-ingest-stock-quotes.ts

@@ -10,8 +10,8 @@ const env = new Env();
 const POLYGON_API_KEY = env.require("POLYGON_API_KEY");
 const CONCURRENCY = Number.parseInt(env.get("CONCURRENCY", "6"));
 const STOCK_QUOTES_TABLE = env.get("STOCK_QUOTES_TABLE", "stock_aggregates");
-const START_DATE = env.get("START_DATE", "2020-01-01");
-const END_DATE = env.get("END_DATE", "2022-03-01");
+const START_DATE = env.get("START_DATE", "2022-03-01");
+const END_DATE = env.get("END_DATE", "2025-01-31");
 
 const polygon = restClient(POLYGON_API_KEY, "https://api.polygon.io", {
   pagination: true,
@@ -65,7 +65,7 @@ for (const date of dateRange(START_DATE, END_DATE)) {
           ? {
               symbol: "SPY",
               ts: new Date(timestamp),
-              price: high + low / 2,
+              price: (high + low) / 2,
             }
           : {
               symbol: "",
@@ -75,7 +75,11 @@ for (const date of dateRange(START_DATE, END_DATE)) {
       ),
       format: "JSONEachRow",
     });
-    // console.log(date, contract.ticker, quotes.results?.length);
+
+    // console.log(
+    //   date,
+    //   quotes.results?.map(({ h: high = 0, l: low = 0 }) => (high + low) / 2)
+    // );
   }
   console.log("finished", date);
 }

02-fill-stock-quotes.ts

@@ -0,0 +1,95 @@
+import { restClient } from "@polygon.io/client-js";
+import { client } from "./clickhouse.js";
+import PQueue from "p-queue";
+import dotenv from "dotenv";
+import { Env } from "@humanwhocodes/env";
+
+dotenv.config();
+
+const env = new Env();
+const POLYGON_API_KEY = env.require("POLYGON_API_KEY");
+const CONCURRENCY = Number.parseInt(env.get("CONCURRENCY", "6"));
+const STOCK_QUOTES_TABLE = env.get("STOCK_QUOTES_TABLE", "stock_aggregates");
+const START_DATE = env.get("START_DATE", "2022-08-01");
+const END_DATE = env.get("END_DATE", "2022-12-31");
+
+const polygon = restClient(POLYGON_API_KEY, "https://api.polygon.io", {
+  pagination: true,
+});
+
+const queue = new PQueue({ concurrency: CONCURRENCY });
+
+function* dateRange(
+  startDateStr: string,
+  endDateStr: string
+): Generator<string, void, unknown> {
+  // Convert the start and end date strings to Date objects
+  const startDate = new Date(startDateStr);
+  const endDate = new Date(endDateStr);
+
+  // Loop from the start date to the end date
+  for (
+    let currentDate = startDate;
+    currentDate <= endDate;
+    currentDate.setDate(currentDate.getDate() + 1)
+  ) {
+    // Format the current date as YYYY-MM-DD
+    const formattedDate = currentDate.toISOString().split("T")[0];
+
+    // Yield the formatted date
+    yield formattedDate;
+  }
+}
+
+for (const date of dateRange(START_DATE, END_DATE)) {
+  /** To check if the market is open on the given date, we just fetch a few (10,
+   * for now) rows from the option_quotes table. If there are not 10 rows, then
+   * the market is closed. */
+  const isMarketOpen =
+    (
+      await (
+        await client.query({
+          query: `
+      SELECT *
+      FROM stock_aggregates
+      WHERE ts >= '${date} 00:00:00'
+        AND ts <= '${date} 23:59:59'
+      LIMIT 10
+    `,
+          format: "JSONEachRow",
+        })
+      ).json()
+    ).length === 10;
+  if (isMarketOpen) {
+    await client.command({
+      query: `
+        INSERT INTO stock_aggregates_filled
+        SELECT
+          symbol,
+          ts,
+          price
+        FROM stock_aggregates
+        WHERE ts >= '${date} 00:00:00'
+        AND ts <= '${date} 23:59:59'
+        ORDER BY symbol, ts 
+        WITH FILL FROM toDateTime('${date} 09:30:00', 'America/New_York') TO toDateTime('${date} 16:15:01', 'America/New_York') INTERPOLATE (price)
+      `,
+    });
+  }
+  console.log("finished", date);
+}
+
+// const data = await polygon.options.quotes(
+//   "O:SPY241220P00600000",
+//   {
+//     "timestamp.gte": "2024-12-15",
+//     "timestamp.lte": "2024-12-16",
+//     sort: "timestamp",
+//     order: "asc",
+//     limit: 50000,
+//   },
+//   { pagination: true }
+// );
+
+// console.log(data.status);
+// console.log(data.results?.length);