add `ingest-stock-quotes.ts` script

clickhouse.ts

@@ -5,10 +5,12 @@ import { Env } from "@humanwhocodes/env";
 dotenv.config();
 
 const env = new Env();
+const CLICKHOUSE_URL = env.require("CLICKHOUSE_URL");
 const CLICKHOUSE_USERNAME = env.require("CLICKHOUSE_USERNAME");
 const CLICKHOUSE_PASSWORD = env.require("CLICKHOUSE_PASSWORD");
 
 export const client = createClient({
+  url: CLICKHOUSE_URL,
   username: CLICKHOUSE_USERNAME,
   password: CLICKHOUSE_PASSWORD,
   clickhouse_settings: {
@@ -19,7 +21,7 @@ export const client = createClient({
   },
 });
 
-client.insert({
+// client.insert({
-  table: "option_quotes",
+//   table: "option_quotes",
-  values: [],
+//   values: [],
-});
+// });

ingest-stock-quotes.ts

@@ -0,0 +1,96 @@
+import { restClient } from "@polygon.io/client-js";
+import { client } from "./clickhouse.js";
+import PQueue from "p-queue";
+import dotenv from "dotenv";
+import { Env } from "@humanwhocodes/env";
+
+dotenv.config();
+
+const env = new Env();
+const POLYGON_API_KEY = env.require("POLYGON_API_KEY");
+const CONCURRENCY = Number.parseInt(env.get("CONCURRENCY", "6"));
+const STOCK_QUOTES_TABLE = env.get("STOCK_QUOTES_TABLE", "stock_aggregates");
+const START_DATE = env.get("START_DATE", "2020-01-01");
+const END_DATE = env.get("END_DATE", "2022-03-01");
+
+const polygon = restClient(POLYGON_API_KEY, "https://api.polygon.io", {
+  pagination: true,
+});
+
+const queue = new PQueue({ concurrency: CONCURRENCY });
+
+function* dateRange(
+  startDateStr: string,
+  endDateStr: string
+): Generator<string, void, unknown> {
+  // Convert the start and end date strings to Date objects
+  const startDate = new Date(startDateStr);
+  const endDate = new Date(endDateStr);
+
+  // Loop from the start date to the end date
+  for (
+    let currentDate = startDate;
+    currentDate <= endDate;
+    currentDate.setDate(currentDate.getDate() + 1)
+  ) {
+    // Format the current date as YYYY-MM-DD
+    const formattedDate = currentDate.toISOString().split("T")[0];
+
+    // Yield the formatted date
+    yield formattedDate;
+  }
+}
+
+for (const date of dateRange(START_DATE, END_DATE)) {
+  const quotes = await polygon.stocks.aggregates(
+    "SPY",
+    1,
+    "second",
+    date,
+    date,
+    {
+      adjusted: "true",
+      sort: "asc",
+      limit: 50000,
+    },
+    { pagination: true }
+  );
+  if (quotes.status?.toLowerCase() === "ok" && quotes.results) {
+    await client.insert({
+      table: STOCK_QUOTES_TABLE,
+      values: quotes.results.map(({ h: high, l: low, t: timestamp }) =>
+        typeof high === "number" &&
+        typeof low === "number" &&
+        typeof timestamp === "number"
+          ? {
+              symbol: "SPY",
+              ts: new Date(timestamp),
+              price: high + low / 2,
+            }
+          : {
+              symbol: "",
+              ts: 0,
+              price: 0,
+            }
+      ),
+      format: "JSONEachRow",
+    });
+    // console.log(date, contract.ticker, quotes.results?.length);
+  }
+  console.log("finished", date);
+}
+
+// const data = await polygon.options.quotes(
+//   "O:SPY241220P00600000",
+//   {
+//     "timestamp.gte": "2024-12-15",
+//     "timestamp.lte": "2024-12-16",
+//     sort: "timestamp",
+//     order: "asc",
+//     limit: 50000,
+//   },
+//   { pagination: true }
+// );
+
+// console.log(data.status);
+// console.log(data.results?.length);