import { restClient } from "@polygon.io/client-js"; import { client } from "./clickhouse.js"; import PQueue from "p-queue"; import dotenv from "dotenv"; import { Env } from "@humanwhocodes/env"; dotenv.config(); const env = new Env(); const POLYGON_API_KEY = env.require("POLYGON_API_KEY"); const CONCURRENCY = Number.parseInt(env.get("CONCURRENCY", "6")); const STOCK_QUOTES_TABLE = env.get("STOCK_QUOTES_TABLE", "stock_aggregates"); const START_DATE = env.get("START_DATE", "2022-03-07"); const END_DATE = env.get("END_DATE", "2022-12-31"); const polygon = restClient(POLYGON_API_KEY, "https://api.polygon.io", { pagination: true, }); const queue = new PQueue({ concurrency: CONCURRENCY }); function* dateRange( startDateStr: string, endDateStr: string ): Generator { // Convert the start and end date strings to Date objects const startDate = new Date(startDateStr); const endDate = new Date(endDateStr); // Loop from the start date to the end date for ( let currentDate = startDate; currentDate <= endDate; currentDate.setDate(currentDate.getDate() + 1) ) { // Format the current date as YYYY-MM-DD const formattedDate = currentDate.toISOString().split("T")[0]; // Yield the formatted date yield formattedDate; } } for (const date of dateRange(START_DATE, END_DATE)) { for (let hour = 13; hour <= 21; hour++) { await client.command({ query: ` INSERT INTO calendar_stats SELECT date, dte, span, moniness, calendar_midpoint as price, count(*) as number_of_quotes FROM ( SELECT toDate('${date}') as date, date_diff('days', toDate(front.ts), front.expiration_date) as dte, date_diff('days', front.expiration_date, back.expiration_date) as span, floor((stock.price - strike) / stock.price, 2) as moniness, back.bid - front.bid as calendar_bid, back.ask - front.ask as calendar_ask, floor((calendar_bid + calendar_ask) / 2, 2) as calendar_midpoint FROM option_quotes_filled AS front INNER JOIN option_quotes_filled AS back ON front.option_type = back.option_type AND front.symbol = back.symbol AND front.strike = back.strike AND front.expiration_date < back.expiration_date AND front.ts = back.ts INNER JOIN stock_aggregates_filled AS stock ON front.symbol = stock.symbol AND front.ts = stock.ts WHERE front.ts >= '${date} ${hour}:00:00' AND front.ts < '${date} ${hour + 1}:00:00' ) GROUP BY date, dte, span, moniness, price ORDER BY date, dte, span, moniness, price SETTINGS allow_experimental_join_condition = 1 ; `, // clickhouse_settings: { // http_receive_timeout: 9999, // receive_timeout: 9999, // stream_poll_timeout_ms: 9999000, // receive_data_timeout_ms: 9999000, // }, }); console.log("finished", date, hour); } } // const data = await polygon.options.quotes( // "O:SPY241220P00600000", // { // "timestamp.gte": "2024-12-15", // "timestamp.lte": "2024-12-16", // sort: "timestamp", // order: "asc", // limit: 50000, // }, // { pagination: true } // ); // console.log(data.status); // console.log(data.results?.length);