diff --git a/clickhouse.ts b/clickhouse.ts new file mode 100644 index 0000000..c49cb95 --- /dev/null +++ b/clickhouse.ts @@ -0,0 +1,26 @@ +import { createClient } from "@clickhouse/client"; +import dotenv from "dotenv"; +import { Env } from "@humanwhocodes/env"; + +dotenv.config(); + +const env = new Env(); +const CLICKHOUSE_USERNAME = env.require("CLICKHOUSE_USERNAME"); +const CLICKHOUSE_PASSWORD = env.require("CLICKHOUSE_PASSWORD"); + +export const client = createClient({ + username: CLICKHOUSE_USERNAME, + password: CLICKHOUSE_PASSWORD, + clickhouse_settings: { + async_insert: 1, + output_format_json_quote_32bit_integers: 0, + output_format_json_quote_64bit_integers: 0, + output_format_json_quote_64bit_floats: 0, + output_format_json_quote_32bit_floats: 0, + }, +}); + +client.insert({ + table: "option_quotes", + values: [], +}); diff --git a/ingest-option-quotes.ts b/ingest-option-quotes.ts index 2188ec4..a879027 100644 --- a/ingest-option-quotes.ts +++ b/ingest-option-quotes.ts @@ -1,4 +1,5 @@ import { restClient } from "@polygon.io/client-js"; +import { client } from "./clickhouse.js"; import PQueue from "p-queue"; import dotenv from "dotenv"; import { Env } from "@humanwhocodes/env"; @@ -59,6 +60,20 @@ for (const date of dateRange("2024-05-02", "2024-05-03")) { { pagination: true } ); if (quotes.status?.toLowerCase() === "ok" && quotes.results) { + await client.insert({ + table: "option_quotes2", + values: quotes.results.map( + ({ bid_price, ask_price, sip_timestamp }) => ({ + option_type: contract.contract_type, + symbol: contract.underlying_ticker, + expiration_date: contract.expiration_date, + strike: contract.strike_price, + ts: sip_timestamp, + bid: bid_price, + ask: ask_price, + }) + ), + }); console.log(date, contract.ticker, quotes.results?.length); } } diff --git a/package.json b/package.json index e5a0840..611a8b0 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,7 @@ "typescript": "^5.7.2" }, "dependencies": { + "@clickhouse/client": "^1.10.0", "@humanwhocodes/env": "^4.0.0", "@polygon.io/client-js": "^7.3.2", "dotenv": "^16.4.7", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 65b66f6..b624f8f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -8,6 +8,9 @@ importers: .: dependencies: + '@clickhouse/client': + specifier: ^1.10.0 + version: 1.10.0 '@humanwhocodes/env': specifier: ^4.0.0 version: 4.0.0 @@ -33,6 +36,13 @@ importers: packages: + '@clickhouse/client-common@1.10.0': + resolution: {integrity: sha512-dxXgJEGxwhOclflDH+GI8z7fQMh7KjJ/r0kSGE7yLKJlW0YHgbfxVufOkrEVg2QJMOmqi+YynkO4w1rXhetC0w==} + + '@clickhouse/client@1.10.0': + resolution: {integrity: sha512-vyLkrE7iqWw3QPTet7no5Rd3qZFV4ujyo01FbspaTpME04F07AkBkRFJ/haUpf6GbZr/LsGIH5I4jy1vzGoRbQ==} + engines: {node: '>=16'} + '@esbuild/aix-ppc64@0.23.1': resolution: {integrity: sha512-6VhYk1diRqrhBAqpJEdjASR/+WVRtfjpqKuNw11cLiaWpAT/Uu+nokB+UJnevzy/P9C/ty6AOe0dwueMrGh/iQ==} engines: {node: '>=18'} @@ -343,6 +353,12 @@ packages: snapshots: + '@clickhouse/client-common@1.10.0': {} + + '@clickhouse/client@1.10.0': + dependencies: + '@clickhouse/client-common': 1.10.0 + '@esbuild/aix-ppc64@0.23.1': optional: true