You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
78 lines
2.5 KiB
TypeScript
78 lines
2.5 KiB
TypeScript
import type { AggregateDatabase } from "../AggregateDatabase/interfaces.js";
|
|
import { database as stockDatabaseClickhouse } from "../AggregateDatabase/Stock/clickhouse.js";
|
|
import { database as stockDatabaseLmdbx } from "../AggregateDatabase/Stock/lmdbx.js";
|
|
// import { optionContractDatabase as optionContractDatabaseClickhouse } from "../optiondb.clickhouse.js";
|
|
// import { optionContractDatabase as optionContractDatabaseLmdbx } from "../optiondb.lmdbx.js";
|
|
import { nextDate } from "../lib/utils/nextDate.js";
|
|
import { retry, retryOnTimeout } from "../lib/utils/retry.js";
|
|
import type { OptionContractKey } from "../AggregateDatabase/OptionContract/interfaces.js";
|
|
import type { StockKey } from "../AggregateDatabase/Stock/interfaces.js";
|
|
|
|
async function syncAggregates<T>({
|
|
fromDatabase,
|
|
toDatabase,
|
|
key,
|
|
date,
|
|
}: {
|
|
fromDatabase: AggregateDatabase<T>;
|
|
toDatabase: AggregateDatabase<T>;
|
|
key: T;
|
|
date: string;
|
|
}) {
|
|
const aggregatesFrom = (await fromDatabase.getAggregates({ key, date })).map(
|
|
(aggregateWithoutKey) => ({ ...aggregateWithoutKey, key })
|
|
);
|
|
await toDatabase.insertAggregates(aggregatesFrom);
|
|
}
|
|
|
|
const symbols = ["AMD", "AAPL", "MSFT", "GOOGL", "NFLX", "NVDA"];
|
|
async function run<T extends StockKey | OptionContractKey>({
|
|
fromDatabase,
|
|
toDatabase,
|
|
}: {
|
|
fromDatabase: AggregateDatabase<T>;
|
|
toDatabase: AggregateDatabase<T>;
|
|
}) {
|
|
const startDate = process.argv[2];
|
|
const endDate = process.argv[3];
|
|
|
|
if (!startDate || !endDate) {
|
|
console.error("Usage: node clickhouse-to-lmdbx.js <startDate> <endDate>");
|
|
console.error("Dates should be in YYYY-MM-DD format");
|
|
process.exit(1);
|
|
}
|
|
for (let date = startDate; date <= endDate; date = nextDate(date)) {
|
|
// const symbols = await stockDatabaseClickhouse.getSymbols({ date });
|
|
for (const symbol of symbols) {
|
|
console.log(date, symbol);
|
|
const keys = await retry(
|
|
() =>
|
|
fromDatabase.getKeys({
|
|
key: { symbol } as T,
|
|
date,
|
|
}),
|
|
{ shouldRetry: retryOnTimeout }
|
|
);
|
|
|
|
for (const key of keys) {
|
|
// console.log(date, symbol, key.expirationDate, key.strike, key.type);
|
|
await retry(
|
|
() =>
|
|
syncAggregates({
|
|
fromDatabase,
|
|
toDatabase,
|
|
key,
|
|
date,
|
|
}),
|
|
{ shouldRetry: retryOnTimeout }
|
|
);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
await run({
|
|
fromDatabase: stockDatabaseClickhouse,
|
|
toDatabase: stockDatabaseLmdbx,
|
|
});
|