avraham/calendar-optimizer
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix: option contract aggregate sync on empty batches or unauthorized fetches

Browse Source
This commit is contained in:
Avraham Sakal
2024-06-30 21:14:37 -04:00
parent e0a2bc395e
commit fc2526a4aa
2 changed files with 42 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/src/lib/polygon.ts
+23 -26
View File
@@ -104,33 +104,30 @@ export async function* makeGetOptionContractAggregatesIterator({
const currentDateAsDateObject = new Date(firstDate); const currentDateAsDateObject = new Date(firstDate);
while (currentDateAsDateObject <= expirationDateAsDateObject) { while (currentDateAsDateObject <= expirationDateAsDateObject) {
const asOfDate = currentDateAsDateObject.toISOString().substring(0, 10); const asOfDate = currentDateAsDateObject.toISOString().substring(0, 10);
let latestBatchResponse = await pRetry( const url = `https://api.polygon.io/v2/aggs/ticker/${optionContractTicker}/range/1/minute/${asOfDate}/${asOfDate}?adjusted=false&sort=asc&limit=50000&apiKey=${await getApiKey()}`;
async () => let latestBatchResponse;
(await ( latestBatchResponse = (await (
await fetch( await fetch(url)
`https://api.polygon.io/v2/aggs/ticker/${optionContractTicker}/range/1/minute/${asOfDate}/${asOfDate}?adjusted=false&sort=asc&limit=50000&apiKey=${await getApiKey()}` ).json()) as PolygonOptionContractAggregatesResponse;
) if (latestBatchResponse.status.toLowerCase() === "ok") {
).json()) as PolygonOptionContractAggregatesResponse, yield latestBatchResponse.results?.map((result) => ({
{ retries: 5, factor: 2, minTimeout: 1000, maxTimeout: 60 * 1000 } symbol,
); expirationDate,
if (latestBatchResponse.status.toLowerCase() !== "ok") { strike,
console.error(latestBatchResponse); type,
throw new Error(
`error fetching option contract aggregate ${optionContractTicker}`
);
}
yield latestBatchResponse.results?.map((result) => ({
symbol,
expirationDate,
strike,
type,
tsStart: (result.t || 0) / 1000, tsStart: (result.t || 0) / 1000,
open: result.o, open: result.o,
close: result.c, close: result.c,
low: result.l, low: result.l,
high: result.h, high: result.h,
})) || []; })) || [];
} else if (latestBatchResponse.status === "NOT_AUTHORIZED") {
console.error("Skipping due to:", latestBatchResponse);
} else {
console.error(latestBatchResponse);
throw new Error(`error fetching option contract aggregate ${url}`);
}
currentDateAsDateObject.setUTCDate( currentDateAsDateObject.setUTCDate(
currentDateAsDateObject.getUTCDate() + 1 currentDateAsDateObject.getUTCDate() + 1
); );
server/src/lib/sync.ts
+19 -8
View File
@@ -148,12 +148,21 @@ export async function pullOptionContractAggregates(
...optionContract, ...optionContract,
firstDate, firstDate,
})) { })) {
console.log(batch.length); if (batch.length > 0) {
await clickhouse.insert({ console.log(
table: "option_contract_aggregates", optionContract.symbol,
values: batch, optionContract.expirationDate,
format: "JSONEachRow", optionContract.strike,
}); optionContract.type,
new Date(batch[0].tsStart * 1000),
new Date(batch[batch.length - 1].tsStart * 1000)
);
await clickhouse.insert({
table: "option_contract_aggregates",
values: batch,
format: "JSONEachRow",
});
}
} }
await setPullOptionContractAggregatesState(ticker, { await setPullOptionContractAggregatesState(ticker, {
status: OptionContractAggregatesSyncStatus.COMPLETED, status: OptionContractAggregatesSyncStatus.COMPLETED,
@@ -210,13 +219,15 @@ export async function pullOptionContractAggregatesSince(
yesterdayAsDateObject.setUTCDate(yesterdayAsDateObject.getUTCDate() - 1); yesterdayAsDateObject.setUTCDate(yesterdayAsDateObject.getUTCDate() - 1);
while (currentDateAsDateObject <= yesterdayAsDateObject) { while (currentDateAsDateObject <= yesterdayAsDateObject) {
const currentDate = currentDateAsDateObject.toISOString().substring(0, 10); const currentDate = currentDateAsDateObject.toISOString().substring(0, 10);
console.log(`Date: ${currentDate}`);
for await (const optionContracts of makeGetOptionContractsIterator( for await (const optionContracts of makeGetOptionContractsIterator(
symbol, symbol,
currentDate currentDate
)) { )) {
await pullOptionContractAggregates(optionContracts); for (const optionContract of optionContracts) {
await pullOptionContractAggregates(optionContract);
}
} }
console.log(`Date: ${currentDate}`);
currentDateAsDateObject.setUTCDate( currentDateAsDateObject.setUTCDate(
currentDateAsDateObject.getUTCDate() + 1 currentDateAsDateObject.getUTCDate() + 1
); );