avraham/calendar-optimizer-deno
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

ingest script

Browse Source
This commit is contained in:
Avraham Sakal
2023-06-14 21:51:40 -04:00
parent b496a35a27
commit f7a35a1bde
1 changed files with 11 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ingest.ts
+11 -11
View File
@@ -32,7 +32,7 @@ contract VARCHAR(20) GENERATED ALWAYS AS (
),
*/
await sql`CREATE TABLE IF NOT EXISTS option_quote (
underlying VARCHAR(4),
underlying VARCHAR,
expiration DATE,
type OPTION_TYPE,
strike FLOAT,
@@ -105,12 +105,11 @@ export async function ingestStocks(sourceDataDir:string):Promise<void>{
export async function ingestOptions(sourceDataDir:string):Promise<void>{
// read each csv, and ingest each row into postgres:
const csvFiles = await fs.readdir(sourceDataDir);
await Promise.all(csvFiles.filter((csvFile)=>csvFile.substring(10,17)==='options').map(async (csvFile)=>{
const csvFiles = (await fs.readdir(sourceDataDir)).filter((csvFile)=>csvFile.substring(10,17)==='options');
for(const csvFile of csvFiles){
const quoteDate = csvFile.substring(0,10);
const rows = parse(await fs.readFile(join(sourceDataDir, csvFile)));
await Promise.all(rows.map(async ([underlying, expiration, type, strike, style, bid, bid_size, ask, ask_size, volume, open_interest, quote_date, delta, gamma, theta, vega, implied_volatility])=>{
expiration=Number(expiration);
const rows = parse(await fs.readFile(join(sourceDataDir, csvFile)), {columns:true});
for(let {underlying, expiration, type, strike, style, bid, bid_size, ask, ask_size, volume, open_interest, quote_date, delta, gamma, theta, vega, implied_volatility} of rows){
strike=Number(strike);
bid=Number(bid);
bid_size=Number(bid_size);
@@ -118,7 +117,6 @@ export async function ingestOptions(sourceDataDir:string):Promise<void>{
ask_size=Number(ask_size);
volume=Number(volume);
open_interest=Number(open_interest);
quote_date=Number(quote_date)
delta=Number(delta);
gamma=Number(gamma);
theta=Number(theta);
@@ -162,11 +160,13 @@ export async function ingestOptions(sourceDataDir:string):Promise<void>{
${vega},
${implied_volatility}
);`;
console.log(`options ${quoteDate} ${underlying}`);
//console.log(`options ${quoteDate} ${underlying}`);
}
catch(err){
console.error(err);
console.error(`${quoteDate} ${underlying}`, err);
}
}));
}));
};
};
console.log("creating index");
sql`CREATE INDEX ON "option_quote" USING btree ("underlying","expiration","type","strike","quote_date");`;
}