ingest script

src/ingest.ts

@@ -32,7 +32,7 @@ contract VARCHAR(20) GENERATED ALWAYS AS (
     ),
  */
 await sql`CREATE TABLE IF NOT EXISTS option_quote (
-    underlying VARCHAR(4),
+    underlying VARCHAR,
     expiration DATE,
     type OPTION_TYPE,
     strike FLOAT,
@@ -105,12 +105,11 @@ export async function ingestStocks(sourceDataDir:string):Promise<void>{
 
 export async function ingestOptions(sourceDataDir:string):Promise<void>{
   // read each csv, and ingest each row into postgres:
-  const csvFiles = await fs.readdir(sourceDataDir);
-  await Promise.all(csvFiles.filter((csvFile)=>csvFile.substring(10,17)==='options').map(async (csvFile)=>{
+  const csvFiles = (await fs.readdir(sourceDataDir)).filter((csvFile)=>csvFile.substring(10,17)==='options');
+  for(const csvFile of csvFiles){
     const quoteDate = csvFile.substring(0,10);
-    const rows = parse(await fs.readFile(join(sourceDataDir, csvFile)));
-    await Promise.all(rows.map(async ([underlying, expiration, type, strike, style, bid, bid_size, ask, ask_size, volume, open_interest, quote_date, delta, gamma, theta, vega, implied_volatility])=>{
-      expiration=Number(expiration); 
+    const rows = parse(await fs.readFile(join(sourceDataDir, csvFile)), {columns:true});
+    for(let {underlying, expiration, type, strike, style, bid, bid_size, ask, ask_size, volume, open_interest, quote_date, delta, gamma, theta, vega, implied_volatility} of rows){
       strike=Number(strike); 
       bid=Number(bid); 
       bid_size=Number(bid_size); 
@@ -118,7 +117,6 @@ export async function ingestOptions(sourceDataDir:string):Promise<void>{
       ask_size=Number(ask_size); 
       volume=Number(volume); 
       open_interest=Number(open_interest); 
-      quote_date=Number(quote_date) 
       delta=Number(delta); 
       gamma=Number(gamma); 
       theta=Number(theta); 
@@ -162,11 +160,13 @@ export async function ingestOptions(sourceDataDir:string):Promise<void>{
             ${vega},
             ${implied_volatility}
           );`;
-        console.log(`options ${quoteDate} ${underlying}`);
+          //console.log(`options ${quoteDate} ${underlying}`);
       }
       catch(err){
-        console.error(err);
+        console.error(`${quoteDate} ${underlying}`, err);
       }
-    }));
-  }));
+    };
+  };
+  console.log("creating index");
+  sql`CREATE INDEX ON "option_quote" USING btree ("underlying","expiration","type","strike","quote_date");`;
 }