From f7a35a1bdec5a5fb0bfb5c0ff501f819bf936ba0 Mon Sep 17 00:00:00 2001
From: Avraham Sakal <briansakal@gmail.com>
Date: Wed, 14 Jun 2023 21:51:40 -0400
Subject: [PATCH] ingest script

---
 src/ingest.ts | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/src/ingest.ts b/src/ingest.ts
index e3af0a5..55ef939 100644
--- a/src/ingest.ts
+++ b/src/ingest.ts
@@ -32,7 +32,7 @@ contract VARCHAR(20) GENERATED ALWAYS AS (
     ),
  */
 await sql`CREATE TABLE IF NOT EXISTS option_quote (
-    underlying VARCHAR(4),
+    underlying VARCHAR,
     expiration DATE,
     type OPTION_TYPE,
     strike FLOAT,
@@ -105,12 +105,11 @@ export async function ingestStocks(sourceDataDir:string):Promise<void>{
 
 export async function ingestOptions(sourceDataDir:string):Promise<void>{
   // read each csv, and ingest each row into postgres:
-  const csvFiles = await fs.readdir(sourceDataDir);
-  await Promise.all(csvFiles.filter((csvFile)=>csvFile.substring(10,17)==='options').map(async (csvFile)=>{
+  const csvFiles = (await fs.readdir(sourceDataDir)).filter((csvFile)=>csvFile.substring(10,17)==='options');
+  for(const csvFile of csvFiles){
     const quoteDate = csvFile.substring(0,10);
-    const rows = parse(await fs.readFile(join(sourceDataDir, csvFile)));
-    await Promise.all(rows.map(async ([underlying, expiration, type, strike, style, bid, bid_size, ask, ask_size, volume, open_interest, quote_date, delta, gamma, theta, vega, implied_volatility])=>{
-      expiration=Number(expiration); 
+    const rows = parse(await fs.readFile(join(sourceDataDir, csvFile)), {columns:true});
+    for(let {underlying, expiration, type, strike, style, bid, bid_size, ask, ask_size, volume, open_interest, quote_date, delta, gamma, theta, vega, implied_volatility} of rows){
       strike=Number(strike); 
       bid=Number(bid); 
       bid_size=Number(bid_size); 
@@ -118,7 +117,6 @@ export async function ingestOptions(sourceDataDir:string):Promise<void>{
       ask_size=Number(ask_size); 
       volume=Number(volume); 
       open_interest=Number(open_interest); 
-      quote_date=Number(quote_date) 
       delta=Number(delta); 
       gamma=Number(gamma); 
       theta=Number(theta); 
@@ -162,11 +160,13 @@ export async function ingestOptions(sourceDataDir:string):Promise<void>{
             ${vega},
             ${implied_volatility}
           );`;
-        console.log(`options ${quoteDate} ${underlying}`);
+          //console.log(`options ${quoteDate} ${underlying}`);
       }
       catch(err){
-        console.error(err);
+        console.error(`${quoteDate} ${underlying}`, err);
       }
-    }));
-  }));
+    };
+  };
+  console.log("creating index");
+  sql`CREATE INDEX ON "option_quote" USING btree ("underlying","expiration","type","strike","quote_date");`;
 }
\ No newline at end of file