import fs from "fs/promises"; import duckdb from "duckdb"; const db = new duckdb.Database("quotes.duckdb"); const pathToCsvs = "/home/brian/Downloads/options-data"; const statements = [ "CREATE TYPE OPTION_TYPE as ENUM ('put', 'call');", "CREATE TYPE OPTION_STYLE as ENUM ('A', 'E');", `CREATE TABLE IF NOT EXISTS option_quote ( contract VARCHAR GENERATED ALWAYS AS ( CONCAT( underlying , RIGHT(YEAR(expiration)::VARCHAR,2) , LPAD(MONTH(expiration)::VARCHAR,2,'0') , LPAD(DAY(expiration)::VARCHAR,2,'0') , (CASE WHEN type = 'call' THEN 'C' ELSE 'P' END) , LPAD(((strike*1000)::INTEGER)::VARCHAR,8,'0') ) ) VIRTUAL, underlying VARCHAR, expiration DATE, type OPTION_TYPE, strike FLOAT, style OPTION_STYLE, bid FLOAT, bid_size INTEGER DEFAULT 0, ask FLOAT, ask_size INTEGER DEFAULT 0, volume INTEGER, open_interest INTEGER, quote_date DATE, delta FLOAT, gamma FLOAT, theta FLOAT, vega FLOAT, implied_volatility FLOAT );`, `CREATE TABLE IF NOT EXISTS stock_quote ( quote_date DATE, symbol VARCHAR, open FLOAT DEFAULT 0.0, high FLOAT DEFAULT 0.0, low FLOAT DEFAULT 0.0, close FLOAT DEFAULT 0.0, volume FLOAT DEFAULT 0.0, adjust_close FLOAT DEFAULT 0.0 );` ]; try { const files = await fs.readdir(pathToCsvs); for (const filename of files) { const fileExtension = filename.substring(filename.length - 11); if (fileExtension === "options.csv" || fileExtension === "options.cvs") { const quoteDate = filename.substring(0, 10); statements.push(`INSERT INTO option_quote ( SELECT underlying, expiration, type, strike, style, bid, bid_size, ask, ask_size, volume, open_interest, quote_date, delta, gamma, theta, vega, implied_volatility FROM read_csv_auto('${pathToCsvs}/${filename}') );`); statements.push(`INSERT INTO stock_quote ( SELECT '${quoteDate}', symbol, open, high, low, close, volume, adjust_close FROM read_csv_auto('${pathToCsvs}/${quoteDate}stocks.cvs') );`); } } console.log(statements); db.exec(statements.join(" "), (err) => { if (err) { console.error(err); return; } db.all("SELECT contract FROM option_quote WHERE underlying = 'TSLA' LIMIT 10", (err2, res) => { if (err2) { console.error(err2); return; } console.log(res[0]); }); }); } catch (err) { console.error(err); }