calendar-optimizer-deno/src/ingest-options-from-polygon.mjs

//import { restClient as PolygonClient } from '@polygon.io/client-js';
import { createClient as createClickhouseClient } from '@clickhouse/client'
import { sleep, getPolygonApiKey } from './util.mjs';

//const polygonClient = PolygonClient(apiKey, "https://api.polygon.io", {pagination: false, trace: true,}); // automatically call `next_url` if there is one

const clickhouse = createClickhouseClient({username:'avraham', password:'buginoo'});

const optionContractsTableName = "option_aggregates";
async function deleteClickhouseTable(){
  await clickhouse.command({
    query: `DROP TABLE IF EXISTS ${optionContractsTableName}`,
  })
}
async function createClickhouseTable(){
  await clickhouse.command({
    query: `
      CREATE TABLE ${optionContractsTableName}
      (
        symbol LowCardinality(String), 
        asOfDate Date,
        expirationDate Date,
        optionType Enum('call', 'put'),
        strike Decimal64(9)
      )
      ENGINE MergeTree()
      ORDER BY (symbol, asOfDate, expirationDate, optionType, strike)
    `,
  });
  /*
  await clickhouse.command({
    query: `
      CREATE TABLE ${optionAggregatesTableName}
      (
        symbol LowCardinality(String), 
        tsStart DateTime32,
        
        expirationDate DateTime32,
        optionType Enum('call', 'put'),
        strike Decimal64(9),

        open Decimal64(9), 
        close Decimal64(9), 
        low Decimal64(9), 
        high Decimal64(9), 
        volume UInt64, 
        volume_weighted_price Decimal64(9)
      )
      ENGINE MergeTree()
      ORDER BY (tsStart, symbol, expirationDate, optionType, strike)
    `,
  });
  */
}

async function insertResultsIntoClickhouse(symbol, asOfDate, optionContractsResult){
  await clickhouse.insert({
    table: optionContractsTableName,
    // structure should match the desired format, JSONEachRow in this example
    values: optionContractsResult.results.map(r=>({
      symbol, 
      asOfDate,
      expirationDate: r.expiration_date,
      optionType: r.contract_type,
      strike: r.strike_price
    })),
    format: 'JSONEachRow',
  });
  console.log('inserted', symbol);
}

/**
 * Get all available option expirations and strikes, from the point-of-view of a given date
 */
async function fetchAvailableOptionExpirationsAndStrikesAsOfDate(symbol, asOfDate){
  const limit = 1000;
  let apiKey = await getPolygonApiKey();
  let optionContractsResult = await (await fetch(`https://api.polygon.io/v3/reference/options/contracts?underlying_ticker=${symbol}&contract_type=call&as_of=${asOfDate}&expired=false&order=asc&limit=${limit}&sort=expiration_date&apiKey=${apiKey}`)).json();
  console.log(optionContractsResult.results.length);
  if(optionContractsResult.status === 'OK' && typeof optionContractsResult.results !== "undefined" && optionContractsResult.results.length > 0){
    await insertResultsIntoClickhouse(symbol, asOfDate, optionContractsResult);
    while(optionContractsResult.hasOwnProperty("next_url")){
      apiKey = await getPolygonApiKey();
      optionContractsResult = await (await fetch(`${optionContractsResult.next_url}&apiKey=${apiKey}`)).json();
      if(optionContractsResult.status === 'OK' && typeof optionContractsResult.results !== "undefined" && optionContractsResult.results.length > 0){
        console.log(optionContractsResult.results.length);
        await insertResultsIntoClickhouse(symbol, asOfDate, optionContractsResult);
      }
      else{
        console.log(optionContractsResult);
      }
    }
  }
  else{
    console.log(optionContractsResult);
  }
}

const stockAggregatesTableName = "stock_aggregates";
async function fetchMarketDates(){
  return (await (await clickhouse.query({
    query: `SELECT DISTINCT(Date(tsStart)) FROM ${stockAggregatesTableName}`,
    format: 'JSONCompactEachRow'
  })).json()).map((r)=>r[0]);
}

await deleteClickhouseTable();
await createClickhouseTable();

const marketDates = await fetchMarketDates();
for(let asOfDate of marketDates){
  for(let symbol of ["AAPL", "GOOGL", "MSFT", "TSLA", "AMD", "NFLX", "SPY"]){
    await fetchAvailableOptionExpirationsAndStrikesAsOfDate(symbol, asOfDate);
  }
}


/**
 * For each day (not aggregate), look-up the available option contracts (i.e.
 * all unique type/strike/exp combinations). Limit is 1000, but it gives you a
 * "nextUrl", so keep calling that until all results are in.
 */

/**
 * For each aggregate and the contracts on that day, get option aggregates
 */



await clickhouse.close();