use Slider for lookback period

gitignore Bruno files
use line on stock price chart; more compact data transfer format
2025-03-03 21:35:51 -05:00 · 2025-03-02 17:54:41 -05:00 · 2025-03-02 17:53:59 -05:00 · 2025-03-02 09:35:43 -05:00 · 2025-02-24 09:15:57 -05:00 · 2025-02-24 09:15:35 -05:00
73 changed files with 8362 additions and 2588 deletions
@@ -0,0 +1,4 @@
 .pnpm-store
 node_modules
 .aider*
 .env
@@ -0,0 +1,3 @@
 nodejs 20.15.1
 python 3.12.4
 pnpm 9.7.1
@@ -0,0 +1,16 @@
 {
 	"$schema": "https://biomejs.dev/schemas/1.8.3/schema.json",
 	"organizeImports": {
 		"enabled": true
 	},
 	"formatter": {
 		"indentWidth": 2,
 		"indentStyle": "space"
 	},
 	"linter": {
 		"enabled": true,
 		"rules": {
 			"recommended": true
 		}
 	}
 }
@@ -0,0 +1 @@
 20
@@ -2,14 +2,14 @@
 FROM node:20-slim AS build
 ENV PNPM_HOME="/pnpm"
 ENV PATH="$PNPM_HOME:$PATH"
-RUN corepack enable
+RUN npm install -g corepack@latest && corepack enable
 WORKDIR /app
 # copy what's necessary to install dependencies:
 COPY package.json pnpm-lock.yaml /app/
 # install dependencies:
 RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --frozen-lockfile
 # copy what's necessary to build:
-COPY tsconfig.json vite.config.ts index.html /app/
+COPY tsconfig.json vite.config.ts index.html index.css /app/
 COPY src /app/src
 # Vite injects envvars at build time, not runtime:
 ENV VITE_SERVER_BASE_URL=https://calendar-optimizer-server.sakal.us
@@ -0,0 +1,37 @@
 :root {
  font-family: Inter, system-ui, Avenir, Helvetica, Arial, sans-serif;
  line-height: 1.5;
  font-weight: 400;
  color: #222;
  background-color: #ffffff;
  font-synthesis: none;
  text-rendering: optimizeLegibility;
  -webkit-font-smoothing: antialiased;
  -moz-osx-font-smoothing: grayscale;
  -webkit-text-size-adjust: 100%;
  box-sizing: border-box;
 }
 *,
 *::before,
 *::after {
  box-sizing: inherit;
 }
 body {
  margin: 0;
 }
 /* Input styling: */
 :root {
  --radius-inputs: 0.25em;
 }
 select,
 textarea,
 input[type="text"] {
  border-radius: var(--radius-inputs);
 }
@@ -1,14 +1,15 @@
 <!DOCTYPE html>
 <html lang="en">
-	<head>
+  <head>
-		<meta charset="UTF-8" />
+    <meta charset="UTF-8" />
-		<link rel="icon" type="image/svg+xml" href="/vite.svg" />
+    <link rel="icon" type="image/svg+xml" href="/vite.svg" />
-		<meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <link rel="stylesheet" type="text/css" href="/index.css" />
-		<meta name="color-scheme" content="light dark" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-		<title>Vite + Preact</title>
+    <meta name="color-scheme" content="light dark" />
-	</head>
+    <title>Options Calendar Spread Research Tool</title>
-	<body>
+  </head>
-		<div id="app"></div>
+  <body>
-		<script type="module" src="/src/index.tsx"></script>
+    <div id="app"></div>
-	</body>
+    <script type="module" src="/src/index.tsx"></script>
  </body>
 </html>
@@ -7,13 +7,19 @@
 		"preview": "vite preview"
 	},
 	"dependencies": {
 		"@mui/icons-material": "^6.1.1",
 		"@mui/material": "^6.1.1",
 		"@mui/system": "^6.1.1",
 		"@mui/x-date-pickers": "^7.18.0",
 		"@preact/signals": "^1.2.2",
 		"@trpc/client": "^10.45.0",
 		"chart.js": "^4.4.1",
 		"date-fns": "^3.6.0",
 		"dotenv": "^16.4.1",
 		"preact": "^10.13.1",
 		"preact-iso": "^2.3.2",
 		"preact-render-to-string": "^6.3.1",
 		"react": "18.3.1",
 		"react-chartjs-2": "^5.2.0"
 	},
 	"devDependencies": {
@@ -1,24 +1,40 @@
-import { useLocation } from 'preact-iso';
+import { useLocation } from "preact-iso";
 import { AppBar, Toolbar, Button, Box } from "@mui/material";
 import { styled } from "@mui/system";
 const StyledToolbar = styled(Toolbar)({
  display: "flex",
  justifyContent: "center",
  backgroundColor: "#E0E7FF", // Indigo-200 equivalent
 });
 const StyledButton = styled(Button)(({ theme, active }) => ({
  color: theme.palette.text.primary,
  fontWeight: active ? "bold" : "normal",
  textDecoration: active ? "underline" : "none",
 }));
 export function Header() {
-	const { url } = useLocation();
+  const { url } = useLocation();
-	return (
+  return (
-		<header>
+    <AppBar position="static" elevation={0}>
-			<nav>
+      <StyledToolbar>
-				<a href="/" class={url == '/' && 'active'}>
+        <Box sx={{ display: "flex", gap: 2 }}>
-					Home
+          <StyledButton
-				</a>
+            href="/"
-				<a href="/calendar-optimizer" class={url == '/calendar-optimizer' && 'active'}>
+            active={url === "/" || url === "/historical-calendar-prices"}
-					Calendar Optimizer
+          >
-				</a>
+            Historical Calendar Prices
-				<a href="/historical-calendar-prices" class={url == '/historical-calendar-prices' && 'active'}>
+          </StyledButton>
-					Historical Calendar Prices
+          <StyledButton
-				</a>
+            href="/calendar-optimizer"
-				<a href="/404" class={url == '/404' && 'active'}>
+            active={url === "/calendar-optimizer"}
-					404
+          >
-				</a>
+            Calendar Optimizer
-			</nav>
+          </StyledButton>
-		</header>
+        </Box>
-	);
+      </StyledToolbar>
-}
+    </AppBar>
  );
 }
@@ -1,28 +1,37 @@
-import _ from './env';
+import { render } from "preact";
-import { render } from 'preact';
+import { LocationProvider, Router, Route } from "preact-iso";
-import { LocationProvider, Router, Route } from 'preact-iso';
+import { ThemeProvider, createTheme } from '@mui/material/styles';
 import CssBaseline from '@mui/material/CssBaseline';
-import { Header } from './components/Header.jsx';
+import { Header } from "./components/Header.jsx";
-import { Home } from './pages/Home/index.jsx';
+import { CalendarOptimizer } from "./pages/CalendarOptimizer.js";
-import { CalendarOptimizer } from './pages/CalendarOptimizer/index.jsx';
+import { NotFound } from "./pages/_404.jsx";
-import { NotFound } from './pages/_404.jsx';
+import { HistoricalCalendarPrices } from "./pages/HistoricalCalendarPrices.js";
-import './style.css';
+
-import { HistoricalCalendarPrices } from './pages/HistoricalCalendarPrices/HistoricalCalendarPrices.js';
+const theme = createTheme();
 export function App() {
-	return (
+  return (
-		<LocationProvider>
+    <ThemeProvider theme={theme}>
-			<Header />
+      <CssBaseline />
-			<main>
+      <LocationProvider>
-				<Router>
+        <div>
-					<Route path="/" component={Home} />
+          <Header />
-					<Route path="/calendar-optimizer" component={CalendarOptimizer} />
+          <main>
-					<Route path="/historical-calendar-prices" component={HistoricalCalendarPrices} />
+            <Router>
-					<Route default component={NotFound} />
+              <Route path="/" component={HistoricalCalendarPrices} />
-				</Router>
+              <Route path="/calendar-optimizer" component={CalendarOptimizer} />
-			</main>
+              <Route
-		</LocationProvider>
+                path="/historical-calendar-prices"
-	);
+                component={HistoricalCalendarPrices}
              />
              <Route default component={NotFound} />
            </Router>
          </main>
        </div>
      </LocationProvider>
    </ThemeProvider>
  );
 }
-render(<App />, document.getElementById('app'));
+render(<App />, document.getElementById("app"));
@@ -0,0 +1,337 @@
 import { signal } from "@preact/signals";
 import { useCallback, useEffect } from "preact/hooks";
 import { trpc } from "../trpc.js";
 import {
  Chart as ChartJS,
  LinearScale,
  CategoryScale,
  PointElement,
  Tooltip,
  Title,
 } from "chart.js";
 import { Scatter } from "react-chartjs-2";
 import {
  Container,
  Grid,
  Typography,
  FormControl,
  InputLabel,
  Select,
  MenuItem,
  Paper,
 } from "@mui/material";
 ChartJS.register(LinearScale, CategoryScale, PointElement, Tooltip, Title);
 const availableUnderlyings = signal([]);
 const chosenUnderlying = signal(null);
 const availableAsOfDates = signal([]);
 const chosenAsOfDate = signal(null);
 const availableExpirations = signal([]);
 const chosenExpiration = signal(null);
 const availableStrikes = signal([]);
 const chosenStrike = signal(null);
 const optionContractUplotData = signal([]);
 const underlyingUplotData = signal([]);
 function chooseUnderlying(underlying: string) {
  chosenUnderlying.value = underlying;
  trpc.getAvailableAsOfDates
    .query({ underlying: underlying })
    .then((getAvailableAsOfDatesResponse) => {
      availableAsOfDates.value = getAvailableAsOfDatesResponse;
      chooseAsOfDate(getAvailableAsOfDatesResponse[0]);
    });
  trpc.getOpensForUnderlying
    .query({ underlying: underlying })
    .then((getOpensForUnderlyingResponse) => {
      underlyingUplotData.value = getOpensForUnderlyingResponse;
    });
 }
 function chooseAsOfDate(asOfDate: string) {
  chosenAsOfDate.value = asOfDate;
  trpc.getExpirationsForUnderlying
    .query({
      underlying: chosenUnderlying.value,
      asOfDate: chosenAsOfDate.value,
    })
    .then((getExpirationsForUnderlyingResponse) => {
      availableExpirations.value = getExpirationsForUnderlyingResponse;
      chooseExpiration(getExpirationsForUnderlyingResponse[0]);
    });
 }
 function chooseExpiration(expiration: string) {
  chosenExpiration.value = expiration;
  trpc.getStrikesForUnderlying
    .query({
      underlying: chosenUnderlying.value,
      asOfDate: chosenAsOfDate.value,
      expirationDate: expiration,
    })
    .then((getStrikesForUnderlyingResponse) => {
      availableStrikes.value = getStrikesForUnderlyingResponse;
      chooseStrike(getStrikesForUnderlyingResponse[0]);
    });
 }
 function chooseStrike(strike: string) {
  chosenStrike.value = strike;
  trpc.getOpensForOptionContract
    .query({
      underlying: chosenUnderlying.value,
      expirationDate: chosenExpiration.value,
      strike: Number.parseFloat(strike),
    })
    .then((getOpensForOptionContractResponse) => {
      optionContractUplotData.value = getOpensForOptionContractResponse;
    });
 }
 export function CalendarOptimizer() {
  const handleInit = useCallback(() => {
    trpc.getAvailableUnderlyings
      .query()
      .then((availableUnderlyingsResponse) => {
        availableUnderlyings.value = availableUnderlyingsResponse;
        // load first underlying in list:
        chooseUnderlying(availableUnderlyingsResponse[0]);
      });
  }, []);
  const handleUnderlyingChange = useCallback((e) => {
    console.log(`Chose Underlying: ${e.target.value}`);
    chooseUnderlying(e.target.value);
  }, []);
  const handleAsOfDateChange = useCallback((e) => {
    console.log(`Chose Date: ${e.target.value}`);
    chooseAsOfDate(e.target.value);
  }, []);
  const handleExpirationChange = useCallback((e) => {
    console.log(`Chose Expiration: ${e.target.value}`);
    chooseExpiration(e.target.value);
  }, []);
  const handleStrikeChange = useCallback((e) => {
    console.log(`Chose Strike: ${e.target.value}`);
    chooseStrike(e.target.value);
  }, []);
  useEffect(handleInit, []);
  return (
    <Container maxWidth="lg">
      <Grid container spacing={4}>
        <Grid item xs={12}>
          <Typography variant="h4" gutterBottom>
            Calendar Optimizer
          </Typography>
        </Grid>
        <Grid item xs={12} md={6}>
          <Paper elevation={3} sx={{ p: 3 }}>
            <Grid container spacing={2}>
              <Grid item xs={12}>
                <FormControl fullWidth>
                  <InputLabel>Available Underlyings</InputLabel>
                  <Select
                    value={chosenUnderlying.value || ""}
                    onChange={handleUnderlyingChange}
                    label="Available Underlyings"
                  >
                    {availableUnderlyings.value.map((underlying) => (
                      <MenuItem key={underlying} value={underlying}>
                        {underlying}
                      </MenuItem>
                    ))}
                  </Select>
                </FormControl>
              </Grid>
              <Grid item xs={12}>
                <FormControl fullWidth>
                  <InputLabel>Available "As-of" Dates</InputLabel>
                  <Select
                    value={chosenAsOfDate.value || ""}
                    onChange={handleAsOfDateChange}
                    label='Available "As-of" Dates'
                  >
                    {availableAsOfDates.value.map((asOfDate) => (
                      <MenuItem key={asOfDate} value={asOfDate}>
                        {asOfDate}
                      </MenuItem>
                    ))}
                  </Select>
                </FormControl>
              </Grid>
              <Grid item xs={12}>
                <FormControl fullWidth>
                  <InputLabel>Available Expirations</InputLabel>
                  <Select
                    value={chosenExpiration.value || ""}
                    onChange={handleExpirationChange}
                    label="Available Expirations"
                  >
                    {availableExpirations.value.map((expiration) => (
                      <MenuItem key={expiration} value={expiration}>
                        {expiration}
                      </MenuItem>
                    ))}
                  </Select>
                </FormControl>
              </Grid>
              <Grid item xs={12}>
                <FormControl fullWidth>
                  <InputLabel>Available Strikes</InputLabel>
                  <Select
                    value={chosenStrike.value || ""}
                    onChange={handleStrikeChange}
                    label="Available Strikes"
                  >
                    {availableStrikes.value.map((strike) => (
                      <MenuItem key={strike} value={strike}>
                        {strike}
                      </MenuItem>
                    ))}
                  </Select>
                </FormControl>
              </Grid>
            </Grid>
          </Paper>
        </Grid>
        <Grid item xs={12} md={6}>
          <Paper elevation={3} sx={{ p: 3, height: "100%" }}>
            {chosenUnderlying.value !== null &&
            underlyingUplotData.value.length > 0 ? (
              <Scatter
                data={{
                  datasets: [
                    {
                      label: "Stock Open Price",
                      data: underlyingUplotData.value,
                    },
                  ],
                }}
                options={{
                  scales: {
                    x: {
                      title: {
                        display: true,
                        text: "Time",
                      },
                      ticks: {
                        callback: (value, index, ticks) => {
                          return new Date((value as number) * 1000)
                            .toISOString()
                            .substring(0, 10);
                        },
                      },
                    },
                    y: {
                      beginAtZero: false,
                      ticks: {
                        callback: (value, index, ticks) => {
                          return `$${value.toString()}`;
                        },
                      },
                    },
                  },
                  elements: {
                    point: {
                      radius: 1,
                      borderWidth: 0,
                    },
                  },
                  plugins: {
                    tooltip: {
                      enabled: false,
                    },
                    legend: {
                      display: false,
                    },
                    title: {
                      display: true,
                      text: "Stock Price",
                    },
                  },
                  animation: false,
                  maintainAspectRatio: false,
                }}
              />
            ) : (
              <Typography>Loading Chart...</Typography>
            )}
          </Paper>
        </Grid>
        <Grid item xs={12}>
          <Paper elevation={3} sx={{ p: 3 }}>
            {chosenUnderlying.value !== null &&
            chosenAsOfDate.value !== null &&
            chosenExpiration.value !== null &&
            chosenStrike.value !== null &&
            optionContractUplotData.value.length > 0 ? (
              <Scatter
                data={{
                  datasets: [
                    {
                      label: "Option Contract Open Price",
                      data: optionContractUplotData.value,
                    },
                  ],
                }}
                options={{
                  scales: {
                    x: {
                      title: {
                        display: true,
                        text: "Time",
                      },
                      ticks: {
                        callback: (value, index, ticks) => {
                          return new Date((value as number) * 1000)
                            .toISOString()
                            .substring(0, 10);
                        },
                      },
                    },
                    y: {
                      beginAtZero: false,
                      ticks: {
                        callback: (value, index, ticks) => {
                          return `$${value.toString()}`;
                        },
                      },
                    },
                  },
                  elements: {
                    point: {
                      radius: 1,
                      borderWidth: 0,
                    },
                  },
                  plugins: {
                    tooltip: {
                      enabled: false,
                    },
                    legend: {
                      display: false,
                    },
                    title: {
                      display: true,
                      text: "Option Contract Price",
                    },
                  },
                  animation: false,
                  maintainAspectRatio: false,
                }}
              />
            ) : (
              <Typography>Loading Chart...</Typography>
            )}
          </Paper>
        </Grid>
      </Grid>
    </Container>
  );
 }
@@ -1,305 +0,0 @@
 import { signal } from "@preact/signals";
 import { useCallback, useEffect } from "preact/hooks";
 import { trpc } from "../../trpc.js";
 import {
  Chart as ChartJS,
  LinearScale,
  CategoryScale,
  PointElement,
  Tooltip,
  Title,
 } from "chart.js";
 import { Scatter } from "react-chartjs-2";
 import "./style.css";
 ChartJS.register(LinearScale, CategoryScale, PointElement, Tooltip, Title);
 const availableUnderlyings = signal([]);
 const chosenUnderlying = signal(null);
 const availableAsOfDates = signal([]);
 const chosenAsOfDate = signal(null);
 const availableExpirations = signal([]);
 const chosenExpiration = signal(null);
 const availableStrikes = signal([]);
 const chosenStrike = signal(null);
 const optionContractUplotData = signal([]);
 const underlyingUplotData = signal([]);
 function chooseUnderlying(underlying: string) {
  chosenUnderlying.value = underlying;
  trpc.getAvailableAsOfDates
    .query({ underlying: underlying })
    .then((getAvailableAsOfDatesResponse) => {
      availableAsOfDates.value = getAvailableAsOfDatesResponse;
      chooseAsOfDate(getAvailableAsOfDatesResponse[0]);
    });
  trpc.getOpensForUnderlying
    .query({ underlying: underlying })
    .then((getOpensForUnderlyingResponse) => {
      underlyingUplotData.value = getOpensForUnderlyingResponse;
    });
 }
 function chooseAsOfDate(asOfDate: string) {
  chosenAsOfDate.value = asOfDate;
  trpc.getExpirationsForUnderlying
    .query({
      underlying: chosenUnderlying.value,
      asOfDate: chosenAsOfDate.value,
    })
    .then((getExpirationsForUnderlyingResponse) => {
      availableExpirations.value = getExpirationsForUnderlyingResponse;
      chooseExpiration(getExpirationsForUnderlyingResponse[0]);
    });
 }
 function chooseExpiration(expiration: string) {
  chosenExpiration.value = expiration;
  trpc.getStrikesForUnderlying
    .query({
      underlying: chosenUnderlying.value,
      asOfDate: chosenAsOfDate.value,
      expirationDate: expiration,
    })
    .then((getStrikesForUnderlyingResponse) => {
      availableStrikes.value = getStrikesForUnderlyingResponse;
      chooseStrike(getStrikesForUnderlyingResponse[0]);
    });
 }
 function chooseStrike(strike: string) {
  chosenStrike.value = strike;
  trpc.getOpensForOptionContract
    .query({
      underlying: chosenUnderlying.value,
      expirationDate: chosenExpiration.value,
      strike: parseFloat(strike),
    })
    .then((getOpensForOptionContractResponse) => {
      optionContractUplotData.value = getOpensForOptionContractResponse;
    });
 }
 export function CalendarOptimizer() {
  const handleInit = useCallback(() => {
    trpc.getAvailableUnderlyings
      .query()
      .then((availableUnderlyingsResponse) => {
        availableUnderlyings.value = availableUnderlyingsResponse;
        // load first underlying in list:
        chooseUnderlying(availableUnderlyingsResponse[0]);
      });
  }, []);
  const handleUnderlyingChange = useCallback((e) => {
    console.log(`Chose Underlying: ${e.target.value}`);
    chooseUnderlying(e.target.value);
  }, []);
  const handleAsOfDateChange = useCallback((e) => {
    console.log(`Chose Date: ${e.target.value}`);
    chooseAsOfDate(e.target.value);
  }, []);
  const handleExpirationChange = useCallback((e) => {
    console.log(`Chose Expiration: ${e.target.value}`);
    chooseExpiration(e.target.value);
  }, []);
  const handleStrikeChange = useCallback((e) => {
    console.log(`Chose Strike: ${e.target.value}`);
    chooseStrike(e.target.value);
  }, []);
  useEffect(handleInit, []);
  return (
    <div>
      <div>
        <label>Available Underlyings</label>
        {availableUnderlyings.value.length === 0 ? (
          "Loading..."
        ) : (
          <select onChange={handleUnderlyingChange}>
            {availableUnderlyings.value.map((availableUnderlying) => (
              <option value={availableUnderlying}>{availableUnderlying}</option>
            ))}
          </select>
        )}
      </div>
      <div>
        <label>Available "As-of" Dates</label>
        {availableAsOfDates.value.length === 0 ? (
          "Loading..."
        ) : (
          <select onChange={handleAsOfDateChange}>
            {availableAsOfDates.value.map((availableAsOfDate) => (
              <option value={availableAsOfDate}>{availableAsOfDate}</option>
            ))}
          </select>
        )}
      </div>
      <div>
        <label>Available Expirations</label>
        {availableExpirations.value.length === 0 ? (
          "Loading..."
        ) : (
          <select onChange={handleExpirationChange}>
            {availableExpirations.value.map((availableExpiration) => (
              <option value={availableExpiration}>{availableExpiration}</option>
            ))}
          </select>
        )}
      </div>
      <div>
        <label>Available Strikes</label>
        {availableStrikes.value.length === 0 ? (
          "Loading..."
        ) : (
          <select onChange={handleStrikeChange}>
            {availableStrikes.value.map((availableStrike) => (
              <option value={availableStrike}>{availableStrike}</option>
            ))}
          </select>
        )}
      </div>
      <div className="chart-container">
        {chosenUnderlying.value !== null &&
        underlyingUplotData.value.length > 0 ? (
          <div className="chart">
            <Scatter
              data={{
                datasets: [
                  {
                    label: "Stock Open Price",
                    data: underlyingUplotData.value,
                  },
                ],
              }}
              options={{
                scales: {
                  x: {
                    title: {
                      display: true,
                      text: "Time",
                    },
                    ticks: {
                      callback: function (value, index, ticks) {
                        return new Date((value as number) * 1000)
                          .toISOString()
                          .substring(0, 10);
                      },
                    },
                    // min: (new Date(chosenLookbackPeriodStart.value)).getTime()/1000,
                    // max: (new Date(chosenLookbackPeriodEnd.value)).getTime()/1000,
                  },
                  y: {
                    beginAtZero: false,
                    ticks: {
                      callback: function (value, index, ticks) {
                        return "$" + value.toString();
                      },
                    },
                    // min: 0,
                    // max: maxChartPrice.value,
                  },
                },
                elements: {
                  point: {
                    radius: 1,
                    borderWidth: 0,
                  },
                },
                plugins: {
                  tooltip: {
                    enabled: false,
                  },
                  legend: {
                    display: false,
                  },
                  title: {
                    display: true,
                    text: "Stock Price",
                  },
                },
                animation: false,
                maintainAspectRatio: false,
              }}
            />
          </div>
        ) : (
          <></>
        )}
        {chosenUnderlying.value !== null &&
        chosenAsOfDate.value !== null &&
        chosenExpiration.value !== null &&
        chosenStrike.value !== null &&
        optionContractUplotData.value.length > 0 ? (
          <div className="chart">
            <Scatter
              data={{
                datasets: [
                  {
                    label: "Option Contract Open Price",
                    data: optionContractUplotData.value,
                  },
                ],
              }}
              options={{
                scales: {
                  x: {
                    title: {
                      display: true,
                      text: "Time",
                    },
                    ticks: {
                      callback: function (value, index, ticks) {
                        return new Date((value as number) * 1000)
                          .toISOString()
                          .substring(0, 10);
                      },
                    },
                    // min: (new Date(chosenLookbackPeriodStart.value)).getTime()/1000,
                    // max: (new Date(chosenLookbackPeriodEnd.value)).getTime()/1000,
                  },
                  y: {
                    beginAtZero: false,
                    ticks: {
                      callback: function (value, index, ticks) {
                        return "$" + value.toString();
                      },
                    },
                    // min: 0,
                    // max: maxChartPrice.value,
                  },
                },
                elements: {
                  point: {
                    radius: 1,
                    borderWidth: 0,
                  },
                },
                plugins: {
                  tooltip: {
                    enabled: false,
                  },
                  legend: {
                    display: false,
                  },
                  title: {
                    display: true,
                    text: "Option Contract Price",
                  },
                },
                animation: false,
                maintainAspectRatio: false,
              }}
            />
          </div>
        ) : (
          <></>
        )}
      </div>
    </div>
  );
 }
@@ -1,11 +0,0 @@
 .chart-container {
  display: flex;
  flex-direction: row;
  flex-wrap: nowrap;
  justify-content: space-around;
  width: 1800px;
  height: 480px;
 }
 .chart-container > .chart {
  width: 880px;
 }
@@ -0,0 +1,360 @@
 import { useEffect } from "preact/hooks";
 import { trpc } from "../trpc.js";
 import {
  Chart as ChartJS,
  LinearScale,
  CategoryScale,
  PointElement,
  LineElement,
  Tooltip,
  Title,
 } from "chart.js";
 import { Scatter } from "react-chartjs-2";
 import {
  Container,
  Grid2,
  Typography,
  Paper,
  Popper,
  ClickAwayListener,
  Stack,
 } from "@mui/material";
 import {
  availableUnderlyings,
  calendarExitPriceChartData,
  isPopperOpen,
  lookbackPeriodEnd,
  lookbackPeriodStart,
  maxChartPrice,
  maxN,
  popperAnchorEl,
  popperContent,
  similarCalendarPriceChartData,
  stockPriceChartData,
  underlying,
 } from "./HistoricalCalendarPrices/state.js";
 import { EditableStrike } from "./HistoricalCalendarPrices/EditableStrike.js";
 import {
  refreshcalendarExitPriceChartData,
  refreshSimilarCalendarPriceChartData,
  refreshStockPriceChartData,
 } from "./HistoricalCalendarPrices/actions.js";
 import { EditableUnderlying } from "./HistoricalCalendarPrices/EditableUnderlying.js";
 import { EditableOpenDTE } from "./HistoricalCalendarPrices/EditableOpenDTE.js";
 import { EditableExitDTE } from "./HistoricalCalendarPrices/EditableExitDTE.js";
 import { EditableSpan } from "./HistoricalCalendarPrices/EditableSpan.js";
 import { EditableLookbackPeriod } from "./HistoricalCalendarPrices/EditableLookbackPeriod.js";
 ChartJS.register(
  LinearScale,
  CategoryScale,
  PointElement,
  LineElement,
  Tooltip,
  Title
 );
 const handleInit = () => {
  trpc.CalendarCharacteristicsForm.getAvailableUnderlyings
    .query()
    .then((availableUnderlyingsResponse) => {
      availableUnderlyings.value = availableUnderlyingsResponse;
      underlying.value = availableUnderlyingsResponse[0];
      refreshStockPriceChartData();
      refreshSimilarCalendarPriceChartData();
      refreshcalendarExitPriceChartData();
    });
 };
 export function HistoricalCalendarPrices() {
  useEffect(handleInit, []);
  return (
    <Container maxWidth="lg">
      <Grid2 container spacing={4} columns={12}>
        {/* <Grid2 size={{ xs: 12 }}>
          <Typography variant="h4" gutterBottom>
            <EditableUnderlying /> :
            <EditableSpan />
            -Day Calendar @ <EditableStrike />
            %-from-the-money
          </Typography>
          <Typography variant="h5" gutterBottom sx={{ pl: 1 }}>
            Opening at <EditableOpenDTE /> DTE, Closing at <EditableExitDTE />
            DTE
          </Typography>
          <Typography variant="h5" gutterBottom>
            <EditableLookbackPeriodStart />-
            <EditableLookbackPeriodEnd />
          </Typography>
          <ClickAwayListener
            onClickAway={() => {
              isPopperOpen.value = false;
              // refreshSimilarCalendarPriceChartData();
              console.log("clicked away");
            }}
          >
            <Popper open={isPopperOpen.value} anchorEl={popperAnchorEl.value}>
              <Paper elevation={3} sx={{ p: 3 }}>
                {popperContent.value}
              </Paper>
            </Popper>
          </ClickAwayListener>
        </Grid2> */}
        <Grid2 size={{ xs: 12 }}>
          <Stack direction="row" spacing={2}>
            <Typography gutterBottom minWidth={"8em"}>
              Underlying
            </Typography>
            <EditableUnderlying />
          </Stack>
          <Stack direction="row" spacing={2}>
            <Typography gutterBottom minWidth={"8em"}>
              Open DTE
            </Typography>
            <EditableOpenDTE />
          </Stack>
          <Stack direction="row" spacing={2}>
            <Typography gutterBottom minWidth={"8em"}>
              Exit DTE
            </Typography>
            <EditableExitDTE />
          </Stack>
          <Stack direction="row" spacing={2}>
            <Typography gutterBottom minWidth={"8em"}>
              Span
            </Typography>
            <EditableSpan />
          </Stack>
          <Stack direction="row" spacing={2}>
            <Typography gutterBottom minWidth={"8em"}>
              Lookback Period
            </Typography>
            <EditableLookbackPeriod />
          </Stack>
          <ClickAwayListener
            onClickAway={() => {
              isPopperOpen.value = false;
              // refreshSimilarCalendarPriceChartData();
              console.log("clicked away");
            }}
          >
            <Popper open={isPopperOpen.value} anchorEl={popperAnchorEl.value}>
              <Paper elevation={3} sx={{ p: 3 }}>
                {popperContent.value}
              </Paper>
            </Popper>
          </ClickAwayListener>
        </Grid2>
        <Grid2 size={{ xs: 12, md: 6 }}>
          <Paper elevation={3} sx={{ p: 3, minHeight: "28em" }}>
            {underlying.value !== null &&
            similarCalendarPriceChartData.value.length > 0 ? (
              <Scatter
                data={{
                  datasets: [
                    {
                      label: "Calendar Open Price",
                      data: similarCalendarPriceChartData.value,
                    },
                  ],
                }}
                options={{
                  scales: {
                    x: {
                      title: {
                        display: true,
                        text: "Time",
                      },
                      ticks: {
                        callback: (value, index, ticks) =>
                          new Date((value as number) * 1000)
                            .toISOString()
                            .substring(0, 10),
                      },
                      min: new Date(lookbackPeriodStart.value).getTime() / 1000,
                      max: new Date(lookbackPeriodEnd.value).getTime() / 1000,
                    },
                    y: {
                      beginAtZero: true,
                      ticks: {
                        callback: (value, index, ticks) =>
                          `$${value.toString()}`,
                      },
                      min: 0,
                      max: maxChartPrice.value,
                    },
                  },
                  plugins: {
                    tooltip: {
                      enabled: false,
                    },
                    legend: {
                      display: false,
                    },
                    title: {
                      display: true,
                      text: "Calendar Price (Under Like Conditions)",
                    },
                  },
                  animation: false,
                  maintainAspectRatio: false,
                  events: [],
                }}
              />
            ) : (
              <Typography>Loading Chart...</Typography>
            )}
          </Paper>
        </Grid2>
        <Grid2 size={{ xs: 12, md: 6 }}>
          <Paper elevation={3} sx={{ p: 3, minHeight: "28em" }}>
            {underlying.value !== null &&
            calendarExitPriceChartData.value.length > 0 ? (
              <Scatter
                data={{
                  datasets: [
                    {
                      label: "Calendar Exit Price",
                      data: calendarExitPriceChartData.value,
                    },
                  ],
                }}
                options={{
                  scales: {
                    x: {
                      type: "linear",
                      beginAtZero: false,
                      title: {
                        display: true,
                        text: "%-From-the-Money",
                      },
                      ticks: {
                        callback: (value, index, ticks) =>
                          `${value.toString()}%`,
                      },
                    },
                    y: {
                      beginAtZero: true,
                      ticks: {
                        callback: (value, index, ticks) =>
                          `$${value.toString()}`,
                      },
                      min: 0,
                      max: maxChartPrice.value,
                    },
                  },
                  elements: {
                    point: {
                      borderWidth: 0,
                      backgroundColor: (context) => {
                        const n = (
                          context.raw as { x: number; y: number; n: number }
                        ).n;
                        const alpha = n / maxN.value;
                        return `rgba(0, 0, 0, ${alpha})`;
                      },
                    },
                  },
                  plugins: {
                    tooltip: {
                      enabled: false,
                    },
                    legend: {
                      display: false,
                    },
                    title: {
                      display: true,
                      text: [
                        "Calendar Prices at Exit",
                        "by %-age from-the-money",
                      ],
                    },
                  },
                  animation: false,
                  maintainAspectRatio: false,
                  events: [],
                }}
              />
            ) : (
              <Typography>Loading Chart...</Typography>
            )}
          </Paper>
        </Grid2>
        <Grid2 size={{ xs: 12 }}>
          <Paper elevation={3} sx={{ p: 3, minHeight: "28em", height: "100%" }}>
            {underlying.value !== null &&
            stockPriceChartData.value.length > 0 ? (
              <Scatter
                data={{
                  datasets: [
                    {
                      label: "Stock Open Price",
                      data: stockPriceChartData.value,
                    },
                  ],
                }}
                options={{
                  showLine: true,
                  normalized: true,
                  scales: {
                    x: {
                      title: {
                        display: true,
                        text: "Time",
                      },
                      ticks: {
                        callback: (value, index, ticks) =>
                          new Date((value as number) * 1000)
                            .toISOString()
                            .substring(0, 10),
                      },
                      min: new Date(lookbackPeriodStart.value).getTime() / 1000,
                      max: new Date(lookbackPeriodEnd.value).getTime() / 1000,
                    },
                    y: {
                      beginAtZero: false,
                      ticks: {
                        callback: (value, index, ticks) =>
                          `$${value.toString()}`,
                      },
                    },
                  },
                  elements: {
                    point: {
                      radius: 2,
                      borderWidth: 0,
                    },
                    line: {
                      borderWidth: 2,
                    },
                  },
                  plugins: {
                    tooltip: {
                      enabled: false,
                    },
                    legend: {
                      display: false,
                    },
                    title: {
                      display: true,
                      text: "Stock Price",
                    },
                  },
                  animation: false,
                  maintainAspectRatio: false,
                  events: [],
                }}
              />
            ) : (
              <Typography>Loading Chart...</Typography>
            )}
          </Paper>
        </Grid2>
      </Grid2>
    </Container>
  );
 }
@@ -0,0 +1,28 @@
 import TextField from "@mui/material/TextField";
 import { EditableValue } from "./EditableValue";
 import { exitDTE } from "./state";
 import { refreshcalendarExitPriceChartData } from "./actions";
 import Slider from "@mui/material/Slider";
 const handleExitDTEChange = (e) => {
  if (exitDTE.value !== Number.parseInt(e.target.value)) {
    exitDTE.value = Number.parseInt(e.target.value);
    refreshcalendarExitPriceChartData();
  }
 };
 export function EditableExitDTE() {
  return (
    <Slider
      value={exitDTE.value}
      onChange={handleExitDTEChange}
      min={0}
      max={5}
      step={1}
      valueLabelDisplay="on"
    />
    // <EditableValue text={exitDTE.value}>
    //   <ExitToFrontExpirationChooser />
    // </EditableValue>
  );
 }
@@ -0,0 +1,66 @@
 import TextField from "@mui/material/TextField";
 import {
  refreshcalendarExitPriceChartData,
  refreshSimilarCalendarPriceChartData,
  refreshStockPriceChartData,
 } from "./actions";
 import { lookbackPeriodEnd, lookbackPeriodStart } from "./state";
 import Slider from "@mui/material/Slider";
 const handleLookbackPeriodChange = (
  e,
  [newLookbackPeriodStart, newLookbackPeriodEnd]: [number, number]
 ) => {
  const [lookbackPeriodStartUnixTime, lookbackPeriodEndUnixTime] = [
    new Date(lookbackPeriodStart.value).getTime(),
    new Date(lookbackPeriodEnd.value).getTime(),
  ];
  if (lookbackPeriodStartUnixTime !== newLookbackPeriodStart) {
    lookbackPeriodStart.value = new Date(newLookbackPeriodStart)
      .toISOString()
      .substring(0, 10);
    refreshStockPriceChartData();
    refreshSimilarCalendarPriceChartData();
    refreshcalendarExitPriceChartData();
  }
  if (lookbackPeriodEndUnixTime !== newLookbackPeriodEnd) {
    lookbackPeriodEnd.value = new Date(newLookbackPeriodEnd)
      .toISOString()
      .substring(0, 10);
    refreshStockPriceChartData();
    refreshSimilarCalendarPriceChartData();
    refreshcalendarExitPriceChartData();
  }
 };
 const earliestDate = new Date("2022-03-07");
 const DAY = 1000 * 60 * 60 * 24;
 function addDays(date, days) {
  const result = new Date(date);
  result.setDate(result.getDate() + days);
  return result.toISOString().substring(0, 10);
 }
 function daysBetween(date1, date2) {
  return Math.round(Math.abs((date2.getTime() - date1.getTime()) / DAY));
 }
 export function EditableLookbackPeriod() {
  return (
    <Slider
      value={[
        new Date(lookbackPeriodStart.value).getTime(),
        new Date(lookbackPeriodEnd.value).getTime(),
      ]}
      onChange={handleLookbackPeriodChange}
      valueLabelFormat={(unixTimeMs) =>
        new Date(unixTimeMs).toISOString().substring(0, 10)
      }
      getAriaValueText={(unixTimeMs) =>
        new Date(unixTimeMs).toISOString().substring(0, 10)
      }
      min={earliestDate.getTime()}
      max={earliestDate.getTime() + 250 * DAY}
      step={DAY}
      valueLabelDisplay="on"
    />
  );
 }
@@ -0,0 +1,38 @@
 import TextField from "@mui/material/TextField";
 import {
  refreshcalendarExitPriceChartData,
  refreshSimilarCalendarPriceChartData,
  refreshStockPriceChartData,
 } from "./actions";
 import { lookbackPeriodEnd } from "./state";
 import { EditableValue } from "./EditableValue";
 const handleLookbackPeriodEndChange = (e) => {
  if (lookbackPeriodEnd.value !== e.target.value) {
    lookbackPeriodEnd.value = e.target.value;
    refreshStockPriceChartData();
    refreshSimilarCalendarPriceChartData();
    refreshcalendarExitPriceChartData();
  }
 };
 function LookbackPeriodEndChooser() {
  return (
    <TextField
      fullWidth
      label="Lookback Period End"
      type="date"
      value={lookbackPeriodEnd.value}
      onChange={handleLookbackPeriodEndChange}
      InputLabelProps={{ shrink: true }}
    />
  );
 }
 export function EditableLookbackPeriodEnd() {
  return (
    <EditableValue text={lookbackPeriodEnd.value}>
      <LookbackPeriodEndChooser />
    </EditableValue>
  );
 }
@@ -0,0 +1,38 @@
 import TextField from "@mui/material/TextField";
 import {
  refreshcalendarExitPriceChartData,
  refreshSimilarCalendarPriceChartData,
  refreshStockPriceChartData,
 } from "./actions";
 import { lookbackPeriodStart } from "./state";
 import { EditableValue } from "./EditableValue";
 const handleLookbackPeriodStartChange = (e) => {
  if (lookbackPeriodStart.value !== e.target.value) {
    lookbackPeriodStart.value = e.target.value;
    refreshStockPriceChartData();
    refreshSimilarCalendarPriceChartData();
    refreshcalendarExitPriceChartData();
  }
 };
 function LookbackPeriodStartChooser() {
  return (
    <TextField
      fullWidth
      label="Lookback Period Start"
      type="date"
      value={lookbackPeriodStart.value}
      onChange={handleLookbackPeriodStartChange}
      InputLabelProps={{ shrink: true }}
    />
  );
 }
 export function EditableLookbackPeriodStart() {
  return (
    <EditableValue text={lookbackPeriodStart.value}>
      <LookbackPeriodStartChooser />
    </EditableValue>
  );
 }
@@ -0,0 +1,26 @@
 import { refreshSimilarCalendarPriceChartData } from "./actions";
 import { openDTE } from "./state";
 import Slider from "@mui/material/Slider";
 const handleOpenDTEChange = (e) => {
  if (openDTE.value !== Number.parseInt(e.target.value)) {
    openDTE.value = Number.parseInt(e.target.value);
    refreshSimilarCalendarPriceChartData();
  }
 };
 export function EditableOpenDTE() {
  return (
    <Slider
      value={openDTE.value}
      onChange={handleOpenDTEChange}
      min={0}
      max={5}
      step={1}
      valueLabelDisplay="on"
    />
    // <EditableValue text={openDTE.value}>
    //   <DaysToFrontExpirationChooser />
    // </EditableValue>
  );
 }
@@ -0,0 +1,45 @@
 import TextField from "@mui/material/TextField";
 import {
  refreshcalendarExitPriceChartData,
  refreshSimilarCalendarPriceChartData,
 } from "./actions";
 import { span } from "./state";
 import { EditableValue } from "./EditableValue";
 import Slider from "@mui/material/Slider";
 const handleSpanChange = (e) => {
  if (span.value !== Number.parseInt(e.target.value)) {
    span.value = Number.parseInt(e.target.value);
    refreshSimilarCalendarPriceChartData();
    refreshcalendarExitPriceChartData();
  }
 };
 function DaysBetweenFrontAndBackExpirationChooser() {
  return (
    <TextField
      fullWidth
      label="Front-to-Back-Month Days to Expiration Difference"
      type="number"
      value={span.value}
      onChange={handleSpanChange}
      InputProps={{ endAdornment: "Days Difference" }}
    />
  );
 }
 export function EditableSpan() {
  return (
    <Slider
      value={span.value}
      onChange={handleSpanChange}
      min={3}
      max={45}
      step={1}
      valueLabelDisplay="on"
    />
    // <EditableValue text={span.value}>
    //   <DaysBetweenFrontAndBackExpirationChooser />
    // </EditableValue>
  );
 }
@@ -0,0 +1,65 @@
 import Box from "@mui/material/Box";
 import { EditableValue } from "./EditableValue";
 import { moniness, moninessRadius } from "./state";
 import Slider from "@mui/material/Slider";
 import { refreshSimilarCalendarPriceChartData } from "./actions";
 function MoninessChooser() {
  return (
    <Slider
      fullWidth
      label="Strike % From Underlying Price"
      value={moniness.value}
      valueLabelDisplay="on"
      min={0}
      max={10}
      step={1}
      onChange={(e, value) => {
        moniness.value = value as number;
      }}
      onChangeCommitted={(e, value) => {
        refreshSimilarCalendarPriceChartData();
      }}
      InputProps={{ endAdornment: "%" }}
    />
  );
 }
 function MoninessRadiusChooser() {
  return (
    <Slider
      fullWidth
      label="Strike % Radius"
      value={moninessRadius.value}
      valueLabelDisplay="on"
      min={0}
      max={10}
      step={1}
      onChange={(e, value) => {
        moninessRadius.value = value as number;
      }}
      onChangeCommitted={(e, value) => {
        refreshSimilarCalendarPriceChartData();
      }}
      InputProps={{ endAdornment: "%" }}
    />
  );
 }
 /** This is its own component so that sliding the slider with the mouse is
 * smoother. Preact detects reads from the "slider" signal values, and
 * associates them with the component that read them and redraws that component.
 * If this was not its own component, it would redraw the entire UI. It was very
 * slow. */
 export function EditableStrike() {
  return (
    <EditableValue
      text={`${moniness.value.toFixed(1)}±${moninessRadius.value.toFixed(2)}`}
    >
      <Box sx={{ minWidth: "20em" }}>
        <MoninessChooser />
        <MoninessRadiusChooser />
      </Box>
    </EditableValue>
  );
 }
@@ -0,0 +1,42 @@
 import Select from "@mui/material/Select";
 import { EditableValue } from "./EditableValue";
 import { availableUnderlyings, underlying } from "./state";
 import MenuItem from "@mui/material/MenuItem";
 import {
  refreshcalendarExitPriceChartData,
  refreshSimilarCalendarPriceChartData,
  refreshStockPriceChartData,
 } from "./actions";
 const handleUnderlyingChange = (e) => {
  if (underlying.value !== e.target.value) {
    underlying.value = e.target.value;
    refreshStockPriceChartData();
    refreshSimilarCalendarPriceChartData();
    refreshcalendarExitPriceChartData();
  }
 };
 function UnderlyingChooser() {
  return (
    <Select
      value={underlying.value || ""}
      onChange={handleUnderlyingChange}
      label="Available Underlyings"
    >
      {availableUnderlyings.value.map((underlying) => (
        <MenuItem key={underlying} value={underlying}>
          {underlying}
        </MenuItem>
      ))}
    </Select>
  );
 }
 export function EditableUnderlying() {
  return (
    <EditableValue text={underlying.value}>
      <UnderlyingChooser />
    </EditableValue>
  );
 }
@@ -0,0 +1,29 @@
 import Button from "@mui/material/Button";
 import { isPopperOpen, popperAnchorEl, popperContent } from "./state";
 export function EditableValue({ text, children }) {
  return (
    <Button
      variant="text"
      size="large"
      sx={{
        textDecoration: "underline",
        textUnderlineOffset: "3px",
        fontSize: "1.0em",
      }}
      onClick={(e) => {
        // stop propagation so it's not caught by the ClickAwayListener:
        e.stopPropagation();
        if (isPopperOpen.value === false) {
          isPopperOpen.value = true;
          popperAnchorEl.value = e.currentTarget;
          popperContent.value = children;
        } else {
          isPopperOpen.value = false;
        }
      }}
    >
      {text}
    </Button>
  );
 }
@@ -1,380 +0,0 @@
 import { signal, computed } from "@preact/signals";
 import { useEffect } from "preact/hooks";
 import {trpc} from '../../trpc.js';
 import {
  Chart as ChartJS,
  LinearScale,
  CategoryScale,
  PointElement,
  Tooltip,
  Title,
 } from 'chart.js';
 import { Scatter } from 'react-chartjs-2';
 import './style.css';
 ChartJS.register(LinearScale, CategoryScale, PointElement, Tooltip, Title,);
 const availableUnderlyings = signal([]);
 const chosenUnderlying = signal(null);
 const chosenDaysToFrontExpiration = signal(14);
 const chosenDaysBetweenFrontAndBackExpiration = signal(14);
 const chosenStrikePercentageFromUnderlyingPrice = signal(1.4);
 const chosenStrikePercentageFromUnderlyingPriceRadius = signal(0.05);
 const chosenExitToFrontExpiration = signal(2);
 const historicalStockQuoteChartData = signal([]);
 const historicalCalendarQuoteChartData = signal([]);
 const historicalCalendarExitQuoteChartData = signal([]);
 const chosenLookbackPeriodStart = signal("2022-01-01");
 const chosenLookbackPeriodEnd = signal("2024-01-01");
 const maxChartPrice = computed(()=>
  Math.max(
    Math.max.apply(null, historicalCalendarQuoteChartData.value.map(d=>d.y)),
    Math.max.apply(null, historicalCalendarExitQuoteChartData.value.map(d=>d.y)),
  )
 );
 const refreshHistoricalStockQuoteChartData = ()=>{
  trpc.getHistoricalStockQuoteChartData
    .query({
      underlying:chosenUnderlying.value,
      lookbackPeriodStart:chosenLookbackPeriodStart.value,
      lookbackPeriodEnd:chosenLookbackPeriodEnd.value,
    })
    .then((getHistoricalStockQuoteChartDataResponse)=>{
      historicalStockQuoteChartData.value = getHistoricalStockQuoteChartDataResponse;
    })
 };
 const refreshHistoricalCalendarQuoteChartData = ()=>{
  trpc.getHistoricalCalendarQuoteChartData
    .query({
      underlying:chosenUnderlying.value,
      daysToFrontExpiration:chosenDaysToFrontExpiration.value,
      daysBetweenFrontAndBackExpiration:chosenDaysBetweenFrontAndBackExpiration.value,
      strikePercentageFromUnderlyingPriceRangeMin:chosenStrikePercentageFromUnderlyingPrice.value - chosenStrikePercentageFromUnderlyingPriceRadius.value,
      strikePercentageFromUnderlyingPriceRangeMax:chosenStrikePercentageFromUnderlyingPrice.value + chosenStrikePercentageFromUnderlyingPriceRadius.value,
      lookbackPeriodStart:chosenLookbackPeriodStart.value,
      lookbackPeriodEnd:chosenLookbackPeriodEnd.value,
    })
    .then((getHistoricalCalendarQuoteChartDataResponse)=>{
      historicalCalendarQuoteChartData.value = getHistoricalCalendarQuoteChartDataResponse;
    })
 };
 const refreshHistoricalCalendarExitQuoteChartData = ()=>{
  trpc.getHistoricalCalendarExitQuoteChartData
    .query({
      underlying:chosenUnderlying.value,
      daysToFrontExpiration:chosenExitToFrontExpiration.value,
      daysBetweenFrontAndBackExpiration:chosenDaysBetweenFrontAndBackExpiration.value,
      lookbackPeriodStart:chosenLookbackPeriodStart.value,
      lookbackPeriodEnd:chosenLookbackPeriodEnd.value,
    })
    .then((getHistoricalCalendarExitQuoteChartDataResponse)=>{
      historicalCalendarExitQuoteChartData.value = getHistoricalCalendarExitQuoteChartDataResponse;
    })
 };
 const handleInit = ()=>{
  trpc.getAvailableUnderlyings
    .query()
    .then((availableUnderlyingsResponse)=>{
      availableUnderlyings.value = availableUnderlyingsResponse;
      chosenUnderlying.value = availableUnderlyingsResponse[0];
      refreshHistoricalStockQuoteChartData();
      refreshHistoricalCalendarQuoteChartData();
      refreshHistoricalCalendarExitQuoteChartData();
    });
 };
 const handleUnderlyingChange = (e)=>{
  if(chosenUnderlying.value !== e.target.value){
    chosenUnderlying.value = e.target.value;
    refreshHistoricalStockQuoteChartData();
    refreshHistoricalCalendarQuoteChartData();
    refreshHistoricalCalendarExitQuoteChartData();
  }
 };
 const handleDaysToFrontExpirationChange = (e)=>{
  if(chosenDaysToFrontExpiration.value !== parseInt(e.target.value)){
    chosenDaysToFrontExpiration.value = parseInt(e.target.value);
    refreshHistoricalCalendarQuoteChartData();
  }
 };
 const handleDaysBetweenFrontAndBackExpirationChange = (e)=>{
  if(chosenDaysBetweenFrontAndBackExpiration.value !== parseInt(e.target.value)){
    chosenDaysBetweenFrontAndBackExpiration.value = parseInt(e.target.value);
    refreshHistoricalCalendarQuoteChartData();
    refreshHistoricalCalendarExitQuoteChartData();
  }
 };
 const handleStrikePercentageFromUnderlyingPriceChange = (e)=>{
  if(chosenStrikePercentageFromUnderlyingPrice.value !== parseFloat(e.target.value)){
    chosenStrikePercentageFromUnderlyingPrice.value = parseFloat(e.target.value);
    refreshHistoricalCalendarQuoteChartData();
  }
 };
 const handleStrikePercentageFromUnderlyingPriceRadiusChange = (e)=>{
  if(chosenStrikePercentageFromUnderlyingPriceRadius.value !== parseFloat(e.target.value)){
    chosenStrikePercentageFromUnderlyingPriceRadius.value = parseFloat(e.target.value);
    refreshHistoricalCalendarQuoteChartData();
  }
 };
 const handleExitToFrontExpirationChange = (e)=>{
  if(chosenExitToFrontExpiration.value !== parseInt(e.target.value)){
    chosenExitToFrontExpiration.value = parseInt(e.target.value);
    refreshHistoricalCalendarExitQuoteChartData();
  }
 };
 const handleLookbackPeriodStartChange = (e)=>{
  if(chosenLookbackPeriodStart.value !== e.target.value){
    chosenLookbackPeriodStart.value = e.target.value;
    refreshHistoricalStockQuoteChartData();
    refreshHistoricalCalendarQuoteChartData();
    refreshHistoricalCalendarExitQuoteChartData();
  }
 };
 const handleLookbackPeriodEndChange = (e)=>{
  if(chosenLookbackPeriodEnd.value !== e.target.value){
    chosenLookbackPeriodEnd.value = e.target.value;
    refreshHistoricalStockQuoteChartData();
    refreshHistoricalCalendarQuoteChartData();
    refreshHistoricalCalendarExitQuoteChartData();
  }
 };
 export function HistoricalCalendarPrices(){
  useEffect(handleInit, []);
  return (
    <div>
      <div>
        <label>Available Underlyings</label>
        {
          availableUnderlyings.value.length === 0 
          ? "Loading..." 
          : <select onChange={handleUnderlyingChange}>
              {availableUnderlyings.value.map((availableUnderlying)=>(
                <option value={availableUnderlying}>{availableUnderlying}</option>
              ))}
            </select>
        }
      </div>
      <div>
        <label>Now-to-Front-Month "Days to Expiration"</label>
        <input type="text" onBlur={handleDaysToFrontExpirationChange} value={chosenDaysToFrontExpiration.value} />
        Days
      </div>
      <div>
        <label>Front-to-Back-Month "Days to Expiration" Difference</label>
        <input type="text" onBlur={handleDaysBetweenFrontAndBackExpirationChange} value={chosenDaysBetweenFrontAndBackExpiration.value} />
        Days Difference
      </div>
      <div>
        <label>"Strike Percentage From Underlying Price" Range</label>
        <input type="text" onBlur={handleStrikePercentageFromUnderlyingPriceChange} value={chosenStrikePercentageFromUnderlyingPrice.value} />
        %
        +/-
        <input type="text" onBlur={handleStrikePercentageFromUnderlyingPriceRadiusChange} value={chosenStrikePercentageFromUnderlyingPriceRadius.value} />
        % from ATM
      </div>
      <div>
        <label>Exit-to-Front-Month "Days to Expiration"</label>
        <input type="text" onBlur={handleExitToFrontExpirationChange} value={chosenExitToFrontExpiration.value} />
        Days
      </div>
      <div>
        <label>Lookback Period</label>
        <input type="text" onBlur={handleLookbackPeriodStartChange} value={chosenLookbackPeriodStart.value} />
        -
        <input type="text" onBlur={handleLookbackPeriodEndChange} value={chosenLookbackPeriodEnd.value} />
      </div>
      <div className="chart-container">
        {chosenUnderlying.value!==null && historicalStockQuoteChartData.value.length>0
        ? <div className="chart">
            <Scatter 
              data={{
                datasets: [
                  {
                    label: "Stock Open Price",
                    data: historicalStockQuoteChartData.value
                  }
                ]
              }}
              options={{
                scales: {
                  x: {
                    title: {
                      display: true,
                      text: "Time"
                    },
                    ticks: {
                      callback: function(value, index, ticks) {
                        return (new Date(value as number*1000)).toISOString().substring(0,10);
                      }
                    },
                    min: (new Date(chosenLookbackPeriodStart.value)).getTime()/1000,
                    max: (new Date(chosenLookbackPeriodEnd.value)).getTime()/1000,
                  },
                  y: {
                    beginAtZero: false,
                    ticks: {
                      callback: function(value, index, ticks) {
                        return "$"+value.toString();
                      }
                    }
                  },
                },
                elements: {
                  point: {
                    radius: 1,
                    borderWidth: 0,
                  }
                },
                plugins: {
                  tooltip: {
                    enabled: false,
                  },
                  legend: {
                    display: false,
                  },
                  title: {
                    display: true,
                    text: "Stock Price"
                  },
                },
                animation: false,
                maintainAspectRatio: false,
              }}
            />
          </div>
        : <div>Loading Chart...</div>}
      </div>
      <div className="chart-container">
        {chosenUnderlying.value!==null && historicalCalendarQuoteChartData.value.length>0
        ? <div className="chart">
            <Scatter 
              data={{
                datasets: [
                  {
                    label: "Calendar Open Price",
                    data: historicalCalendarQuoteChartData.value
                  }
                ]
              }}
              options={{
                scales: {
                  x: {
                    title: {
                      display: true,
                      text: "Time"
                    },
                    ticks: {
                      callback: function(value, index, ticks) {
                        return (new Date(value as number*1000)).toISOString().substring(0,10);
                      }
                    },
                    min: (new Date(chosenLookbackPeriodStart.value)).getTime()/1000,
                    max: (new Date(chosenLookbackPeriodEnd.value)).getTime()/1000,
                  },
                  y: {
                    beginAtZero: true,
                    ticks: {
                      callback: function(value, index, ticks) {
                        return "$"+value.toString();
                      }
                    },
                    min: 0,
                    max: maxChartPrice.value,
                  }
                },
                plugins: {
                  tooltip: {
                    enabled: false,
                  },
                  legend: {
                    display: false,
                  },
                  title: {
                    display: true,
                    text: "Calendar Price (Under Like Conditions)"
                  },
                },
                animation: false,
                maintainAspectRatio: false,
              }}
            />
          </div>
        : <div>Loading Chart...</div>}
        {chosenUnderlying.value!==null && historicalCalendarQuoteChartData.value.length>0
        ? <div className="chart">
            <Scatter 
              data={{
                datasets: [
                  {
                    label: "Calendar Exit Price",
                    data: historicalCalendarExitQuoteChartData.value
                  }
                ]
              }}
              options={{
                scales: {
                  x: {
                    type: 'linear',
                    beginAtZero: false,
                    title: {
                      display: true,
                      text: "%-From-the-Money"
                    },
                    ticks: {
                      callback: function(value, index, ticks) {
                        return value.toString()+"%";
                      }
                    },
                  },
                  y: {
                    beginAtZero: true,
                    ticks: {
                      callback: function(value, index, ticks) {
                        return "$"+value.toString();
                      }
                    },
                    min: 0,
                    max: maxChartPrice.value,
                  },
                },
                elements: {
                  point: {
                    borderWidth: 0,
                  },
                },
                plugins: {
                  tooltip: {
                    enabled: false,
                  },
                  legend: {
                    display: false,
                  },
                  title: {
                    display: true,
                    text: ["Calendar Prices at Exit","by %-age from-the-money"]
                  },
                },
                animation: false,
                maintainAspectRatio: false,
              }}
            />
          </div>
        : <div>Loading Chart...</div>}
      </div>
    </div>
  );
 }
@@ -0,0 +1,81 @@
 import { trpc } from "../../trpc";
 import {
  calendarExitPriceChartData,
  span,
  openDTE,
  exitDTE,
  lookbackPeriodEnd,
  lookbackPeriodStart,
  similarCalendarPriceChartData,
  stockPriceChartData,
  moniness,
  moninessRadius,
  underlying,
 } from "./state";
 function debounce(func, wait) {
  let timeout;
  return function () {
    const context = this;
    const args = arguments;
    clearTimeout(timeout);
    timeout = setTimeout(() => func.apply(context, args), wait);
  };
 }
 function throttle(func, limit) {
  let inThrottle;
  return function () {
    const context = this;
    const args = arguments;
    if (!inThrottle) {
      func.apply(context, args);
      inThrottle = true;
      setTimeout(() => (inThrottle = false), limit);
    }
  };
 }
 export const refreshStockPriceChartData = throttle(() => {
  stockPriceChartData.value = [];
  trpc.StockPriceChart.getChartData
    .query({
      underlying: underlying.value,
      lookbackPeriodStart: lookbackPeriodStart.value,
      lookbackPeriodEnd: lookbackPeriodEnd.value,
    })
    .then((getChartDataResponse) => {
      stockPriceChartData.value = getChartDataResponse;
    });
 }, 400);
 export const refreshSimilarCalendarPriceChartData = throttle(() => {
  similarCalendarPriceChartData.value = [];
  trpc.SimilarCalendarPriceChart.getChartData
    .query({
      underlying: underlying.value,
      daysToFrontExpiration: openDTE.value,
      daysBetweenFrontAndBackExpiration: span.value,
      strikePercentageFromUnderlyingPriceRangeMin:
        (moniness.value - moninessRadius.value) / 100,
      strikePercentageFromUnderlyingPriceRangeMax:
        (moniness.value + moninessRadius.value) / 100,
      lookbackPeriodStart: lookbackPeriodStart.value,
      lookbackPeriodEnd: lookbackPeriodEnd.value,
    })
    .then((getChartDataResponse) => {
      similarCalendarPriceChartData.value = getChartDataResponse;
    });
 }, 400);
 export const refreshcalendarExitPriceChartData = throttle(() => {
  calendarExitPriceChartData.value = [];
  trpc.CalendarExitPriceChart.getChartData
    .query({
      underlying: underlying.value,
      daysToFrontExpiration: exitDTE.value,
      daysBetweenFrontAndBackExpiration: span.value,
      lookbackPeriodStart: lookbackPeriodStart.value,
      lookbackPeriodEnd: lookbackPeriodEnd.value,
    })
    .then((getChartDataResponse) => {
      calendarExitPriceChartData.value = getChartDataResponse;
    });
 }, 400);
@@ -0,0 +1,46 @@
 import { computed, signal } from "@preact/signals";
 export const isPopperOpen = signal(false);
 export const popperAnchorEl = signal(null);
 export const popperContent = signal(null);
 export const availableUnderlyings = signal([]);
 export const underlying = signal(null);
 export const openDTE = signal(14);
 export const span = signal(14);
 export const moniness = signal(1);
 export const moninessRadius = signal(1);
 export const exitDTE = signal(2);
 export const stockPriceChartData = signal<Array<[number, number]>>([]);
 export const similarCalendarPriceChartData = signal([]);
 export const calendarExitPriceChartData = signal([]);
 export const lookbackPeriodStart = signal("2022-03-01");
 export const lookbackPeriodEnd = signal("2022-04-01");
 export const maxChartPrice = computed(() =>
  Math.max(
    Math.max.apply(
      null,
      similarCalendarPriceChartData.value.map((d) => d.y).slice(0, -2)
    ),
    Math.max.apply(
      null,
      calendarExitPriceChartData.value.map((d) => d.y).slice(0, -2)
    )
  )
 );
 export const maxN = computed(() =>
  Math.max.apply(
    null,
    calendarExitPriceChartData.value.map((d) => d.n)
  )
 );
@@ -1,11 +0,0 @@
 .chart-container {
 	display: flex;
 	flex-direction: row;
 	flex-wrap: nowrap;
 	justify-content: space-around;
 	width: 1800px;
 	height: 480px;
 }
 .chart-container > .chart {
 	width: 880px;
 }
@@ -1,47 +0,0 @@
 img {
 	margin-bottom: 1.5rem;
 }
 img:hover {
 	filter: drop-shadow(0 0 2em #673ab8aa);
 }
 .home section {
 	margin-top: 5rem;
 	display: grid;
 	grid-template-columns: repeat(3, 1fr);
 	column-gap: 1.5rem;
 }
 .resource {
 	padding: 0.75rem 1.5rem;
 	border-radius: 0.5rem;
 	text-align: left;
 	text-decoration: none;
 	color: #222;
 	background-color: #f1f1f1;
 	border: 1px solid transparent;
 }
 .resource:hover {
 	border: 1px solid #000;
 	box-shadow: 0 25px 50px -12px #673ab888;
 }
@media (max-width: 639px) {
 	.home section {
 		margin-top: 5rem;
 		grid-template-columns: 1fr;
 		row-gap: 1rem;
 	}
 }
@media (prefers-color-scheme: dark) {
 	.resource {
 		color: #ccc;
 		background-color: #161616;
 	}
 	.resource:hover {
 		border: 1px solid #bbb;
 	}
 }
@@ -1,72 +0,0 @@
 :root {
 	font-family: Inter, system-ui, Avenir, Helvetica, Arial, sans-serif;
 	line-height: 1.5;
 	font-weight: 400;
 	color: #222;
 	background-color: #ffffff;
 	font-synthesis: none;
 	text-rendering: optimizeLegibility;
 	-webkit-font-smoothing: antialiased;
 	-moz-osx-font-smoothing: grayscale;
 	-webkit-text-size-adjust: 100%;
 }
 body {
 	margin: 0;
 }
 #app {
 	display: flex;
 	flex-direction: column;
 	min-height: 100vh;
 	justify-content: start;
 }
 header {
 	display: flex;
 	justify-content: flex-end;
 	background-color: #673ab8;
 }
 header nav {
 	display: flex;
 }
 header a {
 	color: #fff;
 	padding: 0.75rem;
 	text-decoration: none;
 }
 header a.active {
 	background-color: #0005;
 }
 header a:hover {
 	background-color: #0008;
 }
 main {
 	flex: auto;
 	display: flex;
 	flex-direction: column;
 	align-items: start;
 	max-width: 1280px;
 	justify-content: center;
 	text-align: center;
 }
@media (max-width: 639px) {
 	main {
 		margin: 2rem;
 	}
 }
@media (prefers-color-scheme: dark) {
 	:root {
 		color: #ccc;
 		background-color: #1a1a1a;
 	}
 }
@@ -0,0 +1,5 @@
 {
 	"devDependencies": {
 		"@biomejs/biome": "^1.8.3"
 	}
 }
@@ -0,0 +1,105 @@
 lockfileVersion: '9.0'
 settings:
  autoInstallPeers: true
  excludeLinksFromLockfile: false
 importers:
  .:
    devDependencies:
      '@biomejs/biome':
        specifier: ^1.8.3
        version: 1.8.3
 packages:
  '@biomejs/biome@1.8.3':
    resolution: {integrity: sha512-/uUV3MV+vyAczO+vKrPdOW0Iaet7UnJMU4bNMinggGJTAnBPjCoLEYcyYtYHNnUNYlv4xZMH6hVIQCAozq8d5w==}
    engines: {node: '>=14.21.3'}
    hasBin: true
  '@biomejs/cli-darwin-arm64@1.8.3':
    resolution: {integrity: sha512-9DYOjclFpKrH/m1Oz75SSExR8VKvNSSsLnVIqdnKexj6NwmiMlKk94Wa1kZEdv6MCOHGHgyyoV57Cw8WzL5n3A==}
    engines: {node: '>=14.21.3'}
    cpu: [arm64]
    os: [darwin]
  '@biomejs/cli-darwin-x64@1.8.3':
    resolution: {integrity: sha512-UeW44L/AtbmOF7KXLCoM+9PSgPo0IDcyEUfIoOXYeANaNXXf9mLUwV1GeF2OWjyic5zj6CnAJ9uzk2LT3v/wAw==}
    engines: {node: '>=14.21.3'}
    cpu: [x64]
    os: [darwin]
  '@biomejs/cli-linux-arm64-musl@1.8.3':
    resolution: {integrity: sha512-9yjUfOFN7wrYsXt/T/gEWfvVxKlnh3yBpnScw98IF+oOeCYb5/b/+K7YNqKROV2i1DlMjg9g/EcN9wvj+NkMuQ==}
    engines: {node: '>=14.21.3'}
    cpu: [arm64]
    os: [linux]
  '@biomejs/cli-linux-arm64@1.8.3':
    resolution: {integrity: sha512-fed2ji8s+I/m8upWpTJGanqiJ0rnlHOK3DdxsyVLZQ8ClY6qLuPc9uehCREBifRJLl/iJyQpHIRufLDeotsPtw==}
    engines: {node: '>=14.21.3'}
    cpu: [arm64]
    os: [linux]
  '@biomejs/cli-linux-x64-musl@1.8.3':
    resolution: {integrity: sha512-UHrGJX7PrKMKzPGoEsooKC9jXJMa28TUSMjcIlbDnIO4EAavCoVmNQaIuUSH0Ls2mpGMwUIf+aZJv657zfWWjA==}
    engines: {node: '>=14.21.3'}
    cpu: [x64]
    os: [linux]
  '@biomejs/cli-linux-x64@1.8.3':
    resolution: {integrity: sha512-I8G2QmuE1teISyT8ie1HXsjFRz9L1m5n83U1O6m30Kw+kPMPSKjag6QGUn+sXT8V+XWIZxFFBoTDEDZW2KPDDw==}
    engines: {node: '>=14.21.3'}
    cpu: [x64]
    os: [linux]
  '@biomejs/cli-win32-arm64@1.8.3':
    resolution: {integrity: sha512-J+Hu9WvrBevfy06eU1Na0lpc7uR9tibm9maHynLIoAjLZpQU3IW+OKHUtyL8p6/3pT2Ju5t5emReeIS2SAxhkQ==}
    engines: {node: '>=14.21.3'}
    cpu: [arm64]
    os: [win32]
  '@biomejs/cli-win32-x64@1.8.3':
    resolution: {integrity: sha512-/PJ59vA1pnQeKahemaQf4Nyj7IKUvGQSc3Ze1uIGi+Wvr1xF7rGobSrAAG01T/gUDG21vkDsZYM03NAmPiVkqg==}
    engines: {node: '>=14.21.3'}
    cpu: [x64]
    os: [win32]
 snapshots:
  '@biomejs/biome@1.8.3':
    optionalDependencies:
      '@biomejs/cli-darwin-arm64': 1.8.3
      '@biomejs/cli-darwin-x64': 1.8.3
      '@biomejs/cli-linux-arm64': 1.8.3
      '@biomejs/cli-linux-arm64-musl': 1.8.3
      '@biomejs/cli-linux-x64': 1.8.3
      '@biomejs/cli-linux-x64-musl': 1.8.3
      '@biomejs/cli-win32-arm64': 1.8.3
      '@biomejs/cli-win32-x64': 1.8.3
  '@biomejs/cli-darwin-arm64@1.8.3':
    optional: true
  '@biomejs/cli-darwin-x64@1.8.3':
    optional: true
  '@biomejs/cli-linux-arm64-musl@1.8.3':
    optional: true
  '@biomejs/cli-linux-arm64@1.8.3':
    optional: true
  '@biomejs/cli-linux-x64-musl@1.8.3':
    optional: true
  '@biomejs/cli-linux-x64@1.8.3':
    optional: true
  '@biomejs/cli-win32-arm64@1.8.3':
    optional: true
  '@biomejs/cli-win32-x64@1.8.3':
    optional: true
@@ -6,6 +6,7 @@ yarn-debug.log*
 yarn-error.log*
 pnpm-debug.log*
 lerna-debug.log*
 .pnpm-store
 node_modules
 dist
@@ -22,3 +23,8 @@ dist-ssr
 *.njsproj
 *.sln
 *.sw?
 .env
 *.db
 *.db-lck
 Calendar tRPC
@@ -0,0 +1 @@
 20
@@ -2,22 +2,27 @@
 FROM node:20-slim AS base
 ENV PNPM_HOME="/pnpm"
 ENV PATH="$PNPM_HOME:$PATH"
-RUN corepack enable
+RUN npm install -g corepack@latest && corepack enable
 COPY package.json pnpm-lock.yaml /app/
 WORKDIR /app
 FROM base AS prod-deps
 RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --prod --frozen-lockfile
 # install dev dependencies which are needed for building, such as typescript:
 FROM base AS build
 RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --frozen-lockfile
 COPY tsconfig.json /app/
 COPY src /app/src
-RUN pnpm run build
+CMD [ "pnpm", "tsx", "src/index.ts" ] 
-FROM base
+# FROM base AS prod-deps
-COPY --from=prod-deps /app/node_modules /app/node_modules
+# RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --prod --frozen-lockfile
-COPY --from=build /app/dist /app/dist
+
-WORKDIR /app/dist
+# # install dev dependencies which are needed for building, such as typescript:
-CMD [ "node", "index.js" ]
+# FROM base AS build
 # RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --frozen-lockfile
 # COPY tsconfig.json /app/
 # COPY src /app/src
 # RUN pnpm run build
 # FROM base
 # COPY --from=prod-deps /app/node_modules /app/node_modules
 # COPY --from=build /app/dist /app/dist
 # WORKDIR /app/dist
 # CMD [ "node", "index.js" ]
@@ -0,0 +1,2 @@
 - Ingest stock/underlying aggregates from flatfiles
 - Create backtesting function to step through each minute of every day.
@@ -0,0 +1,2 @@
 #!/bin/sh
 kubectl exec -it -n clickhouse clickhouse -- clickhouse-client -u avraham --password buginoo
@@ -0,0 +1,17 @@
 import { open } from 'lmdbx'; // or require
 const MAXIMUM_KEY = Buffer.from([0xff]);
 // or in deno: import { open } from 'https://deno.land/x/lmdbx/mod.ts';
 const myDB = open({
 	path: '/tmp/my.db',
 	// any options go here, we can turn on compression like this:
 	compression: true,
 });
 await myDB.put(["a","b"], "ab");
 await myDB.put(["a","c"], "ac");
 await myDB.put(["a","d"], "ad");
 await myDB.put(["b","a"], "ba");
 await myDB.put(["b","c"], "bc");
 console.log(Array.from(myDB.getRange({start: ["a"], end: ["a", MAXIMUM_KEY]}).asArray))
@@ -2,28 +2,39 @@
 	"private": true,
 	"type": "module",
 	"scripts": {
-		"build": "esbuild src/*.ts src/**/*.ts --platform=node --outdir=dist --format=esm",
+		"build": "esbuild src/*.ts src/*.tsx src/**/*.ts src/**/**/*.ts --platform=node --outdir=dist --format=esm",
 		"dev:node": "node --watch dist/index.js",
 		"dev:esbuild": "pnpm run build --watch",
-		"dev": "run-p dev:*"
+		"dev": "run-p dev:*",
 		"cli": "tsx src/cli.tsx"
 	},
 	"dependencies": {
-		"@clickhouse/client": "^0.2.7",
+		"@clickhouse/client": "^1.4.1",
 		"@humanwhocodes/env": "^3.0.5",
 		"@sinclair/typebox": "^0.32.5",
 		"@trpc/server": "^10.45.0",
 		"cors": "^2.8.5",
-		"dotenv": "^16.4.1",
+		"date-fns": "^3.6.0",
 		"execa": "^9.3.0",
 		"ink": "^4.1.0",
 		"ink-text-input": "^5.0.1",
 		"lmdbx": "^0.5.0",
 		"p-all": "^5.0.0",
 		"p-queue": "^8.0.1",
 		"p-retry": "^6.2.0",
 		"p-series": "^3.0.0",
-		"p-throttle": "^6.1.0"
+		"p-throttle": "^6.1.0",
 		"react": "^18.2.0",
 		"sqlite": "^5.1.1",
 		"sqlite3": "^5.1.7"
 	},
 	"devDependencies": {
 		"@types/cors": "^2.8.17",
 		"@types/node": "^20.10.7",
 		"@types/react": "^18.0.0",
 		"esbuild": "^0.19.11",
 		"npm-run-all": "^4.1.5",
 		"tsx": "^4.17.0",
 		"typescript": "^5.3.3"
 	}
 }
@@ -0,0 +1,191 @@
 import type { CalendarDatabase, CalendarKey } from "./interfaces.js";
 import type { Aggregate } from "../interfaces.js";
 import { query } from "../../lib/clickhouse.js";
 function makeCalendarDatabase(): CalendarDatabase {
  const calendarDatabase: Omit<CalendarDatabase, "getCalendars"> = {
    getKeys: async ({ key: { symbol }, date }) => {
      const calendarsForSymbolOnDate = await query<
        Omit<CalendarKey, "symbol">
      >(`
 				WITH today_option_contracts AS (
 					SELECT expirationDate, strike, type
 					FROM option_contract_existences
 					WHERE symbol = '${symbol}'
 					AND asOfDate = '${date}'
 				)
 				SELECT 
 					front_option_contract.type as type,
 					front_option_contract.strike as strike, 
 					front_option_contract.expirationDate as frontExpirationDate, 
 					back_option_contract.expirationDate as backExpirationDate
 				FROM today_option_contracts AS front_option_contract
 				ASOF INNER JOIN today_option_contracts AS back_option_contract
 				ON front_option_contract.type = back_option_contract.type
 				AND front_option_contract.strike = back_option_contract.strike
 				AND front_option_contract.expirationDate < back_option_contract.expirationDate
 				`);
      return calendarsForSymbolOnDate.map((calendarWithoutSymbol) => ({
        ...calendarWithoutSymbol,
        symbol,
      }));
    },
    getAggregate: async ({
      key: { symbol, frontExpirationDate, backExpirationDate, strike, type },
      tsStart,
    }) => {
      const tsStartString = new Date(tsStart).toISOString();
      return (
        await query<Omit<Aggregate<CalendarKey>, "key">>(`
 				WITH front_option_contract_candlestick AS (
 					SELECT
 						tsStart,
 						open,
 						close,
 						high,
 						low
 					FROM option_contract_aggregates
 					WHERE symbol = '${symbol}'
 					AND type = '${type}'
 					AND strike = '${strike}'
 					AND expirationDate = '${frontExpirationDate}'
 					AND tsStart = '${tsStartString}'
 				),
 				back_option_contract_candlestick AS (
 					SELECT
 						tsStart,
 						open,
 						close,
 						high,
 						low
 					FROM option_contract_aggregates
 					WHERE symbol = '${symbol}'
 					AND type = '${type}'
 					AND strike = '${strike}'
 					AND expirationDate = '${backExpirationDate}'
 					AND tsStart = '${tsStartString}'
 				)
 				SELECT
 						toUnixTimestamp(front_option_contract_candlestick.tsStart) as tsStart,
 						back_option_contract_candlestick.open - front_option_contract_candlestick.open as open,
 						back_option_contract_candlestick.close - front_option_contract_candlestick.close as close
 				FROM front_option_contract_candlestick
 				INNER JOIN back_option_contract_candlestick
 				ON front_option_contract_candlestick.tsStart = back_option_contract_candlestick.tsStart
 				ORDER BY front_option_contract_candlestick.tsStart ASC
 			`)
      ).map((aggregate) => ({
        ...aggregate,
        tsStart: aggregate.tsStart * 1000, // unfortunately, `toUnixTimestamp` only returns second-precision
      }))[0];
    },
    getAggregates: async ({
      key: { symbol, frontExpirationDate, backExpirationDate, strike, type },
      date,
    }) => {
      return (
        await query<Omit<Aggregate<CalendarKey>, "key">>(`
 				WITH front_option_contract_candlestick AS (
 					SELECT
 						tsStart,
 						open,
 						close,
 						high,
 						low
 					FROM option_contract_aggregates
 					WHERE symbol = '${symbol}'
 					AND type = '${type}'
 					AND strike = '${strike}'
 					AND expirationDate = '${frontExpirationDate}'
 					AND toDate(tsStart) = '${date}'
 				),
 				back_option_contract_candlestick AS (
 					SELECT
 						tsStart,
 						open,
 						close,
 						high,
 						low
 					FROM option_contract_aggregates
 					WHERE symbol = '${symbol}'
 					AND type = '${type}'
 					AND strike = '${strike}'
 					AND expirationDate = '${backExpirationDate}'
 					AND toDate(tsStart) = '${date}'
 				)
 				SELECT
 						toUnixTimestamp(front_option_contract_candlestick.tsStart) as tsStart,
 						back_option_contract_candlestick.open - front_option_contract_candlestick.open as open,
 						back_option_contract_candlestick.close - front_option_contract_candlestick.close as close
 				FROM front_option_contract_candlestick
 				INNER JOIN back_option_contract_candlestick
 				ON front_option_contract_candlestick.tsStart = back_option_contract_candlestick.tsStart
 				ORDER BY front_option_contract_candlestick.tsStart ASC
 			`)
      ).map((aggregate) => ({
        ...aggregate,
        tsStart: aggregate.tsStart * 1000, // unfortunately, `toUnixTimestamp` only returns second-precision
      }));
    },
    insertAggregates: async (aggregates) => {
      // no-op: we insert individual option contracts, not calendars
    },
    getClosingPrice: async ({
      key: { symbol, strike, type, frontExpirationDate, backExpirationDate },
    }) => {
      return (
        await query<{ calendarClosingPrice: number }>(`
 					WITH front_option_contract_candlestick AS (
 						SELECT
 								tsStart,
 								open,
 								close,
 								high,
 								low
 						FROM option_contract_aggregates
 						WHERE symbol = '${symbol}'
 						AND type = '${type}'
 						AND strike = '${strike}'
 						AND expirationDate = '${frontExpirationDate}'
 						AND toDate(tsStart) = '${frontExpirationDate}'
 					),
 					back_option_contract_candlestick AS (
 						SELECT
 								tsStart,
 								open,
 								close,
 								high,
 								low
 						FROM option_contract_aggregates
 						WHERE symbol = '${symbol}'
 						AND type = '${type}'
 						AND strike = '${strike}'
 						AND expirationDate = '${backExpirationDate}'
 						AND toDate(tsStart) = '${frontExpirationDate}'
 					)
 					SELECT
 						min(back_option_contract_candlestick.close - front_option_contract_candlestick.close) as calendarClosingPrice
 					FROM front_option_contract_candlestick
 					INNER JOIN back_option_contract_candlestick
 					ON front_option_contract_candlestick.tsStart = back_option_contract_candlestick.tsStart
 				`)
      )[0]?.calendarClosingPrice;
    },
    getTargetPriceByProbability: async ({
      symbol,
      calendarSpan,
      strikePercentageFromTheMoney,
      historicalProbabilityOfSuccess,
    }) => {
      return 0.24;
    },
  };
  return {
    ...calendarDatabase,
    getCalendars: calendarDatabase.getKeys,
  };
 }
 export const database: CalendarDatabase = makeCalendarDatabase();
@@ -0,0 +1,24 @@
 import type { AggregateDatabase } from "../interfaces.js";
 export type CalendarKey = {
  symbol: string;
  type: "call" | "put";
  strike: number;
  frontExpirationDate: string;
  backExpirationDate: string;
 };
 export type CalendarDatabase = AggregateDatabase<CalendarKey> & {
  getCalendars: AggregateDatabase<CalendarKey>["getKeys"];
  getTargetPriceByProbability: ({
    symbol,
    calendarSpan,
    strikePercentageFromTheMoney,
    historicalProbabilityOfSuccess,
  }: {
    symbol: string;
    calendarSpan: number;
    strikePercentageFromTheMoney: number;
    historicalProbabilityOfSuccess: number;
  }) => Promise<number>;
 };
@@ -0,0 +1,166 @@
 import type { CalendarDatabase } from "./interfaces.js";
 import { open } from "lmdbx";
 const calendarAggregatesDb = open({
  path: "./calendar-aggregates.db",
  compression: true,
 });
 const calendarExistenceDb = open({
  path: "./calendar-existence.db",
  compression: true,
 });
 /** Largest possible key according to the `ordered-binary` (used by lmdbx) docs. */
 const MAXIMUM_KEY = Buffer.from([0xff]);
 function makeCalendarDatabase(): CalendarDatabase {
  const calendarDatabase: Omit<CalendarDatabase, "getCalendars"> = {
    getKeys: async ({ key: { symbol }, date }) => {
      return calendarExistenceDb
        .getRange({
          start: [date, symbol],
          end: [date, symbol, MAXIMUM_KEY],
        })
        .map(({ key }) => ({
          symbol,
          frontExpirationDate: key[2],
          backExpirationDate: key[3],
          strike: key[4],
          type: key[5],
        })).asArray;
    },
    getAggregates: async ({
      key: { symbol, frontExpirationDate, backExpirationDate, strike, type },
      date,
    }) => {
      const startOfDayUnix = new Date(`${date}T00:00:00Z`).valueOf();
      const endOfDayUnix = startOfDayUnix + 3600 * 24 * 1000;
      return calendarAggregatesDb
        .getRange({
          start: [
            symbol,
            frontExpirationDate,
            backExpirationDate,
            strike,
            type,
            startOfDayUnix,
          ],
          end: [
            symbol,
            frontExpirationDate,
            backExpirationDate,
            strike,
            type,
            endOfDayUnix,
          ],
        })
        .map(({ value }) => ({
          tsStart: value.tsStart,
          open: value.open,
          close: value.close,
          high: value.high,
          low: value.low,
        })).asArray;
    },
    getAggregate: async ({
      key: { symbol, frontExpirationDate, backExpirationDate, strike, type },
      tsStart,
    }) => {
      return await calendarAggregatesDb.get([
        symbol,
        frontExpirationDate,
        backExpirationDate,
        strike,
        type,
        tsStart,
      ]);
    },
    insertAggregates: async (aggregates) => {
      await calendarExistenceDb.batch(() => {
        for (const aggregate of aggregates) {
          calendarExistenceDb.put(
            [
              new Date(aggregate.tsStart).toISOString().substring(0, 10),
              aggregate.key.symbol,
              aggregate.key.frontExpirationDate,
              aggregate.key.backExpirationDate,
              aggregate.key.strike,
              aggregate.key.type,
            ],
            null
          );
        }
      });
      await calendarAggregatesDb.batch(() => {
        for (const aggregate of aggregates) {
          calendarAggregatesDb.put(
            [
              aggregate.key.symbol,
              aggregate.key.frontExpirationDate,
              aggregate.key.backExpirationDate,
              aggregate.key.strike,
              aggregate.key.type,
              aggregate.tsStart,
            ],
            {
              open: aggregate.open,
              close: aggregate.close,
              high: aggregate.high,
              low: aggregate.low,
            }
          );
        }
      });
    },
    getClosingPrice: async ({
      key: { symbol, strike, type, frontExpirationDate, backExpirationDate },
    }) => {
      const startOfExpirationDateUnix = new Date(
        `${frontExpirationDate}T23:59:59Z`
      ).valueOf();
      const endOfExpirationDateUnix = new Date(
        `${frontExpirationDate}T00:00:00Z`
      ).valueOf();
      for (const { value } of calendarAggregatesDb.getRange({
        start: [
          symbol,
          frontExpirationDate,
          backExpirationDate,
          strike,
          type,
          startOfExpirationDateUnix,
        ],
        end: [
          symbol,
          frontExpirationDate,
          backExpirationDate,
          strike,
          type,
          endOfExpirationDateUnix,
        ],
        reverse: true,
      })) {
        if (value.close > 0) {
          return value.close;
        }
      }
      return 0;
    },
    getTargetPriceByProbability: async ({
      symbol,
      calendarSpan,
      strikePercentageFromTheMoney,
      historicalProbabilityOfSuccess,
    }) => {
      return 0.24;
    },
  };
  return {
    ...calendarDatabase,
    getCalendars: calendarDatabase.getKeys,
  };
 }
 export const database: CalendarDatabase = makeCalendarDatabase();
@@ -0,0 +1,207 @@
 import { database as optionContractDatabase } from "../OptionContract/lmdbx.js";
 import type { CalendarDatabase } from "./interfaces.js";
 /** Largest possible key according to the `ordered-binary` (used by lmdbx) docs. */
 const MAXIMUM_KEY = Buffer.from([0xff]);
 function makeCalendarDatabase(): CalendarDatabase {
  const getAggregatesSync = ({
    key: { symbol, frontExpirationDate, backExpirationDate, strike, type },
    date,
  }) => {
    const frontOptionContractAggregates =
      optionContractDatabase.getAggregatesSync({
        date,
        key: { symbol, expirationDate: frontExpirationDate, strike, type },
      });
    const backOptionContractAggregates =
      optionContractDatabase.getAggregatesSync({
        date,
        key: { symbol, expirationDate: backExpirationDate, strike, type },
      });
    const calendarAggregates = [];
    let i = 0;
    let j = 0;
    while (
      i < frontOptionContractAggregates.length &&
      j < backOptionContractAggregates.length
    ) {
      if (
        frontOptionContractAggregates[i].tsStart ===
        backOptionContractAggregates[j].tsStart
      ) {
        calendarAggregates.push({
          tsStart: frontOptionContractAggregates[i].tsStart,
          open:
            backOptionContractAggregates[j].open -
            frontOptionContractAggregates[i].open,
          close:
            backOptionContractAggregates[j].close -
            frontOptionContractAggregates[i].close,
          // the high and low are not exactly correct since we don't know if each contract's high and low happened at the same moment as the other:
          high:
            backOptionContractAggregates[j].high -
            frontOptionContractAggregates[i].high,
          low:
            backOptionContractAggregates[j].low -
            frontOptionContractAggregates[i].low,
        });
        i++;
        j++;
      } else if (
        frontOptionContractAggregates[i].tsStart >
        backOptionContractAggregates[j].tsStart
      ) {
        j++;
      } else {
        i++;
      }
    }
    return calendarAggregates;
  };
  const calendarDatabase: Omit<CalendarDatabase, "getCalendars"> = {
    getKeys: async ({ key: { symbol }, date }) => {
      const optionContracts = await optionContractDatabase.getOptionContracts({
        date,
        key: { symbol },
      });
      return optionContracts.flatMap(
        (frontOptionContract, i, optionContracts) =>
          optionContracts
            .filter(
              (potientialBackOptionContract) =>
                frontOptionContract.strike ===
                  potientialBackOptionContract.strike &&
                frontOptionContract.type ===
                  potientialBackOptionContract.type &&
                frontOptionContract.expirationDate <
                  potientialBackOptionContract.expirationDate
            )
            .map((backOptionContract) => ({
              symbol,
              frontExpirationDate: frontOptionContract.expirationDate,
              backExpirationDate: backOptionContract.expirationDate,
              strike: frontOptionContract.strike,
              type: frontOptionContract.type,
            }))
      );
    },
    getAggregates: async ({
      key: { symbol, frontExpirationDate, backExpirationDate, strike, type },
      date,
    }) =>
      getAggregatesSync({
        key: { symbol, frontExpirationDate, backExpirationDate, strike, type },
        date,
      }),
    getAggregatesSync,
    insertAggregates: async (aggregates) => {
      // right now, no-op
    },
    getClosingPrice: async ({
      key: { symbol, strike, type, frontExpirationDate, backExpirationDate },
    }) => {
      // get unix timestamp, in milliseconds, of the start of the last hour, which is 03:30PM in the `America/New_York` timezone on the front expiration date:
      const startOfLastHourUnix = new Date(
        `${frontExpirationDate}T19:30:00Z`
      ).getTime();
      const endOfLastHourUnix = startOfLastHourUnix + 3600 * 1000;
      const frontOptionContractAggregates = (
        await optionContractDatabase.getAggregates({
          date: frontExpirationDate,
          key: { symbol, expirationDate: frontExpirationDate, strike, type },
        })
      ).filter(
        ({ tsStart }) =>
          tsStart >= startOfLastHourUnix && tsStart < endOfLastHourUnix
      );
      const backOptionContractAggregates = (
        await optionContractDatabase.getAggregates({
          date: frontExpirationDate,
          key: { symbol, expirationDate: backExpirationDate, strike, type },
        })
      ).filter(
        ({ tsStart }) =>
          tsStart >= startOfLastHourUnix && tsStart < endOfLastHourUnix
      );
      let i = 0;
      let j = 0;
      let minPrice = 0;
      while (
        i < frontOptionContractAggregates.length &&
        j < backOptionContractAggregates.length
      ) {
        if (
          frontOptionContractAggregates[i].tsStart ===
          backOptionContractAggregates[j].tsStart
        ) {
          const calendarClosePrice =
            backOptionContractAggregates[j].close -
            frontOptionContractAggregates[i].close;
          if (calendarClosePrice < minPrice || minPrice === 0) {
            minPrice = calendarClosePrice;
          }
          i++;
          j++;
        } else if (
          frontOptionContractAggregates[i].tsStart >
          backOptionContractAggregates[j].tsStart
        ) {
          j++;
        } else {
          i++;
        }
      }
      return minPrice;
    },
    getAggregate: async ({
      key: { symbol, frontExpirationDate, backExpirationDate, strike, type },
      tsStart,
    }) => {
      const [frontOptionContractAggregate, backOptionContractAggregate] =
        await Promise.all([
          optionContractDatabase.getAggregate({
            key: { symbol, expirationDate: frontExpirationDate, strike, type },
            tsStart,
          }),
          optionContractDatabase.getAggregate({
            key: { symbol, expirationDate: backExpirationDate, strike, type },
            tsStart,
          }),
        ]);
      // only return the calendar aggregate if its constituent front and back option contract aggregates exist:
      if (frontOptionContractAggregate && backOptionContractAggregate) {
        return {
          tsStart,
          open:
            backOptionContractAggregate.open -
            frontOptionContractAggregate.open,
          close:
            backOptionContractAggregate.close -
            frontOptionContractAggregate.close,
          high:
            backOptionContractAggregate.high -
            frontOptionContractAggregate.high,
          low:
            backOptionContractAggregate.low - frontOptionContractAggregate.low,
        };
      }
      return undefined;
    },
    getTargetPriceByProbability: async ({
      symbol,
      calendarSpan,
      strikePercentageFromTheMoney,
      historicalProbabilityOfSuccess,
    }) => {
      return 0.24;
    },
  };
  return {
    ...calendarDatabase,
    getCalendars: calendarDatabase.getKeys,
  };
 }
 export const database: CalendarDatabase = makeCalendarDatabase();
@@ -0,0 +1,114 @@
 import type {
  OptionContractDatabase,
  OptionContractKey,
 } from "./interfaces.js";
 import type { Aggregate } from "../interfaces.js";
 import { clickhouse, query } from "../../lib/clickhouse.js";
 function makeOptionContractDatabase(): OptionContractDatabase {
  const optionContractDatabase: Omit<
    OptionContractDatabase,
    "getOptionContracts"
  > = {
    getKeys: async ({ key: { symbol }, date }) => {
      return (
        await query<Omit<OptionContractKey, "symbol">>(`
          SELECT expirationDate, strike, type
 					FROM option_contract_existences
 					WHERE symbol = '${symbol}'
 					AND asOfDate = '${date}'
        `)
      ).map((optionContractWithoutKey) => ({
        ...optionContractWithoutKey,
        symbol,
      }));
    },
    getAggregates: async ({
      key: { symbol, expirationDate, strike, type },
      date,
    }) => {
      return (
        await query<Omit<Aggregate<OptionContractKey>, "key">>(`
        SELECT
 						toUnixTimestamp(tsStart) as tsStart,
 						open,
 						close,
 						high,
 						low
 					FROM option_contract_aggregates
 					WHERE symbol = '${symbol}'
 					AND type = '${type}'
 					AND strike = '${strike}'
 					AND expirationDate = '${expirationDate}'
 					AND toDate(tsStart) = '${date}'
        ORDER BY tsStart ASC
      `)
      ).map((aggregate) => ({
        ...aggregate,
        tsStart: aggregate.tsStart * 1000, // unfortunately, `toUnixTimestamp` only returns second-precision
      }));
    },
    getAggregate: async ({
      key: { symbol, expirationDate, strike, type },
      tsStart,
    }) => {
      const tsStartString = new Date(tsStart).toISOString();
      return (
        await query<Omit<Aggregate<OptionContractKey>, "key">>(`
          SELECT
 						open,
 						close,
 						high,
 						low
 					FROM option_contract_aggregates
 					WHERE symbol = '${symbol}'
 					AND type = '${type}'
 					AND strike = '${strike}'
 					AND expirationDate = '${expirationDate}'
 					AND tsStart = '${tsStartString}'
          ORDER BY tsStart ASC
        `)
      ).map((aggregate) => ({
        ...aggregate,
        tsStart,
      }))[0];
    },
    insertAggregates: async (aggregates) => {
      // stock existence is taken care of by clickhouse materialized view
      await clickhouse.insert({
        table: "option_contract_aggregates",
        values: aggregates.map(
          ({
            key: { symbol, expirationDate, strike, type },
            tsStart,
            open,
            close,
            high,
            low,
          }) => ({
            symbol,
            expirationDate,
            strike,
            type,
            tsStart,
            open,
            close,
            high,
            low,
          })
        ),
      });
    },
    getClosingPrice: async ({ key }) => {
      // no-op: not used since stocks don't have a "closing" price, unlike options.
      return 0;
    },
  };
  return {
    ...optionContractDatabase,
    getOptionContracts: optionContractDatabase.getKeys,
  };
 }
 export const database: OptionContractDatabase = makeOptionContractDatabase();
@@ -0,0 +1,12 @@
 import type { AggregateDatabase } from "../interfaces.js";
 export type OptionContractKey = {
  symbol: string;
  expirationDate: string;
  strike: number;
  type: "call" | "put";
 };
 export type OptionContractDatabase = AggregateDatabase<OptionContractKey> & {
  getOptionContracts: AggregateDatabase<OptionContractKey>["getKeys"];
 };
@@ -0,0 +1,138 @@
 import type { OptionContractDatabase } from "./interfaces.js";
 import { open } from "lmdbx";
 const optionContractAggregatesDb = open({
  path: "./option-contract-aggregates.db",
  // any options go here, we can turn on compression like this:
  compression: true,
 });
 const optionContractExistenceDb = open({
  path: "./option-contract-existence.db",
  // any options go here, we can turn on compression like this:
  compression: true,
 });
 /** Largest possible key according to the `ordered-binary` (used by lmdbx) docs. */
 const MAXIMUM_KEY = Buffer.from([0xff]);
 function makeOptionContractDatabase(): OptionContractDatabase {
  const getAggregatesSync = ({
    key: { symbol, expirationDate, strike, type },
    date,
  }) => {
    const startOfDayUnix = new Date(`${date}T00:00:00Z`).valueOf();
    const endOfDayUnix = startOfDayUnix + 3600 * 24 * 1000;
    return optionContractAggregatesDb
      .getRange({
        start: [symbol, expirationDate, strike, type, startOfDayUnix],
        end: [symbol, expirationDate, strike, type, endOfDayUnix],
      })
      .map(({ key, value }) => ({
        tsStart: key[4],
        open: value.open,
        close: value.close,
        high: value.high,
        low: value.low,
      })).asArray;
  };
  const optionContractDatabase: Omit<
    OptionContractDatabase,
    "getOptionContracts"
  > = {
    getKeys: async ({ key: { symbol }, date }) => {
      return optionContractExistenceDb
        .getRange({
          start: [date, symbol],
          end: [date, symbol, MAXIMUM_KEY],
        })
        .map(({ key }) => ({
          symbol,
          expirationDate: key[2],
          strike: key[3],
          type: key[4],
        })).asArray;
    },
    getAggregatesSync,
    getAggregates: async ({
      key: { symbol, expirationDate, strike, type },
      date,
    }) =>
      getAggregatesSync({
        key: { symbol, expirationDate, strike, type },
        date,
      }),
    insertAggregates: async (aggregates) => {
      await optionContractExistenceDb.batch(() => {
        for (const aggregate of aggregates) {
          optionContractExistenceDb.put(
            [
              new Date(aggregate.tsStart).toISOString().substring(0, 10),
              aggregate.key.symbol,
              aggregate.key.expirationDate,
              aggregate.key.strike,
              aggregate.key.type,
            ],
            null
          );
        }
      });
      await optionContractAggregatesDb.batch(() => {
        for (const aggregate of aggregates) {
          optionContractAggregatesDb.put(
            [
              aggregate.key.symbol,
              aggregate.key.expirationDate,
              aggregate.key.strike,
              aggregate.key.type,
              aggregate.tsStart,
            ],
            {
              open: aggregate.open,
              close: aggregate.close,
              high: aggregate.high,
              low: aggregate.low,
            }
          );
        }
      });
    },
    getClosingPrice: async ({
      key: { symbol, strike, type, expirationDate },
    }) => {
      const startOfLastHourUnix = new Date(
        `${expirationDate}T00:00:00Z`
      ).valueOf();
      const endOfLastHourUnix = startOfLastHourUnix + 3600 * 1000;
      let minPrice = 0;
      for (const { value } of optionContractAggregatesDb.getRange({
        start: [symbol, expirationDate, strike, type, startOfLastHourUnix],
        end: [symbol, expirationDate, strike, type, endOfLastHourUnix],
      })) {
        if (value.close < minPrice || minPrice === 0) {
          minPrice = value.close;
        }
      }
      return minPrice;
    },
    getAggregate: async ({
      key: { symbol, expirationDate, strike, type },
      tsStart,
    }) => {
      return await optionContractAggregatesDb.get([
        symbol,
        expirationDate,
        strike,
        type,
        tsStart,
      ]);
    },
  };
  return {
    ...optionContractDatabase,
    getOptionContracts: optionContractDatabase.getKeys,
  };
 }
 export const database: OptionContractDatabase = makeOptionContractDatabase();
@@ -0,0 +1,79 @@
 import type { StockDatabase, StockKey } from "./interfaces.js";
 import type { Aggregate } from "../interfaces.js";
 import { clickhouse, query } from "../../lib/clickhouse.js";
 function makeStockDatabase(): StockDatabase {
  const stockDatabase: Omit<StockDatabase, "getSymbols"> = {
    getKeys: async ({ date, key }) => {
      if (key?.symbol) {
        return [key as StockKey];
      }
      return await query(`
          SELECT DISTINCT symbol FROM stock_aggregates WHERE toDate(tsStart) = '${date}'
        `);
    },
    getAggregates: async ({ key: { symbol }, date }) => {
      return (
        await query<Omit<Aggregate<StockKey>, "key">>(`
        SELECT 
            toUnixTimestamp(tsStart) as tsStart,
            open,
            close,
            high,
            low
        FROM stock_aggregates
        WHERE symbol = '${symbol}'
        AND toDate(tsStart) = '${date}'
        ORDER BY tsStart ASC
      `)
      ).map((aggregate) => ({
        ...aggregate,
        tsStart: aggregate.tsStart * 1000, // unfortunately, `toUnixTimestamp` only returns second-precision
      }));
    },
    getAggregate: async ({ key: { symbol }, tsStart }) => {
      return (
        await query<Omit<Aggregate<StockKey>, "key">>(`
          SELECT 
            open,
            close,
            high,
            low
          FROM stock_aggregates
          WHERE symbol = '${symbol}'
          AND tsStart = '${tsStart}'
        `)
      ).map((aggregate) => ({
        ...aggregate,
        tsStart,
      }))[0];
    },
    insertAggregates: async (aggregates) => {
      // stock existence is taken care of by clickhouse materialized view
      await clickhouse.insert({
        table: "stock_aggregates",
        values: aggregates.map(
          ({ key: { symbol }, tsStart, open, close, high, low }) => ({
            symbol,
            tsStart,
            open,
            close,
            high,
            low,
          })
        ),
      });
    },
    getClosingPrice: async ({ key }) => {
      // no-op: not used since stocks don't have a "closing" price, unlike options.
      return 0;
    },
  };
  return {
    ...stockDatabase,
    getSymbols: stockDatabase.getKeys,
  };
 }
 export const database: StockDatabase = makeStockDatabase();
@@ -0,0 +1,7 @@
 import type { AggregateDatabase } from "../interfaces.js";
 export type StockKey = { symbol: string };
 export type StockDatabase = AggregateDatabase<StockKey> & {
  getSymbols: AggregateDatabase<StockKey>["getKeys"];
 };
@@ -0,0 +1,86 @@
 import type { StockDatabase, StockKey } from "./interfaces.js";
 import { open } from "lmdbx";
 const stockAggregatesDb = open({
  path: "./stock-aggregates.db",
  // any options go here, we can turn on compression like this:
  compression: true,
 });
 const stockExistenceDb = open({
  path: "./stock-existence.db",
  // any options go here, we can turn on compression like this:
  compression: true,
 });
 /** Largest possible key according to the `ordered-binary` (used by lmdbx) docs. */
 const MAXIMUM_KEY = Buffer.from([0xff]);
 function makeStockDatabase(): StockDatabase {
  const stockDatabase: Omit<StockDatabase, "getSymbols"> = {
    getKeys: async ({ date, key }) => {
      if (key?.symbol) {
        return [key as StockKey];
      }
      return stockExistenceDb
        .getRange({
          start: [date],
          end: [date, MAXIMUM_KEY],
        })
        .map(({ key }) => ({ symbol: key[1] })).asArray;
    },
    getAggregates: async ({ key: { symbol }, date }) => {
      const startOfDayUnix = new Date(`${date}T00:00:00Z`).valueOf();
      const endOfDayUnix = startOfDayUnix + 3600 * 24 * 1000;
      return stockAggregatesDb
        .getRange({
          start: [symbol, startOfDayUnix],
          end: [symbol, endOfDayUnix],
        })
        .map(({ key, value }) => ({
          tsStart: key[1],
          open: value.open,
          close: value.close,
          high: value.high,
          low: value.low,
        })).asArray;
    },
    getAggregate: async ({ key: { symbol }, tsStart }) => {
      return stockAggregatesDb.get([symbol, tsStart]);
    },
    insertAggregates: async (aggregates) => {
      await stockExistenceDb.batch(() => {
        for (const aggregate of aggregates) {
          stockExistenceDb.put(
            [
              new Date(aggregate.tsStart).toISOString().substring(0, 10),
              aggregate.key.symbol,
            ],
            null
          );
        }
      });
      await stockAggregatesDb.batch(() => {
        for (const aggregate of aggregates) {
          stockAggregatesDb.put([aggregate.key.symbol, aggregate.tsStart], {
            open: aggregate.open,
            close: aggregate.close,
            high: aggregate.high,
            low: aggregate.low,
          });
        }
      });
    },
    getClosingPrice: async ({ key }) => {
      // no-op: not used since stocks don't have a "closing" price, unlike options.
      return 0;
    },
  };
  return {
    ...stockDatabase,
    getSymbols: stockDatabase.getKeys,
  };
 }
 export const database: StockDatabase = makeStockDatabase();
@@ -0,0 +1,46 @@
 export type Candlestick = {
  open: number;
  close: number;
  high: number;
  low: number;
 };
 export type Aggregate<T> = {
  key: T;
  /** UNIX time in milliseconds */
  tsStart: number;
 } & Candlestick;
 export type AggregateDatabase<T> = {
  getKeys: ({
    key,
    date,
  }: {
    key?: T | Partial<T>;
    date?: string;
  }) => Promise<Array<T>>;
  getAggregates: ({
    key,
    date,
  }: {
    key: T;
    date: string;
  }) => Promise<Array<Omit<Aggregate<T>, "key">>>;
  /** Since an aggregate may not exist at the specified `tsStart`, return `undefined` if it doesn't exist. */
  getAggregate: ({
    key,
    tsStart,
  }: {
    key: T;
    tsStart: number;
  }) => Promise<Omit<Aggregate<T>, "key"> | undefined>;
  getAggregatesSync?: ({
    key,
    date,
  }: {
    key: T;
    date: string;
  }) => Array<Omit<Aggregate<T>, "key">>;
  insertAggregates: (aggregates: Array<Aggregate<T>>) => Promise<void>;
  getClosingPrice: ({ key }: { key: T }) => Promise<number>;
 };
@@ -0,0 +1,87 @@
 import { query } from "./lib/clickhouse.js";
 import { publicProcedure, RpcType, router } from "./trpc.js";
 import {
  Object as ObjectT,
  String as StringT,
  Number as NumberT,
 } from "@sinclair/typebox";
 /** Gets a list of symbols that have at least one option contract */
 export const getAvailableUnderlyings = publicProcedure.query(async (opts) => {
  // return (
  //   await query<{ symbol: string }>(`
  //   SELECT DISTINCT(symbol) as symbol FROM option_contract_existences WHERE asOfDate = (SELECT max(asOfDate) FROM option_contract_existences)
  // `)
  // ).map(({ symbol }) => symbol);
  return ["SPY"];
 });
 export const getAvailableAsOfDates = publicProcedure
  .input(RpcType(ObjectT({ underlying: StringT() })))
  .query(async (opts) => {
    const underlying = opts.input.underlying;
    return (
      await query<{ asOfDate: string }>(`
 		SELECT 
 			DISTINCT(asOfDate) as asOfDate 
 		FROM option_contract_existences 
 		WHERE symbol = '${underlying}'
 		ORDER BY asOfDate
 	`)
    ).map(({ asOfDate }) => asOfDate);
  });
 export const getExpirationsForUnderlying = publicProcedure
  .input(
    RpcType(
      ObjectT({
        underlying: StringT({ maxLength: 5 }),
        asOfDate: StringT(),
      })
    )
  )
  .query(async (opts) => {
    const { underlying, asOfDate } = opts.input;
    return (
      await query<{ expirationDate: string }>(`
 		SELECT
 			DISTINCT(expirationDate) as expirationDate
 		FROM option_contract_existences
 		WHERE symbol = '${underlying}'
 		AND asOfDate = '${asOfDate}'
 		ORDER BY expirationDate
 	`)
    ).map(({ expirationDate }) => expirationDate);
  });
 export const getStrikesForUnderlying = publicProcedure
  .input(
    RpcType(
      ObjectT({
        underlying: StringT({ maxLength: 5 }),
        asOfDate: StringT(),
        expirationDate: StringT(),
      })
    )
  )
  .query(async (opts) => {
    const { underlying, asOfDate, expirationDate } = opts.input;
    return (
      await query<{ strike: string }>(`
 		SELECT
 			DISTINCT(strike) as strike
 		FROM option_contract_existences
 		WHERE symbol = '${underlying}'
 		AND asOfDate = '${asOfDate}'
 		AND expirationDate = '${expirationDate}'
 		ORDER BY strike
 	`)
    ).map(({ strike }) => strike);
  });
 export default router({
  getAvailableUnderlyings,
  getAvailableAsOfDates,
  getExpirationsForUnderlying,
  getStrikesForUnderlying,
 });
@@ -0,0 +1,53 @@
 import { query } from "./lib/clickhouse.js";
 import { publicProcedure, RpcType, router } from "./trpc.js";
 import {
  Object as ObjectT,
  String as StringT,
  Number as NumberT,
 } from "@sinclair/typebox";
 export const getChartData = publicProcedure
  .input(
    RpcType(
      ObjectT({
        underlying: StringT({ maxLength: 5 }),
        daysToFrontExpiration: NumberT(),
        daysBetweenFrontAndBackExpiration: NumberT(),
        lookbackPeriodStart: StringT({
          pattern: "[0-9]{4}-[0-9]{2}-[0-9]{2}",
        }),
        lookbackPeriodEnd: StringT({ pattern: "[0-9]{4}-[0-9]{2}-[0-9]{2}" }),
      })
    )
  )
  .query(async (opts) => {
    const {
      underlying,
      daysToFrontExpiration,
      daysBetweenFrontAndBackExpiration,
      lookbackPeriodStart,
      lookbackPeriodEnd,
    } = opts.input;
    return await query<[number, number, number]>(
      `
     SELECT
 			 moniness*100 as x,
 			 FLOOR(price, 1) as y,
 			 sum(number_of_quotes) as n
 		 FROM calendar_stats
 		 WHERE dte = ${daysToFrontExpiration}
 		 AND moniness >= -0.05
 		 AND moniness <= 0.05
 		 AND span = ${daysBetweenFrontAndBackExpiration}
 		 AND date >= '${lookbackPeriodStart}'
 		 AND date <= '${lookbackPeriodEnd}'
 		 GROUP BY x, y
 		 ORDER BY x ASC, y ASC
 	 `,
      "JSONEachRow"
    );
  });
 export default router({
  getChartData,
 });
@@ -0,0 +1,54 @@
 import { query } from "./lib/clickhouse.js";
 import { publicProcedure, RpcType, router } from "./trpc.js";
 import {
  Object as ObjectT,
  String as StringT,
  Number as NumberT,
 } from "@sinclair/typebox";
 /** Returns prices for all matching calendars (i.e. those with similar
 * characteristics to those given) */
 export const getChartData = publicProcedure
  .input(
    RpcType(
      ObjectT({
        underlying: StringT({ maxLength: 5 }),
        daysToFrontExpiration: NumberT(),
        daysBetweenFrontAndBackExpiration: NumberT(),
        strikePercentageFromUnderlyingPriceRangeMin: NumberT(),
        strikePercentageFromUnderlyingPriceRangeMax: NumberT(),
        lookbackPeriodStart: StringT(),
        lookbackPeriodEnd: StringT(),
      })
    )
  )
  .query(async (opts) => {
    const {
      underlying,
      daysToFrontExpiration,
      daysBetweenFrontAndBackExpiration,
      strikePercentageFromUnderlyingPriceRangeMin,
      strikePercentageFromUnderlyingPriceRangeMax,
      lookbackPeriodStart,
      lookbackPeriodEnd,
    } = opts.input;
    return await query<[number, number]>(
      `
     SELECT
 			 toUnixTimestamp(date) as x,
 			 price as y
 		 FROM calendar_stats
 		 WHERE dte = ${daysToFrontExpiration}
 		 AND moniness >= ${strikePercentageFromUnderlyingPriceRangeMin}
 		 AND moniness <= ${strikePercentageFromUnderlyingPriceRangeMax}
 		 AND span = ${daysBetweenFrontAndBackExpiration}
 		 AND date >= '${lookbackPeriodStart}'
 		 AND date <= '${lookbackPeriodEnd}'
 	 `,
      "JSONEachRow"
    );
  });
 export default router({
  getChartData,
 });
@@ -0,0 +1,36 @@
 import { query } from "./lib/clickhouse.js";
 import { publicProcedure, RpcType, router } from "./trpc.js";
 import { Object as ObjectT, String as StringT } from "@sinclair/typebox";
 export const getChartData = publicProcedure
  .input(
    RpcType(
      ObjectT({
        underlying: StringT({ maxLength: 5 }),
        lookbackPeriodStart: StringT(),
        lookbackPeriodEnd: StringT(),
      })
    )
  )
  .query(async (opts) => {
    const { underlying, lookbackPeriodStart, lookbackPeriodEnd } = opts.input;
    return await query<[number, number]>(
      `
 		 SELECT
 			 toUnixTimestamp(toStartOfHour(ts)) as x,
 			 avg(price) as y
 		 FROM stock_aggregates_filled
 		 WHERE symbol = '${underlying}'
 		 AND ts >= '${lookbackPeriodStart} 00:00:00'
 		 AND ts <= '${lookbackPeriodEnd} 00:00:00'
     GROUP BY x
 		 ORDER BY x ASC
 	 `,
      "JSONCompactEachRow"
      // "JSONEachRow"
    );
  });
 export default router({
  getChartData,
 });
@@ -0,0 +1,146 @@
 import { database as stockDatabase } from "./AggregateDatabase/Stock/lmdbx.js";
 import { database as calendarDatabase } from "./AggregateDatabase/Calendar/optiondb-lmdbx.js";
 import type { CalendarKey } from "./AggregateDatabase/Calendar/interfaces.js";
 import { nextDate } from "./lib/utils/nextDate.js";
 export type BacktestInput = {
  symbol: string;
  startDate: string;
  endDate: string;
  /** Between 0 and 1. The frequency that similar calendars have historically ended (i.e. within the last hour) at a higher price than the current calendar's price. */
  historicalProbabilityOfSuccess?: number;
  initialBuyingPower?: number;
 };
 export async function backtest({
  symbol,
  startDate,
  endDate,
  historicalProbabilityOfSuccess = 0.8,
  initialBuyingPower = 2000,
 }: BacktestInput) {
  let buyingPower = initialBuyingPower;
  const portfolio = new Set<CalendarKey>();
  // for each day:
  for (
    let date = startDate, didBuyCalendar = false;
    date <= endDate;
    date = nextDate(date), didBuyCalendar = false
  ) {
    const calendars = await calendarDatabase.getCalendars({
      key: { symbol },
      date,
    });
    const stockAggregates = await stockDatabase.getAggregates({
      key: { symbol },
      date,
    });
    // for each minute of that day for which we have a stock candlestick:
    for (const stockAggregate of stockAggregates) {
      // console.log("Current Time:", new Date(stockAggregate.tsStart));
      // filter-out calendars that are far-from-the-money (5%)
      const calendarsNearTheMoney = calendars.filter(
        ({ strike }) =>
          Math.abs((stockAggregate.open - strike) / stockAggregate.open) < 0.05
      );
      console.log(
        "Current Date:",
        new Intl.DateTimeFormat("en-US", {
          timeZone: "America/New_York",
          dateStyle: "full",
          timeStyle: "long",
        }).format(new Date(stockAggregate.tsStart)),
        ";",
        `${calendarsNearTheMoney.length} Calendars Near The Money`
      );
      // for each relevant calendar on that day:
      for (const calendar of calendarsNearTheMoney) {
        const strikePercentageFromTheMoney = Math.abs(
          (stockAggregate.open - calendar.strike) / stockAggregate.open
        );
        /** Number of days between the back and front expiration dates. */
        const calendarSpanInDays =
          (new Date(calendar.backExpirationDate).valueOf() -
            new Date(calendar.frontExpirationDate).valueOf()) /
          (1000 * 60 * 60 * 24);
        const targetCalendarPrice =
          await calendarDatabase.getTargetPriceByProbability({
            symbol,
            calendarSpan: calendarSpanInDays,
            strikePercentageFromTheMoney,
            historicalProbabilityOfSuccess,
          });
        const calendarAggregateAtCurrentTime =
          await calendarDatabase.getAggregate({
            key: {
              ...calendar,
            },
            tsStart: stockAggregate.tsStart,
          });
        // if there exists a matching calendar candlestick for the current minute:
        if (calendarAggregateAtCurrentTime) {
          // if the current candlestick is a good price (i.e. less than the target price):
          const minCalendarPriceInCandlestick = Math.min(
            calendarAggregateAtCurrentTime.open,
            calendarAggregateAtCurrentTime.close
          );
          if (
            minCalendarPriceInCandlestick < targetCalendarPrice &&
            minCalendarPriceInCandlestick >
              0.07 /* sometimes the calendar price is zero or negative, which is of course impossible; some institution got a good deal */
          ) {
            // if we can afford to buy the calendar:
            if (buyingPower > minCalendarPriceInCandlestick) {
              // buy the calendar, and continue to the next day:
              portfolio.add(calendar);
              buyingPower = buyingPower - minCalendarPriceInCandlestick * 100;
              console.log(
                "Bought",
                calendar,
                "for",
                minCalendarPriceInCandlestick * 100,
                "...$",
                buyingPower,
                "left"
              );
              didBuyCalendar = true;
            }
          }
        }
        if (didBuyCalendar) {
          break;
        }
      }
      if (didBuyCalendar) {
        break;
      }
    }
    // for each calendar in portfolio, if today is the last day, close the position:
    for (const calendar of portfolio.values()) {
      if (calendar.frontExpirationDate === date) {
        const calendarClosingPrice = await calendarDatabase.getClosingPrice({
          key: {
            ...calendar,
          },
        });
        portfolio.delete(calendar);
        buyingPower = buyingPower + calendarClosingPrice * 100;
        console.log(
          "Sold",
          calendar,
          "for",
          calendarClosingPrice,
          "...$",
          buyingPower,
          "left"
        );
      }
    }
  }
  return {
    endingBuyingPower: buyingPower,
    portfolio: Array.from(portfolio.values()),
  };
 }
@@ -0,0 +1,85 @@
 import React, { useState, useEffect } from "react";
 import { render, Box, Text, useInput, useApp } from "ink";
 import TextInput from "ink-text-input";
 import type { BacktestInput } from "./backtest.js";
 import { backtest } from "./backtest.js";
 const App = () => {
  const [step, setStep] = useState(0);
  const [input, setInput] = useState<Partial<BacktestInput>>({});
  const [result, setResult] = useState<string>("");
  const { exit } = useApp();
  const steps = [
    { prompt: "Enter symbol:", key: "symbol" },
    { prompt: "Enter start date (YYYY-MM-DD):", key: "startDate" },
    { prompt: "Enter end date (YYYY-MM-DD):", key: "endDate" },
    {
      prompt: "Enter historical probability of success (0-1, default 0.8):",
      key: "historicalProbabilityOfSuccess",
    },
    {
      prompt: "Enter initial buying power (default 2000):",
      key: "initialBuyingPower",
    },
  ];
  useInput((input, key) => {
    if (key.escape) {
      exit();
    }
  });
  useEffect(() => {
    if (step === steps.length) {
      const runBacktest = async () => {
        try {
          const backtestResult = await backtest(input as BacktestInput);
          setResult(JSON.stringify(backtestResult, null, 2));
        } catch (error) {
          setResult(`Error: ${error.message}`);
        }
      };
      runBacktest();
    }
  }, [step, input]);
  const handleSubmit = (value: string) => {
    const currentStep = steps[step];
    let parsedValue: string | number = value;
    if (
      currentStep.key === "historicalProbabilityOfSuccess" ||
      currentStep.key === "initialBuyingPower"
    ) {
      parsedValue = Number.parseFloat(value) || undefined;
    }
    setInput({ ...input, [currentStep.key]: parsedValue });
    setStep(step + 1);
  };
  if (step < steps.length) {
    return (
      <Box flexDirection="column">
        <Text>{steps[step].prompt}</Text>
        <TextInput
          value={(input[steps[step].key] as string) || ""}
          onChange={(value) =>
            setInput((prev) => ({ ...prev, [steps[step].key]: value }))
          }
          onSubmit={handleSubmit}
        />
      </Box>
    );
  }
  return (
    <Box flexDirection="column">
      <Text>Backtest Results:</Text>
      <Text>{result}</Text>
    </Box>
  );
 };
 render(<App />);
@@ -1,18 +0,0 @@
 import _ from './env.js';
 import { createClient as createClickhouseClient } from '@clickhouse/client';
 import type { DataFormat } from '@clickhouse/client';
 // prevent from tree-shaking:
 console.log(_);
 export const clickhouse = createClickhouseClient({
  host: process.env.CLICKHOUSE_HOST || "http://localhost:8123",
  username:'avraham', 
  password:'buginoo'
 });
 export async function query<T>(queryString:string, format:DataFormat='JSONEachRow') : Promise<Array<T>>{
  return await (await clickhouse.query({
    query: queryString,
    format
  })).json()
 }
@@ -1,21 +0,0 @@
 import path from 'path';
 import { fileURLToPath } from 'url';
 import dotenv from 'dotenv';
 /** ES modules cannot use `__dirname`, so we have to mimic its functionality.
 *  Taken from [https://flaviocopes.com/fix-dirname-not-defined-es-module-scope/]
 */
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
 if(process.env.NODE_ENV==="development"){
  const ret = dotenv.config({ path:`${__dirname}/../.env.development` });
  if(ret.parsed){
    console.log("parsed!", process.env)
  }
  else{
    console.log("not parsed ;-(", ret.error)
  }
 }
 export default null;
@@ -1,304 +1,82 @@
-import _ from "./env";
+import { publicProcedure, router, RpcType } from "./trpc.js";
-import { publicProcedure, router } from "./trpc.js";
+import { query } from "./lib/clickhouse.js";
 import { query } from "./clickhouse.js";
 import { createHTTPHandler } from "@trpc/server/adapters/standalone";
 import cors from "cors";
 import {
  Object as ObjectT,
  String as StringT,
  TSchema,
  Number as NumberT,
 } from "@sinclair/typebox";
-import { TypeCompiler } from "@sinclair/typebox/compiler";
+import { createServer } from "node:http";
-import { TRPCError } from "@trpc/server";
+import { Env } from "@humanwhocodes/env";
-import { createServer } from "http";
+import StockPriceChart from "./StockPriceChart.js";
 import SimilarCalendarPriceChart from "./SimilarCalendarPriceChart.js";
 import CalendarExitPriceChart from "./CalendarExitPriceChart.js";
 import CalendarCharacteristicsForm from "./CalendarCharacteristicsForm.js";
-/**
+const env = new Env();
- * Generate a TRPC-compatible validator function given a Typebox schema.
+
- * This was copied from [https://github.com/sinclairzx81/typebox/blob/6cfcdc02cc813af2f1be57407c771fc4fadfc34a/example/trpc/readme.md].
+const LISTEN_PORT = env.get("LISTEN_PORT", 3005);
- * @param schema A Typebox schema
+
- * @returns A TRPC-compatible validator function
+export const getOpensForUnderlying = publicProcedure
- */
+  .input(
-export function RpcType<T extends TSchema>(schema: T) {
+    RpcType(
-  const check = TypeCompiler.Compile(schema);
+      ObjectT({
-  return (value: unknown) => {
+        underlying: StringT({ maxLength: 5 }),
-    if (check.Check(value)) return value;
+      })
-    const { path, message } = check.Errors(value).First()!;
+    )
-    throw new TRPCError({
+  )
-      message: `${message} for ${path}`,
+  .query(async (opts) => {
-      code: "BAD_REQUEST",
+    const { underlying } = opts.input;
-    });
+    return await query<{ x: number; y: number }>(
-  };
+      `
-}
+		SELECT
 			toUnixTimestamp(tsStart) as x,
 			open as y
 		FROM stock_aggregates
 		WHERE symbol = '${underlying}'
 		ORDER BY tsStart ASC
 	`,
      "JSONEachRow"
    );
  });
 export const getOpensForOptionContract = publicProcedure
  .input(
    RpcType(
      ObjectT({
        underlying: StringT({ maxLength: 5 }),
        expirationDate: StringT(),
        strike: NumberT(),
      })
    )
  )
  .query(async (opts) => {
    const { underlying, expirationDate, strike } = opts.input;
    return await query<{ x: number; y: number }>(
      `
 		SELECT
 			toUnixTimestamp(tsStart) as x,
 			open as y
 		FROM option_contract_aggregates
 		WHERE symbol = '${underlying}'
 		AND expirationDate = '${expirationDate}'
 		AND strike = ${strike}
 		AND type = 'call'
 		ORDER BY tsStart ASC
 	`,
      "JSONEachRow"
    );
  });
 const appRouter = router({
-  getAvailableUnderlyings: publicProcedure.query(async (opts) => {
+  CalendarCharacteristicsForm,
-    // return (await query<{symbol:string}>(`
+  StockPriceChart,
-    //   SELECT DISTINCT(symbol) as symbol FROM option_contracts
+  SimilarCalendarPriceChart,
-    // `))
+  CalendarExitPriceChart,
-    // .map(({symbol})=>symbol);
+
-    return ["AAPL", "AMD", "GOOGL", "MSFT", "NFLX"];
+  getOpensForUnderlying,
-  }),
+  getOpensForOptionContract,
  getAvailableAsOfDates: publicProcedure
    .input(RpcType(ObjectT({ underlying: StringT() })))
    .query(async (opts) => {
      const underlying = opts.input.underlying;
      return (
        await query<{ asOfDate: string }>(`
        SELECT 
          DISTINCT(asOfDate) as asOfDate 
        FROM option_contracts 
        WHERE symbol = '${underlying}'
        ORDER BY asOfDate
      `)
      ).map(({ asOfDate }) => asOfDate);
    }),
  getExpirationsForUnderlying: publicProcedure
    .input(
      RpcType(
        ObjectT({
          underlying: StringT({ maxLength: 5 }),
          asOfDate: StringT(),
        })
      )
    )
    .query(async (opts) => {
      const { underlying, asOfDate } = opts.input;
      return (
        await query<{ expirationDate: string }>(`
        SELECT
          DISTINCT(expirationDate) as expirationDate
        FROM option_contracts
        WHERE symbol = '${underlying}'
        AND asOfDate = '${asOfDate}'
        ORDER BY expirationDate
      `)
      ).map(({ expirationDate }) => expirationDate);
    }),
  getStrikesForUnderlying: publicProcedure
    .input(
      RpcType(
        ObjectT({
          underlying: StringT({ maxLength: 5 }),
          asOfDate: StringT(),
          expirationDate: StringT(),
        })
      )
    )
    .query(async (opts) => {
      const { underlying, asOfDate, expirationDate } = opts.input;
      return (
        await query<{ strike: string }>(`
        SELECT
          DISTINCT(strike) as strike
        FROM option_contracts
        WHERE symbol = '${underlying}'
        AND asOfDate = '${asOfDate}'
        AND expirationDate = '${expirationDate}'
        ORDER BY strike
      `)
      ).map(({ strike }) => strike);
    }),
  getOpensForUnderlying: publicProcedure
    .input(
      RpcType(
        ObjectT({
          underlying: StringT({ maxLength: 5 }),
        })
      )
    )
    .query(async (opts) => {
      const { underlying } = opts.input;
      return await query<{ x: number; y: number }>(
        `
        SELECT
          toUnixTimestamp(tsStart) as x,
          open as y
        FROM stock_aggregates
        WHERE symbol = '${underlying}'
        ORDER BY tsStart ASC
      `,
        "JSONEachRow"
      );
    }),
  getOpensForOptionContract: publicProcedure
    .input(
      RpcType(
        ObjectT({
          underlying: StringT({ maxLength: 5 }),
          expirationDate: StringT(),
          strike: NumberT(),
        })
      )
    )
    .query(async (opts) => {
      const { underlying, expirationDate, strike } = opts.input;
      return await query<{ x: number; y: number }>(
        `
        SELECT
          toUnixTimestamp(tsStart) as x,
          open as y
        FROM option_aggregates
        WHERE symbol = '${underlying}'
        AND expirationDate = '${expirationDate}'
        AND strike = ${strike}
        AND type = 'call'
        ORDER BY tsStart ASC
      `,
        "JSONEachRow"
      );
    }),
  getHistoricalCalendarPrices: publicProcedure
    .input(
      RpcType(
        ObjectT({
          underlying: StringT({ maxLength: 5 }),
          daysToFrontExpiration: NumberT(),
          daysBetweenFrontAndBackExpiration: NumberT(),
          strikePercentageFromUnderlyingPriceRangeMin: NumberT(),
          strikePercentageFromUnderlyingPriceRangeMax: NumberT(),
        })
      )
    )
    .query(async (opts) => {
      const {
        underlying,
        daysToFrontExpiration,
        daysBetweenFrontAndBackExpiration,
        strikePercentageFromUnderlyingPriceRangeMin,
        strikePercentageFromUnderlyingPriceRangeMax,
      } = opts.input;
      return (
        await query<[number, number]>(
          `
        SELECT
          toUnixTimestamp(tsStart) as asOfTs,
          calendarPrice
        FROM calendar_histories
        WHERE symbol = '${underlying}'
        AND daysToFrontExpiration = ${daysToFrontExpiration}
        AND strikePercentageFromUnderlyingPrice >= ${strikePercentageFromUnderlyingPriceRangeMin}
        AND strikePercentageFromUnderlyingPrice <= ${strikePercentageFromUnderlyingPriceRangeMax}
        AND daysBetweenFrontAndBackExpiration = ${daysBetweenFrontAndBackExpiration}
      `,
          "JSONCompactEachRow"
        )
      ).reduce(
        (columns, row) => {
          columns[0].push(row[0]);
          columns[1].push(row[1]);
          return columns;
        },
        [[], []]
      );
    }),
  getHistoricalStockQuoteChartData: publicProcedure
    .input(
      RpcType(
        ObjectT({
          underlying: StringT({ maxLength: 5 }),
          lookbackPeriodStart: StringT(),
          lookbackPeriodEnd: StringT(),
        })
      )
    )
    .query(async (opts) => {
      const { underlying, lookbackPeriodStart, lookbackPeriodEnd } = opts.input;
      return await query<[number, number]>(
        `
        SELECT
          toUnixTimestamp(tsStart) as x,
          open as y
        FROM stock_aggregates
        WHERE symbol = '${underlying}'
        AND tsStart >= '${lookbackPeriodStart} 00:00:00'
        AND tsStart <= '${lookbackPeriodEnd} 00:00:00'
        ORDER BY x ASC
      `,
        "JSONEachRow"
      );
    }),
  getHistoricalCalendarQuoteChartData: publicProcedure
    .input(
      RpcType(
        ObjectT({
          underlying: StringT({ maxLength: 5 }),
          daysToFrontExpiration: NumberT(),
          daysBetweenFrontAndBackExpiration: NumberT(),
          strikePercentageFromUnderlyingPriceRangeMin: NumberT(),
          strikePercentageFromUnderlyingPriceRangeMax: NumberT(),
          lookbackPeriodStart: StringT(),
          lookbackPeriodEnd: StringT(),
        })
      )
    )
    .query(async (opts) => {
      const {
        underlying,
        daysToFrontExpiration,
        daysBetweenFrontAndBackExpiration,
        strikePercentageFromUnderlyingPriceRangeMin,
        strikePercentageFromUnderlyingPriceRangeMax,
        lookbackPeriodStart,
        lookbackPeriodEnd,
      } = opts.input;
      return await query<[number, number]>(
        `
        SELECT
          toUnixTimestamp(tsStart) as x,
          calendarPrice as y
        FROM calendar_histories
        WHERE symbol = '${underlying}'
        AND daysToFrontExpiration = ${daysToFrontExpiration}
        AND strikePercentageFromUnderlyingPrice >= ${strikePercentageFromUnderlyingPriceRangeMin}
        AND strikePercentageFromUnderlyingPrice <= ${strikePercentageFromUnderlyingPriceRangeMax}
        AND daysBetweenFrontAndBackExpiration = ${daysBetweenFrontAndBackExpiration}
        AND tsStart >= '${lookbackPeriodStart} 00:00:00'
        AND tsStart <= '${lookbackPeriodEnd} 00:00:00'
      `,
        "JSONEachRow"
      );
    }),
  getHistoricalCalendarExitQuoteChartData: publicProcedure
    .input(
      RpcType(
        ObjectT({
          underlying: StringT({ maxLength: 5 }),
          daysToFrontExpiration: NumberT(),
          daysBetweenFrontAndBackExpiration: NumberT(),
          lookbackPeriodStart: StringT({
            pattern: "[0-9]{4}-[0-9]{2}-[0-9]{2}",
          }),
          lookbackPeriodEnd: StringT({ pattern: "[0-9]{4}-[0-9]{2}-[0-9]{2}" }),
        })
      )
    )
    .query(async (opts) => {
      const {
        underlying,
        daysToFrontExpiration,
        daysBetweenFrontAndBackExpiration,
        lookbackPeriodStart,
        lookbackPeriodEnd,
      } = opts.input;
      return await query<[number, number]>(
        `
        SELECT
          FLOOR(strikePercentageFromUnderlyingPrice, 1) as x,
          calendarPrice as y
        FROM calendar_histories
        WHERE symbol = '${underlying}'
        AND daysToFrontExpiration = ${daysToFrontExpiration}
        AND strikePercentageFromUnderlyingPrice >= -5.0
        AND strikePercentageFromUnderlyingPrice <= 5.0
        AND daysBetweenFrontAndBackExpiration = ${daysBetweenFrontAndBackExpiration}
        AND tsStart >= '${lookbackPeriodStart} 00:00:00'
        AND tsStart <= '${lookbackPeriodEnd} 00:00:00'
        ORDER BY x ASC
      `,
        "JSONEachRow"
      );
    }),
 });
 // Export type router type signature,
@@ -322,4 +100,4 @@ const server = createServer((req, res) => {
  }
 });
-server.listen(parseInt(process.env.LISTEN_PORT) || 3005);
+server.listen(Number.parseInt(LISTEN_PORT));
@@ -0,0 +1,38 @@
 import { createClient as createClickhouseClient } from "@clickhouse/client";
 import type { DataFormat } from "@clickhouse/client";
 import { Env } from "@humanwhocodes/env";
 import { retry } from "./utils/retry.js";
 const env = new Env();
 const { CLICKHOUSE_USER, CLICKHOUSE_PASS } = env.required;
 const CLICKHOUSE_URL = env.get("CLICKHOUSE_URL", "http://localhost:8123");
 export const clickhouse = createClickhouseClient({
  url: CLICKHOUSE_URL,
  username: CLICKHOUSE_USER,
  password: CLICKHOUSE_PASS,
  keep_alive: {
    enabled: true,
    // socket_ttl: 2500,
  },
 });
 export async function query<T>(
  queryString: string,
  format: DataFormat = "JSONEachRow"
 ): Promise<Array<T>> {
  return await retry(
    async () => {
      const result = await clickhouse.query({
        query: queryString,
        format,
        clickhouse_settings: {
          output_format_json_quote_64bit_integers: 0,
        },
      });
      return await result.json();
    },
    { maxRetries: 5 }
  );
 }
@@ -0,0 +1,237 @@
 import pThrottle from "p-throttle";
 import pRetry from "p-retry";
 import { Env } from "@humanwhocodes/env";
 const env = new Env();
 const { POLYGON_API_KEY } = env.required;
 const apiKey = POLYGON_API_KEY;
 export const getApiKey = pThrottle({ limit: 5, interval: 72000 })(() => apiKey);
 // export const getApiKey = () => apiKey;
 export const optionContractToTicker = ({
  symbol,
  expirationDate,
  strike,
  type,
 }: {
  symbol: string;
  expirationDate: string;
  strike: number;
  type: "call" | "put";
 }) =>
  `O:${symbol}${expirationDate.substring(2, 4)}${expirationDate.substring(
    5,
    7
  )}${expirationDate.substring(8, 10)}${
    type === "call" ? "C" : "P"
  }${Math.floor(strike * 1000)
    .toString()
    .padStart(8, "0")}`;
 type PolygonOptionContractsResponse = {
  next_url?: string;
  results: Array<{
    ticker: string;
    expiration_date: string;
    strike_price: number;
    contract_type: "call" | "put";
  }>;
 };
 export async function* makeGetOptionContractsIterator(
  symbol: string,
  date: string
 ) {
  let latestBatchResponse = await pRetry(
    async () =>
      (await (
        await fetch(
          `https://api.polygon.io/v3/reference/options/contracts?underlying_ticker=${symbol}&as_of=${date}&sort=ticker&limit=1000&apiKey=${await getApiKey()}`
        )
      ).json()) as PolygonOptionContractsResponse,
    { forever: true, factor: 2, maxTimeout: 120000 }
  );
  yield latestBatchResponse.results.map((result) => ({
    asOfDate: date,
    symbol,
    expirationDate: result.expiration_date,
    strike: result.strike_price,
    type: result.contract_type,
  }));
  // as long as there's a `next_url`, call that:
  while (latestBatchResponse.hasOwnProperty("next_url")) {
    latestBatchResponse = await pRetry(
      async () =>
        (await (
          await fetch(
            `${latestBatchResponse.next_url}&apiKey=${await getApiKey()}`
          )
        ).json()) as PolygonOptionContractsResponse,
      { forever: true, factor: 2, maxTimeout: 120000 }
    );
    yield latestBatchResponse.results?.map((result) => ({
      asOfDate: date,
      symbol,
      expirationDate: result.expiration_date,
      strike: result.strike_price,
      type: result.contract_type,
    })) || [];
  }
 }
 type PolygonOptionContractAggregatesResponse = {
  next_url?: string;
  status: string;
  resultsCount: number;
  results: Array<{
    c: number;
    h: number;
    n: number;
    l: number;
    o: number;
    t: number;
    v: number;
    vw: number;
  }>;
 };
 export type OptionContract = {
  symbol: string;
  expirationDate: string;
  strike: number;
  type: "call" | "put";
 };
 export async function* makeGetOptionContractAggregatesIterator({
  symbol,
  expirationDate,
  strike,
  type,
  firstDate,
 }: OptionContract & { firstDate: string }) {
  const optionContractTicker = optionContractToTicker({
    symbol: symbol,
    expirationDate,
    strike,
    type,
  });
  const expirationDateAsDateObject = new Date(expirationDate);
  const currentDateAsDateObject = new Date(firstDate);
  while (currentDateAsDateObject <= expirationDateAsDateObject) {
    const asOfDate = currentDateAsDateObject.toISOString().substring(0, 10);
    const url = `https://api.polygon.io/v2/aggs/ticker/${optionContractTicker}/range/1/minute/${asOfDate}/${asOfDate}?adjusted=false&sort=asc&limit=50000&apiKey=${await getApiKey()}`;
    let latestBatchResponse;
    latestBatchResponse = await pRetry(
      async () =>
        (await (
          await fetch(url)
        ).json()) as PolygonOptionContractAggregatesResponse,
      { forever: true, factor: 2, maxTimeout: 120000 }
    );
    if (latestBatchResponse.status.toLowerCase() === "ok") {
      yield latestBatchResponse.results?.map((result) => ({
        symbol,
        expirationDate,
        strike,
        type,
        tsStart: (result.t || 0) / 1000,
        open: result.o,
        close: result.c,
        low: result.l,
        high: result.h,
      })) || [];
    } else if (latestBatchResponse.status === "NOT_AUTHORIZED") {
      console.error("Skipping due to:", latestBatchResponse);
    } else if (latestBatchResponse.status === "DELAYED") {
      console.error("Skipping due to:", latestBatchResponse);
    } else {
      console.error(latestBatchResponse);
      throw new Error(`error fetching option contract aggregate ${url}`);
    }
    currentDateAsDateObject.setUTCDate(
      currentDateAsDateObject.getUTCDate() + 1
    );
  }
 }
 type PolygonStockAggregatesResponse = {
  next_url?: string;
  status: string;
  resultsCount: number;
  results: Array<{
    c: number;
    h: number;
    n: number;
    l: number;
    o: number;
    t: number;
    v: number;
    vw: number;
  }>;
 };
 export async function* makeGetStockAggregatesIterator({
  symbol,
  firstDate,
  lastDate,
 }: {
  symbol: string;
  firstDate: string;
  lastDate: string;
 }) {
  const url = `https://api.polygon.io/v2/aggs/ticker/${symbol}/range/1/minute/${firstDate}/${lastDate}?adjusted=false&sort=asc&limit=10000&apiKey=${await getApiKey()}`;
  let latestBatchResponse = await pRetry(
    async () =>
      (await (await fetch(url)).json()) as PolygonStockAggregatesResponse,
    { forever: true, factor: 2, maxTimeout: 120000 }
  );
  if (latestBatchResponse.status.toLowerCase() === "ok") {
    yield latestBatchResponse.results?.map((result) => ({
      symbol,
      tsStart: (result.t || 0) / 1000,
      open: result.o,
      close: result.c,
      low: result.l,
      high: result.h,
    })) || [];
  } else if (latestBatchResponse.status === "NOT_AUTHORIZED") {
    console.error("Skipping due to:", latestBatchResponse);
  } else if (latestBatchResponse.status === "DELAYED") {
    console.error("Skipping due to:", latestBatchResponse);
  } else {
    console.error(latestBatchResponse);
    throw new Error(`error fetching option contract aggregate ${url}`);
  }
  // as long as there's a `next_url`, call that:
  while (latestBatchResponse.hasOwnProperty("next_url")) {
    latestBatchResponse = await pRetry(
      async () =>
        (await (
          await fetch(
            `${latestBatchResponse.next_url}&apiKey=${await getApiKey()}`
          )
        ).json()) as PolygonStockAggregatesResponse,
      { forever: true, factor: 2, maxTimeout: 120000 }
    );
    if (latestBatchResponse.status.toLowerCase() === "ok") {
      yield latestBatchResponse.results?.map((result) => ({
        symbol,
        tsStart: (result.t || 0) / 1000,
        open: result.o,
        close: result.c,
        low: result.l,
        high: result.h,
      })) || [];
    } else if (latestBatchResponse.status === "NOT_AUTHORIZED") {
      console.error("Skipping due to:", latestBatchResponse);
    } else if (latestBatchResponse.status === "DELAYED") {
      console.error("Skipping due to:", latestBatchResponse);
    } else {
      console.error(latestBatchResponse);
      throw new Error(`error fetching option contract aggregate ${url}`);
    }
  }
 }
@@ -0,0 +1,495 @@
 import * as Polygon from "./polygon.js";
 import sqlite3 from "sqlite3";
 import { open } from "sqlite";
 import { clickhouse, query } from "./clickhouse.js";
 import { OptionContract } from "./polygon.js";
 import pRetry from "p-retry";
 import { createReadStream } from "node:fs";
 import zlib from "node:zlib";
 import { Transform, Writable } from "stream";
 import { pipeline } from "stream/promises";
 import { execa } from "execa";
 import { rm } from "node:fs/promises";
 import { Env } from "@humanwhocodes/env";
 const env = new Env();
 const { POLYGON_S3_ACCESS_KEY_ID, POLYGON_S3_SECRET_ACCESS_KEY } = env.required;
 const sqliteDb = await open({
  filename: "/tmp/sync-state.db",
  driver: sqlite3.Database,
 });
 await sqliteDb.exec(`
  CREATE TABLE IF NOT EXISTS option_contract_sync_states (
    symbol TEXT,
    date TEXT,
    status TINYINT,
    UNIQUE (symbol, date) ON CONFLICT REPLACE
  )
 `);
 await sqliteDb.exec(`
  CREATE TABLE IF NOT EXISTS option_contract_aggregates_sync_states (
    ticker TEXT,
    status TINYINT,
    UNIQUE (ticker) ON CONFLICT REPLACE
  )
 `);
 export async function getPullOptionContractsState(
  symbol: string,
  date: string
 ) {
  const state = await sqliteDb.get(
    `
    SELECT
      *
    FROM option_contract_sync_states
    WHERE symbol = :symbol
    AND date = :date
  `,
    {
      ":symbol": symbol,
      ":date": date,
    }
  );
  return state;
 }
 const enum OptionContractSyncStatus {
  STARTED = 0,
  COMPLETED = 1,
 }
 type OptionContractSyncState = {
  status: OptionContractSyncStatus;
 };
 export async function setPullOptionContractsState(
  symbol: string,
  date: string,
  state: OptionContractSyncState
 ) {
  await sqliteDb.run(
    `
    INSERT INTO option_contract_sync_states (symbol, date, status) VALUES (:symbol, :date, :status)
  `,
    {
      ":symbol": symbol,
      ":date": date,
      ":status": state.status,
    }
  );
 }
 export async function getPullAggregatesState(ticker: string) {
  const state = await sqliteDb.get(
    `
    SELECT
      *
    FROM option_contract_aggregates_sync_states
    WHERE ticker = :ticker
  `,
    {
      ":ticker": ticker,
    }
  );
  return state;
 }
 const enum OptionContractAggregatesSyncStatus {
  STARTED = 0,
  COMPLETED = 1,
 }
 type OptionContractAggregatesSyncState = {
  status: OptionContractAggregatesSyncStatus;
 };
 export async function setPullAggregatesState(
  ticker: string,
  state: OptionContractAggregatesSyncState
 ) {
  await sqliteDb.run(
    `
    INSERT INTO option_contract_aggregates_sync_states (ticker, status) VALUES (:ticker, :status)
  `,
    {
      ":ticker": ticker,
      ":status": state.status,
    }
  );
 }
 export async function pullOptionContracts(symbol: string, date: string) {
  // check if sync was completed:
  if (
    (await getPullOptionContractsState(symbol, date))?.status !==
    OptionContractSyncStatus.COMPLETED
  ) {
    await setPullOptionContractsState(symbol, date, {
      status: OptionContractSyncStatus.STARTED,
    });
    for await (const batch of Polygon.makeGetOptionContractsIterator(
      symbol,
      date
    )) {
      console.log(batch.length);
      await pRetry(
        async () => {
          await clickhouse.insert({
            table: "option_contract_existences",
            values: batch,
            format: "JSONEachRow",
          });
        },
        { forever: true, factor: 2, maxTimeout: 120000 }
      );
    }
    await setPullOptionContractsState(symbol, date, {
      status: OptionContractSyncStatus.COMPLETED,
    });
  }
 }
 export async function pullOptionContractAggregates(
  optionContract: OptionContract
 ) {
  const ticker = Polygon.optionContractToTicker(optionContract);
  // check if sync was completed:
  if (
    (await getPullAggregatesState(ticker))?.status !==
    OptionContractAggregatesSyncStatus.COMPLETED
  ) {
    await setPullAggregatesState(ticker, {
      status: OptionContractAggregatesSyncStatus.STARTED,
    });
    const { firstDate } = await getOptionContractDateRange(optionContract);
    for await (const batch of Polygon.makeGetOptionContractAggregatesIterator({
      ...optionContract,
      firstDate,
    })) {
      if (batch.length > 0) {
        console.log(
          optionContract.symbol,
          optionContract.expirationDate,
          optionContract.strike,
          optionContract.type,
          new Date(batch[0].tsStart * 1000),
          new Date(batch[batch.length - 1].tsStart * 1000)
        );
        await pRetry(
          async () => {
            await clickhouse.insert({
              table: "option_contract_aggregates",
              values: batch,
              format: "JSONEachRow",
            });
          },
          { forever: true, factor: 2, maxTimeout: 120000 }
        );
      }
    }
    await setPullAggregatesState(ticker, {
      status: OptionContractAggregatesSyncStatus.COMPLETED,
    });
  }
 }
 export async function pullStockAggregates(
  symbol: string,
  firstDate: string,
  lastDate: string
 ) {
  // check if sync was completed:
  if (
    (await getPullAggregatesState(symbol))?.status !==
    OptionContractAggregatesSyncStatus.COMPLETED
  ) {
    await setPullAggregatesState(symbol, {
      status: OptionContractAggregatesSyncStatus.STARTED,
    });
    for await (const batch of Polygon.makeGetStockAggregatesIterator({
      symbol,
      firstDate,
      lastDate,
    })) {
      if (batch.length > 0) {
        console.log(
          symbol,
          new Date(batch[0].tsStart * 1000),
          new Date(batch[batch.length - 1].tsStart * 1000)
        );
        await clickhouse.insert({
          table: "stock_aggregates",
          values: batch,
          format: "JSONEachRow",
        });
        // await pRetry(
        //   async () => {
        //     console.log(`inserting ${batch.length} rows`);
        //     await clickhouse.insert({
        //       table: "stock_aggregates",
        //       values: batch,
        //       format: "JSONEachRow",
        //     });
        //     console.log("inserted");
        //   },
        //   { forever: true, factor: 2, maxTimeout: 120000 }
        // );
      }
    }
    await setPullAggregatesState(symbol, {
      status: OptionContractAggregatesSyncStatus.COMPLETED,
    });
  }
 }
 export async function pullAllStockAggregates(firstDate, lastDate) {
  const symbols = (
    await query(
      `SELECT DISTINCT(symbol) as symbol FROM option_contract_existences WHERE asOfDate >= '${firstDate}' AND asOfDate <= '${lastDate}'`
    )
  ).map(({ symbol }) => symbol);
  for (const symbol of symbols) {
    await pullStockAggregates(symbol, firstDate, lastDate);
  }
 }
 export async function* makeGetOptionContractsIterator(
  symbol: string,
  asOfDate: string
 ) {
  const limit = 2000;
  let offset = 0;
  let batch;
  do {
    batch = await query(`
      SELECT
        *
      FROM option_contract_existences
      WHERE asOfDate = '${asOfDate}' 
      AND symbol = '${symbol}'
      ORDER BY expirationDate ASC, strike ASC, type ASC
      LIMIT ${limit}
      OFFSET ${offset}
    `);
    yield batch;
    offset = offset + limit;
  } while (batch.length === limit);
 }
 export async function pullOptionContractsSince(
  symbol: string,
  firstDate: string
 ) {
  const currentDateAsDateObject = new Date(firstDate);
  const yesterdayAsDateObject = new Date();
  yesterdayAsDateObject.setUTCDate(yesterdayAsDateObject.getUTCDate() - 1);
  while (currentDateAsDateObject <= yesterdayAsDateObject) {
    const currentDate = currentDateAsDateObject.toISOString().substring(0, 10);
    console.log(`Date: ${currentDate}:`);
    await pullOptionContracts(symbol, currentDate);
    currentDateAsDateObject.setUTCDate(
      currentDateAsDateObject.getUTCDate() + 1
    );
  }
 }
 export async function pullOptionContractAggregatesSince(
  symbol: string,
  firstDate: string
 ) {
  const currentDateAsDateObject = new Date(firstDate);
  const yesterdayAsDateObject = new Date();
  yesterdayAsDateObject.setUTCDate(yesterdayAsDateObject.getUTCDate() - 1);
  while (currentDateAsDateObject <= yesterdayAsDateObject) {
    const currentDate = currentDateAsDateObject.toISOString().substring(0, 10);
    console.log(`Date: ${currentDate}`);
    for await (const optionContracts of makeGetOptionContractsIterator(
      symbol,
      currentDate
    )) {
      for (const optionContract of optionContracts) {
        await pullOptionContractAggregates(optionContract);
      }
    }
    currentDateAsDateObject.setUTCDate(
      currentDateAsDateObject.getUTCDate() + 1
    );
  }
 }
 export async function getOptionContractDateRange({
  symbol,
  expirationDate,
  strike,
  type,
 }: OptionContract) {
  const rows = await query<{ firstDate: string; lastDate: string }>(`
    SELECT
      min(asOfDate) AS firstDate,
      max(asOfDate) AS lastDate
    FROM option_contract_existences
    WHERE symbol = '${symbol}'
    AND expirationDate = '${expirationDate}'
    AND strike = ${strike}
    AND type = '${type}'
  `);
  return rows[0] || { firstDate: null, lastDate: null };
 }
 function transformCsvLineToObject(line: string) {
  const [
    ticker,
    volume,
    open,
    close,
    high,
    low,
    window_start,
    numberOfTransactions,
  ] = line.split(",");
  const symbol = ticker.substring(2, ticker.length - 15);
  const tickerDate = ticker.substring(ticker.length - 15, ticker.length - 9);
  const expirationDate = `20${tickerDate.substring(
    0,
    2
  )}-${tickerDate.substring(2, 4)}-${tickerDate.substring(4)}`;
  const type =
    ticker.substring(ticker.length - 9, ticker.length - 8) === "C"
      ? "call"
      : "put";
  const strike = parseInt(ticker.substring(ticker.length - 8)) / 1000;
  /** UNIX time in seconds */
  const tsStart = parseInt(window_start.substring(0, window_start.length - 9));
  return {
    symbol,
    expirationDate,
    strike,
    type,
    tsStart,
    open,
    close,
    low,
    high,
    volume,
    volumeWeightedPrice: 0,
  };
 }
 export async function uploadCsvToClickhouse(filename: string) {
  let buf = "";
  for await (const chunk of createReadStream(filename, {
    start: 60 /* skip header */,
    highWaterMark: 1024 * 1024,
  })) {
    const lines = buf.concat(chunk).split(/\r?\n/);
    buf = lines.pop() ?? "";
    await clickhouse.insert({
      table: "option_contract_aggregates",
      values: lines.map(transformCsvLineToObject),
      format: "JSONEachRow",
    });
  }
  if (buf.length) {
    // last line, if file does not end with newline
    await clickhouse.insert({
      table: "option_contract_aggregates",
      values: [transformCsvLineToObject(buf)],
      format: "JSONEachRow",
    });
  }
 }
 const accessKeyId = POLYGON_S3_ACCESS_KEY_ID;
 const secretAccessKey = POLYGON_S3_SECRET_ACCESS_KEY;
 // const X_Amz_Algorithm = "AWS4-HMAC-SHA256";
 // const X_Amz_Credential =
 //   "bfe011b0-01e7-4c16-aedf-52cb83722c36/20240702/us-east-1/s3/aws4_request";
 // const X_Amz_Expires = "900";
 // const X_Amz_SignedHeaders = "host";
 // const X_Amz_Signature =
 //   "e9fd0ac569c4d5fd5757ba4104e95ce4cfd0c17b16648ec32f978265d4188f37";
 export async function ingestOptionContractAggregateFlatfile(date: string) {
  let buf = "";
  let skippedFirstLine = false;
  const [year, month, day] = date.split("-");
  const localFilename = `/tmp/${date}.csv.gz`;
  try {
    await execa({
      env: {
        AWS_ACCESS_KEY_ID: accessKeyId,
        AWS_SECRET_ACCESS_KEY: secretAccessKey,
        S3_ENDPOINT_URL: "https://files.polygon.io",
        AWS_REGION: "us-east-1",
      },
    })("s5cmd", [
      "cp",
      `s3://flatfiles/us_options_opra/minute_aggs_v1/${year}/${month}/${date}.csv.gz`,
      localFilename,
    ]);
  } catch (err) {
    return;
    // if (err.includes("status code: 404")) {
    //   return;
    // } else {
    //   console.error("error downloading flatfile from polygon s3", err);
    //   throw err;
    // }
  }
  try {
    await pipeline(
      createReadStream(localFilename, {
        highWaterMark: 1024 * 1024,
      }),
      zlib.createGunzip(),
      new Transform({
        transform(chunk, encoding, next) {
          const lines = buf.concat(chunk).split(/\r?\n/);
          if (!skippedFirstLine) {
            lines.shift();
            skippedFirstLine = true;
          }
          buf = lines.pop();
          next(null, lines);
        },
        objectMode: true,
      }),
      new Writable({
        objectMode: true,
        async write(lines, encoding, next) {
          // console.log(lines.map(transformCsvLineToObject));
          await clickhouse.insert({
            table: "option_contract_aggregates",
            values: lines.map(transformCsvLineToObject),
            format: "JSONEachRow",
          });
          next();
        },
      })
    );
  } catch (err) {
    console.error(err);
  }
  await rm(localFilename);
  console.log("done");
 }
 export async function pullOptionContractAggregatesFromFlatFileSince(
  firstDate: string,
  lastDate?: string
 ) {
  const currentDateAsDateObject = new Date(firstDate);
  const yesterdayAsDateObject = new Date();
  yesterdayAsDateObject.setUTCDate(yesterdayAsDateObject.getUTCDate() - 1);
  const lastDateAsDateObject = lastDate
    ? new Date(lastDate)
    : yesterdayAsDateObject;
  while (currentDateAsDateObject <= lastDateAsDateObject) {
    const currentDate = currentDateAsDateObject.toISOString().substring(0, 10);
    console.log(`Date: ${currentDate}`);
    await ingestOptionContractAggregateFlatfile(currentDate);
    currentDateAsDateObject.setUTCDate(
      currentDateAsDateObject.getUTCDate() + 1
    );
  }
 }
@@ -0,0 +1,6 @@
 export function nextDate(date: string) {
  const [year, month, day] = date.split('-').map(Number);
  const nextDay = new Date(Date.UTC(year, month - 1, day + 1));
  const nextDateString = nextDay.toISOString().substring(0, 10);
  return nextDateString;
 }
@@ -0,0 +1,61 @@
 type RetryDecision = {
  shouldRetry: boolean;
  maxRetries?: number;
  delay?: number;
 };
 type RetryOptions = {
  maxRetries?: number;
  delay?: number;
  shouldRetry?: (error: unknown) => RetryDecision;
 };
 export async function retry<T>(
  fn: () => Promise<T>,
  options: RetryOptions = {},
 ): Promise<T> {
  const {
    maxRetries: defaultMaxRetries = 3,
    delay: defaultDelay = 1000,
    shouldRetry = retryOnAnyError,
  } = options;
  let attempt = 1;
  while (true) {
    try {
      return await fn();
    } catch (error) {
      const decision = shouldRetry(error);
      if (!decision.shouldRetry) throw error;
      const currentMaxRetries = decision.maxRetries ?? defaultMaxRetries;
      const currentDelay = decision.delay ?? defaultDelay;
      if (attempt >= currentMaxRetries) throw error;
      console.warn(
        `Error occurred, retrying (attempt ${attempt}/${currentMaxRetries})...`,
      );
      console.error(error);
      await new Promise((resolve) => setTimeout(resolve, currentDelay));
      attempt++;
    }
  }
 }
 export const retryOnAnyError = (): RetryDecision => ({ shouldRetry: true });
 export const retryOnTimeout = (error: unknown): RetryDecision =>
  error instanceof Error && error.message.includes("timeout")
    ? { shouldRetry: true, maxRetries: 5, delay: 2000 }
    : { shouldRetry: false };
 export const retryOnErrorType =
  (errorType: new () => Error, options?: Partial<RetryDecision>) =>
  (error: unknown) =>
    error instanceof errorType
      ? { shouldRetry: true, ...options }
      : { shouldRetry: false };
 export const retryOnErrorSubstring =
  (substring: string, options?: Partial<RetryDecision>) => (error: unknown) =>
    error instanceof Error && error.message.includes(substring)
      ? { shouldRetry: true, ...options }
      : { shouldRetry: false };
@@ -0,0 +1,77 @@
 import type { AggregateDatabase } from "../AggregateDatabase/interfaces.js";
 import { database as stockDatabaseClickhouse } from "../AggregateDatabase/Stock/clickhouse.js";
 import { database as stockDatabaseLmdbx } from "../AggregateDatabase/Stock/lmdbx.js";
 // import { optionContractDatabase as optionContractDatabaseClickhouse } from "../optiondb.clickhouse.js";
 // import { optionContractDatabase as optionContractDatabaseLmdbx } from "../optiondb.lmdbx.js";
 import { nextDate } from "../lib/utils/nextDate.js";
 import { retry, retryOnTimeout } from "../lib/utils/retry.js";
 import type { OptionContractKey } from "../AggregateDatabase/OptionContract/interfaces.js";
 import type { StockKey } from "../AggregateDatabase/Stock/interfaces.js";
 async function syncAggregates<T>({
  fromDatabase,
  toDatabase,
  key,
  date,
 }: {
  fromDatabase: AggregateDatabase<T>;
  toDatabase: AggregateDatabase<T>;
  key: T;
  date: string;
 }) {
  const aggregatesFrom = (await fromDatabase.getAggregates({ key, date })).map(
    (aggregateWithoutKey) => ({ ...aggregateWithoutKey, key })
  );
  await toDatabase.insertAggregates(aggregatesFrom);
 }
 const symbols = ["AMD", "AAPL", "MSFT", "GOOGL", "NFLX", "NVDA"];
 async function run<T extends StockKey | OptionContractKey>({
  fromDatabase,
  toDatabase,
 }: {
  fromDatabase: AggregateDatabase<T>;
  toDatabase: AggregateDatabase<T>;
 }) {
  const startDate = process.argv[2];
  const endDate = process.argv[3];
  if (!startDate || !endDate) {
    console.error("Usage: node clickhouse-to-lmdbx.js <startDate> <endDate>");
    console.error("Dates should be in YYYY-MM-DD format");
    process.exit(1);
  }
  for (let date = startDate; date <= endDate; date = nextDate(date)) {
    // const symbols = await stockDatabaseClickhouse.getSymbols({ date });
    for (const symbol of symbols) {
      console.log(date, symbol);
      const keys = await retry(
        () =>
          fromDatabase.getKeys({
            key: { symbol } as T,
            date,
          }),
        { shouldRetry: retryOnTimeout }
      );
      for (const key of keys) {
        // console.log(date, symbol, key.expirationDate, key.strike, key.type);
        await retry(
          () =>
            syncAggregates({
              fromDatabase,
              toDatabase,
              key,
              date,
            }),
          { shouldRetry: retryOnTimeout }
        );
      }
    }
  }
 }
 await run({
  fromDatabase: stockDatabaseClickhouse,
  toDatabase: stockDatabaseLmdbx,
 });
@@ -1,5 +1,5 @@
-import { clickhouse, query } from "../clickhouse.js";
+import { clickhouse, query } from "../lib/clickhouse.js";
-import { getApiKey } from "./polygon.js";
+import { getApiKey } from "../lib/polygon.js";
 import pAll from "p-all";
 import pQueue from "p-queue";
 import pSeries from "p-series";
@@ -1,5 +1,5 @@
-import { clickhouse, query } from "../clickhouse.js";
+import { clickhouse, query } from "../lib/clickhouse.js";
-import { getApiKey } from "./polygon.js";
+import { getApiKey } from "../lib/polygon.js";
 import pAll from "p-all";
 import pQueue from "p-queue";
 import pSeries from "p-series";
@@ -27,6 +27,8 @@ const optionContractToTicker = ({
 type PolygonResponse = {
  next_url?: string;
  status: string;
  resultsCount: number;
  results: Array<{
    c: number;
    h: number;
@@ -80,31 +82,33 @@ async function getOptionAggregates(
      ).json()) as PolygonResponse,
    { retries: 5, factor: 2, minTimeout: 1000, maxTimeout: 60 * 1000 }
  );
-  if (!latestBatchResponse.results) {
+  if (latestBatchResponse.status.toLowerCase() !== "ok") {
    console.log(latestBatchResponse);
    return;
  }
-  let latestBatch = latestBatchResponse.results.map((result) => ({
+  if (latestBatchResponse.resultsCount > 0) {
-    symbol: underlyingSymbol,
+    let latestBatch = latestBatchResponse.results.map((result) => ({
-    expirationDate,
+      symbol: underlyingSymbol,
-    strike,
+      expirationDate,
-    type,
+      strike,
      type,
-    tsStart: (result.t || 0) / 1000,
+      tsStart: (result.t || 0) / 1000,
-    open: result.o,
+      open: result.o,
-    close: result.c,
+      close: result.c,
-    low: result.l,
+      low: result.l,
-    high: result.h,
+      high: result.h,
-  }));
+    }));
-  await pRetry(
+    await pRetry(
-    () =>
+      () =>
-      clickhouse.insert({
+        clickhouse.insert({
-        table: "option_aggregates",
+          table: "option_aggregates",
-        values: latestBatch,
+          values: latestBatch,
-        format: "JSONEachRow",
+          format: "JSONEachRow",
-      }),
+        }),
-    { retries: 5, factor: 2, minTimeout: 1000, maxTimeout: 60 * 1000 }
+      { retries: 5, factor: 2, minTimeout: 1000, maxTimeout: 60 * 1000 }
-  );
+    );
  }
  await pRetry(
    () =>
      clickhouse.insert({
@@ -137,19 +141,19 @@ async function getNextBatchOfUnstartedOptionAggregates(
  limit: number
 ): Promise<Array<OptionContractDay>> {
  if (typeof previousUnstartedOptionContract === "undefined") {
-    return;
+    return [];
  }
-  const optionContractsWithoutAggregates = await pRetry(
+  const queryContents = `
    () =>
      query<OptionContractDay>(`
    SELECT 
      asOfDate,
      symbol, 
      expirationDate, 
      strike, 
-      type
+      type,
      argMax(status, ts) as status
    FROM amg_option_aggregate_sync_statuses
-    WHERE (
+    WHERE symbol IN ['AAPL','AMD','GOOGL','MSFT','NFLX']
    AND (
      (
        asOfDate = '${previousUnstartedOptionContract.asOfDate}'
        AND symbol = '${previousUnstartedOptionContract.symbol}'
@@ -177,13 +181,18 @@ async function getNextBatchOfUnstartedOptionAggregates(
        asOfDate > '${previousUnstartedOptionContract.asOfDate}'
      )
    )
-    AND status = 'not-started'
+    GROUP BY asOfDate, symbol, expirationDate, strike, type
    HAVING status = 'not-started'
    ORDER BY asOfDate, symbol, expirationDate, strike, type
    LIMIT ${limit}
-  `),
+  `;
  //console.log(queryContents);
  const optionContractsWithoutAggregates = await pRetry(
    () => query<OptionContractDay>(queryContents),
    { retries: 5, factor: 2, minTimeout: 1000, maxTimeout: 60 * 1000 }
  );
-  return optionContractsWithoutAggregates;
+  console.log(`Got ${optionContractsWithoutAggregates.length} records`);
  return optionContractsWithoutAggregates || [];
 }
 /**
@@ -194,91 +203,97 @@ async function getNextBatchOfUnstartedOptionAggregates(
 * so as to start afresh.
 */
 async function revertPendingSyncs() {
-  const pendingOptionContracts = await query<{
+  const batchSize = 1000;
-    asOfDate: string;
+  let pendingOptionContracts;
-    symbol: string;
+  do {
-    expirationDate: string;
+    pendingOptionContracts = await query<{
-    strike: number;
+      asOfDate: string;
-    type: "call" | "put";
+      symbol: string;
-    latestStatus: "not-started" | "pending" | "done";
+      expirationDate: string;
-  }>(`
+      strike: number;
-    SELECT 
+      type: "call" | "put";
      latestStatus: "not-started" | "pending" | "done";
    }>(`
    SELECT
      asOfDate,
      symbol,
      expirationDate,
      strike,
-      type
+      type,
      argMax(status, ts) as status
    FROM amg_option_aggregate_sync_statuses
-    WHERE status = 'pending'
+    WHERE symbol IN ['AAPL','AMD','GOOGL','MSFT','NFLX']
-    ORDER BY asOfDate, symbol, expirationDate, strike, type
+    GROUP BY asOfDate, symbol, expirationDate, strike, type
-  `);
+    HAVING status = 'pending'
-  console.log(
+    LIMIT ${batchSize}
-    "Pending operations:",
+    `);
-    pendingOptionContracts.map(
+    console.log(
-      ({ asOfDate, symbol, expirationDate, strike, type }) =>
+      "Pending operations:",
-        `${symbol} ${expirationDate} ${strike} ${type}  @  ${asOfDate}`
+      pendingOptionContracts.map(
-    )
+        ({ asOfDate, symbol, expirationDate, strike, type }) =>
-  );
+          `${symbol} ${expirationDate} ${strike} ${type}  @  ${asOfDate}`
-  await pAll(
+      )
-    pendingOptionContracts.map(
+    );
-      ({ asOfDate, symbol, expirationDate, strike, type }) =>
+    await pSeries([
-        () =>
+      // Delete option_contracts first, in case this `pAll` operation fails and we need to restart; so `option_contract_sync_statuses` "pending" rows are still there for the restart
-          pSeries([
+      () =>
-            // Delete option_contracts first, in case this `pAll` operation fails and we need to restart; so `option_contract_sync_statuses` "pending" rows are still there for the restart
+        clickhouse
-            () =>
+          .command({
-              clickhouse
+            query: `
-                .command({
+                    DELETE FROM option_aggregates 
-                  query: `
+                    WHERE (symbol, expirationDate, strike, type, toDate(tsStart)) 
-                  DELETE FROM option_aggregates 
+                    IN [${pendingOptionContracts
-                  WHERE symbol = '${symbol}' 
+                      .map(
-                  AND expirationDate = '${expirationDate}'
+                        ({ asOfDate, symbol, expirationDate, strike, type }) =>
-                  AND strike = ${strike}
+                          `('${symbol}', '${expirationDate}', ${strike}, '${type}', '${asOfDate}')`
-                  AND type = '${type}'
+                      )
-                  AND toDate(tsStart) = '${asOfDate}'
+                      .join(",")}
-                `,
+                    ]
-                })
+                  `,
-                .then(() => {
+          })
-                  console.log(`Deleted aggregates for `);
+          .then(() => {
-                }),
+            console.log(`Deleted ${pendingOptionContracts.length} aggregates`);
-            () =>
+          }),
-              clickhouse
+      () =>
-                .insert({
+        clickhouse
-                  table: "amg_option_aggregate_sync_statuses",
+          .insert({
-                  values: [
+            table: "amg_option_aggregate_sync_statuses",
-                    {
+            values: pendingOptionContracts.map(
-                      asOfDate,
+              ({ asOfDate, symbol, expirationDate, strike, type }) => ({
-                      symbol,
+                asOfDate,
-                      expirationDate,
+                symbol,
-                      strike,
+                expirationDate,
-                      type,
+                strike,
-                      status: "not-started",
+                type,
-                    },
+                status: "not-started",
-                  ],
+              })
-                  format: "JSONEachRow",
+            ),
-                })
+            format: "JSONEachRow",
-                .then(() => {
+          })
-                  console.log();
+          .then(() => {}),
-                }),
+    ]);
-          ])
+  } while (pendingOptionContracts.length === batchSize);
-    ),
+  await clickhouse.command({
-    { concurrency: 1 }
+    query: `
-  );
+      OPTIMIZE TABLE amg_option_aggregate_sync_statuses FINAL
    `,
  });
 }
 // First, revert 'pending' syncs:
-//await revertPendingSyncs();
+await revertPendingSyncs();
 /** Second, for each option contract, get all of its quotes.
 *
- * This queries Polygon with a concurrency of 6.
+ * This queries Polygon with a concurrency of 16.
 */
-const q = new pQueue({ concurrency: 6 });
+const q = new pQueue({ concurrency: 16 });
 /** Initialized with the lowest possible option contract.
 *  It's passed into `getNextUnstartedSymbolAndAsOfDate()`.
 */
 let nextBatchOfUnstartedOptionContracts: Array<OptionContractDay> = [
  {
-    asOfDate: "2022-04-05",
+    asOfDate: "2022-03-27",
    symbol: "A",
    expirationDate: "2022-02-01",
    strike: 0,
@@ -289,8 +304,8 @@ while (
  (nextBatchOfUnstartedOptionContracts =
    await getNextBatchOfUnstartedOptionAggregates(
      nextBatchOfUnstartedOptionContracts.pop(),
-      200
+      100
-    )) !== null
+    )).length !== 0
 ) {
  await pAll(
    nextBatchOfUnstartedOptionContracts.map(
@@ -1,5 +0,0 @@
 //import pThrottle from 'p-throttle';
 const apiKey = "H95NTsatM1iTWLUwDLxM2J5zhUVYdCEz";
 //export const getApiKey = pThrottle({limit: 5, interval: 60000})(()=>apiKey);
 export const getApiKey = () => apiKey;
@@ -1,14 +1,35 @@
-import { initTRPC } from '@trpc/server';
+import { initTRPC } from "@trpc/server";
- 
+import { TypeCompiler } from "@sinclair/typebox/compiler";
 import { TRPCError } from "@trpc/server";
 import type { TSchema } from "@sinclair/typebox";
 /**
 * Initialization of tRPC backend
 * Should be done only once per backend!
 */
 const t = initTRPC.create();
- 
+
 /**
 * Export reusable router and procedure helpers
 * that can be used throughout the router
 */
 export const router = t.router;
-export const publicProcedure = t.procedure;
+export const publicProcedure = t.procedure;
 /**
 * Generate a TRPC-compatible validator function given a Typebox schema.
 * This was copied from [https://github.com/sinclairzx81/typebox/blob/6cfcdc02cc813af2f1be57407c771fc4fadfc34a/example/trpc/readme.md].
 * @param schema A Typebox schema
 * @returns A TRPC-compatible validator function
 */
 export function RpcType<T extends TSchema>(schema: T) {
  const check = TypeCompiler.Compile(schema);
  return (value: unknown) => {
    if (check.Check(value)) return value;
    const { path, message } = check.Errors(value).First();
    throw new TRPCError({
      message: `${message} for ${path}`,
      code: "BAD_REQUEST",
    });
  };
 }
@@ -12,16 +12,7 @@ CREATE TABLE symbols
 ENGINE MergeTree()
 ORDER BY (symbol);
-CREATE TABLE option_contract_sync_statuses
+CREATE TABLE option_contract_existences
 (
  symbol String,
  asOfDate Date,
  status ENUM('not-started','pending','done')
 )
 ENGINE MergeTree()
 ORDER BY (asOfDate, symbol);
 CREATE TABLE option_contracts
 (
  asOfDate Date,
  symbol LowCardinality(String),
@@ -29,60 +20,54 @@ CREATE TABLE option_contracts
  strike Float32,
  type ENUM('call', 'put')
 )
-ENGINE MergeTree()
+ENGINE ReplacingMergeTree()
 PRIMARY KEY (asOfDate, symbol)
 ORDER BY (asOfDate, symbol, expirationDate, strike, type);
 CREATE MATERIALIZED VIEW option_contract_existences_mv TO option_contract_existences
 AS
 SELECT 
  toDate(tsStart) as asOfDate,
  symbol,
  expirationDate,
  strike,
  type
 FROM option_contract_aggregates
 GROUP BY 
  asOfDate,
  symbol,
  expirationDate,
  strike,
  type;
-
+CREATE TABLE option_contracts
 -- BEGIN: Option Contract Quotes
 CREATE TABLE option_aggregate_sync_statuses
 (
  asOfDate Date,
  symbol LowCardinality(String),
  expirationDate Date,
  strike Float32,
-  type ENUM('call', 'put'),
+  type ENUM('call', 'put')
  status ENUM('not-started','pending','done'),
  ts DateTime64 DEFAULT now()
 )
-ENGINE MergeTree()
+ENGINE ReplacingMergeTree()
-ORDER BY (asOfDate, symbol, expirationDate, strike, type, ts);
+PRIMARY KEY (symbol, expirationDate)
-CREATE MATERIALIZED VIEW option_aggregate_sync_statuses_mv
+ORDER BY (symbol, expirationDate, strike, type);
-TO option_aggregate_sync_statuses
+
 CREATE MATERIALIZED VIEW option_contracts_mv
 TO option_contracts
 AS
 SELECT
  DISTINCT ON (
    asOfDate,
    symbol,
    expirationDate,
    strike,
    type
  )
  asOfDate,
  symbol,
  expirationDate,
  strike,
-  type,
+  type
-  'not-started' as status,
+FROM option_contract_existences;
  now() as ts
 FROM option_contracts;
 CREATE TABLE amg_option_aggregate_sync_statuses (
  asOfDate Date,
  symbol LowCardinality(String),
  expirationDate Date,
  strike Float32,
  type ENUM('call', 'put'),
  status SimpleAggregateFunction(anyLast, ENUM('not-started','pending','done')),
  ts DateTime64 DEFAULT now()
 ) 
 ENGINE=AggregatingMergeTree 
 ORDER BY (asOfDate, symbol, expirationDate, strike, type, ts);
-INSERT INTO amg_option_aggregate_sync_statuses
+-- BEGIN: Option Contract Quotes
 SELECT asOfDate, symbol, expirationDate, strike, type, status, ts 
 FROM option_aggregate_sync_statuses 
 ORDER BY asOfDate, symbol, expirationDate, strike, type, ts;
 -- END: Option Contract Quotes
@@ -97,10 +82,10 @@ CREATE TABLE stock_aggregates
  volume UInt64, 
  volume_weighted_price Float64
 )
-ENGINE MergeTree()
+ENGINE ReplacingMergeTree()
 ORDER BY (symbol, tsStart)
-CREATE TABLE option_aggregates
+CREATE TABLE option_contract_aggregates
 (
  symbol LowCardinality(String), 
  expirationDate Date,
@@ -115,12 +100,54 @@ CREATE TABLE option_aggregates
  volume UInt32 CODEC(T64),
  volumeWeightedPrice Float32 CODEC(Delta(2), ZSTD)
 )
-ENGINE MergeTree()
+ENGINE ReplacingMergeTree()
 ORDER BY (symbol, expirationDate, strike, type, tsStart)
-ALTER TABLE option_aggregates ADD INDEX idx_expirationDate expirationDate TYPE minmax GRANULARITY 2;
+-- For stats about the character of this stock's options given a certain distance-from-the-money and time-to-expiration:
-ALTER TABLE option_aggregates ADD INDEX idx_strike strike TYPE minmax GRANULARITY 2;
+CREATE TABLE calendar_stats_by_symbol
-ALTER TABLE option_aggregates ADD INDEX idx_tsStart tsStart TYPE minmax GRANULARITY 2;
+(
  symbol LowCardinality(String),
  calendarSpanInDays UInt16,
  tsStart DateTime32 CODEC(Delta, ZSTD), -- included so as to assess the character of the stock's options within a given range of time; for example, if the stock got really hot for a few months.
  minutesToExpiration UInt32,
  frontMonthOpen Float32, 
  backMonthOpen Float32, 
  strikePercentageFromUnderlyingOpen Float64,
  frontMonthClose Float32, 
  backMonthClose Float32, 
  strikePercentageFromUnderlyingClose Float64
 )
 ENGINE MergeTree()
 PRIMARY KEY (symbol, calendarSpanInDays, tsStart)
 ORDER BY (symbol, calendarSpanInDays, tsStart, minutesToExpiration);
 -- Populate `calendar_stats_by_symbol` by:
 -- INSERT INTO calendar_stats_by_symbol
 SELECT
  frontMonth.symbol,
  dateDiff('day', frontMonth.expirationDate, backMonth.expirationDate) as calendarSpanInDays,
  frontMonth.tsStart,
  dateDiff('minute', frontMonth.tsStart, addMinutes(toDateTime(expirationDate, 'America/New_York'), 60 * 16)) as minutesToExpiration,
  frontMonth.open as frontMonthOpen,
  backMonth.open as backMonthOpen,
  (frontMonth.strike-stock_aggregates.open)/stock_aggregates.open*100.0 as strikePercentageFromUnderlyingOpen,
  frontMonth.close as frontMonthClose,
  backMonth.close as backMonthClose,
  (frontMonth.strike-stock_aggregates.close)/stock_aggregates.close*100.0 as strikePercentageFromUnderlyingClose
 FROM option_contract_aggregates as frontMonth
 INNER JOIN stock_aggregates
 ON option_contract_aggregates.symbol = stock_aggregates.symbol
 AND option_contract_aggregates.tsStart = stock_aggregates.tsStart
 INNER JOIN option_contract_aggregates as backMonth
 ON frontMonth.symbol = backMonth.symbol
 AND frontMonth.strike = backMonth.strike
 AND frontMonth.type = backMonth.type
 AND frontMonth.tsStart = backMonth.tsStart
 WHERE backMonth.expirationDate > frontMonth.expirationDate
 AND frontMonth.symbol = 'AAPL'
 AND calendarSpanInDays = 14
 CREATE TABLE option_histories
 (
@@ -1,12 +1,13 @@
 {
-	"compilerOptions": {
+  "compilerOptions": {
-		"target": "ES2020",
+    "jsx": "react",
-		"module": "ESNext",
+    "target": "ES2020",
-		"moduleResolution": "bundler",
+    "module": "ESNext",
-		"noEmit": true,
+    "moduleResolution": "bundler",
-		"allowJs": true,
+    "noEmit": true,
-		"checkJs": true,
+    "allowJs": true,
-		"lib": ["es2022"]
+    "checkJs": false,
-	},
+    "lib": ["es2022"]
-	"include": ["**/*"]
+  },
  "include": ["src/**/*"]
 }
Author	SHA1	Message	Date
avraham	c83f2ce0a0	use Slider for lookback period	2025-03-03 21:35:51 -05:00
avraham	4d989f10eb	gitignore Bruno files	2025-03-02 17:54:41 -05:00
avraham	906e63eb91	use line on stock price chart; more compact data transfer format	2025-03-02 17:53:59 -05:00
avraham	3732680d52	use `CLICKHOUSE_URL` envvar instead of `CLICKHOUSE_HOST`	2025-03-02 09:35:43 -05:00
avraham	3c841f488c	attempt to remove outliers from chart data	2025-02-24 09:15:57 -05:00
avraham	b6cc25b769	begin with narrower lookback period	2025-02-24 09:15:35 -05:00
avraham	14dff05785	fix: case where calendar exit price data is empty	2025-02-24 09:15:09 -05:00
avraham	f05831b5f1	return moniness in percentage, not ratio	2025-02-23 13:20:05 -05:00
avraham	b7f9d60715	friendlier variable names; use sliders for input	2025-02-23 13:11:34 -05:00
avraham	b2169e1da7	adjust moniness step in form	2025-02-07 09:36:05 -05:00
avraham	aeac0e1042	update Dockerfile to use latest corepack to fix npm keys	2025-02-07 09:31:34 -05:00
avraham	904d2da84a	send `strikePercentageFromUnderlyingPriceRange*` as ratio not percentage	2025-02-07 09:30:50 -05:00
avraham	694bb38536	update Dockerfile to use latest corepack to fix npm keys	2025-02-07 09:25:33 -05:00
avraham	64a5172ea8	use new clickhouse tables	2025-02-07 09:23:47 -05:00
avraham	ac9554ba21	migrated to mui `Grid2`	2024-10-06 15:14:30 -04:00
avraham	2597c0f6ac	attempt fix: trpc routes not found	2024-09-29 20:19:51 -04:00
avraham	add21288be	fix: Dockerfile - no longer using postcss and tailwind	2024-09-29 20:03:58 -04:00
avraham	db809d7b57	remove unused imports	2024-09-29 20:02:55 -04:00
avraham	8986dc4ea9	fully adopt `@mui/material`; refactor	2024-09-29 20:01:53 -04:00
avraham	ea9bd307f3	refactor	2024-09-26 21:00:58 -04:00
avraham	7bca5e701d	refactor	2024-09-26 20:14:11 -04:00
avraham	82915fb0b5	aider: implement simple cli in `ink` for running backtests	2024-08-11 19:58:40 -04:00
avraham	d134385bd7	use `tsc` for type-checking	2024-08-11 19:52:29 -04:00
avraham	3bc976b63a	use standard export name for AggregateDatabase subtype instances	2024-08-11 19:51:32 -04:00
avraham	72f45b81a5	fix: `clickhouse-to-lmdbx` script imports	2024-08-11 19:16:30 -04:00
avraham	d6762fdae5	re-organized code; implemented `getAggregate()` where it was missing	2024-08-11 18:08:54 -04:00
avraham	15a5d7c67b	use lmdbx for stockdb in backtest, and limit to calendars within 5% of the money	2024-08-11 17:39:34 -04:00
avraham	fe1265810d	fix: the "optiondb-lmdbx" calendardb was returning too many possible strike-front-back permutations due to faulty programming	2024-08-11 17:38:04 -04:00
avraham	8d908521fd	fix: `getClosingPrice()` for two calendardb modules	2024-08-11 17:36:31 -04:00
avraham	5b3e9f85f6	fix: lmdbx optiondb `getAggregateSync()` returned `tsStart = undefined`	2024-08-11 17:35:00 -04:00
avraham	666ff16583	add `getAggregate` method	2024-08-11 17:33:31 -04:00
avraham	35b3278d08	add `aider`	2024-08-11 17:30:45 -04:00
avraham	704d59a363	use `tsx` to run	2024-08-11 17:29:08 -04:00
avraham	93d5ac8a30	refactor; backtest using lmdbx	2024-08-07 21:57:03 -04:00
avraham	1d83cd419a	fix: Update clickhouse-to-lmdbx script to use retry mechanism and updated utility functions.	2024-08-04 19:26:27 -04:00
avraham	cfb207aae8	Revert "feat: Implement retry mechanism for clickhouse timeout errors with exponential backoff strategy in `clickhouse-to-lmdbx.ts` script" This reverts commit `c749321fe9`.	2024-08-04 18:24:28 -04:00
avraham	c749321fe9	feat: Implement retry mechanism for clickhouse timeout errors with exponential backoff strategy in `clickhouse-to-lmdbx.ts` script	2024-08-04 18:23:19 -04:00
avraham	eba5344b15	fix biome kvetches	2024-08-02 17:00:35 -04:00
avraham	39bb6c85f8	fix biome kvetches	2024-08-02 17:00:01 -04:00
avraham	bf094de461	improve and extract `nextDate()` function; improve clickhouse-to-lmdbx sync script	2024-08-02 16:41:42 -04:00
avraham	85cafd985d	don't use `/tmp` dir for lmdbx dbs	2024-08-02 16:40:28 -04:00
avraham	70b690ab9d	converted frontend from tailwind to material-ui using AI	2024-08-02 13:37:08 -04:00
avraham	3e5e728d92	well-interfaced pluggable databases	2024-07-30 03:46:41 +00:00
avraham	9e1a5906e4	ignore asdf's pnpm's `pnpm-store`, which ends up in the repo, not in `~`	2024-07-17 01:48:38 +00:00
avraham	99a32230ce	use asdf instead of nvm	2024-07-17 01:47:57 +00:00
Avraham Sakal	742414697f	add stock aggregate fetching functions	2024-07-12 19:39:13 -04:00
Avraham Sakal	df4925e3a4	add materialized view to keep individual contract existences	2024-07-05 16:47:15 -04:00
Avraham Sakal	cd9dd9fefc	I don't have the underlying data yet	2024-07-05 16:42:04 -04:00
Avraham Sakal	37363030ec	use envs for secrets	2024-07-05 15:45:29 -04:00
Avraham Sakal	cb7dbc29e8	stream-ingest flat files from polygon; add a few tables	2024-07-05 15:33:54 -04:00
Avraham Sakal	c6e5c76952	handle newly-encountered `DELAYED` error	2024-07-01 08:31:40 -04:00
Avraham Sakal	71f72eb474	handle any socket hangups	2024-07-01 08:16:41 -04:00
Avraham Sakal	f8279d4932	retry clickhouse insert in case of socket hangup	2024-06-30 21:45:51 -04:00
Avraham Sakal	fc2526a4aa	fix: option contract aggregate sync on empty batches or unauthorized fetches	2024-06-30 21:14:37 -04:00
Avraham Sakal	e0a2bc395e	use sqlite for sync state; use sync.ts lib instead of scripts	2024-06-30 17:31:10 -04:00
Avraham Sakal	ad66397639	reduce data points for historical exit quotes by aggregating and applying weighted transparency clean frontend html structure	2024-06-23 21:07:36 -04:00
Avraham Sakal	28caba57ca	add .nvmrc to frontend and server	2024-06-23 17:51:03 -04:00
Avraham Sakal	60e09b261a	proper html title	2024-06-23 17:50:50 -04:00
Avraham Sakal	8a2aa8478e	truncate calendar prices to two decimal places	2024-06-23 17:48:40 -04:00
Avraham Sakal	7dd43b0a34	second route for historical calendar page	2024-06-23 17:48:11 -04:00
Avraham Sakal	819ca5733f	oops. now need to include postcss.config.js in Dockerfile	2024-06-17 22:01:42 -04:00
Avraham Sakal	767691902c	oops. now need to include tailwind.config.js in Dockerfile	2024-06-17 21:58:39 -04:00
Avraham Sakal	c53b8515da	oops. now need to include index.css in Dockerfile	2024-06-17 21:54:53 -04:00
Avraham Sakal	6d8b874a82	revamp frontend with tailwind	2024-06-17 21:49:32 -04:00
Avraham Sakal	5614586b66	fix ingestion script not marking `done` when no results are returned	2024-03-27 21:58:04 -04:00
		`@@ -0,0 +1,2 @@`
							`- Ingest stock/underlying aggregates from flatfiles`
							`- Create backtesting function to step through each minute of every day.`
		`@@ -0,0 +1,2 @@`
							`#!/bin/sh`
							`kubectl exec -it -n clickhouse clickhouse -- clickhouse-client -u avraham --password buginoo`