From ad66397639266f7c414e1260a5c2c55168a05517 Mon Sep 17 00:00:00 2001
From: Avraham Sakal <avraham@torahanytime.com>
Date: Sun, 23 Jun 2024 21:07:36 -0400
Subject: [PATCH] reduce data points for historical exit quotes by aggregating
 and applying weighted transparency clean frontend html structure

---
 .../src/pages/HistoricalCalendarPrices.tsx    | 41 ++++++++++++++-----
 server/src/clickhouse.ts                      | 32 ++++++++++-----
 server/src/index.ts                           |  8 ++--
 3 files changed, 57 insertions(+), 24 deletions(-)

diff --git a/frontend/src/pages/HistoricalCalendarPrices.tsx b/frontend/src/pages/HistoricalCalendarPrices.tsx
index 3122fff..27b1301 100644
--- a/frontend/src/pages/HistoricalCalendarPrices.tsx
+++ b/frontend/src/pages/HistoricalCalendarPrices.tsx
@@ -48,7 +48,15 @@ const maxChartPrice = computed(() =>
   )
 );
 
+const maxN = computed(() =>
+  Math.max.apply(
+    null,
+    historicalCalendarExitQuoteChartData.value.map((d) => d.n)
+  )
+);
+
 const refreshHistoricalStockQuoteChartData = () => {
+  historicalStockQuoteChartData.value = [];
   trpc.getHistoricalStockQuoteChartData
     .query({
       underlying: chosenUnderlying.value,
@@ -61,6 +69,7 @@ const refreshHistoricalStockQuoteChartData = () => {
     });
 };
 const refreshHistoricalCalendarQuoteChartData = () => {
+  historicalCalendarQuoteChartData.value = [];
   trpc.getHistoricalCalendarQuoteChartData
     .query({
       underlying: chosenUnderlying.value,
@@ -82,6 +91,7 @@ const refreshHistoricalCalendarQuoteChartData = () => {
     });
 };
 const refreshHistoricalCalendarExitQuoteChartData = () => {
+  historicalCalendarExitQuoteChartData.value = [];
   trpc.getHistoricalCalendarExitQuoteChartData
     .query({
       underlying: chosenUnderlying.value,
@@ -289,11 +299,11 @@ export function HistoricalCalendarPrices() {
           </div>
         </div>
         {/* charts container: */}
-        <div>
-          <div className="chart-container">
+        <div className="flex flex-col justify-start gap-3">
+          <div className="min-h-96">
             {chosenUnderlying.value !== null &&
             historicalStockQuoteChartData.value.length > 0 ? (
-              <div className="chart">
+              <div className="h-full">
                 <Scatter
                   data={{
                     datasets: [
@@ -353,17 +363,18 @@ export function HistoricalCalendarPrices() {
                     },
                     animation: false,
                     maintainAspectRatio: false,
+                    events: [],
                   }}
                 />
               </div>
             ) : (
-              <div>Loading Chart...</div>
+              <div className="h-full">Loading Chart...</div>
             )}
           </div>
-          <div className="chart-container">
+          <div className="min-h-96">
             {chosenUnderlying.value !== null &&
             historicalCalendarQuoteChartData.value.length > 0 ? (
-              <div className="chart">
+              <div className="h-full">
                 <Scatter
                   data={{
                     datasets: [
@@ -419,16 +430,18 @@ export function HistoricalCalendarPrices() {
                     },
                     animation: false,
                     maintainAspectRatio: false,
+                    events: [],
                   }}
                 />
               </div>
             ) : (
-              <div>Loading Chart...</div>
+              <div className="h-full">Loading Chart...</div>
             )}
-
+          </div>
+          <div className="min-h-96">
             {chosenUnderlying.value !== null &&
             historicalCalendarQuoteChartData.value.length > 0 ? (
-              <div className="chart">
+              <div className="h-full">
                 <Scatter
                   data={{
                     datasets: [
@@ -467,6 +480,13 @@ export function HistoricalCalendarPrices() {
                     elements: {
                       point: {
                         borderWidth: 0,
+                        backgroundColor: function (context) {
+                          const n = (
+                            context.raw as { x: number; y: number; n: number }
+                          ).n;
+                          const alpha = n / maxN.value;
+                          return `rgba(0, 0, 0, ${alpha})`;
+                        },
                       },
                     },
                     plugins: {
@@ -486,11 +506,12 @@ export function HistoricalCalendarPrices() {
                     },
                     animation: false,
                     maintainAspectRatio: false,
+                    events: [],
                   }}
                 />
               </div>
             ) : (
-              <div>Loading Chart...</div>
+              <div className="h-full">Loading Chart...</div>
             )}
           </div>
         </div>
diff --git a/server/src/clickhouse.ts b/server/src/clickhouse.ts
index c3fed9e..ab59772 100644
--- a/server/src/clickhouse.ts
+++ b/server/src/clickhouse.ts
@@ -1,18 +1,28 @@
-import _ from './env.js';
-import { createClient as createClickhouseClient } from '@clickhouse/client';
-import type { DataFormat } from '@clickhouse/client';
+import _ from "./env.js";
+import { createClient as createClickhouseClient } from "@clickhouse/client";
+import type { DataFormat } from "@clickhouse/client";
 
 // prevent from tree-shaking:
 console.log(_);
 export const clickhouse = createClickhouseClient({
   host: process.env.CLICKHOUSE_HOST || "http://localhost:8123",
-  username:'avraham', 
-  password:'buginoo'
+  username: "avraham",
+  password: "buginoo",
 });
 
-export async function query<T>(queryString:string, format:DataFormat='JSONEachRow') : Promise<Array<T>>{
-  return await (await clickhouse.query({
-    query: queryString,
-    format
-  })).json()
-}
\ No newline at end of file
+export async function query<T>(
+  queryString: string,
+  format: DataFormat = "JSONEachRow"
+): Promise<Array<T>> {
+  return await (
+    await clickhouse.query({
+      query: queryString,
+      format,
+      clickhouse_settings: {
+        output_format_json_quote_64bit_integers: 0,
+        //output_format_json_quote_64bit_floats: false,
+        //output_format_json_quote_64bit_decimals: false,
+      },
+    })
+  ).json();
+}
diff --git a/server/src/index.ts b/server/src/index.ts
index 7cb733a..b9d8009 100644
--- a/server/src/index.ts
+++ b/server/src/index.ts
@@ -281,11 +281,12 @@ const appRouter = router({
         lookbackPeriodStart,
         lookbackPeriodEnd,
       } = opts.input;
-      return await query<[number, number]>(
+      return await query<[number, number, number]>(
         `
         SELECT
           FLOOR(strikePercentageFromUnderlyingPrice, 1) as x,
-          truncate(calendarPrice, 2) as y
+          FLOOR(calendarPrice, 1) as y,
+          count(*) as n
         FROM calendar_histories
         WHERE symbol = '${underlying}'
         AND daysToFrontExpiration = ${daysToFrontExpiration}
@@ -294,7 +295,8 @@ const appRouter = router({
         AND daysBetweenFrontAndBackExpiration = ${daysBetweenFrontAndBackExpiration}
         AND tsStart >= '${lookbackPeriodStart} 00:00:00'
         AND tsStart <= '${lookbackPeriodEnd} 00:00:00'
-        ORDER BY x ASC
+        GROUP BY x, y
+        ORDER BY x ASC, y ASC
       `,
         "JSONEachRow"
       );