refactor; backtest using lmdbx

server/src/lib/clickhouse.ts

@@ -1,6 +1,7 @@
 import { createClient as createClickhouseClient } from "@clickhouse/client";
 import type { DataFormat } from "@clickhouse/client";
 import { Env } from "@humanwhocodes/env";
+import { retry } from "./utils/retry.js";
 
 const env = new Env();
 
@@ -11,21 +12,27 @@ export const clickhouse = createClickhouseClient({
   host: CLICKHOUSE_HOST,
   username: CLICKHOUSE_USER,
   password: CLICKHOUSE_PASS,
+  keep_alive: {
+    enabled: true,
+    socket_ttl: 2500,
+  },
 });
 
 export async function query<T>(
   queryString: string,
-  format: DataFormat = "JSONEachRow"
+  format: DataFormat = "JSONEachRow",
 ): Promise<Array<T>> {
-  return await (
-    await clickhouse.query({
-      query: queryString,
-      format,
-      clickhouse_settings: {
-        output_format_json_quote_64bit_integers: 0,
-        //output_format_json_quote_64bit_floats: false,
-        //output_format_json_quote_64bit_decimals: false,
-      },
-    })
-  ).json();
+  return await retry(
+    async () => {
+      const result = await clickhouse.query({
+        query: queryString,
+        format,
+        clickhouse_settings: {
+          output_format_json_quote_64bit_integers: 0,
+        },
+      });
+      return await result.json();
+    },
+    { maxRetries: 5 },
+  );
 }

server/src/lib/utils/retry.ts

@@ -0,0 +1,61 @@
+type RetryDecision = {
+  shouldRetry: boolean;
+  maxRetries?: number;
+  delay?: number;
+};
+
+type RetryOptions = {
+  maxRetries?: number;
+  delay?: number;
+  shouldRetry?: (error: unknown) => RetryDecision;
+};
+
+export async function retry<T>(
+  fn: () => Promise<T>,
+  options: RetryOptions = {},
+): Promise<T> {
+  const {
+    maxRetries: defaultMaxRetries = 3,
+    delay: defaultDelay = 1000,
+    shouldRetry = retryOnAnyError,
+  } = options;
+
+  let attempt = 1;
+  while (true) {
+    try {
+      return await fn();
+    } catch (error) {
+      const decision = shouldRetry(error);
+      if (!decision.shouldRetry) throw error;
+
+      const currentMaxRetries = decision.maxRetries ?? defaultMaxRetries;
+      const currentDelay = decision.delay ?? defaultDelay;
+
+      if (attempt >= currentMaxRetries) throw error;
+
+      console.warn(
+        `Error occurred, retrying (attempt ${attempt}/${currentMaxRetries})...`,
+      );
+      console.error(error);
+      await new Promise((resolve) => setTimeout(resolve, currentDelay));
+      attempt++;
+    }
+  }
+}
+
+export const retryOnAnyError = (): RetryDecision => ({ shouldRetry: true });
+export const retryOnTimeout = (error: unknown): RetryDecision =>
+  error instanceof Error && error.message.includes("timeout")
+    ? { shouldRetry: true, maxRetries: 5, delay: 2000 }
+    : { shouldRetry: false };
+export const retryOnErrorType =
+  (errorType: new () => Error, options?: Partial<RetryDecision>) =>
+  (error: unknown) =>
+    error instanceof errorType
+      ? { shouldRetry: true, ...options }
+      : { shouldRetry: false };
+export const retryOnErrorSubstring =
+  (substring: string, options?: Partial<RetryDecision>) => (error: unknown) =>
+    error instanceof Error && error.message.includes(substring)
+      ? { shouldRetry: true, ...options }
+      : { shouldRetry: false };