refactor(sql): corrections to the way we reconnect so the app can error out and be reactivated later

backend/prodSql/prodSqlConnection.controller.ts

@@ -7,12 +7,17 @@ import { returnFunc } from "../utils/returnHelper.utils.js";
 export let pool: sql.ConnectionPool;
 export let connected: boolean = false;
 export let reconnecting = false;
+// start the delay out as 2 seconds
+let delayStart = 2000;
+let attempt = 0;
+const maxAttempts = 10;
 
 export const connectProdSql = async () => {
 	const serverUp = await checkHostnamePort(`${process.env.PROD_SERVER}:1433`);
 	if (!serverUp) {
 		// we will try to reconnect
 		connected = false;
+		reconnectToSql();
 		return returnFunc({
 			success: false,
 			level: "error",
@@ -48,6 +53,7 @@ export const connectProdSql = async () => {
 			notify: false,
 		});
 	} catch (error) {
+		reconnectToSql();
 		return returnFunc({
 			success: false,
 			level: "error",
@@ -104,11 +110,6 @@ export const reconnectToSql = async () => {
 	//set reconnecting to true while we try to reconnect
 	reconnecting = true;
 
-	// start the delay out as 2 seconds
-	let delayStart = 2000;
-	let attempt = 0;
-	const maxAttempts = 10;
-
 	while (!connected && attempt < maxAttempts) {
 		attempt++;
 		log.info(
@@ -121,7 +122,7 @@ export const reconnectToSql = async () => {
 
 		if (!serverUp) {
 			delayStart = Math.min(delayStart * 2, 30000); // exponential backoff until up to 30000
-			return;
+			continue;
 		}
 
 		try {
@@ -133,19 +134,12 @@ export const reconnectToSql = async () => {
 			);
 		} catch (error) {
 			delayStart = Math.min(delayStart * 2, 30000);
-			return returnFunc({
-				success: false,
-				level: "error",
-				module: "system",
-				subModule: "db",
-				message: "Failed to reconnect to the prod sql server.",
-				data: [error],
-				notify: false,
-			});
+			delayStart = Math.min(delayStart * 2, 30000);
+			log.error({ error }, "Failed to reconnect to the prod sql server.");
 		}
 	}
 
-	if (!connected) {
+	if (!connected && attempt >= maxAttempts) {
 		log.error(
 			{ notify: true },
 			"Max reconnect attempts reached on the prodSql server. Stopping retries.",

backend/prodSql/prodSqlQuery.controller.ts

@@ -1,10 +1,5 @@
 import { returnFunc } from "../utils/returnHelper.utils.js";
-import {
-	connected,
-	pool,
-	reconnecting,
-	reconnectToSql,
-} from "./prodSqlConnection.controller.js";
+import { connected, pool } from "./prodSqlConnection.controller.js";
 
 interface SqlError extends Error {
 	code?: string;
@@ -22,29 +17,15 @@ interface SqlError extends Error {
  */
 export const prodQuery = async (queryToRun: string, name: string) => {
 	if (!connected) {
-		reconnectToSql();
-
-		if (reconnecting) {
-			return returnFunc({
-				success: false,
-				level: "error",
-				module: "system",
-				subModule: "prodSql",
-				message: `The sql ${process.env.PROD_PLANT_TOKEN} is trying to reconnect already`,
-				data: [],
-				notify: false,
-			});
-		} else {
-			return returnFunc({
-				success: false,
-				level: "error",
-				module: "system",
-				subModule: "prodSql",
-				message: `${process.env.PROD_PLANT_TOKEN} is not connected, and failed to connect.`,
-				data: [],
-				notify: true,
-			});
-		}
+		return returnFunc({
+			success: false,
+			level: "error",
+			module: "system",
+			subModule: "prodSql",
+			message: `${process.env.PROD_PLANT_TOKEN} is offline or attempting to reconnect`,
+			data: [],
+			notify: false,
+		});
 	}
 
 	//change to the correct server