reafactored data mart and added better job monitor

backend/datamart/datamart.controller.ts

@@ -13,49 +13,72 @@
  *
  * when a criteria is password over we will handle it by counting how many were passed up to 3 then deal with each one respectively
  */
-
-import { eq } from "drizzle-orm";
-import { db } from "../db/db.controller.js";
-import { datamart } from "../db/schema/datamart.schema.js";
 import { prodQuery } from "../prodSql/prodSqlQuery.controller.js";
+import {
+	type SqlQuery,
+	sqlQuerySelector,
+} from "../prodSql/prodSqlQuerySelector.utils.js";
 import { returnFunc } from "../utils/returnHelper.utils.js";
 import { tryCatch } from "../utils/trycatch.utils.js";
+import { datamartData } from "./datamartData.utlis.js";
 
+type Options = {
+	name: string;
+	value: string;
+};
 type Data = {
 	name: string;
-	options: string;
+	options: Options;
+	optionsRequired?: boolean;
+	howManyOptionsRequired?: number;
 };
 
 export const runDatamartQuery = async (data: Data) => {
 	// search the query db for the query by name
-	const { data: queryInfo, error: qIe } = await tryCatch(
-		db.select().from(datamart).where(eq(datamart.name, data.name)),
-	);
+	const sqlQuery = sqlQuerySelector(`${data.name}`) as SqlQuery;
 
-	if (qIe) {
+	const getDataMartInfo = datamartData.filter((x) => x.endpoint === data.name);
+
+	// const optionsMissing =
+	// 	!data.options || Object.keys(data.options).length === 0;
+
+	const optionCount =
+		Object.keys(data.options).length ===
+		getDataMartInfo[0]?.howManyOptionsRequired;
+
+	if (getDataMartInfo[0]?.optionsRequired && !optionCount) {
+		return returnFunc({
+			success: false,
+			level: "error",
+			module: "datamart",
+			subModule: "query",
+			message: `This query is required to have the ${getDataMartInfo[0]?.howManyOptionsRequired} options set in order use it.`,
+			data: [getDataMartInfo[0].options],
+			notify: false,
+		});
+	}
+
+	if (!sqlQuery.success) {
 		return returnFunc({
 			success: false,
 			level: "error",
 			module: "datamart",
 			subModule: "query",
 			message: `Error getting ${data.name} info`,
-			data: [qIe],
+			data: [sqlQuery.message],
 			notify: false,
 		});
 	}
 
 	// create the query with no changed just to have it here
-	let datamartQuery = queryInfo[0]?.query || "";
+	let datamartQuery = sqlQuery?.query || "";
 
 	// split the criteria by "," then and then update the query
-	if (data.options !== "") {
-		const params = new URLSearchParams(data.options);
-
-		for (const [rawKey, rawValue] of params.entries()) {
-			const key = rawKey.trim();
-			const value = rawValue.trim();
-			datamartQuery = datamartQuery.replaceAll(`[${key}]`, value);
-		}
+	if (data.options) {
+		Object.entries(data.options ?? {}).forEach(([key, value]) => {
+			const pattern = new RegExp(`\\[${key.trim()}\\]`, "g");
+			datamartQuery = datamartQuery.replace(pattern, String(value).trim());
+		});
 	}
 
 	const { data: queryRun, error } = await tryCatch(