feat(datamart): get, add, update queries

backend/src/configs/scaler.config.ts

@@ -9,6 +9,7 @@ import { apiReference } from "@scalar/express-api-reference";
 // const port = 3000;
 import type { OpenAPIV3_1 } from "openapi-types";
 import { datamartAddSpec } from "../scaler/datamartAdd.spec.js";
+import { datamartUpdateSpec } from "../scaler/datamartUpdate.spec.js";
 import { getDatamartSpec } from "../scaler/getDatamart.spec.js";
 import { prodLoginSpec } from "../scaler/login.spec.js";
 import { prodRestartSpec } from "../scaler/prodSqlRestart.spec.js";
@@ -82,6 +83,15 @@ export const openApiBase: OpenAPIV3_1.Document = {
 };
 
 export const setupApiDocsRoutes = (baseUrl: string, app: Express) => {
+	const mergedDatamart = {
+		"/api/datamart": {
+			...(getDatamartSpec["/api/datamart"] ?? {}),
+			...(datamartAddSpec["/api/datamart"] ?? {}),
+			...(datamartUpdateSpec["/api/datamart"] ?? {}),
+		},
+		"/api/datamart/{name}": getDatamartSpec["/api/datamart/{name}"],
+	};
+
 	const fullSpec = {
 		...openApiBase,
 		paths: {
@@ -91,8 +101,7 @@ export const setupApiDocsRoutes = (baseUrl: string, app: Express) => {
 			...prodRestartSpec,
 			...prodLoginSpec,
 			...prodRegisterSpec,
-			...getDatamartSpec,
-			...datamartAddSpec,
+			...mergedDatamart,
 
 			// Add more specs here as you build features
 		},

backend/src/datamart/datamart.controller.ts

@@ -14,38 +14,83 @@
  * when a criteria is password over we will handle it by counting how many were passed up to 3 then deal with each one respectively
  */
 
+import { eq } from "drizzle-orm";
+import { db } from "../db/db.controller.js";
+import { datamart } from "../db/schema/datamart.schema.js";
+import { prodQuery } from "../prodSql/prodSqlQuery.controller.js";
 import { returnFunc } from "../utils/returnHelper.utils.js";
+import { tryCatch } from "../utils/trycatch.utils.js";
 
 type Data = {
 	name: string;
-	criteria: string;
+	options: string;
 };
 
 export const runDatamartQuery = async (data: Data) => {
 	// search the query db for the query by name
-	const dummyquery = {
-		name: "something",
-		query: "select * from tableA where start=[start] and end=[end]",
-	};
+	const { data: queryInfo, error: qIe } = await tryCatch(
+		db.select().from(datamart).where(eq(datamart.name, data.name)),
+	);
+
+	if (qIe) {
+		return returnFunc({
+			success: false,
+			level: "error",
+			module: "datamart",
+			subModule: "query",
+			message: `Error getting ${data.name} info`,
+			data: [qIe],
+			notify: false,
+		});
+	}
 
 	// create the query with no changed just to have it here
-	let datamartQuery = dummyquery.query;
+	let datamartQuery = queryInfo[0]?.query || "";
 	// split the criteria by "," then and then update the query
-	if (data.criteria) {
-		const params = new URLSearchParams(data.criteria);
+	if (data.options) {
+		const params = new URLSearchParams(data.options);
 
-		for (const [key, value] of params.entries()) {
+		for (const [rawKey, rawValue] of params.entries()) {
+			const key = rawKey.trim();
+			const value = rawValue.trim();
 			datamartQuery = datamartQuery.replaceAll(`[${key}]`, value);
 		}
 	}
 
+	const { data: queryRun, error } = await tryCatch(
+		prodQuery(datamartQuery, `Running datamart query: ${data.name}`),
+	);
+
+	if (error) {
+		return returnFunc({
+			success: false,
+			level: "error",
+			module: "datamart",
+			subModule: "query",
+			message: `Data for: ${data.name} encountered an error while trying to get it`,
+			data: [error],
+			notify: false,
+		});
+	}
+
+	if (!queryRun.success) {
+		return returnFunc({
+			success: false,
+			level: "error",
+			module: "datamart",
+			subModule: "query",
+			message: queryRun.message,
+			data: queryRun.data,
+			notify: false,
+		});
+	}
 	return returnFunc({
 		success: true,
 		level: "info",
 		module: "datamart",
 		subModule: "query",
 		message: `Data for: ${data.name}`,
-		data: [{ data: datamartQuery }],
+		data: queryRun.data,
 		notify: false,
 	});
 };

backend/src/datamart/datamart.routes.ts

@@ -4,6 +4,7 @@ import { db } from "../db/db.controller.js";
 import { datamart } from "../db/schema/datamart.schema.js";
 import { apiReturn } from "../utils/returnHelper.utils.js";
 import addQuery from "./datamartAdd.route.js";
+import updateQuery from "./datamartUpdate.route.js";
 import runQuery from "./getDatamart.route.js";
 
 export const setupDatamartRoutes = (baseUrl: string, app: Express) => {
@@ -11,6 +12,7 @@ export const setupDatamartRoutes = (baseUrl: string, app: Express) => {
 
 	app.use(`${baseUrl}/api/datamart`, runQuery);
 	app.use(`${baseUrl}/api/datamart`, addQuery);
+	app.use(`${baseUrl}/api/datamart`, updateQuery);
 
 	// just sending a get on datamart will return all the queries that we can call.
 	app.get(`${baseUrl}/api/datamart`, async (_, res) => {

backend/src/datamart/datamartAdd.route.ts

@@ -1,27 +1,96 @@
+import fs from "node:fs";
 import { Router } from "express";
+import multer from "multer";
 import z from "zod";
-import type { NewDatamart } from "../db/schema/datamart.schema.js";
+import { db } from "../db/db.controller.js";
+import { datamart, type NewDatamart } from "../db/schema/datamart.schema.js";
 import { apiReturn } from "../utils/returnHelper.utils.js";
+import { tryCatch } from "../utils/trycatch.utils.js";
 
 const r = Router();
+const upload = multer({ dest: "uploads/" });
 
 const newQuery = z.object({
 	name: z.string().min(5),
 	description: z.string().min(30),
-	query: z.string().min(10),
+	query: z.string().min(10).optional(),
 	options: z
 		.string()
 		.describe("This should be a set of keys separated by a comma")
 		.optional(),
 });
 
-r.post("/", async (req, res) => {
+r.post("/", upload.single("queryFile"), async (req, res) => {
 	try {
 		const v = newQuery.parse(req.body);
 
-		const query: NewDatamart = { ...v };
+		const query: NewDatamart = {
+			...v,
+			name: v.name?.trim().replaceAll(" ", "_"),
+		};
 
-		console.log(query);
+		//console.log(query);
+		if (req.file) {
+			const sqlContents = fs.readFileSync(req.file.path, "utf8");
+			query.query = sqlContents;
+
+			// optional: delete temp file afterwards
+			fs.unlink(req.file.path, () => {});
+		}
+
+		// if we forget the file crash out
+		if (!query.query) {
+			// no query text anywhere
+			return apiReturn(res, {
+				success: true,
+				level: "info", //connect.success ? "info" : "error",
+				module: "routes",
+				subModule: "datamart",
+				message: `${query.name} missing sql file to parse`,
+				data: [],
+				status: 400, //connect.success ? 200 : 400,
+			});
+		}
+
+		// // if we didn't replace the test1 stuff crash out
+		// if (!query.query.includes("test1")) {
+		// 	return apiReturn(res, {
+		// 		success: true,
+		// 		level: "info", //connect.success ? "info" : "error",
+		// 		module: "routes",
+		// 		subModule: "datamart",
+		// 		message:
+		// 			"Query must include the 'test1' or everything switched to test1",
+		// 		data: [],
+		// 		status: 400, //connect.success ? 200 : 400,
+		// 	});
+		// }
+
+		const { data, error } = await tryCatch(db.insert(datamart).values(query));
+
+		if (error) {
+			return apiReturn(res, {
+				success: true,
+				level: "error", //connect.success ? "info" : "error",
+				module: "routes",
+				subModule: "datamart",
+				message: `${query.name} encountered an error while being added`,
+				data: [error.cause],
+				status: 200, //connect.success ? 200 : 400,
+			});
+		}
+
+		if (data) {
+			return apiReturn(res, {
+				success: true,
+				level: "info", //connect.success ? "info" : "error",
+				module: "routes",
+				subModule: "datamart",
+				message: `${query.name} was just added`,
+				data: [query],
+				status: 200, //connect.success ? 200 : 400,
+			});
+		}
 	} catch (err) {
 		if (err instanceof z.ZodError) {
 			const flattened = z.flattenError(err);

backend/src/datamart/datamartUpdate.route.ts

@@ -0,0 +1,156 @@
+import fs from "node:fs";
+import { eq, sql } from "drizzle-orm";
+import { Router } from "express";
+import multer from "multer";
+import z from "zod";
+import { db } from "../db/db.controller.js";
+import { datamart } from "../db/schema/datamart.schema.js";
+import { apiReturn } from "../utils/returnHelper.utils.js";
+import { tryCatch } from "../utils/trycatch.utils.js";
+
+const r = Router();
+const upload = multer({ dest: "uploads/" });
+
+const newQuery = z.object({
+	name: z.string().min(5).optional(),
+	description: z.string().min(30).optional(),
+	query: z.string().min(10).optional(),
+	options: z
+		.string()
+		.describe("This should be a set of keys separated by a comma")
+		.optional(),
+	setActive: z.string().optional(),
+	active: z.boolean().optional(),
+});
+
+r.patch("/:id", upload.single("queryFile"), async (req, res) => {
+	const { id } = req.params;
+
+	try {
+		const v = newQuery.parse(req.body);
+
+		const query = {
+			...v,
+		};
+
+		//console.log(query);
+		if (req.file) {
+			const sqlContents = fs.readFileSync(req.file.path, "utf8");
+			query.query = sqlContents;
+
+			// optional: delete temp file afterwards
+			fs.unlink(req.file.path, () => {});
+		}
+
+		if (v.name) {
+			query.name = v.name.trim().replaceAll(" ", "_");
+		}
+
+		if (v.description) {
+			query.options = v.description;
+		}
+
+		if (v.options) {
+			query.options = v.options;
+		}
+
+		if (v.setActive) {
+			query.active = v.setActive === "true";
+		}
+
+		// if we forget the file crash out
+		// if (!query.query) {
+		// 	// no query text anywhere
+		// 	return apiReturn(res, {
+		// 		success: true,
+		// 		level: "info", //connect.success ? "info" : "error",
+		// 		module: "routes",
+		// 		subModule: "datamart",
+		// 		message: `${query.name} missing sql file to parse`,
+		// 		data: [],
+		// 		status: 400, //connect.success ? 200 : 400,
+		// 	});
+		// }
+
+		// // if we didn't replace the test1 stuff crash out
+
+		if (query.query && !query.query.includes("test1")) {
+			return apiReturn(res, {
+				success: true,
+				level: "error", //connect.success ? "info" : "error",
+				module: "routes",
+				subModule: "datamart",
+				message:
+					"All queries must point to test1 this way we can keep it dynamic.",
+				data: [],
+				status: 400, //connect.success ? 200 : 400,
+			});
+		}
+
+		const { data, error } = await tryCatch(
+			db
+				.update(datamart)
+				.set({
+					...query,
+					version: sql`${datamart.version} + 1`,
+					upd_date: sql`NOW()`,
+					upd_user: "lst_user",
+				})
+				.where(eq(datamart.id, id as string)),
+		);
+
+		if (error) {
+			return apiReturn(res, {
+				success: true,
+				level: "error", //connect.success ? "info" : "error",
+				module: "routes",
+				subModule: "datamart",
+				message: `${query.name} encountered an error while being updated`,
+				data: [error.cause],
+				status: 200, //connect.success ? 200 : 400,
+			});
+		}
+
+		if (data) {
+			return apiReturn(res, {
+				success: true,
+				level: "info", //connect.success ? "info" : "error",
+				module: "routes",
+				subModule: "datamart",
+				message: `${query.name} was just updated`,
+				data: [],
+				status: 200, //connect.success ? 200 : 400,
+			});
+		}
+	} catch (err) {
+		if (err instanceof z.ZodError) {
+			const flattened = z.flattenError(err);
+			// return res.status(400).json({
+			// 	error: "Validation failed",
+			// 	details: flattened,
+			// });
+
+			return apiReturn(res, {
+				success: false,
+				level: "error", //connect.success ? "info" : "error",
+				module: "routes",
+				subModule: "auth",
+				message: "Validation failed",
+				data: [flattened],
+				status: 400, //connect.success ? 200 : 400,
+			});
+		}
+
+		return apiReturn(res, {
+			success: false,
+			level: "error",
+			module: "routes",
+			subModule: "datamart",
+			message: "There was an error updating the query",
+			data: [err],
+			status: 200,
+		});
+	}
+});
+
+export default r;

backend/src/datamart/getDatamart.route.ts

@@ -6,11 +6,11 @@ const r = Router();
 
 r.get("/:name", async (req, res) => {
 	const { name } = req.params;
-	const criteria = new URLSearchParams(
+	const options = new URLSearchParams(
 		req.query as Record<string, string>,
 	).toString();
 
-	const dataRan = await runDatamartQuery({ name, criteria });
+	const dataRan = await runDatamartQuery({ name, options });
 	return apiReturn(res, {
 		success: dataRan.success,
 		level: "info",

backend/src/db/schema/datamart.schema.ts

@@ -11,16 +11,16 @@ import type { z } from "zod";
 
 export const datamart = pgTable("datamart", {
 	id: uuid("id").defaultRandom().primaryKey(),
-	name: text("name"),
+	name: text("name").unique(),
 	description: text("description").notNull(),
 	query: text("query"),
 	version: integer("version").default(1).notNull(),
 	active: boolean("active").default(true),
-	options: text("checked").default(""),
+	options: text("options").default(""),
 	add_date: timestamp("add_date").defaultNow(),
 	add_user: text("add_user").default("lst-system"),
 	upd_date: timestamp("upd_date").defaultNow(),
-	upd_user: text("upd_date").default("lst-system"),
+	upd_user: text("upd_user").default("lst-system"),
 });
 
 export const datamartSchema = createSelectSchema(datamart);

backend/src/scaler/datamartAdd.spec.ts

@@ -1,37 +1,40 @@
 import type { OpenAPIV3_1 } from "openapi-types";
 
 export const datamartAddSpec: OpenAPIV3_1.PathsObject = {
-	"/api/datamart/add": {
+	"/api/datamart": {
 		post: {
-			summary: "Creates the new query",
-			description: "Queries can only be created on the main server.",
+			summary: "New datamart query",
+			description:
+				"Creates a new query entry in the datamart. Must be called on the main server. The SQL can be provided as a file upload.",
 			tags: ["Datamart"],
 			requestBody: {
 				required: true,
 				content: {
-					"application/json": {
+					"multipart/form-data": {
 						schema: {
 							type: "object",
-							required: ["username", "password", "email"],
+							required: ["name", "description", "queryFile"],
 							properties: {
-								username: {
-									type: "string",
-									example: "jdoe",
-								},
 								name: {
 									type: "string",
-									format: "string",
-									example: "joe",
+									example: "active_av",
+									description: "Unique name for the query",
 								},
-								email: {
+								description: {
 									type: "string",
-									format: "email",
-									example: "joe.doe@alpla.net",
+									example: "Gets active audio/visual records",
+									description: "Short explanation of what this query does",
 								},
-								password: {
+								options: {
 									type: "string",
-									format: "password",
-									example: "superSecretPassword",
+									example: "foo,baz",
+									description:
+										"Optional comma separated options string passed to the query",
+								},
+								queryFile: {
+									type: "string",
+									format: "binary",
+									description: "SQL file containing the query text",
 								},
 							},
 						},
@@ -40,20 +43,16 @@ export const datamartAddSpec: OpenAPIV3_1.PathsObject = {
 			},
 			responses: {
 				"200": {
-					description: "User info",
+					description: "Query successfully created",
 					content: {
 						"application/json": {
 							schema: {
 								type: "object",
 								properties: {
-									success: {
-										type: "boolean",
-										format: "true",
-										example: true,
-									},
+									success: { type: "boolean", example: true },
 									message: {
 										type: "string",
-										example: "User was created",
+										example: "active_av was just added",
 									},
 								},
 							},
@@ -61,20 +60,16 @@ export const datamartAddSpec: OpenAPIV3_1.PathsObject = {
 					},
 				},
 				"400": {
-					description: "Invalid Data was sent over",
+					description: "Validation or input error",
 					content: {
 						"application/json": {
 							schema: {
 								type: "object",
 								properties: {
-									success: {
-										type: "boolean",
-										format: "false",
-										example: false,
-									},
+									success: { type: "boolean", example: false },
 									message: {
 										type: "string",
-										format: "Invalid Data was sent over.",
+										example: "Validation failed",
 									},
 								},
 							},

backend/src/scaler/datamartUpdate.spec.ts

@@ -0,0 +1,81 @@
+import type { OpenAPIV3_1 } from "openapi-types";
+
+export const datamartUpdateSpec: OpenAPIV3_1.PathsObject = {
+	"/api/datamart": {
+		patch: {
+			summary: "Update datamart query",
+			description:
+				"Update query entry in the datamart. Must be called on the main server. The SQL must be provided as a file upload.",
+			tags: ["Datamart"],
+			requestBody: {
+				required: true,
+				content: {
+					"multipart/form-data": {
+						schema: {
+							type: "object",
+							properties: {
+								name: {
+									type: "string",
+									example: "active_av",
+									description: "Unique name for the query",
+								},
+								description: {
+									type: "string",
+									example: "Gets active articles",
+									description: "Short explanation of what this query does",
+								},
+								options: {
+									type: "string",
+									example: "foo,baz",
+									description:
+										"Optional comma separated options string passed to the query",
+								},
+								queryFile: {
+									type: "string",
+									format: "binary",
+									description: "SQL file containing the query text",
+								},
+							},
+						},
+					},
+				},
+			},
+			responses: {
+				"200": {
+					description: "Query successfully created",
+					content: {
+						"application/json": {
+							schema: {
+								type: "object",
+								properties: {
+									success: { type: "boolean", example: true },
+									message: {
+										type: "string",
+										example: "active_av was just added",
+									},
+								},
+							},
+						},
+					},
+				},
+				"400": {
+					description: "Validation or input error",
+					content: {
+						"application/json": {
+							schema: {
+								type: "object",
+								properties: {
+									success: { type: "boolean", example: false },
+									message: {
+										type: "string",
+										example: "Validation failed",
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	},
+};

backend/src/scaler/getDatamart.spec.ts

@@ -12,7 +12,7 @@ export const getDatamartSpec: OpenAPIV3_1.PathsObject = {
 				{
 					name: "name",
 					in: "path",
-					required: true,
+					required: false,
 					description: "Name to look up",
 					schema: {
 						type: "string",