feat(notification): error monitoring

if there are more than 10 errors in a 15min window sends email to alert someone

LogisticsSupportTool_API_DOCS/LstV2/Notifcations/Error logging.bru

@@ -0,0 +1,16 @@
+meta {
+  name: Error logging
+  type: http
+  seq: 4
+}
+
+get {
+  url: {{urlv2}}/api/notify/materialperday
+  body: none
+  auth: inherit
+}
+
+settings {
+  encodeUrl: true
+  timeout: 0
+}

LogisticsSupportTool_API_DOCS/environments/lst.bru

@@ -1,7 +1,7 @@
 vars {
   url: https://uslim1prod.alpla.net
   session_cookie: 
-  urlv2: http://usbow1vms006:3000
+  urlv2: http://localhost:3000
   jwtV2: 
   userID: 
 }

lstV2/server/services/notifications/controller/notifications/tooManyErrors.ts

@@ -0,0 +1,118 @@
+// SELECT count(*) FROM V_EtikettenGedruckt where AnzahlGedruckterKopien > 2 and CONVERT(varchar(5), Add_Date,108) not like CONVERT(varchar(5), Upd_Date,108) and Upd_Date > DATEADD(SECOND, -30,getdate()) and VpkVorschriftBez not like '%$%'
+
+import { errorMonitor } from "node:events";
+import { eq, sql } from "drizzle-orm";
+import { db } from "../../../../../database/dbclient.js";
+import { notifications } from "../../../../../database/schema/notifications.js";
+import { settings } from "../../../../../database/schema/settings.js";
+import { tryCatch } from "../../../../globalUtils/tryCatch.js";
+import { createLog } from "../../../logger/logger.js";
+import { sendEmail } from "../sendMail.js";
+
+export interface DownTime {
+	downTimeId?: number;
+	machineAlias?: string;
+}
+export default async function tooManyErrors(notifyData: any) {
+	// we will over ride this with users that want to sub to this
+	// a new table will be called subalerts and link to the do a kinda linkn where the user wants it then it dose subId: 1, userID: x, notificationId: y. then in here we look up the userid to get the email :D
+	// this could then leave the emails in the notificaion blank and let users sub to it.
+	//console.log(notifyData);
+	if (notifyData.emails === "") {
+		createLog(
+			"error",
+			"notify",
+			"notify",
+			`There are no emails set for ${notifyData.name}`,
+		);
+
+		return;
+	}
+
+	// console.log(data.secondarySetting[0].duration);
+
+	const plant = await db
+		.select()
+		.from(settings)
+		.where(eq(settings.name, "plantToken"));
+	console.log(plant[0].value);
+	// console.log(
+	// 	errorQuery
+	// 		.replace("[time]", notifyData.checkInterval)
+	// 		.replace("[errorCount]", notifyData.notifiySettings.errorCount),
+	// 	errorLogQuery.replace("[time]", notifyData.checkInterval),
+	// );
+
+	let errorLogData: any = [];
+	try {
+		const errorData = await db.execute(sql`
+            SELECT 'error' AS level, COUNT(*) AS error_count
+            FROM public.logs
+            WHERE level = 'error'
+                AND "add_Date" > now() - INTERVAL ${sql.raw(`'${notifyData.checkInterval} minutes'`)}
+            GROUP BY level
+            HAVING COUNT(*) >= ${notifyData.notifiySettings.errorCount}
+            `);
+		if (
+			errorData.length > 0
+			// && downTime[0]?.downTimeId > notifyData.notifiySettings.prodID
+		) {
+			const errorLogs = await db.execute(sql`
+            select* from public.logs where level = 'error' and "add_Date" > now() - INTERVAL ${sql.raw(`'${notifyData.checkInterval} minutes'`)} order by "add_Date" desc;
+            `);
+
+			errorLogData = errorLogs;
+			//send the email :D
+			const emailSetup = {
+				email: notifyData.emails,
+				subject: `Alert! ${plant[0].value} has encountered ${
+					errorData.length
+				} ${errorData.length > 1 ? "errors" : "error"} in the last ${notifyData.checkInterval} min`,
+				template: "tooManyErrors",
+				context: {
+					data: errorLogData,
+					count: notifyData.notifiySettings.errorCount,
+					time: notifyData.checkInterval,
+				},
+			};
+
+			//console.log(emailSetup);
+
+			const sentEmail = await sendEmail(emailSetup);
+
+			if (!sentEmail.success) {
+				createLog(
+					"error",
+					"notify",
+					"notify",
+					"Failed to send email, will try again on next interval",
+				);
+				return {
+					success: false,
+					message: "Failed to send email, will try again on next interval",
+					data: sentEmail,
+				};
+			}
+		}
+	} catch (err) {
+		console.log(err);
+		createLog(
+			"error",
+			"notify",
+			"notify",
+			`Error from running the downtimeCheck query: ${err}`,
+		);
+
+		return {
+			success: false,
+			message: "Error running error data",
+			data: err,
+		};
+	}
+
+	return {
+		success: true,
+		message: "Error log checking ran",
+		data: errorLogData ?? [],
+	};
+}

lstV2/server/services/notifications/notifyService.ts

@@ -10,6 +10,7 @@ import tiTrigger from "./routes/manualTiggerTi.js";
 import materialCheck from "./routes/materialPerDay.js";
 import blocking from "./routes/qualityBlocking.js";
 import sendemail from "./routes/sendMail.js";
+import errorHandling from "./routes/tooManyErrors.js";
 import { note, notificationCreate } from "./utils/masterNotifications.js";
 import { startNotificationMonitor } from "./utils/processNotifications.js";
 
@@ -23,6 +24,7 @@ const routes = [
 	notify,
 	fifoIndex,
 	materialCheck,
+	errorHandling,
 ] as const;
 
 const appRoutes = routes.forEach((route) => {

lstV2/server/services/notifications/routes/tooManyErrors.ts

@@ -0,0 +1,50 @@
+// an external way to creating logs
+import { createRoute, OpenAPIHono, z } from "@hono/zod-openapi";
+import { eq } from "drizzle-orm";
+import { db } from "../../../../database/dbclient.js";
+import { notifications } from "../../../../database/schema/notifications.js";
+import { apiHit } from "../../../globalUtils/apiHits.js";
+import { responses } from "../../../globalUtils/routeDefs/responses.js";
+import { tryCatch } from "../../../globalUtils/tryCatch.js";
+import { authMiddleware } from "../../auth/middleware/authMiddleware.js";
+import hasCorrectRole from "../../auth/middleware/roleCheck.js";
+import tooManyErrors from "../controller/notifications/tooManyErrors.js";
+import { getAllJobs } from "../utils/processNotifications.js";
+
+const app = new OpenAPIHono({ strict: false });
+
+app.openapi(
+	createRoute({
+		tags: ["server"],
+		summary: "Returns current active notifications.",
+		method: "get",
+		path: "/toomanyerrors",
+		middleware: [authMiddleware, hasCorrectRole(["systemAdmin"], "admin")],
+		responses: responses(),
+	}),
+	async (c) => {
+		apiHit(c, { endpoint: "/toomanyerrors" });
+
+		const { data, error } = await tryCatch(
+			db
+				.select()
+				.from(notifications)
+				.where(eq(notifications.name, "tooManyErrors")),
+		);
+
+		if (error) {
+			return c.json({
+				success: false,
+				message: "Error Getting Notification Settings.",
+				data: error,
+			});
+		}
+		const errorData = await tooManyErrors(data[0]);
+		return c.json({
+			success: true,
+			message: "Current Error log data",
+			data: errorData?.data,
+		});
+	},
+);
+export default app;

lstV2/server/services/notifications/utils/masterNotifications.ts

@@ -15,8 +15,7 @@ export const note: any = [
 	},
 	{
 		name: "downTimeCheck",
-        description:
-            "Checks for specific downtimes that are greater than 105 min.",
+		description: "Checks for specific downtimes that are greater than 105 min.",
 		checkInterval: 30,
 		timeType: "min",
 		emails: "",
@@ -141,6 +140,18 @@ export const note: any = [
 			avType: 1,
 		},
 	},
+	{
+		name: "tooManyErrors",
+		description:
+			"Checks to see how many errors in the last x time and sends an email based on this.",
+		checkInterval: 15,
+		timeType: "min",
+		emails: "blake.matthes@alpla.com",
+		active: true,
+		notifiySettings: {
+			errorCount: 10, // change this to something else or leave blank to use the av type
+		},
+	},
 ];
 
 export const notificationCreate = async () => {
@@ -163,8 +174,8 @@ export const notificationCreate = async () => {
 				"notify",
 				"notify",
 				`There was an error getting the notifications: ${JSON.stringify(
-                    error
-                )}`
+					error,
+				)}`,
 			);
 		}
 	}
@@ -172,6 +183,6 @@ export const notificationCreate = async () => {
 		"info",
 		"lst",
 		"nofity",
-        "notifications were just added/updated due to server startup"
+		"notifications were just added/updated due to server startup",
 	);
 };

lstV2/server/services/notifications/utils/views/tooManyErrors.hbs

@@ -0,0 +1,42 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    {{!-- <link rel="stylesheet" href="styles/styles.css" /> --}}
+    {{> styles}}
+</head>
+<body>
+        <p>All,</p>  
+        <p>The plant has encountered more than {{count}} errors in the last {{time}} mins, please see below errors and address as needed. </p>        
+        <table > 
+        <thead> 
+            <tr> 
+            <th>Username</th> 
+            <th>Service</th> 
+            <th>Message</th> 
+            <th>Checked</th> 
+            <th>LogTime</th> 
+            {{!-- <th>Downtime finish</th>  --}}
+            </tr> 
+        </thead> 
+        <tbody>
+            {{#each data}}
+                    <tr>
+                        <td>{{username}}</td>
+                        <td>{{service}}</td>
+                        <td>{{message}}</td>
+                        <td>{{checked}}</td>
+                        <td>{{add_Date}}</td>
+                        {{!-- <td>{{dtEnd}}</td> --}}
+                    </tr>
+                {{/each}}
+        </tbody>
+        </table>
+        <div>
+           <p>Thank you,</p>
+        <p>LST Team</p> 
+        </div>
+        
+</body>
+</html>