feat(notification): error monitoring

if there are more than 10 errors in a 15min window sends email to alert someone
2025-12-30 10:54:09 -06:00
parent 09f16f4e62
commit 6cbffa4ac5
7 changed files with 408 additions and 169 deletions
--- a/LogisticsSupportTool_API_DOCS/LstV2/Notifcations/Error
+++ b/LogisticsSupportTool_API_DOCS/LstV2/Notifcations/Error
@@ -0,0 +1,16 @@
+meta {
+  name: Error logging
+  type: http
+  seq: 4
+}
+
+get {
+  url: {{urlv2}}/api/notify/materialperday
+  body: none
+  auth: inherit
+}
+
+settings {
+  encodeUrl: true
+  timeout: 0
+}
--- a/LogisticsSupportTool_API_DOCS/environments/lst.bru
+++ b/LogisticsSupportTool_API_DOCS/environments/lst.bru
@@ -1,7 +1,7 @@
 vars {
  url: https://uslim1prod.alpla.net
  session_cookie: 
-  urlv2: http://usbow1vms006:3000
+  urlv2: http://localhost:3000
  jwtV2: 
  userID: 
 }
--- a/lstV2/server/services/notifications/controller/notifications/tooManyErrors.ts
+++ b/lstV2/server/services/notifications/controller/notifications/tooManyErrors.ts
@@ -0,0 +1,118 @@
+// SELECT count(*) FROM V_EtikettenGedruckt where AnzahlGedruckterKopien > 2 and CONVERT(varchar(5), Add_Date,108) not like CONVERT(varchar(5), Upd_Date,108) and Upd_Date > DATEADD(SECOND, -30,getdate()) and VpkVorschriftBez not like '%$%'
+
+import { errorMonitor } from "node:events";
+import { eq, sql } from "drizzle-orm";
+import { db } from "../../../../../database/dbclient.js";
+import { notifications } from "../../../../../database/schema/notifications.js";
+import { settings } from "../../../../../database/schema/settings.js";
+import { tryCatch } from "../../../../globalUtils/tryCatch.js";
+import { createLog } from "../../../logger/logger.js";
+import { sendEmail } from "../sendMail.js";
+
+export interface DownTime {
+	downTimeId?: number;
+	machineAlias?: string;
+}
+export default async function tooManyErrors(notifyData: any) {
+	// we will over ride this with users that want to sub to this
+	// a new table will be called subalerts and link to the do a kinda linkn where the user wants it then it dose subId: 1, userID: x, notificationId: y. then in here we look up the userid to get the email :D
+	// this could then leave the emails in the notificaion blank and let users sub to it.
+	//console.log(notifyData);
+	if (notifyData.emails === "") {
+		createLog(
+			"error",
+			"notify",
+			"notify",
+			`There are no emails set for ${notifyData.name}`,
+		);
+
+		return;
+	}
+
+	// console.log(data.secondarySetting[0].duration);
+
+	const plant = await db
+		.select()
+		.from(settings)
+		.where(eq(settings.name, "plantToken"));
+	console.log(plant[0].value);
+	// console.log(
+	// 	errorQuery
+	// 		.replace("[time]", notifyData.checkInterval)
+	// 		.replace("[errorCount]", notifyData.notifiySettings.errorCount),
+	// 	errorLogQuery.replace("[time]", notifyData.checkInterval),
+	// );
+
+	let errorLogData: any = [];
+	try {
+		const errorData = await db.execute(sql`
+            SELECT 'error' AS level, COUNT(*) AS error_count
+            FROM public.logs
+            WHERE level = 'error'
+                AND "add_Date" > now() - INTERVAL ${sql.raw(`'${notifyData.checkInterval} minutes'`)}
+            GROUP BY level
+            HAVING COUNT(*) >= ${notifyData.notifiySettings.errorCount}
+            `);
+		if (
+			errorData.length > 0
+			// && downTime[0]?.downTimeId > notifyData.notifiySettings.prodID
+		) {
+			const errorLogs = await db.execute(sql`
+            select* from public.logs where level = 'error' and "add_Date" > now() - INTERVAL ${sql.raw(`'${notifyData.checkInterval} minutes'`)} order by "add_Date" desc;
+            `);
+
+			errorLogData = errorLogs;
+			//send the email :D
+			const emailSetup = {
+				email: notifyData.emails,
+				subject: `Alert! ${plant[0].value} has encountered ${
+					errorData.length
+				} ${errorData.length > 1 ? "errors" : "error"} in the last ${notifyData.checkInterval} min`,
+				template: "tooManyErrors",
+				context: {
+					data: errorLogData,
+					count: notifyData.notifiySettings.errorCount,
+					time: notifyData.checkInterval,
+				},
+			};
+
+			//console.log(emailSetup);
+
+			const sentEmail = await sendEmail(emailSetup);
+
+			if (!sentEmail.success) {
+				createLog(
+					"error",
+					"notify",
+					"notify",
+					"Failed to send email, will try again on next interval",
+				);
+				return {
+					success: false,
+					message: "Failed to send email, will try again on next interval",
+					data: sentEmail,
+				};
+			}
+		}
+	} catch (err) {
+		console.log(err);
+		createLog(
+			"error",
+			"notify",
+			"notify",
+			`Error from running the downtimeCheck query: ${err}`,
+		);
+
+		return {
+			success: false,
+			message: "Error running error data",
+			data: err,
+		};
+	}
+
+	return {
+		success: true,
+		message: "Error log checking ran",
+		data: errorLogData ?? [],
+	};
+}
--- a/lstV2/server/services/notifications/notifyService.ts
+++ b/lstV2/server/services/notifications/notifyService.ts
@@ -10,6 +10,7 @@ import tiTrigger from "./routes/manualTiggerTi.js";
 import materialCheck from "./routes/materialPerDay.js";
 import blocking from "./routes/qualityBlocking.js";
 import sendemail from "./routes/sendMail.js";
+import errorHandling from "./routes/tooManyErrors.js";
 import { note, notificationCreate } from "./utils/masterNotifications.js";
 import { startNotificationMonitor } from "./utils/processNotifications.js";

@@ -23,6 +24,7 @@ const routes = [
 	notify,
 	fifoIndex,
 	materialCheck,
+	errorHandling,
 ] as const;

 const appRoutes = routes.forEach((route) => {
--- a/lstV2/server/services/notifications/routes/tooManyErrors.ts
+++ b/lstV2/server/services/notifications/routes/tooManyErrors.ts
@@ -0,0 +1,50 @@
+// an external way to creating logs
+import { createRoute, OpenAPIHono, z } from "@hono/zod-openapi";
+import { eq } from "drizzle-orm";
+import { db } from "../../../../database/dbclient.js";
+import { notifications } from "../../../../database/schema/notifications.js";
+import { apiHit } from "../../../globalUtils/apiHits.js";
+import { responses } from "../../../globalUtils/routeDefs/responses.js";
+import { tryCatch } from "../../../globalUtils/tryCatch.js";
+import { authMiddleware } from "../../auth/middleware/authMiddleware.js";
+import hasCorrectRole from "../../auth/middleware/roleCheck.js";
+import tooManyErrors from "../controller/notifications/tooManyErrors.js";
+import { getAllJobs } from "../utils/processNotifications.js";
+
+const app = new OpenAPIHono({ strict: false });
+
+app.openapi(
+	createRoute({
+		tags: ["server"],
+		summary: "Returns current active notifications.",
+		method: "get",
+		path: "/toomanyerrors",
+		middleware: [authMiddleware, hasCorrectRole(["systemAdmin"], "admin")],
+		responses: responses(),
+	}),
+	async (c) => {
+		apiHit(c, { endpoint: "/toomanyerrors" });
+
+		const { data, error } = await tryCatch(
+			db
+				.select()
+				.from(notifications)
+				.where(eq(notifications.name, "tooManyErrors")),
+		);
+
+		if (error) {
+			return c.json({
+				success: false,
+				message: "Error Getting Notification Settings.",
+				data: error,
+			});
+		}
+		const errorData = await tooManyErrors(data[0]);
+		return c.json({
+			success: true,
+			message: "Current Error log data",
+			data: errorData?.data,
+		});
+	},
+);
+export default app;
--- a/lstV2/server/services/notifications/utils/masterNotifications.ts
+++ b/lstV2/server/services/notifications/utils/masterNotifications.ts
@@ -3,175 +3,186 @@ import { notifications } from "../../../../database/schema/notifications.js";
 import { createLog } from "../../logger/logger.js";

 export const note: any = [
-    {
-        name: "reprintLabels",
-        description:
-            "Monitors the labels that are printed and returns a value if one falls withing the time frame defined below.",
-        checkInterval: 1,
-        timeType: "min",
-        emails: "",
-        active: false,
-        notifiySettings: { prodID: 1 },
-    },
-    {
-        name: "downTimeCheck",
-        description:
-            "Checks for specific downtimes that are greater than 105 min.",
-        checkInterval: 30,
-        timeType: "min",
-        emails: "",
-        active: false,
-        notifiySettings: { prodID: 1, daysInPast: 5, duration: 105 },
-    },
-    {
-        name: "qualityBlocking",
-        description:
-            "Checks for new blocking orders that have been entered, recommened to get the most recent order in here before activating.",
-        checkInterval: 30,
-        timeType: "min",
-        emails: "",
-        active: false,
-        notifiySettings: {
-            prodID: 1,
-            sentBlockingOrders: [{ timeStamp: "0", blockingOrder: 1 }],
-        },
-    },
-    {
-        name: "productionCheck",
-        description: "Checks ppoo",
-        checkInterval: 2,
-        timeType: "hour",
-        emails: "",
-        active: false,
-        notifiySettings: {
-            prodID: 1,
-            count: 0,
-            weekend: false,
-            locations: "0",
-        },
-    },
-    {
-        name: "stagingCheck",
-        description:
-            "Checks staging based on locations, locations need to be seperated by a ,",
-        checkInterval: 2,
-        timeType: "hour",
-        emails: "",
-        active: false,
-        notifiySettings: {
-            prodID: 1,
-            count: 0,
-            weekend: false,
-            locations: "0",
-        },
-    },
-    {
-        name: "tiIntergration",
-        description: "Checks for new releases to be put into ti",
-        checkInterval: 60,
-        timeType: "min",
-        emails: "",
-        active: false,
-        notifiySettings: {
-            prodID: 1,
-            start: 36,
-            end: 36,
-            releases: [{ timeStamp: "0", releaseNumber: 1 }],
-        },
-    },
-    {
-        name: "exampleNotification",
-        description: "Checks for new releases to be put into ti",
-        checkInterval: 2,
-        timeType: "min",
-        emails: "",
-        active: true,
-        notifiySettings: {
-            prodID: 1,
-            start: 36,
-            end: 36,
-            releases: [1, 2, 3],
-        },
-    },
-    {
-        name: "fifoIndex",
-        description: "Checks for pallets that were shipped out of fifo",
-        checkInterval: 1,
-        timeType: "hour",
-        emails: "blake.matthes@alpla.com",
-        active: false,
-        notifiySettings: {
-            prodID: 1,
-            start: 36,
-            end: 36,
-            releases: [1, 2, 3],
-        },
-    },
-    {
-        name: "bow2henkelincoming",
-        description:
-            "Checks for new incoming goods orders to be completed and sends an email for what truck and carrier it was",
-        checkInterval: 15,
-        timeType: "min",
-        emails: "blake.matthes@alpla.com",
-        active: false,
-        notifiySettings: { processTime: 15 },
-    },
-    {
-        name: "palletsRemovedAsWaste",
-        description:
-            "Validates stock to make sure, there are no pallets released that have been removed as waste already ",
-        checkInterval: 15,
-        timeType: "min",
-        emails: "blake.matthes@alpla.com",
-        active: false,
-        notifiySettings: { prodID: 1 },
-    },
-    {
-        name: "shortageBookings",
-        description:
-            "Checks for material shortage bookings by single av type or all types ",
-        checkInterval: 15,
-        timeType: "min",
-        emails: "blake.matthes@alpla.com",
-        active: false,
-        notifiySettings: {
-            time: 15,
-            type: "all", // change this to something else or leave blank to use the av type
-            avType: 1,
-        },
-    },
+	{
+		name: "reprintLabels",
+		description:
+			"Monitors the labels that are printed and returns a value if one falls withing the time frame defined below.",
+		checkInterval: 1,
+		timeType: "min",
+		emails: "",
+		active: false,
+		notifiySettings: { prodID: 1 },
+	},
+	{
+		name: "downTimeCheck",
+		description: "Checks for specific downtimes that are greater than 105 min.",
+		checkInterval: 30,
+		timeType: "min",
+		emails: "",
+		active: false,
+		notifiySettings: { prodID: 1, daysInPast: 5, duration: 105 },
+	},
+	{
+		name: "qualityBlocking",
+		description:
+			"Checks for new blocking orders that have been entered, recommened to get the most recent order in here before activating.",
+		checkInterval: 30,
+		timeType: "min",
+		emails: "",
+		active: false,
+		notifiySettings: {
+			prodID: 1,
+			sentBlockingOrders: [{ timeStamp: "0", blockingOrder: 1 }],
+		},
+	},
+	{
+		name: "productionCheck",
+		description: "Checks ppoo",
+		checkInterval: 2,
+		timeType: "hour",
+		emails: "",
+		active: false,
+		notifiySettings: {
+			prodID: 1,
+			count: 0,
+			weekend: false,
+			locations: "0",
+		},
+	},
+	{
+		name: "stagingCheck",
+		description:
+			"Checks staging based on locations, locations need to be seperated by a ,",
+		checkInterval: 2,
+		timeType: "hour",
+		emails: "",
+		active: false,
+		notifiySettings: {
+			prodID: 1,
+			count: 0,
+			weekend: false,
+			locations: "0",
+		},
+	},
+	{
+		name: "tiIntergration",
+		description: "Checks for new releases to be put into ti",
+		checkInterval: 60,
+		timeType: "min",
+		emails: "",
+		active: false,
+		notifiySettings: {
+			prodID: 1,
+			start: 36,
+			end: 36,
+			releases: [{ timeStamp: "0", releaseNumber: 1 }],
+		},
+	},
+	{
+		name: "exampleNotification",
+		description: "Checks for new releases to be put into ti",
+		checkInterval: 2,
+		timeType: "min",
+		emails: "",
+		active: true,
+		notifiySettings: {
+			prodID: 1,
+			start: 36,
+			end: 36,
+			releases: [1, 2, 3],
+		},
+	},
+	{
+		name: "fifoIndex",
+		description: "Checks for pallets that were shipped out of fifo",
+		checkInterval: 1,
+		timeType: "hour",
+		emails: "blake.matthes@alpla.com",
+		active: false,
+		notifiySettings: {
+			prodID: 1,
+			start: 36,
+			end: 36,
+			releases: [1, 2, 3],
+		},
+	},
+	{
+		name: "bow2henkelincoming",
+		description:
+			"Checks for new incoming goods orders to be completed and sends an email for what truck and carrier it was",
+		checkInterval: 15,
+		timeType: "min",
+		emails: "blake.matthes@alpla.com",
+		active: false,
+		notifiySettings: { processTime: 15 },
+	},
+	{
+		name: "palletsRemovedAsWaste",
+		description:
+			"Validates stock to make sure, there are no pallets released that have been removed as waste already ",
+		checkInterval: 15,
+		timeType: "min",
+		emails: "blake.matthes@alpla.com",
+		active: false,
+		notifiySettings: { prodID: 1 },
+	},
+	{
+		name: "shortageBookings",
+		description:
+			"Checks for material shortage bookings by single av type or all types ",
+		checkInterval: 15,
+		timeType: "min",
+		emails: "blake.matthes@alpla.com",
+		active: false,
+		notifiySettings: {
+			time: 15,
+			type: "all", // change this to something else or leave blank to use the av type
+			avType: 1,
+		},
+	},
+	{
+		name: "tooManyErrors",
+		description:
+			"Checks to see how many errors in the last x time and sends an email based on this.",
+		checkInterval: 15,
+		timeType: "min",
+		emails: "blake.matthes@alpla.com",
+		active: true,
+		notifiySettings: {
+			errorCount: 10, // change this to something else or leave blank to use the av type
+		},
+	},
 ];

 export const notificationCreate = async () => {
-    for (let i = 0; i < note.length; i++) {
-        try {
-            const notify = await db
-                .insert(notifications)
-                .values(note[i])
-                .onConflictDoUpdate({
-                    target: notifications.name,
-                    set: {
-                        name: note[i].name,
-                        description: note[i].description,
-                        //notifiySettings: note[i].notifiySettings,
-                    },
-                });
-        } catch (error) {
-            createLog(
-                "error",
-                "notify",
-                "notify",
-                `There was an error getting the notifications: ${JSON.stringify(
-                    error
-                )}`
-            );
-        }
-    }
-    createLog(
-        "info",
-        "lst",
-        "nofity",
-        "notifications were just added/updated due to server startup"
-    );
+	for (let i = 0; i < note.length; i++) {
+		try {
+			const notify = await db
+				.insert(notifications)
+				.values(note[i])
+				.onConflictDoUpdate({
+					target: notifications.name,
+					set: {
+						name: note[i].name,
+						description: note[i].description,
+						//notifiySettings: note[i].notifiySettings,
+					},
+				});
+		} catch (error) {
+			createLog(
+				"error",
+				"notify",
+				"notify",
+				`There was an error getting the notifications: ${JSON.stringify(
+					error,
+				)}`,
+			);
+		}
+	}
+	createLog(
+		"info",
+		"lst",
+		"nofity",
+		"notifications were just added/updated due to server startup",
+	);
 };
--- a/lstV2/server/services/notifications/utils/views/tooManyErrors.hbs
+++ b/lstV2/server/services/notifications/utils/views/tooManyErrors.hbs
@@ -0,0 +1,42 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    {{!-- <link rel="stylesheet" href="styles/styles.css" /> --}}
+    {{> styles}}
+</head>
+<body>
+        <p>All,</p>  
+        <p>The plant has encountered more than {{count}} errors in the last {{time}} mins, please see below errors and address as needed. </p>        
+        <table > 
+        <thead> 
+            <tr> 
+            <th>Username</th> 
+            <th>Service</th> 
+            <th>Message</th> 
+            <th>Checked</th> 
+            <th>LogTime</th> 
+            {{!-- <th>Downtime finish</th>  --}}
+            </tr> 
+        </thead> 
+        <tbody>
+            {{#each data}}
+                    <tr>
+                        <td>{{username}}</td>
+                        <td>{{service}}</td>
+                        <td>{{message}}</td>
+                        <td>{{checked}}</td>
+                        <td>{{add_Date}}</td>
+                        {{!-- <td>{{dtEnd}}</td> --}}
+                    </tr>
+                {{/each}}
+        </tbody>
+        </table>
+        <div>
+           <p>Thank you,</p>
+        <p>LST Team</p> 
+        </div>
+        
+</body>
+</html>