From 6cbffa4ac5b0b86314f68f555e8489b2ebbd8f05 Mon Sep 17 00:00:00 2001 From: Blake Matthes Date: Tue, 30 Dec 2025 10:54:09 -0600 Subject: [PATCH] feat(notification): error monitoring if there are more than 10 errors in a 15min window sends email to alert someone --- .../LstV2/Notifcations/Error logging.bru | 16 + .../environments/lst.bru | 2 +- .../controller/notifications/tooManyErrors.ts | 118 ++++++ .../services/notifications/notifyService.ts | 2 + .../notifications/routes/tooManyErrors.ts | 50 +++ .../utils/masterNotifications.ts | 347 +++++++++--------- .../utils/views/tooManyErrors.hbs | 42 +++ 7 files changed, 408 insertions(+), 169 deletions(-) create mode 100644 LogisticsSupportTool_API_DOCS/LstV2/Notifcations/Error logging.bru create mode 100644 lstV2/server/services/notifications/controller/notifications/tooManyErrors.ts create mode 100644 lstV2/server/services/notifications/routes/tooManyErrors.ts create mode 100644 lstV2/server/services/notifications/utils/views/tooManyErrors.hbs diff --git a/LogisticsSupportTool_API_DOCS/LstV2/Notifcations/Error logging.bru b/LogisticsSupportTool_API_DOCS/LstV2/Notifcations/Error logging.bru new file mode 100644 index 0000000..a23b3b7 --- /dev/null +++ b/LogisticsSupportTool_API_DOCS/LstV2/Notifcations/Error logging.bru @@ -0,0 +1,16 @@ +meta { + name: Error logging + type: http + seq: 4 +} + +get { + url: {{urlv2}}/api/notify/materialperday + body: none + auth: inherit +} + +settings { + encodeUrl: true + timeout: 0 +} diff --git a/LogisticsSupportTool_API_DOCS/environments/lst.bru b/LogisticsSupportTool_API_DOCS/environments/lst.bru index f605e85..efd761c 100644 --- a/LogisticsSupportTool_API_DOCS/environments/lst.bru +++ b/LogisticsSupportTool_API_DOCS/environments/lst.bru @@ -1,7 +1,7 @@ vars { url: https://uslim1prod.alpla.net session_cookie: - urlv2: http://usbow1vms006:3000 + urlv2: http://localhost:3000 jwtV2: userID: } diff --git a/lstV2/server/services/notifications/controller/notifications/tooManyErrors.ts b/lstV2/server/services/notifications/controller/notifications/tooManyErrors.ts new file mode 100644 index 0000000..2834375 --- /dev/null +++ b/lstV2/server/services/notifications/controller/notifications/tooManyErrors.ts @@ -0,0 +1,118 @@ +// SELECT count(*) FROM V_EtikettenGedruckt where AnzahlGedruckterKopien > 2 and CONVERT(varchar(5), Add_Date,108) not like CONVERT(varchar(5), Upd_Date,108) and Upd_Date > DATEADD(SECOND, -30,getdate()) and VpkVorschriftBez not like '%$%' + +import { errorMonitor } from "node:events"; +import { eq, sql } from "drizzle-orm"; +import { db } from "../../../../../database/dbclient.js"; +import { notifications } from "../../../../../database/schema/notifications.js"; +import { settings } from "../../../../../database/schema/settings.js"; +import { tryCatch } from "../../../../globalUtils/tryCatch.js"; +import { createLog } from "../../../logger/logger.js"; +import { sendEmail } from "../sendMail.js"; + +export interface DownTime { + downTimeId?: number; + machineAlias?: string; +} +export default async function tooManyErrors(notifyData: any) { + // we will over ride this with users that want to sub to this + // a new table will be called subalerts and link to the do a kinda linkn where the user wants it then it dose subId: 1, userID: x, notificationId: y. then in here we look up the userid to get the email :D + // this could then leave the emails in the notificaion blank and let users sub to it. + //console.log(notifyData); + if (notifyData.emails === "") { + createLog( + "error", + "notify", + "notify", + `There are no emails set for ${notifyData.name}`, + ); + + return; + } + + // console.log(data.secondarySetting[0].duration); + + const plant = await db + .select() + .from(settings) + .where(eq(settings.name, "plantToken")); + console.log(plant[0].value); + // console.log( + // errorQuery + // .replace("[time]", notifyData.checkInterval) + // .replace("[errorCount]", notifyData.notifiySettings.errorCount), + // errorLogQuery.replace("[time]", notifyData.checkInterval), + // ); + + let errorLogData: any = []; + try { + const errorData = await db.execute(sql` + SELECT 'error' AS level, COUNT(*) AS error_count + FROM public.logs + WHERE level = 'error' + AND "add_Date" > now() - INTERVAL ${sql.raw(`'${notifyData.checkInterval} minutes'`)} + GROUP BY level + HAVING COUNT(*) >= ${notifyData.notifiySettings.errorCount} + `); + if ( + errorData.length > 0 + // && downTime[0]?.downTimeId > notifyData.notifiySettings.prodID + ) { + const errorLogs = await db.execute(sql` + select* from public.logs where level = 'error' and "add_Date" > now() - INTERVAL ${sql.raw(`'${notifyData.checkInterval} minutes'`)} order by "add_Date" desc; + `); + + errorLogData = errorLogs; + //send the email :D + const emailSetup = { + email: notifyData.emails, + subject: `Alert! ${plant[0].value} has encountered ${ + errorData.length + } ${errorData.length > 1 ? "errors" : "error"} in the last ${notifyData.checkInterval} min`, + template: "tooManyErrors", + context: { + data: errorLogData, + count: notifyData.notifiySettings.errorCount, + time: notifyData.checkInterval, + }, + }; + + //console.log(emailSetup); + + const sentEmail = await sendEmail(emailSetup); + + if (!sentEmail.success) { + createLog( + "error", + "notify", + "notify", + "Failed to send email, will try again on next interval", + ); + return { + success: false, + message: "Failed to send email, will try again on next interval", + data: sentEmail, + }; + } + } + } catch (err) { + console.log(err); + createLog( + "error", + "notify", + "notify", + `Error from running the downtimeCheck query: ${err}`, + ); + + return { + success: false, + message: "Error running error data", + data: err, + }; + } + + return { + success: true, + message: "Error log checking ran", + data: errorLogData ?? [], + }; +} diff --git a/lstV2/server/services/notifications/notifyService.ts b/lstV2/server/services/notifications/notifyService.ts index e990010..9f9c788 100644 --- a/lstV2/server/services/notifications/notifyService.ts +++ b/lstV2/server/services/notifications/notifyService.ts @@ -10,6 +10,7 @@ import tiTrigger from "./routes/manualTiggerTi.js"; import materialCheck from "./routes/materialPerDay.js"; import blocking from "./routes/qualityBlocking.js"; import sendemail from "./routes/sendMail.js"; +import errorHandling from "./routes/tooManyErrors.js"; import { note, notificationCreate } from "./utils/masterNotifications.js"; import { startNotificationMonitor } from "./utils/processNotifications.js"; @@ -23,6 +24,7 @@ const routes = [ notify, fifoIndex, materialCheck, + errorHandling, ] as const; const appRoutes = routes.forEach((route) => { diff --git a/lstV2/server/services/notifications/routes/tooManyErrors.ts b/lstV2/server/services/notifications/routes/tooManyErrors.ts new file mode 100644 index 0000000..9334bb6 --- /dev/null +++ b/lstV2/server/services/notifications/routes/tooManyErrors.ts @@ -0,0 +1,50 @@ +// an external way to creating logs +import { createRoute, OpenAPIHono, z } from "@hono/zod-openapi"; +import { eq } from "drizzle-orm"; +import { db } from "../../../../database/dbclient.js"; +import { notifications } from "../../../../database/schema/notifications.js"; +import { apiHit } from "../../../globalUtils/apiHits.js"; +import { responses } from "../../../globalUtils/routeDefs/responses.js"; +import { tryCatch } from "../../../globalUtils/tryCatch.js"; +import { authMiddleware } from "../../auth/middleware/authMiddleware.js"; +import hasCorrectRole from "../../auth/middleware/roleCheck.js"; +import tooManyErrors from "../controller/notifications/tooManyErrors.js"; +import { getAllJobs } from "../utils/processNotifications.js"; + +const app = new OpenAPIHono({ strict: false }); + +app.openapi( + createRoute({ + tags: ["server"], + summary: "Returns current active notifications.", + method: "get", + path: "/toomanyerrors", + middleware: [authMiddleware, hasCorrectRole(["systemAdmin"], "admin")], + responses: responses(), + }), + async (c) => { + apiHit(c, { endpoint: "/toomanyerrors" }); + + const { data, error } = await tryCatch( + db + .select() + .from(notifications) + .where(eq(notifications.name, "tooManyErrors")), + ); + + if (error) { + return c.json({ + success: false, + message: "Error Getting Notification Settings.", + data: error, + }); + } + const errorData = await tooManyErrors(data[0]); + return c.json({ + success: true, + message: "Current Error log data", + data: errorData?.data, + }); + }, +); +export default app; diff --git a/lstV2/server/services/notifications/utils/masterNotifications.ts b/lstV2/server/services/notifications/utils/masterNotifications.ts index 677d87f..6dcf28c 100644 --- a/lstV2/server/services/notifications/utils/masterNotifications.ts +++ b/lstV2/server/services/notifications/utils/masterNotifications.ts @@ -3,175 +3,186 @@ import { notifications } from "../../../../database/schema/notifications.js"; import { createLog } from "../../logger/logger.js"; export const note: any = [ - { - name: "reprintLabels", - description: - "Monitors the labels that are printed and returns a value if one falls withing the time frame defined below.", - checkInterval: 1, - timeType: "min", - emails: "", - active: false, - notifiySettings: { prodID: 1 }, - }, - { - name: "downTimeCheck", - description: - "Checks for specific downtimes that are greater than 105 min.", - checkInterval: 30, - timeType: "min", - emails: "", - active: false, - notifiySettings: { prodID: 1, daysInPast: 5, duration: 105 }, - }, - { - name: "qualityBlocking", - description: - "Checks for new blocking orders that have been entered, recommened to get the most recent order in here before activating.", - checkInterval: 30, - timeType: "min", - emails: "", - active: false, - notifiySettings: { - prodID: 1, - sentBlockingOrders: [{ timeStamp: "0", blockingOrder: 1 }], - }, - }, - { - name: "productionCheck", - description: "Checks ppoo", - checkInterval: 2, - timeType: "hour", - emails: "", - active: false, - notifiySettings: { - prodID: 1, - count: 0, - weekend: false, - locations: "0", - }, - }, - { - name: "stagingCheck", - description: - "Checks staging based on locations, locations need to be seperated by a ,", - checkInterval: 2, - timeType: "hour", - emails: "", - active: false, - notifiySettings: { - prodID: 1, - count: 0, - weekend: false, - locations: "0", - }, - }, - { - name: "tiIntergration", - description: "Checks for new releases to be put into ti", - checkInterval: 60, - timeType: "min", - emails: "", - active: false, - notifiySettings: { - prodID: 1, - start: 36, - end: 36, - releases: [{ timeStamp: "0", releaseNumber: 1 }], - }, - }, - { - name: "exampleNotification", - description: "Checks for new releases to be put into ti", - checkInterval: 2, - timeType: "min", - emails: "", - active: true, - notifiySettings: { - prodID: 1, - start: 36, - end: 36, - releases: [1, 2, 3], - }, - }, - { - name: "fifoIndex", - description: "Checks for pallets that were shipped out of fifo", - checkInterval: 1, - timeType: "hour", - emails: "blake.matthes@alpla.com", - active: false, - notifiySettings: { - prodID: 1, - start: 36, - end: 36, - releases: [1, 2, 3], - }, - }, - { - name: "bow2henkelincoming", - description: - "Checks for new incoming goods orders to be completed and sends an email for what truck and carrier it was", - checkInterval: 15, - timeType: "min", - emails: "blake.matthes@alpla.com", - active: false, - notifiySettings: { processTime: 15 }, - }, - { - name: "palletsRemovedAsWaste", - description: - "Validates stock to make sure, there are no pallets released that have been removed as waste already ", - checkInterval: 15, - timeType: "min", - emails: "blake.matthes@alpla.com", - active: false, - notifiySettings: { prodID: 1 }, - }, - { - name: "shortageBookings", - description: - "Checks for material shortage bookings by single av type or all types ", - checkInterval: 15, - timeType: "min", - emails: "blake.matthes@alpla.com", - active: false, - notifiySettings: { - time: 15, - type: "all", // change this to something else or leave blank to use the av type - avType: 1, - }, - }, + { + name: "reprintLabels", + description: + "Monitors the labels that are printed and returns a value if one falls withing the time frame defined below.", + checkInterval: 1, + timeType: "min", + emails: "", + active: false, + notifiySettings: { prodID: 1 }, + }, + { + name: "downTimeCheck", + description: "Checks for specific downtimes that are greater than 105 min.", + checkInterval: 30, + timeType: "min", + emails: "", + active: false, + notifiySettings: { prodID: 1, daysInPast: 5, duration: 105 }, + }, + { + name: "qualityBlocking", + description: + "Checks for new blocking orders that have been entered, recommened to get the most recent order in here before activating.", + checkInterval: 30, + timeType: "min", + emails: "", + active: false, + notifiySettings: { + prodID: 1, + sentBlockingOrders: [{ timeStamp: "0", blockingOrder: 1 }], + }, + }, + { + name: "productionCheck", + description: "Checks ppoo", + checkInterval: 2, + timeType: "hour", + emails: "", + active: false, + notifiySettings: { + prodID: 1, + count: 0, + weekend: false, + locations: "0", + }, + }, + { + name: "stagingCheck", + description: + "Checks staging based on locations, locations need to be seperated by a ,", + checkInterval: 2, + timeType: "hour", + emails: "", + active: false, + notifiySettings: { + prodID: 1, + count: 0, + weekend: false, + locations: "0", + }, + }, + { + name: "tiIntergration", + description: "Checks for new releases to be put into ti", + checkInterval: 60, + timeType: "min", + emails: "", + active: false, + notifiySettings: { + prodID: 1, + start: 36, + end: 36, + releases: [{ timeStamp: "0", releaseNumber: 1 }], + }, + }, + { + name: "exampleNotification", + description: "Checks for new releases to be put into ti", + checkInterval: 2, + timeType: "min", + emails: "", + active: true, + notifiySettings: { + prodID: 1, + start: 36, + end: 36, + releases: [1, 2, 3], + }, + }, + { + name: "fifoIndex", + description: "Checks for pallets that were shipped out of fifo", + checkInterval: 1, + timeType: "hour", + emails: "blake.matthes@alpla.com", + active: false, + notifiySettings: { + prodID: 1, + start: 36, + end: 36, + releases: [1, 2, 3], + }, + }, + { + name: "bow2henkelincoming", + description: + "Checks for new incoming goods orders to be completed and sends an email for what truck and carrier it was", + checkInterval: 15, + timeType: "min", + emails: "blake.matthes@alpla.com", + active: false, + notifiySettings: { processTime: 15 }, + }, + { + name: "palletsRemovedAsWaste", + description: + "Validates stock to make sure, there are no pallets released that have been removed as waste already ", + checkInterval: 15, + timeType: "min", + emails: "blake.matthes@alpla.com", + active: false, + notifiySettings: { prodID: 1 }, + }, + { + name: "shortageBookings", + description: + "Checks for material shortage bookings by single av type or all types ", + checkInterval: 15, + timeType: "min", + emails: "blake.matthes@alpla.com", + active: false, + notifiySettings: { + time: 15, + type: "all", // change this to something else or leave blank to use the av type + avType: 1, + }, + }, + { + name: "tooManyErrors", + description: + "Checks to see how many errors in the last x time and sends an email based on this.", + checkInterval: 15, + timeType: "min", + emails: "blake.matthes@alpla.com", + active: true, + notifiySettings: { + errorCount: 10, // change this to something else or leave blank to use the av type + }, + }, ]; export const notificationCreate = async () => { - for (let i = 0; i < note.length; i++) { - try { - const notify = await db - .insert(notifications) - .values(note[i]) - .onConflictDoUpdate({ - target: notifications.name, - set: { - name: note[i].name, - description: note[i].description, - //notifiySettings: note[i].notifiySettings, - }, - }); - } catch (error) { - createLog( - "error", - "notify", - "notify", - `There was an error getting the notifications: ${JSON.stringify( - error - )}` - ); - } - } - createLog( - "info", - "lst", - "nofity", - "notifications were just added/updated due to server startup" - ); + for (let i = 0; i < note.length; i++) { + try { + const notify = await db + .insert(notifications) + .values(note[i]) + .onConflictDoUpdate({ + target: notifications.name, + set: { + name: note[i].name, + description: note[i].description, + //notifiySettings: note[i].notifiySettings, + }, + }); + } catch (error) { + createLog( + "error", + "notify", + "notify", + `There was an error getting the notifications: ${JSON.stringify( + error, + )}`, + ); + } + } + createLog( + "info", + "lst", + "nofity", + "notifications were just added/updated due to server startup", + ); }; diff --git a/lstV2/server/services/notifications/utils/views/tooManyErrors.hbs b/lstV2/server/services/notifications/utils/views/tooManyErrors.hbs new file mode 100644 index 0000000..48b0ca9 --- /dev/null +++ b/lstV2/server/services/notifications/utils/views/tooManyErrors.hbs @@ -0,0 +1,42 @@ + + + + + + {{!-- --}} + {{> styles}} + + +

All,

+

The plant has encountered more than {{count}} errors in the last {{time}} mins, please see below errors and address as needed.

+ + + + + + + + + {{!-- --}} + + + + {{#each data}} + + + + + + + {{!-- --}} + + {{/each}} + +
UsernameServiceMessageCheckedLogTimeDowntime finish
{{username}}{{service}}{{message}}{{checked}}{{add_Date}}{{dtEnd}}
+
+

Thank you,

+

LST Team

+
+ + + \ No newline at end of file