refactor(ocp): plc reading changes to disconnect and reconnect

it was found that there were some errors that spammed the log and caused the server to actually stop
responding and crash weirdly so added a disconnect and reconnect back. so we can figure out whats
going on.

LogisticsSupportTool_API_DOCS/LstV2/Notifcations/Error logging.bru

@@ -0,0 +1,16 @@
+meta {
+  name: Error logging
+  type: http
+  seq: 4
+}
+
+get {
+  url: {{urlv2}}/api/notify/materialperday
+  body: none
+  auth: inherit
+}
+
+settings {
+  encodeUrl: true
+  timeout: 0
+}

LogisticsSupportTool_API_DOCS/environments/lst.bru

@@ -1,7 +1,7 @@
 vars {
   url: https://uslim1prod.alpla.net
   session_cookie: 
-  urlv2: http://usbow1vms006:3000
+  urlv2: http://localhost:3000
   jwtV2: 
   userID: 
 }

lstV2/server/services/notifications/controller/notifications/tooManyErrors.ts

@@ -0,0 +1,118 @@
+// SELECT count(*) FROM V_EtikettenGedruckt where AnzahlGedruckterKopien > 2 and CONVERT(varchar(5), Add_Date,108) not like CONVERT(varchar(5), Upd_Date,108) and Upd_Date > DATEADD(SECOND, -30,getdate()) and VpkVorschriftBez not like '%$%'
+
+import { errorMonitor } from "node:events";
+import { eq, sql } from "drizzle-orm";
+import { db } from "../../../../../database/dbclient.js";
+import { notifications } from "../../../../../database/schema/notifications.js";
+import { settings } from "../../../../../database/schema/settings.js";
+import { tryCatch } from "../../../../globalUtils/tryCatch.js";
+import { createLog } from "../../../logger/logger.js";
+import { sendEmail } from "../sendMail.js";
+
+export interface DownTime {
+	downTimeId?: number;
+	machineAlias?: string;
+}
+export default async function tooManyErrors(notifyData: any) {
+	// we will over ride this with users that want to sub to this
+	// a new table will be called subalerts and link to the do a kinda linkn where the user wants it then it dose subId: 1, userID: x, notificationId: y. then in here we look up the userid to get the email :D
+	// this could then leave the emails in the notificaion blank and let users sub to it.
+	//console.log(notifyData);
+	if (notifyData.emails === "") {
+		createLog(
+			"error",
+			"notify",
+			"notify",
+			`There are no emails set for ${notifyData.name}`,
+		);
+
+		return;
+	}
+
+	// console.log(data.secondarySetting[0].duration);
+
+	const plant = await db
+		.select()
+		.from(settings)
+		.where(eq(settings.name, "plantToken"));
+	console.log(plant[0].value);
+	// console.log(
+	// 	errorQuery
+	// 		.replace("[time]", notifyData.checkInterval)
+	// 		.replace("[errorCount]", notifyData.notifiySettings.errorCount),
+	// 	errorLogQuery.replace("[time]", notifyData.checkInterval),
+	// );
+
+	let errorLogData: any = [];
+	try {
+		const errorData = await db.execute(sql`
+            SELECT 'error' AS level, COUNT(*) AS error_count
+            FROM public.logs
+            WHERE level = 'error'
+                AND "add_Date" > now() - INTERVAL ${sql.raw(`'${notifyData.checkInterval} minutes'`)}
+            GROUP BY level
+            HAVING COUNT(*) >= ${notifyData.notifiySettings.errorCount}
+            `);
+		if (
+			errorData.length > 0
+			// && downTime[0]?.downTimeId > notifyData.notifiySettings.prodID
+		) {
+			const errorLogs = await db.execute(sql`
+            select* from public.logs where level = 'error' and "add_Date" > now() - INTERVAL ${sql.raw(`'${notifyData.checkInterval} minutes'`)} order by "add_Date" desc;
+            `);
+
+			errorLogData = errorLogs;
+			//send the email :D
+			const emailSetup = {
+				email: notifyData.emails,
+				subject: `Alert! ${plant[0].value} has encountered ${
+					errorData.length
+				} ${errorData.length > 1 ? "errors" : "error"} in the last ${notifyData.checkInterval} min`,
+				template: "tooManyErrors",
+				context: {
+					data: errorLogData,
+					count: notifyData.notifiySettings.errorCount,
+					time: notifyData.checkInterval,
+				},
+			};
+
+			//console.log(emailSetup);
+
+			const sentEmail = await sendEmail(emailSetup);
+
+			if (!sentEmail.success) {
+				createLog(
+					"error",
+					"notify",
+					"notify",
+					"Failed to send email, will try again on next interval",
+				);
+				return {
+					success: false,
+					message: "Failed to send email, will try again on next interval",
+					data: sentEmail,
+				};
+			}
+		}
+	} catch (err) {
+		console.log(err);
+		createLog(
+			"error",
+			"notify",
+			"notify",
+			`Error from running the downtimeCheck query: ${err}`,
+		);
+
+		return {
+			success: false,
+			message: "Error running error data",
+			data: err,
+		};
+	}
+
+	return {
+		success: true,
+		message: "Error log checking ran",
+		data: errorLogData ?? [],
+	};
+}

lstV2/server/services/notifications/notifyService.ts

@@ -10,6 +10,7 @@ import tiTrigger from "./routes/manualTiggerTi.js";
 import materialCheck from "./routes/materialPerDay.js";
 import blocking from "./routes/qualityBlocking.js";
 import sendemail from "./routes/sendMail.js";
+import errorHandling from "./routes/tooManyErrors.js";
 import { note, notificationCreate } from "./utils/masterNotifications.js";
 import { startNotificationMonitor } from "./utils/processNotifications.js";
 
@@ -23,6 +24,7 @@ const routes = [
 	notify,
 	fifoIndex,
 	materialCheck,
+	errorHandling,
 ] as const;
 
 const appRoutes = routes.forEach((route) => {

lstV2/server/services/notifications/routes/tooManyErrors.ts

@@ -0,0 +1,50 @@
+// an external way to creating logs
+import { createRoute, OpenAPIHono, z } from "@hono/zod-openapi";
+import { eq } from "drizzle-orm";
+import { db } from "../../../../database/dbclient.js";
+import { notifications } from "../../../../database/schema/notifications.js";
+import { apiHit } from "../../../globalUtils/apiHits.js";
+import { responses } from "../../../globalUtils/routeDefs/responses.js";
+import { tryCatch } from "../../../globalUtils/tryCatch.js";
+import { authMiddleware } from "../../auth/middleware/authMiddleware.js";
+import hasCorrectRole from "../../auth/middleware/roleCheck.js";
+import tooManyErrors from "../controller/notifications/tooManyErrors.js";
+import { getAllJobs } from "../utils/processNotifications.js";
+
+const app = new OpenAPIHono({ strict: false });
+
+app.openapi(
+	createRoute({
+		tags: ["server"],
+		summary: "Returns current active notifications.",
+		method: "get",
+		path: "/toomanyerrors",
+		middleware: [authMiddleware, hasCorrectRole(["systemAdmin"], "admin")],
+		responses: responses(),
+	}),
+	async (c) => {
+		apiHit(c, { endpoint: "/toomanyerrors" });
+
+		const { data, error } = await tryCatch(
+			db
+				.select()
+				.from(notifications)
+				.where(eq(notifications.name, "tooManyErrors")),
+		);
+
+		if (error) {
+			return c.json({
+				success: false,
+				message: "Error Getting Notification Settings.",
+				data: error,
+			});
+		}
+		const errorData = await tooManyErrors(data[0]);
+		return c.json({
+			success: true,
+			message: "Current Error log data",
+			data: errorData?.data,
+		});
+	},
+);
+export default app;

lstV2/server/services/notifications/utils/masterNotifications.ts

@@ -3,175 +3,186 @@ import { notifications } from "../../../../database/schema/notifications.js";
 import { createLog } from "../../logger/logger.js";
 
 export const note: any = [
-    {
-        name: "reprintLabels",
-        description:
-            "Monitors the labels that are printed and returns a value if one falls withing the time frame defined below.",
-        checkInterval: 1,
-        timeType: "min",
-        emails: "",
-        active: false,
-        notifiySettings: { prodID: 1 },
-    },
-    {
-        name: "downTimeCheck",
-        description:
-            "Checks for specific downtimes that are greater than 105 min.",
-        checkInterval: 30,
-        timeType: "min",
-        emails: "",
-        active: false,
-        notifiySettings: { prodID: 1, daysInPast: 5, duration: 105 },
-    },
-    {
-        name: "qualityBlocking",
-        description:
-            "Checks for new blocking orders that have been entered, recommened to get the most recent order in here before activating.",
-        checkInterval: 30,
-        timeType: "min",
-        emails: "",
-        active: false,
-        notifiySettings: {
-            prodID: 1,
-            sentBlockingOrders: [{ timeStamp: "0", blockingOrder: 1 }],
-        },
-    },
-    {
-        name: "productionCheck",
-        description: "Checks ppoo",
-        checkInterval: 2,
-        timeType: "hour",
-        emails: "",
-        active: false,
-        notifiySettings: {
-            prodID: 1,
-            count: 0,
-            weekend: false,
-            locations: "0",
-        },
-    },
-    {
-        name: "stagingCheck",
-        description:
-            "Checks staging based on locations, locations need to be seperated by a ,",
-        checkInterval: 2,
-        timeType: "hour",
-        emails: "",
-        active: false,
-        notifiySettings: {
-            prodID: 1,
-            count: 0,
-            weekend: false,
-            locations: "0",
-        },
-    },
-    {
-        name: "tiIntergration",
-        description: "Checks for new releases to be put into ti",
-        checkInterval: 60,
-        timeType: "min",
-        emails: "",
-        active: false,
-        notifiySettings: {
-            prodID: 1,
-            start: 36,
-            end: 36,
-            releases: [{ timeStamp: "0", releaseNumber: 1 }],
-        },
-    },
-    {
-        name: "exampleNotification",
-        description: "Checks for new releases to be put into ti",
-        checkInterval: 2,
-        timeType: "min",
-        emails: "",
-        active: true,
-        notifiySettings: {
-            prodID: 1,
-            start: 36,
-            end: 36,
-            releases: [1, 2, 3],
-        },
-    },
-    {
-        name: "fifoIndex",
-        description: "Checks for pallets that were shipped out of fifo",
-        checkInterval: 1,
-        timeType: "hour",
-        emails: "blake.matthes@alpla.com",
-        active: false,
-        notifiySettings: {
-            prodID: 1,
-            start: 36,
-            end: 36,
-            releases: [1, 2, 3],
-        },
-    },
-    {
-        name: "bow2henkelincoming",
-        description:
-            "Checks for new incoming goods orders to be completed and sends an email for what truck and carrier it was",
-        checkInterval: 15,
-        timeType: "min",
-        emails: "blake.matthes@alpla.com",
-        active: false,
-        notifiySettings: { processTime: 15 },
-    },
-    {
-        name: "palletsRemovedAsWaste",
-        description:
-            "Validates stock to make sure, there are no pallets released that have been removed as waste already ",
-        checkInterval: 15,
-        timeType: "min",
-        emails: "blake.matthes@alpla.com",
-        active: false,
-        notifiySettings: { prodID: 1 },
-    },
-    {
-        name: "shortageBookings",
-        description:
-            "Checks for material shortage bookings by single av type or all types ",
-        checkInterval: 15,
-        timeType: "min",
-        emails: "blake.matthes@alpla.com",
-        active: false,
-        notifiySettings: {
-            time: 15,
-            type: "all", // change this to something else or leave blank to use the av type
-            avType: 1,
-        },
-    },
+	{
+		name: "reprintLabels",
+		description:
+			"Monitors the labels that are printed and returns a value if one falls withing the time frame defined below.",
+		checkInterval: 1,
+		timeType: "min",
+		emails: "",
+		active: false,
+		notifiySettings: { prodID: 1 },
+	},
+	{
+		name: "downTimeCheck",
+		description: "Checks for specific downtimes that are greater than 105 min.",
+		checkInterval: 30,
+		timeType: "min",
+		emails: "",
+		active: false,
+		notifiySettings: { prodID: 1, daysInPast: 5, duration: 105 },
+	},
+	{
+		name: "qualityBlocking",
+		description:
+			"Checks for new blocking orders that have been entered, recommened to get the most recent order in here before activating.",
+		checkInterval: 30,
+		timeType: "min",
+		emails: "",
+		active: false,
+		notifiySettings: {
+			prodID: 1,
+			sentBlockingOrders: [{ timeStamp: "0", blockingOrder: 1 }],
+		},
+	},
+	{
+		name: "productionCheck",
+		description: "Checks ppoo",
+		checkInterval: 2,
+		timeType: "hour",
+		emails: "",
+		active: false,
+		notifiySettings: {
+			prodID: 1,
+			count: 0,
+			weekend: false,
+			locations: "0",
+		},
+	},
+	{
+		name: "stagingCheck",
+		description:
+			"Checks staging based on locations, locations need to be seperated by a ,",
+		checkInterval: 2,
+		timeType: "hour",
+		emails: "",
+		active: false,
+		notifiySettings: {
+			prodID: 1,
+			count: 0,
+			weekend: false,
+			locations: "0",
+		},
+	},
+	{
+		name: "tiIntergration",
+		description: "Checks for new releases to be put into ti",
+		checkInterval: 60,
+		timeType: "min",
+		emails: "",
+		active: false,
+		notifiySettings: {
+			prodID: 1,
+			start: 36,
+			end: 36,
+			releases: [{ timeStamp: "0", releaseNumber: 1 }],
+		},
+	},
+	{
+		name: "exampleNotification",
+		description: "Checks for new releases to be put into ti",
+		checkInterval: 2,
+		timeType: "min",
+		emails: "",
+		active: true,
+		notifiySettings: {
+			prodID: 1,
+			start: 36,
+			end: 36,
+			releases: [1, 2, 3],
+		},
+	},
+	{
+		name: "fifoIndex",
+		description: "Checks for pallets that were shipped out of fifo",
+		checkInterval: 1,
+		timeType: "hour",
+		emails: "blake.matthes@alpla.com",
+		active: false,
+		notifiySettings: {
+			prodID: 1,
+			start: 36,
+			end: 36,
+			releases: [1, 2, 3],
+		},
+	},
+	{
+		name: "bow2henkelincoming",
+		description:
+			"Checks for new incoming goods orders to be completed and sends an email for what truck and carrier it was",
+		checkInterval: 15,
+		timeType: "min",
+		emails: "blake.matthes@alpla.com",
+		active: false,
+		notifiySettings: { processTime: 15 },
+	},
+	{
+		name: "palletsRemovedAsWaste",
+		description:
+			"Validates stock to make sure, there are no pallets released that have been removed as waste already ",
+		checkInterval: 15,
+		timeType: "min",
+		emails: "blake.matthes@alpla.com",
+		active: false,
+		notifiySettings: { prodID: 1 },
+	},
+	{
+		name: "shortageBookings",
+		description:
+			"Checks for material shortage bookings by single av type or all types ",
+		checkInterval: 15,
+		timeType: "min",
+		emails: "blake.matthes@alpla.com",
+		active: false,
+		notifiySettings: {
+			time: 15,
+			type: "all", // change this to something else or leave blank to use the av type
+			avType: 1,
+		},
+	},
+	{
+		name: "tooManyErrors",
+		description:
+			"Checks to see how many errors in the last x time and sends an email based on this.",
+		checkInterval: 15,
+		timeType: "min",
+		emails: "blake.matthes@alpla.com",
+		active: true,
+		notifiySettings: {
+			errorCount: 10, // change this to something else or leave blank to use the av type
+		},
+	},
 ];
 
 export const notificationCreate = async () => {
-    for (let i = 0; i < note.length; i++) {
-        try {
-            const notify = await db
-                .insert(notifications)
-                .values(note[i])
-                .onConflictDoUpdate({
-                    target: notifications.name,
-                    set: {
-                        name: note[i].name,
-                        description: note[i].description,
-                        //notifiySettings: note[i].notifiySettings,
-                    },
-                });
-        } catch (error) {
-            createLog(
-                "error",
-                "notify",
-                "notify",
-                `There was an error getting the notifications: ${JSON.stringify(
-                    error
-                )}`
-            );
-        }
-    }
-    createLog(
-        "info",
-        "lst",
-        "nofity",
-        "notifications were just added/updated due to server startup"
-    );
+	for (let i = 0; i < note.length; i++) {
+		try {
+			const notify = await db
+				.insert(notifications)
+				.values(note[i])
+				.onConflictDoUpdate({
+					target: notifications.name,
+					set: {
+						name: note[i].name,
+						description: note[i].description,
+						//notifiySettings: note[i].notifiySettings,
+					},
+				});
+		} catch (error) {
+			createLog(
+				"error",
+				"notify",
+				"notify",
+				`There was an error getting the notifications: ${JSON.stringify(
+					error,
+				)}`,
+			);
+		}
+	}
+	createLog(
+		"info",
+		"lst",
+		"nofity",
+		"notifications were just added/updated due to server startup",
+	);
 };

lstV2/server/services/notifications/utils/views/tooManyErrors.hbs

@@ -0,0 +1,42 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    {{!-- <link rel="stylesheet" href="styles/styles.css" /> --}}
+    {{> styles}}
+</head>
+<body>
+        <p>All,</p>  
+        <p>The plant has encountered more than {{count}} errors in the last {{time}} mins, please see below errors and address as needed. </p>        
+        <table > 
+        <thead> 
+            <tr> 
+            <th>Username</th> 
+            <th>Service</th> 
+            <th>Message</th> 
+            <th>Checked</th> 
+            <th>LogTime</th> 
+            {{!-- <th>Downtime finish</th>  --}}
+            </tr> 
+        </thead> 
+        <tbody>
+            {{#each data}}
+                    <tr>
+                        <td>{{username}}</td>
+                        <td>{{service}}</td>
+                        <td>{{message}}</td>
+                        <td>{{checked}}</td>
+                        <td>{{add_Date}}</td>
+                        {{!-- <td>{{dtEnd}}</td> --}}
+                    </tr>
+                {{/each}}
+        </tbody>
+        </table>
+        <div>
+           <p>Thank you,</p>
+        <p>LST Team</p> 
+        </div>
+        
+</body>
+</html>

lstV2/server/services/ocp/controller/specialProcesses/dyco/plcConnection.ts

@@ -20,84 +20,90 @@ const palletSend = new Tag("Zone_6.Ready_to_Send");
 const strapperError = new Tag("Zone_3.Strapper_Faulted");
 
 export const dycoConnect = async () => {
-    // if we crash or start over reset the timers so we dont get duplicates
-    clearInterval(plcCycle);
-    if (isDycoRunning)
-        return { success: false, message: "Dyco is already connected." };
+	// if we crash or start over reset the timers so we dont get duplicates
+	clearInterval(plcCycle);
+	if (isDycoRunning)
+		return { success: false, message: "Dyco is already connected." };
 
-    // Remove all listeners before adding a new one to prevent memory leaks
-    PLC.removeAllListeners("error");
+	// Remove all listeners before adding a new one to prevent memory leaks
+	PLC.removeAllListeners("error");
 
-    try {
-        await PLC.connect(plcAddress, 0).then(async () => {
-            createLog("info", "dyco", "ocp", `We are connected to the dyco.`);
-            isDycoRunning = true;
+	try {
+		await PLC.connect(plcAddress, 0).then(async () => {
+			createLog("info", "dyco", "ocp", `We are connected to the dyco.`);
+			isDycoRunning = true;
 
-            plcCycle = setInterval(async () => {
-                if (isReading) {
-                    createLog(
-                        "debug",
-                        "dyco",
-                        "ocp",
-                        "Skipping cycle: previous read still in progress."
-                    );
-                    return;
-                }
-                isReading = true; // Set flag
-                try {
-                    await PLC.readTag(labelerTag);
-                    await PLC.readTag(palletSend);
-                    await PLC.readTag(strapperError);
+			plcCycle = setInterval(async () => {
+				if (isReading) {
+					createLog(
+						"debug",
+						"dyco",
+						"ocp",
+						"Skipping cycle: previous read still in progress.",
+					);
+					return;
+				}
+				isReading = true; // Set flag
+				try {
+					await PLC.readTag(labelerTag);
+					await PLC.readTag(palletSend);
+					await PLC.readTag(strapperError);
 
-                    // strapper check
-                    strapperFaults(strapperError);
+					// strapper check
+					strapperFaults(strapperError);
 
-                    // send the labeler tag data over
-                    labelerTagRead(labelerTag);
+					// send the labeler tag data over
+					labelerTagRead(labelerTag);
 
-                    // send the end of line check over.
-                    palletSendTag(palletSend);
-                } catch (error: any) {
-                    createLog(
-                        "error",
-                        "dyco",
-                        "ocp",
-                        `Error reading PLC tag: ${error.message}`
-                    );
-                } finally {
-                    isReading = false; // Reset flag
-                }
-            }, plcInterval);
-        });
-    } catch (error) {
-        createLog(
-            "error",
-            "dyco",
-            "ocp",
-            `There was an error in the dyco: ${error}`
-        );
-        await PLC.disconnect();
-        isDycoRunning = false;
-    }
+					// send the end of line check over.
+					palletSendTag(palletSend);
+				} catch (error: any) {
+					createLog(
+						"error",
+						"dyco",
+						"ocp",
+						`Error reading PLC tag: ${error.message}`,
+					);
+					// if we error out we want to disconnect and reconnect
+					closeDyco();
+					setTimeout(() => {
+						createLog("info", "dyco", "ocp", `Reconnecting to the dyco`);
+						dycoConnect();
+					}, 2 * 1000);
+				} finally {
+					isReading = false; // Reset flag
+				}
+			}, plcInterval);
+		});
+	} catch (error) {
+		createLog(
+			"error",
+			"dyco",
+			"ocp",
+			`There was an error in the dyco: ${error}`,
+		);
+		await PLC.disconnect();
+		isDycoRunning = false;
+	}
 };
 
 export const closeDyco = async () => {
-    if (!isDycoRunning)
-        return { success: false, message: "Dyco is not connected." };
+	if (!isDycoRunning)
+		return { success: false, message: "Dyco is not connected." };
 
-    console.log(`Closing the connection`);
-    try {
-        await PLC.disconnect();
-        isDycoRunning = false;
-        return {
-            success: true,
-            message: "Dyco Connection is now closed.",
-        };
-    } catch (error) {
-        console.log(error);
-        return {
-            success: false,
-            message: "There was an error closing the dyco connection.",
-        };
-    }
+	console.log(`Closing the connection`);
+	try {
+		await PLC.disconnect();
+		isDycoRunning = false;
+		return {
+			success: true,
+			message: "Dyco Connection is now closed.",
+		};
+	} catch (error) {
+		console.log(error);
+		return {
+			success: false,
+			message: "There was an error closing the dyco connection.",
+		};
+	}
 };