refactor(ocp): plc reading changes to disconnect and reconnect

it was found that there were some errors that spammed the log and caused the server to actually stop
responding and crash weirdly so added a disconnect and reconnect back. so we can figure out whats
going on.

LogisticsSupportTool_API_DOCS/LstV2/Notifcations/Error logging.bru

@@ -0,0 +1,16 @@
+meta {
+  name: Error logging
+  type: http
+  seq: 4
+}
+
+get {
+  url: {{urlv2}}/api/notify/materialperday
+  body: none
+  auth: inherit
+}
+
+settings {
+  encodeUrl: true
+  timeout: 0
+}

LogisticsSupportTool_API_DOCS/environments/lst.bru

@@ -1,7 +1,7 @@
 vars {
   url: https://uslim1prod.alpla.net
   session_cookie: 
-  urlv2: http://usbow1vms006:3000
+  urlv2: http://localhost:3000
   jwtV2: 
   userID: 
 }

lstV2/server/services/notifications/controller/notifications/tooManyErrors.ts

@@ -0,0 +1,118 @@
+// SELECT count(*) FROM V_EtikettenGedruckt where AnzahlGedruckterKopien > 2 and CONVERT(varchar(5), Add_Date,108) not like CONVERT(varchar(5), Upd_Date,108) and Upd_Date > DATEADD(SECOND, -30,getdate()) and VpkVorschriftBez not like '%$%'
+
+import { errorMonitor } from "node:events";
+import { eq, sql } from "drizzle-orm";
+import { db } from "../../../../../database/dbclient.js";
+import { notifications } from "../../../../../database/schema/notifications.js";
+import { settings } from "../../../../../database/schema/settings.js";
+import { tryCatch } from "../../../../globalUtils/tryCatch.js";
+import { createLog } from "../../../logger/logger.js";
+import { sendEmail } from "../sendMail.js";
+
+export interface DownTime {
+	downTimeId?: number;
+	machineAlias?: string;
+}
+export default async function tooManyErrors(notifyData: any) {
+	// we will over ride this with users that want to sub to this
+	// a new table will be called subalerts and link to the do a kinda linkn where the user wants it then it dose subId: 1, userID: x, notificationId: y. then in here we look up the userid to get the email :D
+	// this could then leave the emails in the notificaion blank and let users sub to it.
+	//console.log(notifyData);
+	if (notifyData.emails === "") {
+		createLog(
+			"error",
+			"notify",
+			"notify",
+			`There are no emails set for ${notifyData.name}`,
+		);
+
+		return;
+	}
+
+	// console.log(data.secondarySetting[0].duration);
+
+	const plant = await db
+		.select()
+		.from(settings)
+		.where(eq(settings.name, "plantToken"));
+	console.log(plant[0].value);
+	// console.log(
+	// 	errorQuery
+	// 		.replace("[time]", notifyData.checkInterval)
+	// 		.replace("[errorCount]", notifyData.notifiySettings.errorCount),
+	// 	errorLogQuery.replace("[time]", notifyData.checkInterval),
+	// );
+
+	let errorLogData: any = [];
+	try {
+		const errorData = await db.execute(sql`
+            SELECT 'error' AS level, COUNT(*) AS error_count
+            FROM public.logs
+            WHERE level = 'error'
+                AND "add_Date" > now() - INTERVAL ${sql.raw(`'${notifyData.checkInterval} minutes'`)}
+            GROUP BY level
+            HAVING COUNT(*) >= ${notifyData.notifiySettings.errorCount}
+            `);
+		if (
+			errorData.length > 0
+			// && downTime[0]?.downTimeId > notifyData.notifiySettings.prodID
+		) {
+			const errorLogs = await db.execute(sql`
+            select* from public.logs where level = 'error' and "add_Date" > now() - INTERVAL ${sql.raw(`'${notifyData.checkInterval} minutes'`)} order by "add_Date" desc;
+            `);
+
+			errorLogData = errorLogs;
+			//send the email :D
+			const emailSetup = {
+				email: notifyData.emails,
+				subject: `Alert! ${plant[0].value} has encountered ${
+					errorData.length
+				} ${errorData.length > 1 ? "errors" : "error"} in the last ${notifyData.checkInterval} min`,
+				template: "tooManyErrors",
+				context: {
+					data: errorLogData,
+					count: notifyData.notifiySettings.errorCount,
+					time: notifyData.checkInterval,
+				},
+			};
+
+			//console.log(emailSetup);
+
+			const sentEmail = await sendEmail(emailSetup);
+
+			if (!sentEmail.success) {
+				createLog(
+					"error",
+					"notify",
+					"notify",
+					"Failed to send email, will try again on next interval",
+				);
+				return {
+					success: false,
+					message: "Failed to send email, will try again on next interval",
+					data: sentEmail,
+				};
+			}
+		}
+	} catch (err) {
+		console.log(err);
+		createLog(
+			"error",
+			"notify",
+			"notify",
+			`Error from running the downtimeCheck query: ${err}`,
+		);
+
+		return {
+			success: false,
+			message: "Error running error data",
+			data: err,
+		};
+	}
+
+	return {
+		success: true,
+		message: "Error log checking ran",
+		data: errorLogData ?? [],
+	};
+}

lstV2/server/services/notifications/notifyService.ts

@@ -10,6 +10,7 @@ import tiTrigger from "./routes/manualTiggerTi.js";
 import materialCheck from "./routes/materialPerDay.js";
 import blocking from "./routes/qualityBlocking.js";
 import sendemail from "./routes/sendMail.js";
+import errorHandling from "./routes/tooManyErrors.js";
 import { note, notificationCreate } from "./utils/masterNotifications.js";
 import { startNotificationMonitor } from "./utils/processNotifications.js";
 
@@ -23,6 +24,7 @@ const routes = [
 	notify,
 	fifoIndex,
 	materialCheck,
+	errorHandling,
 ] as const;
 
 const appRoutes = routes.forEach((route) => {

lstV2/server/services/notifications/routes/tooManyErrors.ts

@@ -0,0 +1,50 @@
+// an external way to creating logs
+import { createRoute, OpenAPIHono, z } from "@hono/zod-openapi";
+import { eq } from "drizzle-orm";
+import { db } from "../../../../database/dbclient.js";
+import { notifications } from "../../../../database/schema/notifications.js";
+import { apiHit } from "../../../globalUtils/apiHits.js";
+import { responses } from "../../../globalUtils/routeDefs/responses.js";
+import { tryCatch } from "../../../globalUtils/tryCatch.js";
+import { authMiddleware } from "../../auth/middleware/authMiddleware.js";
+import hasCorrectRole from "../../auth/middleware/roleCheck.js";
+import tooManyErrors from "../controller/notifications/tooManyErrors.js";
+import { getAllJobs } from "../utils/processNotifications.js";
+
+const app = new OpenAPIHono({ strict: false });
+
+app.openapi(
+	createRoute({
+		tags: ["server"],
+		summary: "Returns current active notifications.",
+		method: "get",
+		path: "/toomanyerrors",
+		middleware: [authMiddleware, hasCorrectRole(["systemAdmin"], "admin")],
+		responses: responses(),
+	}),
+	async (c) => {
+		apiHit(c, { endpoint: "/toomanyerrors" });
+
+		const { data, error } = await tryCatch(
+			db
+				.select()
+				.from(notifications)
+				.where(eq(notifications.name, "tooManyErrors")),
+		);
+
+		if (error) {
+			return c.json({
+				success: false,
+				message: "Error Getting Notification Settings.",
+				data: error,
+			});
+		}
+		const errorData = await tooManyErrors(data[0]);
+		return c.json({
+			success: true,
+			message: "Current Error log data",
+			data: errorData?.data,
+		});
+	},
+);
+export default app;

lstV2/server/services/notifications/utils/masterNotifications.ts

@@ -15,8 +15,7 @@ export const note: any = [
 	},
 	{
 		name: "downTimeCheck",
-        description:
-            "Checks for specific downtimes that are greater than 105 min.",
+		description: "Checks for specific downtimes that are greater than 105 min.",
 		checkInterval: 30,
 		timeType: "min",
 		emails: "",
@@ -141,6 +140,18 @@ export const note: any = [
 			avType: 1,
 		},
 	},
+	{
+		name: "tooManyErrors",
+		description:
+			"Checks to see how many errors in the last x time and sends an email based on this.",
+		checkInterval: 15,
+		timeType: "min",
+		emails: "blake.matthes@alpla.com",
+		active: true,
+		notifiySettings: {
+			errorCount: 10, // change this to something else or leave blank to use the av type
+		},
+	},
 ];
 
 export const notificationCreate = async () => {
@@ -163,8 +174,8 @@ export const notificationCreate = async () => {
 				"notify",
 				"notify",
 				`There was an error getting the notifications: ${JSON.stringify(
-                    error
-                )}`
+					error,
+				)}`,
 			);
 		}
 	}
@@ -172,6 +183,6 @@ export const notificationCreate = async () => {
 		"info",
 		"lst",
 		"nofity",
-        "notifications were just added/updated due to server startup"
+		"notifications were just added/updated due to server startup",
 	);
 };

lstV2/server/services/notifications/utils/views/tooManyErrors.hbs

@@ -0,0 +1,42 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    {{!-- <link rel="stylesheet" href="styles/styles.css" /> --}}
+    {{> styles}}
+</head>
+<body>
+        <p>All,</p>  
+        <p>The plant has encountered more than {{count}} errors in the last {{time}} mins, please see below errors and address as needed. </p>        
+        <table > 
+        <thead> 
+            <tr> 
+            <th>Username</th> 
+            <th>Service</th> 
+            <th>Message</th> 
+            <th>Checked</th> 
+            <th>LogTime</th> 
+            {{!-- <th>Downtime finish</th>  --}}
+            </tr> 
+        </thead> 
+        <tbody>
+            {{#each data}}
+                    <tr>
+                        <td>{{username}}</td>
+                        <td>{{service}}</td>
+                        <td>{{message}}</td>
+                        <td>{{checked}}</td>
+                        <td>{{add_Date}}</td>
+                        {{!-- <td>{{dtEnd}}</td> --}}
+                    </tr>
+                {{/each}}
+        </tbody>
+        </table>
+        <div>
+           <p>Thank you,</p>
+        <p>LST Team</p> 
+        </div>
+        
+</body>
+</html>

lstV2/server/services/ocp/controller/specialProcesses/dyco/plcConnection.ts

@@ -39,7 +39,7 @@ export const dycoConnect = async () => {
 						"debug",
 						"dyco",
 						"ocp",
-                        "Skipping cycle: previous read still in progress."
+						"Skipping cycle: previous read still in progress.",
 					);
 					return;
 				}
@@ -62,8 +62,14 @@ export const dycoConnect = async () => {
 						"error",
 						"dyco",
 						"ocp",
-                        `Error reading PLC tag: ${error.message}`
+						`Error reading PLC tag: ${error.message}`,
 					);
+					// if we error out we want to disconnect and reconnect
+					closeDyco();
+					setTimeout(() => {
+						createLog("info", "dyco", "ocp", `Reconnecting to the dyco`);
+						dycoConnect();
+					}, 2 * 1000);
 				} finally {
 					isReading = false; // Reset flag
 				}
@@ -74,7 +80,7 @@ export const dycoConnect = async () => {
 			"error",
 			"dyco",
 			"ocp",
-            `There was an error in the dyco: ${error}`
+			`There was an error in the dyco: ${error}`,
 		);
 		await PLC.disconnect();
 		isDycoRunning = false;