feat(dm): migrated all the dm topics

backend/db/db.controller.ts

@@ -3,6 +3,8 @@ import postgres from "postgres";
 import * as dockScans from "./schema/dockdoor.scans.schema.js";
 import * as logs from "./schema/logs.schema.js";
 import * as opendockAVCheck from "./schema/opendock_articleSetup.js";
+import * as prodAuditLogId from "./schema/prodAuditlog.lastProcessed.schema.js";
+import * as prodAuditLog from "./schema/prodAuditlog.schema.js";
 import * as scanUserSchema from "./schema/scanUsers.js";
 import * as settingsSchema from "./schema/settings.schema.js";
 
@@ -27,5 +29,7 @@ export const db = drizzle(queryClient, {
 		...opendockAVCheck,
 		...logs,
 		...dockScans,
+		...prodAuditLogId,
+		...prodAuditLog,
 	},
 });

backend/db/db.listener.ts

@@ -1,5 +1,6 @@
 import postgres from "postgres";
 import { createLogger } from "../logger/logger.controller.js";
+import { bufferProcess } from "../system/system.prodAuditLog.utils.js";
 import { handleDbNotification } from "./db.router.js";
 
 const log = createLogger({
@@ -37,6 +38,11 @@ export async function startDbNotificationListener() {
 
 		log.info({ stack: { channel } }, `Listening for ${channel}`);
 	}
+
+	// server side only listeners
+	await sql.listen("auditLog_inserted", async (rawPayload) => {
+		bufferProcess(rawPayload);
+	});
 }
 
 async function processNotification(

backend/db/db.setupNotifications.ts

@@ -19,6 +19,7 @@ export async function setupDbNotifications() {
 
 	await setupLogsNotifications();
 	await setupDockScansNotifications();
+	await setupProdAuditNotifications();
 
 	log.info({}, "DB notifications setup complete");
 }
@@ -97,3 +98,36 @@ async function setupDockScansNotifications() {
 
 	log.info({}, "Dock scan DB notification trigger ready");
 }
+
+async function setupProdAuditNotifications() {
+	await db.execute(sql`
+		CREATE OR REPLACE FUNCTION notify_auditLog_inserted()
+		RETURNS trigger AS $$
+		BEGIN
+			PERFORM pg_notify(
+				'auditLog_inserted',
+				json_build_object(
+					'table', TG_TABLE_NAME,
+					'action', TG_OP,
+					'id', NEW.id
+				)::text
+			);
+
+			RETURN NEW;
+		END;
+		$$ LANGUAGE plpgsql;
+	`);
+
+	await db.execute(sql`
+		DROP TRIGGER IF EXISTS auditLog_inserted_notify_trigger ON prod_audit_log;
+	`);
+
+	await db.execute(sql`
+		CREATE TRIGGER auditLog_inserted_notify_trigger
+		AFTER INSERT ON prod_audit_log
+		FOR EACH ROW
+		EXECUTE FUNCTION notify_auditLog_inserted();
+	`);
+
+	log.info({}, "Audit Log DB notification trigger ready");
+}

backend/db/schema/ordersImports.schema.ts

@@ -0,0 +1,23 @@
+import { jsonb, pgTable, text, timestamp, uuid } from "drizzle-orm/pg-core";
+import { createInsertSchema, createSelectSchema } from "drizzle-zod";
+import type { z } from "zod";
+
+export const orderImport = pgTable("order_import", {
+	id: uuid("id").defaultRandom().primaryKey(),
+	receivingPlantId: text("receiving_plant_id").notNull(),
+	documentName: text("documentName"),
+	sender: text("sender"),
+	customerId: text("customer_id"),
+	invoiceAddressId: text("invoice_address_id"),
+	rawData: jsonb("raw_data").default([]),
+	add_date: timestamp("add_date", { withTimezone: true }).defaultNow(),
+	add_user: text("add_user").default("lst-system"),
+	upd_date: timestamp("upd_date", { withTimezone: true }).defaultNow(),
+	upd_user: text("upd_user").default("lst-system"),
+});
+
+export const orderImportSchema = createSelectSchema(orderImport);
+export const newOrderImportSchema = createInsertSchema(orderImport);
+
+export type OrderImport = z.infer<typeof orderImportSchema>;
+export type NewOrderImport = z.infer<typeof newOrderImportSchema>;

backend/db/schema/prodAuditlog.lastProcessed.schema.ts

@@ -0,0 +1,39 @@
+import { integer, pgTable, timestamp } from "drizzle-orm/pg-core";
+
+export const prodAuditLogState = pgTable("prod_audit_log_state", {
+	id: integer("id").primaryKey().default(1),
+
+	lastImportedAuditId: integer("last_imported_audit_id").notNull().default(0),
+	lastProcessedAuditId: integer("last_processed_audit_id").notNull().default(0),
+
+	createdAt: timestamp("created_at", { withTimezone: true }).defaultNow(),
+	updatedAt: timestamp("updated_at", { withTimezone: true }).defaultNow(),
+});
+
+/*
+if the system fails do the process we do 
+and increase the retry to x max of 5 tries
+const nextRetryAt = new Date(Date.now() + Math.min(30 * retryCount, 600) * 1000);
+
+Cron every 30s
+  ↓
+Pull ERP AuditLog by Id > lastAuditId
+  ↓
+Insert into prod_audit_log
+  ↓
+Postgres NOTIFY wakes worker
+  ↓
+Worker processes pending rows
+  ↓
+Success = success
+Failure = error + retryCount + nextRetryAt
+20 failures = dead + email
+
+
+for the check we want to do 
+
+status IN ('pending', 'error')
+AND retry_count < 20
+AND (next_retry_at IS NULL OR next_retry_at <= NOW())
+
+*/

backend/db/schema/prodAuditlog.schema.ts

@@ -0,0 +1,70 @@
+import {
+	boolean,
+	integer,
+	jsonb,
+	pgTable,
+	serial,
+	text,
+	timestamp,
+} from "drizzle-orm/pg-core";
+import { createInsertSchema, createSelectSchema } from "drizzle-zod";
+import type { z } from "zod";
+
+export const prodAuditLog = pgTable("prod_audit_log", {
+	id: serial("id").primaryKey(),
+	auditId: integer("audit_id").notNull().unique(),
+	actorName: text("actor_name").notNull(),
+	auditCreatedDate: timestamp("audit_created_date", {
+		withTimezone: true,
+	}).notNull(),
+	message: text("message").notNull(), // mirrors how prod sends it over basically this is where the domain its coming from is. well split by "." and then pass it to what it needs to go to later.
+	content: jsonb("content").notNull(),
+
+	status: text("status").notNull().default("pending"), // pending | processing | success | error | dead
+	processed: boolean("processed").default(false),
+
+	retryCount: integer("retry_count").notNull().default(0),
+	nextRetryAt: timestamp("next_retry_at", { withTimezone: true }),
+
+	errorMessage: text("error_message"),
+	errorStack: text("error_stack"),
+
+	processedAt: timestamp("processed_at", { withTimezone: true }),
+
+	createdAt: timestamp("created_at", { withTimezone: true }).defaultNow(),
+	updatedAt: timestamp("updated_at", { withTimezone: true }).defaultNow(),
+});
+
+export const prodAuditLogSchema = createSelectSchema(prodAuditLog);
+export const newProdAuditLogSchema = createInsertSchema(prodAuditLog);
+
+export type ProdAuditLog = z.infer<typeof prodAuditLogSchema>;
+export type NewProdAuditLog = z.infer<typeof newProdAuditLogSchema>;
+
+/*
+if the system fails do the process we do 
+and increase the retry to x max of 5 tries
+const nextRetryAt = new Date(Date.now() + Math.min(30 * retryCount, 600) * 1000);
+
+Cron every 30s
+  ↓
+Pull ERP AuditLog by Id > lastAuditId
+  ↓
+Insert into prod_audit_log
+  ↓
+Postgres NOTIFY wakes worker
+  ↓
+Worker processes pending rows
+  ↓
+Success = success
+Failure = error + retryCount + nextRetryAt
+20 failures = dead + email
+
+
+for the check we want to do 
+
+status IN ('pending', 'error')
+AND retry_count < 20
+AND (next_retry_at IS NULL OR next_retry_at <= NOW())
+
+*/