db cleanups and logging for od

backend/db/dbCleanup.controller.ts

@@ -0,0 +1,72 @@
+import { createLogger } from "../logger/logger.controller.js";
+import { delay } from "../utils/delay.utils.js";
+import { db } from "./db.controller.js";
+
+type DBCount = {
+	count: string;
+};
+
+const tableMap = {
+	logs: "logs",
+	jobs: "job_audit_log",
+	opendockApt: "opendock_apt",
+} as const;
+
+type CleanupTable = keyof typeof tableMap;
+
+/**
+ * We will clean up the db based on age.
+ * @param name database to run the cleanup on
+ * @param daysToKeep optional default will be 90 days
+ */
+export const dbCleanup = async (name: CleanupTable, daysToKeep?: number) => {
+	const log = createLogger({ module: "db", subModule: "cleanup" });
+	// TODO: send backup of this to another server, via post or something maybe have to reduce the limit but well tackle that later.
+
+	if (!daysToKeep) {
+		daysToKeep = 90;
+	}
+	const limit = 1000;
+	const delayTime = 250;
+	let rowsDeleted: number;
+
+	const dbCount = (await db.execute(
+		`select count(*) from public.${tableMap[name]} WHERE created_at < NOW() - INTERVAL '${daysToKeep} days'`,
+	)) as DBCount[];
+
+	const loopCount = Math.ceil(
+		parseInt(dbCount[0]?.count ?? `${limit}`, 10) / limit,
+	);
+
+	if (parseInt(dbCount[0]?.count ?? `${limit}`, 10) > 1) {
+		log.info(
+			`Table clean up for: ${name}, that are older than ${daysToKeep} day, will be removed, There is ${loopCount} loops to be completed, Approx time: ${((loopCount * delayTime) / 1000 / 60).toFixed(2)} min(s).`,
+		);
+	} else {
+		log.info(`Table clean up for: ${name}, Currently has nothing to clean up.`);
+		return;
+	}
+
+	do {
+		// cleanup logs
+		const deleted = await db.execute(`
+            DELETE FROM public.${tableMap[name]}
+            WHERE id IN (
+                SELECT id
+                FROM public.${tableMap[name]}
+                WHERE created_at < NOW() - INTERVAL '${daysToKeep} days'
+                ORDER BY created_at
+                LIMIT ${limit}
+            )
+            RETURNING id;
+        `);
+
+		rowsDeleted = deleted.length;
+
+		if (rowsDeleted > 0) {
+			await delay(delayTime);
+		}
+	} while (rowsDeleted === limit);
+
+	log.info(`Table clean up for: ${name}, Has completed.`);
+};

backend/db/schema/auditLog.schema.ts

@@ -1,4 +1,5 @@
 import {
+	index,
 	integer,
 	jsonb,
 	pgTable,
@@ -9,17 +10,29 @@ import {
 import { createInsertSchema, createSelectSchema } from "drizzle-zod";
 import type { z } from "zod";
 
-export const jobAuditLog = pgTable("job_audit_log", {
-	id: uuid("id").defaultRandom().primaryKey(),
-	jobName: text("job_name"),
-	startedAt: timestamp("start_at"),
-	finishedAt: timestamp("finished_at"),
-	durationMs: integer("duration_ms"),
-	status: text("status"), //success | error
-	errorMessage: text("error_message"),
-	errorStack: text("error_stack"),
-	metadata: jsonb("meta_data"),
-});
+export const jobAuditLog = pgTable(
+	"job_audit_log",
+	{
+		id: uuid("id").defaultRandom().primaryKey(),
+		jobName: text("job_name"),
+		startedAt: timestamp("start_at"),
+		finishedAt: timestamp("finished_at"),
+		durationMs: integer("duration_ms"),
+		status: text("status"), //success | error
+		errorMessage: text("error_message"),
+		errorStack: text("error_stack"),
+		metadata: jsonb("meta_data"),
+		createdAt: timestamp("created_at").defaultNow(),
+	},
+	(table) => {
+		return {
+			cleanupIdx: index("idx_job_audit_logs_cleanup").on(
+				table.startedAt,
+				table.id,
+			),
+		};
+	},
+);
 
 export const jobAuditLogSchema = createSelectSchema(jobAuditLog);
 export const newJobAuditLogSchema = createInsertSchema(jobAuditLog);

backend/db/schema/opendock.schema.ts

@@ -9,7 +9,7 @@ import {
 import { createInsertSchema, createSelectSchema } from "drizzle-zod";
 import type { z } from "zod";
 
-export const opendockApt = pgTable("opendock.apt", {
+export const opendockApt = pgTable("opendock_apt", {
 	id: uuid("id").defaultRandom().primaryKey(),
 	release: integer("release").unique(),
 	openDockAptId: text("open_dock_apt_id").notNull(),