feat(lstv2 move): moved lstv2 into this app to keep them combined and easier to maintain

lstV2/server/services/logger/controller/clearLog.ts

@@ -0,0 +1,30 @@
+import { eq, sql } from "drizzle-orm";
+import { db } from "../../../../database/dbclient.js";
+import { logs } from "../../../../database/schema/logs.js";
+import { createLog } from "../logger.js";
+
+export const clearLog = async (id: string) => {
+    /**
+     * mark the log as cleared
+     */
+
+    try {
+        const clear = await db
+            .update(logs)
+            .set({ checked: true, created_at: sql`NOW()` })
+            .where(eq(logs.log_id, id));
+        createLog("info", "lst", "logger", "Log just cleared.");
+        return { success: true, message: "Log was just cleared." };
+    } catch (error) {
+        createLog(
+            "error",
+            "lst",
+            "logger",
+            "There was an error clearing the log."
+        );
+        return {
+            success: false,
+            message: "There was an error clearing the log.",
+        };
+    }
+};

lstV2/server/services/logger/controller/getLogs.ts

@@ -0,0 +1,42 @@
+import { and, desc, eq, gte, inArray, lte, sql } from "drizzle-orm";
+import { db } from "../../../../database/dbclient.js";
+import { logs } from "../../../../database/schema/logs.js";
+import { createLog } from "../logger.js";
+
+export const getLogs = async (data: any) => {
+    try {
+        // clear all remaining logs ne to info.
+        const checked =
+            data.checked && data.checked[0] === "true" ? true : false || false;
+        const logData = await db
+            .select()
+            .from(logs)
+            .where(
+                and(
+                    gte(
+                        logs.created_at,
+                        sql.raw(`NOW() - INTERVAL '${data.hours ?? "4"} hours'`)
+                    ),
+                    inArray(logs.service, data.service),
+                    inArray(logs.level, data.level),
+                    eq(logs.checked, checked)
+                )
+            )
+            .orderBy(desc(logs.created_at));
+
+        return { success: true, message: "logs returned", data: logData };
+    } catch (error) {
+        console.log(error);
+        createLog(
+            "error",
+            "lst",
+            "logger",
+            `There was an error deleteing server logs. ${error}`
+        );
+        return {
+            success: false,
+            message: "An error occured while trying to get the logs",
+            error,
+        };
+    }
+};

lstV2/server/services/logger/controller/logCleanup.ts

@@ -0,0 +1,65 @@
+import { and, eq, inArray, lte, ne, sql } from "drizzle-orm";
+import { db } from "../../../../database/dbclient.js";
+import { logs } from "../../../../database/schema/logs.js";
+import { createLog } from "../logger.js";
+
+export const logCleanup = async () => {
+    /**
+     * We will run the clean logger where we have aged logs that do not need to be here flooding the db
+     */
+
+    // clear the server logs older than 3 days
+    try {
+        // clear info logs older than 3 days
+        const delLogs = await db
+            .delete(logs)
+            .where(
+                and(
+                    lte(logs.created_at, sql`NOW() - INTERVAL '3 days'`),
+                    //inArray(logs.service, ["server", "tcp", "sqlProd", "globalutils","notify", "logger", "serverupdater"]),
+                    eq(logs.level, "info")
+                )
+            )
+            .returning({ name: logs.message });
+        createLog(
+            "info",
+            "lst",
+            "logger",
+            `${delLogs.length} Server logs were just deleted that were older than 3 days`
+        );
+    } catch (error) {
+        createLog(
+            "error",
+            "lst",
+            "logger",
+            `There was an error deleteing server logs. ${error}`
+        );
+    }
+
+    try {
+        // clear all remaining logs ne to info.
+        const delLogs = await db
+            .delete(logs)
+            .where(
+                and(
+                    lte(logs.created_at, sql`NOW() - INTERVAL '7 days'`),
+                    //inArray(logs.service, ["server", "tcp", "sqlProd", "globalutils", "notify", "logger", "serverupdater"]),
+                    ne(logs.level, "info")
+                )
+            )
+            .returning({ name: logs.message });
+        createLog(
+            "info",
+            "lst",
+            "logger",
+            `${delLogs.length} Server logs were just deleted that were older than 7 days`
+        );
+    } catch (error) {
+        createLog(
+            "error",
+            "lst",
+            "logger",
+            `There was an error deleteing server logs. ${error}`
+        );
+    }
+};

lstV2/server/services/logger/controller/streamLogs.ts

@@ -0,0 +1,29 @@
+import type {Context} from "hono";
+import {db} from "../../../../database/dbclient.js";
+import {and, eq, gt} from "drizzle-orm";
+import {streamSSE, streamText} from "hono/streaming";
+import {logs} from "../../../../database/schema/logs.js";
+
+export async function streamLogs(c: Context) {
+    let id = 0;
+    let running = true;
+    // c.header("Content-Type", "text/event-stream");
+    // c.header("Cache-Control", "no-cache");
+    // c.header("Connection", "keep-alive");
+
+    const getLogs = async () => {};
+    return streamSSE(c, async (stream) => {
+        while (running) {
+            const message = `It is ${new Date().toISOString()}`;
+            await stream.writeSSE({
+                data: message,
+                event: "time-update",
+                id: String(id++),
+            });
+            await stream.sleep(1000);
+            if (id === 5) {
+                running = false;
+            }
+        }
+    });
+}

lstV2/server/services/logger/dbTransport.ts

@@ -0,0 +1,41 @@
+import {db} from "../../../database/dbclient.js";
+import {logs} from "../../../database/schema/logs.js";
+import build from "pino-abstract-transport";
+
+type Log = {
+    username: string | null;
+    service: string;
+    level: string;
+    msg: string;
+};
+
+const pinoLogLevels: any = {
+    10: "trace",
+    20: "debug",
+    30: "info",
+    40: "warn",
+    50: "error",
+    60: "fatal",
+};
+// Create a custom transport function
+export default async function (log: Log) {
+    //const {username, service, level, msg, ...extra} = log;
+    try {
+        return build(async function (source) {
+            for await (let obj of source) {
+                // Insert log entry into the PostgreSQL database using Drizzle ORM
+
+                // convert to the name to make it more easy to find later :P
+                const levelName = pinoLogLevels[obj.level] || "unknown";
+                await db.insert(logs).values({
+                    level: levelName,
+                    username: obj?.username.toLowerCase(),
+                    service: obj?.service.toLowerCase(),
+                    message: obj.msg,
+                });
+            }
+        });
+    } catch (err) {
+        console.error("Error inserting log into database:", err);
+    }
+}

lstV2/server/services/logger/goTransport.ts

@@ -0,0 +1,50 @@
+process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0";
+import axios from "axios";
+import { pino } from "pino";
+import build from "pino-abstract-transport";
+import { tryCatch } from "../../globalUtils/tryCatch.js";
+
+const pinoLogLevels: any = {
+    10: "trace",
+    20: "debug",
+    30: "info",
+    40: "warn",
+    50: "error",
+    60: "fatal",
+};
+
+export default async function buildGoTransport() {
+    try {
+        return build(async function (source) {
+            for await (let obj of source) {
+                // Insert log entry into the PostgreSQL database using Drizzle ORM
+
+                // convert to the name to make it more easy to find later :P
+                const levelName = pinoLogLevels[obj.level] || "unknown";
+                // await db.insert(logs).values({
+                //     level: levelName,
+                //     username: obj?.username.toLowerCase(),
+                //     service: obj?.service.toLowerCase(),
+                //     message: obj.msg,
+                // });
+                const { data, error } = (await tryCatch(
+                    axios.post(`${process.env.LST_BASE_URL}/api/v1/log`, {
+                        service: obj?.service.toLowerCase(),
+                        level: levelName,
+                        message: obj.msg,
+                    })
+                )) as any;
+
+                if (error) {
+                    console.log(
+                        "The go server must be offline so we cant post the new logs."
+                    );
+                }
+
+                // console.log(`Go log level: ${levelName}`);
+            }
+        });
+    } catch (err) {
+        console.error("Error inserting log into database:", err);
+    }
+}

lstV2/server/services/logger/logger.ts

@@ -0,0 +1,62 @@
+import { pino, type LogFn, type Logger } from "pino";
+
+export let logLevel = process.env.LOG_LEVEL || "info";
+
+const transport = pino.transport({
+    targets: [
+        {
+            target: "pino-pretty",
+            options: {
+                colorize: true,
+                singleLine: true,
+                // customPrettifiers: {
+                //     time: (time) => `🕰 ${time}`,
+                // },
+                destination: process.stdout.fd,
+            },
+        },
+        {
+            target: "./dbTransport.js",
+        },
+        // Only log to Go if LST_USE_GO=true
+        ...(process.env.LST_USE_GO === "true"
+            ? [
+                  {
+                      target: "./goTransport.js", // New transport for Go
+                  },
+              ]
+            : []),
+    ],
+});
+
+const log: Logger = pino(
+    {
+        level: process.env.LOG_LEVEL || logLevel,
+        //level: "debug",
+        // formatters: {
+        //     level: (label) => {
+        //         return {level: label.toUpperCase()};
+        //     },
+        // },
+        //customLevels: {death: 70},
+        // removes data from the logs that we dont want to be shown :D
+        redact: { paths: ["email", "password"], remove: true },
+    },
+    transport
+);
+
+export const createLog = (
+    level: "info" | "fatal" | "error" | "warn" | "debug" | "trace",
+    username: string,
+    service: string,
+    message: string
+) => {
+    if (level in log) {
+        log[level]({ username, service }, message);
+    } else {
+        log.warn(
+            { username, service },
+            `Invalid log level '${level}', falling back to warn: ${message}`
+        );
+    }
+};

lstV2/server/services/logger/loggerService.ts

@@ -0,0 +1,34 @@
+import { OpenAPIHono } from "@hono/zod-openapi";
+
+// routes
+import clearLog from "./routes/clearLog.js";
+import { db } from "../../../database/dbclient.js";
+import { settings } from "../../../database/schema/settings.js";
+import { logCleanup } from "./controller/logCleanup.js";
+import createNewLog from "./routes/createLog.js";
+import getLogs from "./routes/getLogs.js";
+import stream from "./routes/streamLogs.js";
+
+const app = new OpenAPIHono();
+
+const routes = [clearLog, createNewLog, getLogs, stream] as const;
+//const setting = await db.select().from(settings);
+
+const appRoutes = routes.forEach((route) => {
+    app.route("/logger", route);
+});
+
+app.all("/logger/*", (c) => {
+    return c.json({
+        success: false,
+        message: "You have encounters a log route that dose not exist.",
+    });
+});
+
+// run the clean up job ones on server restart/crash/update and then once a date
+logCleanup();
+setInterval(async () => {
+    logCleanup();
+}, 60 * 1000 * 60 * 24);
+
+export default app;

lstV2/server/services/logger/routes/clearLog.ts

@@ -0,0 +1,44 @@
+import {createRoute, OpenAPIHono, z} from "@hono/zod-openapi";
+import {apiHit} from "../../../globalUtils/apiHits.js";
+import {responses} from "../../../globalUtils/routeDefs/responses.js";
+import {clearLog} from "../controller/clearLog.js";
+
+const app = new OpenAPIHono({strict: false});
+const ParamsSchema = z.object({
+    id: z
+    .string()
+    .min(3)
+    .openapi({
+        param: {
+            name: "id",
+            in: "path",
+        },
+        example: "1212121",
+    }),
+});
+
+app.openapi(
+    createRoute({
+        tags: ["server:logger"],
+        summary: "Marks the select log id as cleared out.",
+        method: "patch",
+        path: "/logs/{id}",
+        request: {
+            params: ParamsSchema,
+        },
+        responses: responses(),
+    }),
+    async (c) => {
+        const {id} = c.req.valid("param");
+        //const body = await c.req.json();
+        // make sure we have a vaid user being accessed thats really logged in
+        apiHit(c, {endpoint: `api/logger/logs/id`});
+        try {
+            const clear = await clearLog(id);
+            return c.json({success: clear.success, message: clear.message, data: []}, 200);
+        } catch (error) {
+            return c.json({success: false, message: "There was an error clearing the log.", data: error}, 400);
+        }
+    }
+);
+export default app;

lstV2/server/services/logger/routes/createLog.ts

@@ -0,0 +1,38 @@
+// an external way to creating logs
+import {createRoute, OpenAPIHono, z} from "@hono/zod-openapi";
+import {apiHit} from "../../../globalUtils/apiHits.js";
+import {responses} from "../../../globalUtils/routeDefs/responses.js";
+import {createLog} from "../logger.js";
+
+const app = new OpenAPIHono({strict: false});
+const CreateLog = z.object({
+    level: z.string().openapi({example: "info"}),
+    service: z.string().openapi({example: "server"}),
+    message: z.string().openapi({example: "This is a new log posted"}),
+});
+
+app.openapi(
+    createRoute({
+        tags: ["server:logger"],
+        summary: "Post a log to the db.",
+        method: "post",
+        path: "/logs",
+        description: "This might be a temp soltuin during the transtion between versions",
+        request: {
+            body: {content: {"application/json": {schema: CreateLog}}},
+        },
+        responses: responses(),
+    }),
+    async (c) => {
+        const body = await c.req.json();
+
+        apiHit(c, {endpoint: `api/logger/logs/id`});
+        try {
+            createLog(body.level, "logger", body.service, body.message);
+            return c.json({success: true, message: "A new log was created.", data: []}, 200);
+        } catch (error) {
+            return c.json({success: false, message: "There was an error clearing the log.", data: error}, 400);
+        }
+    }
+);
+export default app;

lstV2/server/services/logger/routes/getLogs.ts

@@ -0,0 +1,39 @@
+// an external way to creating logs
+import {createRoute, OpenAPIHono, z} from "@hono/zod-openapi";
+import {apiHit} from "../../../globalUtils/apiHits.js";
+import {responses} from "../../../globalUtils/routeDefs/responses.js";
+import {createLog} from "../logger.js";
+import {getLogs} from "../controller/getLogs.js";
+
+const app = new OpenAPIHono({strict: false});
+const CreateLog = z.object({
+    level: z.string().openapi({example: "info"}),
+    service: z.string().openapi({example: "server"}),
+    message: z.string().openapi({example: "This is a new log posted"}),
+});
+
+app.openapi(
+    createRoute({
+        tags: ["server:logger"],
+        summary: "Gets logs.",
+        method: "get",
+        path: "/logs",
+        description: "This might be a temp soltuin during the transtion between versions",
+        request: {
+            body: {content: {"application/json": {schema: CreateLog}}},
+        },
+        responses: responses(),
+    }),
+    async (c) => {
+        const query = await c.req.queries();
+
+        apiHit(c, {endpoint: `api/logger/logs`});
+        try {
+            const logData = await getLogs(query);
+            return c.json({success: logData?.success, message: logData?.message, data: logData?.data}, 200);
+        } catch (error) {
+            return c.json({success: false, message: "There was an error clearing the log.", data: error}, 400);
+        }
+    }
+);
+export default app;

lstV2/server/services/logger/routes/streamLogs.ts

@@ -0,0 +1,51 @@
+// an external way to creating logs
+//@ts-nocheck
+import {createRoute, OpenAPIHono, z} from "@hono/zod-openapi";
+import {apiHit} from "../../../globalUtils/apiHits.js";
+import {responses} from "../../../globalUtils/routeDefs/responses.js";
+import {createLog} from "../logger.js";
+import {getLogs} from "../controller/getLogs.js";
+import {streamLogs} from "../controller/streamLogs.js";
+import {streamSSE} from "hono/streaming";
+
+const app = new OpenAPIHono({strict: false});
+
+app.openapi(
+    createRoute({
+        tags: ["server:logger"],
+        summary: "Streams the logs to the frontend.",
+        method: "get",
+        path: "/logs/stream",
+        description: "This should only be used on the event you need to monitor logs.",
+
+        responses: {
+            200: {
+                content: {
+                    "application/json": {schema: z.object({message: z.string().optional()})},
+                },
+                description: "Response message",
+            },
+        },
+    }),
+    async (c) => {
+        apiHit(c, {endpoint: `api/logger/logs`});
+        c.header("Content-Type", "text/event-stream");
+        c.header("Cache-Control", "no-cache");
+        c.header("Connection", "keep-alive");
+        return streamSSE(c, async (stream) => {
+            let id = 0;
+            const encoder = new TextEncoder();
+            while (true) {
+                const message = `It is ${new Date().toISOString()}`;
+                await stream.writeSSE({
+                    data: message,
+                    event: "time-update",
+                    id: String(id++),
+                });
+                encoder.encode(`data: ${JSON.stringify({type: "progress", data: id})}\n\n`);
+                await stream.sleep(1000);
+            }
+        });
+    }
+);
+export default app;