From 1e78188c7111585afc6b66625caae5e0db8ec01c Mon Sep 17 00:00:00 2001 From: Owen Date: Sat, 8 Mar 2025 11:58:48 -0500 Subject: [PATCH] Remove json_group_array from queries Also improve handling tcp/udp resources in newt function so it does not loop twice --- server/routers/newt/handleRegisterMessage.ts | 130 +++++++++++-------- server/routers/traefik/getTraefikConfig.ts | 123 +++++++++++------- 2 files changed, 148 insertions(+), 105 deletions(-) diff --git a/server/routers/newt/handleRegisterMessage.ts b/server/routers/newt/handleRegisterMessage.ts index 0f086698..11b33478 100644 --- a/server/routers/newt/handleRegisterMessage.ts +++ b/server/routers/newt/handleRegisterMessage.ts @@ -7,7 +7,7 @@ import { Target, targets } from "@server/db/schema"; -import { eq, and, sql } from "drizzle-orm"; +import { eq, and, sql, inArray } from "drizzle-orm"; import { addPeer, deletePeer } from "../gerbil/peers"; import logger from "@server/logger"; @@ -75,68 +75,84 @@ export const handleRegisterMessage: MessageHandler = async (context) => { allowedIps: [site.subnet] }); - const allResources = await db - .select({ - // Resource fields - resourceId: resources.resourceId, - subdomain: resources.subdomain, - fullDomain: resources.fullDomain, - ssl: resources.ssl, - blockAccess: resources.blockAccess, - sso: resources.sso, - emailWhitelistEnabled: resources.emailWhitelistEnabled, - http: resources.http, - proxyPort: resources.proxyPort, - protocol: resources.protocol, - // Targets as a subquery - targets: sql`json_group_array(json_object( - 'targetId', ${targets.targetId}, - 'ip', ${targets.ip}, - 'method', ${targets.method}, - 'port', ${targets.port}, - 'internalPort', ${targets.internalPort}, - 'enabled', ${targets.enabled} - ))`.as("targets") - }) - .from(resources) - .leftJoin( - targets, - and( - eq(targets.resourceId, resources.resourceId), - eq(targets.enabled, true) + // Improved version + const allResources = await db.transaction(async (tx) => { + // First get all resources for the site + const resourcesList = await tx + .select({ + resourceId: resources.resourceId, + subdomain: resources.subdomain, + fullDomain: resources.fullDomain, + ssl: resources.ssl, + blockAccess: resources.blockAccess, + sso: resources.sso, + emailWhitelistEnabled: resources.emailWhitelistEnabled, + http: resources.http, + proxyPort: resources.proxyPort, + protocol: resources.protocol + }) + .from(resources) + .where(eq(resources.siteId, siteId)); + + // Get all enabled targets for these resources in a single query + const resourceIds = resourcesList.map((r) => r.resourceId); + const allTargets = + resourceIds.length > 0 + ? await tx + .select({ + resourceId: targets.resourceId, + targetId: targets.targetId, + ip: targets.ip, + method: targets.method, + port: targets.port, + internalPort: targets.internalPort, + enabled: targets.enabled + }) + .from(targets) + .where( + and( + inArray(targets.resourceId, resourceIds), + eq(targets.enabled, true) + ) + ) + : []; + + // Combine the data in JS instead of using SQL for the JSON + return resourcesList.map((resource) => ({ + ...resource, + targets: allTargets.filter( + (target) => target.resourceId === resource.resourceId ) - ) - .where(eq(resources.siteId, siteId)) - .groupBy(resources.resourceId); + })); + }); - let tcpTargets: string[] = []; - let udpTargets: string[] = []; + const { tcpTargets, udpTargets } = allResources.reduce( + (acc, resource) => { + // Skip resources with no targets + if (!resource.targets?.length) return acc; - for (const resource of allResources) { - const targets = JSON.parse(resource.targets); - if (!targets || targets.length === 0) { - continue; - } - if (resource.protocol === "tcp") { - tcpTargets = tcpTargets.concat( - targets.map( + // Format valid targets into strings + const formattedTargets = resource.targets + .filter( (target: Target) => - `${ - target.internalPort ? target.internalPort + ":" : "" - }${target.ip}:${target.port}` + target?.internalPort && target?.ip && target?.port ) - ); - } else { - udpTargets = tcpTargets.concat( - targets.map( + .map( (target: Target) => - `${ - target.internalPort ? target.internalPort + ":" : "" - }${target.ip}:${target.port}` - ) - ); - } - } + `${target.internalPort}:${target.ip}:${target.port}` + ); + + // Add to the appropriate protocol array + if (resource.protocol === "tcp") { + acc.tcpTargets.push(...formattedTargets); + } else { + acc.udpTargets.push(...formattedTargets); + } + + return acc; + }, + { tcpTargets: [] as string[], udpTargets: [] as string[] } + ); return { message: { diff --git a/server/routers/traefik/getTraefikConfig.ts b/server/routers/traefik/getTraefikConfig.ts index 5f6f194f..c6488412 100644 --- a/server/routers/traefik/getTraefikConfig.ts +++ b/server/routers/traefik/getTraefikConfig.ts @@ -1,6 +1,6 @@ import { Request, Response } from "express"; import db from "@server/db"; -import { and, eq } from "drizzle-orm"; +import { and, eq, inArray } from "drizzle-orm"; import logger from "@server/logger"; import HttpCode from "@server/types/HttpCode"; import config from "@server/lib/config"; @@ -12,52 +12,79 @@ export async function traefikConfigProvider( res: Response ): Promise { try { - const allResources = await db - .select({ - // Resource fields - resourceId: resources.resourceId, - subdomain: resources.subdomain, - fullDomain: resources.fullDomain, - ssl: resources.ssl, - blockAccess: resources.blockAccess, - sso: resources.sso, - emailWhitelistEnabled: resources.emailWhitelistEnabled, - http: resources.http, - proxyPort: resources.proxyPort, - protocol: resources.protocol, - isBaseDomain: resources.isBaseDomain, - domainId: resources.domainId, - // Site fields - site: { - siteId: sites.siteId, - type: sites.type, - subnet: sites.subnet - }, - // Org fields - org: { - orgId: orgs.orgId - }, - // Targets as a subquery - targets: sql`json_group_array(json_object( - 'targetId', ${targets.targetId}, - 'ip', ${targets.ip}, - 'method', ${targets.method}, - 'port', ${targets.port}, - 'internalPort', ${targets.internalPort}, - 'enabled', ${targets.enabled} - ))`.as("targets") - }) - .from(resources) - .innerJoin(sites, eq(sites.siteId, resources.siteId)) - .innerJoin(orgs, eq(resources.orgId, orgs.orgId)) - .leftJoin( - targets, - and( - eq(targets.resourceId, resources.resourceId), - eq(targets.enabled, true) - ) - ) - .groupBy(resources.resourceId); + // Get all resources with related data + const allResources = await db.transaction(async (tx) => { + // First query to get resources with site and org info + const resourcesWithRelations = await tx + .select({ + // Resource fields + resourceId: resources.resourceId, + subdomain: resources.subdomain, + fullDomain: resources.fullDomain, + ssl: resources.ssl, + blockAccess: resources.blockAccess, + sso: resources.sso, + emailWhitelistEnabled: resources.emailWhitelistEnabled, + http: resources.http, + proxyPort: resources.proxyPort, + protocol: resources.protocol, + isBaseDomain: resources.isBaseDomain, + domainId: resources.domainId, + // Site fields + site: { + siteId: sites.siteId, + type: sites.type, + subnet: sites.subnet + }, + // Org fields + org: { + orgId: orgs.orgId + } + }) + .from(resources) + .innerJoin(sites, eq(sites.siteId, resources.siteId)) + .innerJoin(orgs, eq(resources.orgId, orgs.orgId)); + + // Get all resource IDs from the first query + const resourceIds = resourcesWithRelations.map((r) => r.resourceId); + + // Second query to get all enabled targets for these resources + const allTargets = + resourceIds.length > 0 + ? await tx + .select({ + resourceId: targets.resourceId, + targetId: targets.targetId, + ip: targets.ip, + method: targets.method, + port: targets.port, + internalPort: targets.internalPort, + enabled: targets.enabled + }) + .from(targets) + .where( + and( + inArray(targets.resourceId, resourceIds), + eq(targets.enabled, true) + ) + ) + : []; + + // Create a map for fast target lookup by resourceId + const targetsMap = allTargets.reduce((map, target) => { + if (!map.has(target.resourceId)) { + map.set(target.resourceId, []); + } + map.get(target.resourceId).push(target); + return map; + }, new Map()); + + // Combine the data + return resourcesWithRelations.map((resource) => ({ + ...resource, + targets: targetsMap.get(resource.resourceId) || [] + })); + }); if (!allResources.length) { return res.status(HttpCode.OK).json({}); @@ -101,7 +128,7 @@ export async function traefikConfigProvider( }; for (const resource of allResources) { - const targets = JSON.parse(resource.targets); + const targets = resource.targets as Target[]; const site = resource.site; const org = resource.org;