From fb49fb8ddd226b912ebf3fc44e82b00b1850ee45 Mon Sep 17 00:00:00 2001 From: Owen Date: Thu, 20 Feb 2025 18:10:52 -0500 Subject: [PATCH 001/135] Initial schema --- server/db/schema.ts | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/server/db/schema.ts b/server/db/schema.ts index 3380cdbf..6671152a 100644 --- a/server/db/schema.ts +++ b/server/db/schema.ts @@ -25,7 +25,14 @@ export const sites = sqliteTable("sites", { megabytesOut: integer("bytesOut"), lastBandwidthUpdate: text("lastBandwidthUpdate"), type: text("type").notNull(), // "newt" or "wireguard" - online: integer("online", { mode: "boolean" }).notNull().default(false) + online: integer("online", { mode: "boolean" }).notNull().default(false), + + // exit node stuff that is how to connect to the site when it has a gerbil + address: text("address"), // this is the address of the wireguard interface in gerbil + endpoint: text("endpoint"), // this is how to reach gerbil externally - gets put into the wireguard config + publicKey: text("pubicKey"), + listenPort: integer("listenPort"), + reachableAt: text("reachableAt") // this is the internal address of the gerbil http server for command control }); export const resources = sqliteTable("resources", { @@ -108,6 +115,15 @@ export const newts = sqliteTable("newt", { }) }); +export const clients = sqliteTable("clients", { + clientId: text("id").primaryKey(), + secretHash: text("secretHash").notNull(), + dateCreated: text("dateCreated").notNull(), + siteId: integer("siteId").references(() => sites.siteId, { + onDelete: "cascade" + }) +}); + export const twoFactorBackupCodes = sqliteTable("twoFactorBackupCodes", { codeId: integer("id").primaryKey({ autoIncrement: true }), userId: text("userId") @@ -132,6 +148,14 @@ export const newtSessions = sqliteTable("newtSession", { expiresAt: integer("expiresAt").notNull() }); +export const clientSessions = sqliteTable("clientSession", { + sessionId: text("id").primaryKey(), + clientId: text("clientId") + .notNull() + .references(() => newts.newtId, { onDelete: "cascade" }), + expiresAt: integer("expiresAt").notNull() +}); + export const userOrgs = sqliteTable("userOrgs", { userId: text("userId") .notNull() @@ -393,6 +417,8 @@ export type Target = InferSelectModel; export type Session = InferSelectModel; export type Newt = InferSelectModel; export type NewtSession = InferSelectModel; +export type Client = InferSelectModel; +export type ClientSession = InferSelectModel; export type EmailVerificationCode = InferSelectModel< typeof emailVerificationCodes >; From 41983ce3560f0af0d64cdc339db45300075a7404 Mon Sep 17 00:00:00 2001 From: miloschwartz Date: Thu, 20 Feb 2025 22:34:51 -0500 Subject: [PATCH 002/135] add wg site get config and pick client defaults --- server/auth/actions.ts | 1 + server/db/schema.ts | 14 +- server/lib/config.ts | 4 + server/routers/client/index.ts | 1 + server/routers/client/pickClientDefaults.ts | 128 +++++++++++++++ server/routers/external.ts | 9 ++ server/routers/gerbil/getConfig.ts | 2 +- server/routers/messageHandlers.ts | 4 +- server/routers/newt/handleGetConfigMessage.ts | 147 ++++++++++++++++++ 9 files changed, 305 insertions(+), 5 deletions(-) create mode 100644 server/routers/client/index.ts create mode 100644 server/routers/client/pickClientDefaults.ts create mode 100644 server/routers/newt/handleGetConfigMessage.ts diff --git a/server/auth/actions.ts b/server/auth/actions.ts index 001b9a6c..972ff1a7 100644 --- a/server/auth/actions.ts +++ b/server/auth/actions.ts @@ -62,6 +62,7 @@ export enum ActionsEnum { deleteResourceRule = "deleteResourceRule", listResourceRules = "listResourceRules", updateResourceRule = "updateResourceRule", + createClient = "createClient" } export async function checkUserActionPermission( diff --git a/server/db/schema.ts b/server/db/schema.ts index 6671152a..70817573 100644 --- a/server/db/schema.ts +++ b/server/db/schema.ts @@ -31,8 +31,7 @@ export const sites = sqliteTable("sites", { address: text("address"), // this is the address of the wireguard interface in gerbil endpoint: text("endpoint"), // this is how to reach gerbil externally - gets put into the wireguard config publicKey: text("pubicKey"), - listenPort: integer("listenPort"), - reachableAt: text("reachableAt") // this is the internal address of the gerbil http server for command control + listenPort: integer("listenPort") }); export const resources = sqliteTable("resources", { @@ -121,7 +120,16 @@ export const clients = sqliteTable("clients", { dateCreated: text("dateCreated").notNull(), siteId: integer("siteId").references(() => sites.siteId, { onDelete: "cascade" - }) + }), + + // wgstuff + pubKey: text("pubKey"), + subnet: text("subnet").notNull(), + megabytesIn: integer("bytesIn"), + megabytesOut: integer("bytesOut"), + lastBandwidthUpdate: text("lastBandwidthUpdate"), + type: text("type").notNull(), // "newt" or "wireguard" + online: integer("online", { mode: "boolean" }).notNull().default(false), }); export const twoFactorBackupCodes = sqliteTable("twoFactorBackupCodes", { diff --git a/server/lib/config.ts b/server/lib/config.ts index 7c5ad227..fc1c0531 100644 --- a/server/lib/config.ts +++ b/server/lib/config.ts @@ -109,6 +109,10 @@ const configSchema = z.object({ block_size: z.number().positive().gt(0), site_block_size: z.number().positive().gt(0) }), + wg_site: z.object({ + block_size: z.number().positive().gt(0), + subnet_group: z.string(), + }), rate_limits: z.object({ global: z.object({ window_minutes: z.number().positive().gt(0), diff --git a/server/routers/client/index.ts b/server/routers/client/index.ts new file mode 100644 index 00000000..5b493724 --- /dev/null +++ b/server/routers/client/index.ts @@ -0,0 +1 @@ +export * from "./pickClientDefaults"; diff --git a/server/routers/client/pickClientDefaults.ts b/server/routers/client/pickClientDefaults.ts new file mode 100644 index 00000000..eb765fc2 --- /dev/null +++ b/server/routers/client/pickClientDefaults.ts @@ -0,0 +1,128 @@ +import { Request, Response, NextFunction } from "express"; +import { db } from "@server/db"; +import { clients, sites } from "@server/db/schema"; +import { eq } from "drizzle-orm"; +import response from "@server/lib/response"; +import HttpCode from "@server/types/HttpCode"; +import createHttpError from "http-errors"; +import logger from "@server/logger"; +import { findNextAvailableCidr } from "@server/lib/ip"; +import { generateId } from "@server/auth/sessions/app"; +import config from "@server/lib/config"; +import { z } from "zod"; +import { fromError } from "zod-validation-error"; + +const getSiteSchema = z + .object({ + siteId: z.number().int().positive() + }) + .strict(); + +export type PickClientDefaultsResponse = { + siteId: number; + address: string; + publicKey: string; + name: string; + listenPort: number; + endpoint: string; + subnet: string; + clientId: string; + clientSecret: string; +}; + +export async function pickClientDefaults( + req: Request, + res: Response, + next: NextFunction +): Promise { + try { + const parsedParams = getSiteSchema.safeParse(req.params); + if (!parsedParams.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedParams.error).toString() + ) + ); + } + + const { siteId } = parsedParams.data; + + const [site] = await db + .select() + .from(sites) + .where(eq(sites.siteId, siteId)); + + if (!site) { + return next(createHttpError(HttpCode.NOT_FOUND, "Site not found")); + } + + // make sure all the required fields are present + if ( + !site.address || + !site.publicKey || + !site.listenPort || + !site.endpoint + ) { + return next( + createHttpError(HttpCode.BAD_REQUEST, "Site has no address") + ); + } + + const clientsQuery = await db + .select({ + subnet: clients.subnet + }) + .from(clients) + .where(eq(clients.siteId, site.siteId)); + + let subnets = clientsQuery.map((client) => client.subnet); + + // exclude the exit node address by replacing after the / with a site block size + subnets.push( + site.address.replace( + /\/\d+$/, + `/${config.getRawConfig().wg_site.block_size}` + ) + ); + const newSubnet = findNextAvailableCidr( + subnets, + config.getRawConfig().wg_site.block_size, + site.address + ); + if (!newSubnet) { + return next( + createHttpError( + HttpCode.INTERNAL_SERVER_ERROR, + "No available subnets" + ) + ); + } + + const clientId = generateId(15); + const secret = generateId(48); + + return response(res, { + data: { + siteId: site.siteId, + address: site.address, + publicKey: site.publicKey, + name: site.name, + listenPort: site.listenPort, + endpoint: site.endpoint, + subnet: newSubnet, + clientId, + clientSecret: secret + }, + success: true, + error: false, + message: "Organization retrieved successfully", + status: HttpCode.OK + }); + } catch (error) { + logger.error(error); + return next( + createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred") + ); + } +} diff --git a/server/routers/external.ts b/server/routers/external.ts index 19c57008..778bf288 100644 --- a/server/routers/external.ts +++ b/server/routers/external.ts @@ -7,6 +7,7 @@ import * as target from "./target"; import * as user from "./user"; import * as auth from "./auth"; import * as role from "./role"; +import * as client from "./client"; import * as accessToken from "./accessToken"; import HttpCode from "@server/types/HttpCode"; import { @@ -94,6 +95,14 @@ authenticated.get( verifyUserHasAction(ActionsEnum.getSite), site.getSite ); + +authenticated.get( + "/site/:siteId/pick-client-defaults", + verifyOrgAccess, + verifyUserHasAction(ActionsEnum.createClient), + client.pickClientDefaults +); + // authenticated.get( // "/site/:siteId/roles", // verifySiteAccess, diff --git a/server/routers/gerbil/getConfig.ts b/server/routers/gerbil/getConfig.ts index 28b576d8..95e0df6b 100644 --- a/server/routers/gerbil/getConfig.ts +++ b/server/routers/gerbil/getConfig.ts @@ -86,7 +86,7 @@ export async function getConfig(req: Request, res: Response, next: NextFunction) const peers = await Promise.all(sitesRes.map(async (site) => { return { publicKey: site.pubKey, - allowedIps: await getAllowedIps(site.siteId) + allowedIps: await getAllowedIps(site.siteId) // put 0.0.0.0/0 for now }; })); diff --git a/server/routers/messageHandlers.ts b/server/routers/messageHandlers.ts index 9dd7756f..262f9869 100644 --- a/server/routers/messageHandlers.ts +++ b/server/routers/messageHandlers.ts @@ -1,6 +1,8 @@ import { handleRegisterMessage } from "./newt"; +import { handleGetConfigMessage } from "./newt/handleGetConfigMessage"; import { MessageHandler } from "./ws"; export const messageHandlers: Record = { "newt/wg/register": handleRegisterMessage, -}; \ No newline at end of file + "newt/wg/get-config": handleGetConfigMessage, +}; diff --git a/server/routers/newt/handleGetConfigMessage.ts b/server/routers/newt/handleGetConfigMessage.ts new file mode 100644 index 00000000..17ac63dd --- /dev/null +++ b/server/routers/newt/handleGetConfigMessage.ts @@ -0,0 +1,147 @@ +import { z } from "zod"; +import { MessageHandler } from "../ws"; +import logger from "@server/logger"; +import { fromError } from "zod-validation-error"; +import db from "@server/db"; +import { clients, Site, sites } from "@server/db/schema"; +import { eq, isNotNull } from "drizzle-orm"; +import { findNextAvailableCidr } from "@server/lib/ip"; +import config from "@server/lib/config"; + +const inputSchema = z.object({ + publicKey: z.string(), + endpoint: z.string(), + listenPort: z.number() +}); + +type Input = z.infer; + +export const handleGetConfigMessage: MessageHandler = async (context) => { + const { message, newt, sendToClient } = context; + + logger.debug("Handling Newt get config message!"); + + if (!newt) { + logger.warn("Newt not found"); + return; + } + + if (!newt.siteId) { + logger.warn("Newt has no site!"); // TODO: Maybe we create the site here? + return; + } + + const parsed = inputSchema.safeParse(message.data); + if (!parsed.success) { + logger.error( + "handleGetConfigMessage: Invalid input: " + + fromError(parsed.error).toString() + ); + return; + } + + const { publicKey, endpoint, listenPort } = message.data as Input; + + const siteId = newt.siteId; + + const [siteRes] = await db + .select() + .from(sites) + .where(eq(sites.siteId, siteId)); + + if (!siteRes) { + logger.warn("handleGetConfigMessage: Site not found"); + return; + } + + let site: Site | undefined; + if (!site) { + const address = await getNextAvailableSubnet(); + + // create a new exit node + const [updateRes] = await db + .update(sites) + .set({ + publicKey, + endpoint, + address, + listenPort + }) + .where(eq(sites.siteId, siteId)) + .returning(); + + site = updateRes; + + logger.info(`Updated site ${siteId} with new WG Newt info`); + } else { + site = siteRes; + } + + if (!site) { + logger.error("handleGetConfigMessage: Failed to update site"); + return; + } + + const clientsRes = await db + .select() + .from(clients) + .where(eq(clients.siteId, siteId)); + + const peers = await Promise.all( + clientsRes.map(async (client) => { + return { + publicKey: client.pubKey, + allowedIps: "0.0.0.0/0" + }; + }) + ); + + const configResponse = { + listenPort: site.listenPort, // ????? + // ipAddress: exitNode[0].address, + peers + }; + + logger.debug("Sending config: ", configResponse); + + return { + message: { + type: "newt/wg/connect", // what to make the response type? + data: { + config: configResponse + } + }, + broadcast: false, // Send to all clients + excludeSender: false // Include sender in broadcast + }; +}; + +async function getNextAvailableSubnet(): Promise { + const existingAddresses = await db + .select({ + address: sites.address + }) + .from(sites) + .where(isNotNull(sites.address)); + + const addresses = existingAddresses + .map((a) => a.address) + .filter((a) => a) as string[]; + + let subnet = findNextAvailableCidr( + addresses, + config.getRawConfig().wg_site.block_size, + config.getRawConfig().wg_site.subnet_group + ); + if (!subnet) { + throw new Error("No available subnets remaining in space"); + } + + // replace the last octet with 1 + subnet = + subnet.split(".").slice(0, 3).join(".") + + ".1" + + "/" + + subnet.split("/")[1]; + return subnet; +} From e112fcba29ab8b3b7cfc59ae07ed9d39da7b3f65 Mon Sep 17 00:00:00 2001 From: Owen Date: Fri, 21 Feb 2025 10:13:41 -0500 Subject: [PATCH 003/135] Move things around; rename to olm --- server/auth/sessions/olm.ts | 72 ++++++++ server/db/schema.ts | 12 +- server/routers/client/index.ts | 1 - server/routers/external.ts | 4 +- server/routers/messageHandlers.ts | 6 +- ...essage.ts => handleNewtRegisterMessage.ts} | 6 +- server/routers/olm/createOlm.ts | 106 ++++++++++++ server/routers/olm/getToken.ts | 115 +++++++++++++ server/routers/olm/handleGetConfigMessage.ts | 147 ++++++++++++++++ .../routers/olm/handleOlmRegisterMessage.ts | 93 ++++++++++ server/routers/olm/index.ts | 1 + .../pickOlmDefaults.ts} | 10 +- server/routers/ws.ts | 162 ++++++++++-------- 13 files changed, 642 insertions(+), 93 deletions(-) create mode 100644 server/auth/sessions/olm.ts delete mode 100644 server/routers/client/index.ts rename server/routers/newt/{handleRegisterMessage.ts => handleNewtRegisterMessage.ts} (96%) create mode 100644 server/routers/olm/createOlm.ts create mode 100644 server/routers/olm/getToken.ts create mode 100644 server/routers/olm/handleGetConfigMessage.ts create mode 100644 server/routers/olm/handleOlmRegisterMessage.ts create mode 100644 server/routers/olm/index.ts rename server/routers/{client/pickClientDefaults.ts => olm/pickOlmDefaults.ts} (94%) diff --git a/server/auth/sessions/olm.ts b/server/auth/sessions/olm.ts new file mode 100644 index 00000000..8d24c16f --- /dev/null +++ b/server/auth/sessions/olm.ts @@ -0,0 +1,72 @@ +import { + encodeHexLowerCase, +} from "@oslojs/encoding"; +import { sha256 } from "@oslojs/crypto/sha2"; +import { Olm, olms, olmSessions, OlmSession } from "@server/db/schema"; +import db from "@server/db"; +import { eq } from "drizzle-orm"; + +export const EXPIRES = 1000 * 60 * 60 * 24 * 30; + +export async function createOlmSession( + token: string, + olmId: string, +): Promise { + const sessionId = encodeHexLowerCase( + sha256(new TextEncoder().encode(token)), + ); + const session: OlmSession = { + sessionId: sessionId, + olmId, + expiresAt: new Date(Date.now() + EXPIRES).getTime(), + }; + await db.insert(olmSessions).values(session); + return session; +} + +export async function validateOlmSessionToken( + token: string, +): Promise { + const sessionId = encodeHexLowerCase( + sha256(new TextEncoder().encode(token)), + ); + const result = await db + .select({ olm: olms, session: olmSessions }) + .from(olmSessions) + .innerJoin(olms, eq(olmSessions.olmId, olms.olmId)) + .where(eq(olmSessions.sessionId, sessionId)); + if (result.length < 1) { + return { session: null, olm: null }; + } + const { olm, session } = result[0]; + if (Date.now() >= session.expiresAt) { + await db + .delete(olmSessions) + .where(eq(olmSessions.sessionId, session.sessionId)); + return { session: null, olm: null }; + } + if (Date.now() >= session.expiresAt - (EXPIRES / 2)) { + session.expiresAt = new Date( + Date.now() + EXPIRES, + ).getTime(); + await db + .update(olmSessions) + .set({ + expiresAt: session.expiresAt, + }) + .where(eq(olmSessions.sessionId, session.sessionId)); + } + return { session, olm }; +} + +export async function invalidateOlmSession(sessionId: string): Promise { + await db.delete(olmSessions).where(eq(olmSessions.sessionId, sessionId)); +} + +export async function invalidateAllOlmSessions(olmId: string): Promise { + await db.delete(olmSessions).where(eq(olmSessions.olmId, olmId)); +} + +export type SessionValidationResult = + | { session: OlmSession; olm: Olm } + | { session: null; olm: null }; diff --git a/server/db/schema.ts b/server/db/schema.ts index 70817573..9564fdff 100644 --- a/server/db/schema.ts +++ b/server/db/schema.ts @@ -114,8 +114,8 @@ export const newts = sqliteTable("newt", { }) }); -export const clients = sqliteTable("clients", { - clientId: text("id").primaryKey(), +export const olms = sqliteTable("olms", { + olmId: text("id").primaryKey(), secretHash: text("secretHash").notNull(), dateCreated: text("dateCreated").notNull(), siteId: integer("siteId").references(() => sites.siteId, { @@ -156,9 +156,9 @@ export const newtSessions = sqliteTable("newtSession", { expiresAt: integer("expiresAt").notNull() }); -export const clientSessions = sqliteTable("clientSession", { +export const olmSessions = sqliteTable("clientSession", { sessionId: text("id").primaryKey(), - clientId: text("clientId") + olmId: text("olmId") .notNull() .references(() => newts.newtId, { onDelete: "cascade" }), expiresAt: integer("expiresAt").notNull() @@ -425,8 +425,8 @@ export type Target = InferSelectModel; export type Session = InferSelectModel; export type Newt = InferSelectModel; export type NewtSession = InferSelectModel; -export type Client = InferSelectModel; -export type ClientSession = InferSelectModel; +export type Olm = InferSelectModel; +export type OlmSession = InferSelectModel; export type EmailVerificationCode = InferSelectModel< typeof emailVerificationCodes >; diff --git a/server/routers/client/index.ts b/server/routers/client/index.ts deleted file mode 100644 index 5b493724..00000000 --- a/server/routers/client/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from "./pickClientDefaults"; diff --git a/server/routers/external.ts b/server/routers/external.ts index 778bf288..cb5fbfd7 100644 --- a/server/routers/external.ts +++ b/server/routers/external.ts @@ -7,7 +7,7 @@ import * as target from "./target"; import * as user from "./user"; import * as auth from "./auth"; import * as role from "./role"; -import * as client from "./client"; +import * as olm from "./olm"; import * as accessToken from "./accessToken"; import HttpCode from "@server/types/HttpCode"; import { @@ -100,7 +100,7 @@ authenticated.get( "/site/:siteId/pick-client-defaults", verifyOrgAccess, verifyUserHasAction(ActionsEnum.createClient), - client.pickClientDefaults + olm.pickOlmDefaults ); // authenticated.get( diff --git a/server/routers/messageHandlers.ts b/server/routers/messageHandlers.ts index 262f9869..bf8f357c 100644 --- a/server/routers/messageHandlers.ts +++ b/server/routers/messageHandlers.ts @@ -1,8 +1,10 @@ -import { handleRegisterMessage } from "./newt"; +import { handleNewtRegisterMessage } from "./newt"; +import { handleOlmRegisterMessage } from "./olm"; import { handleGetConfigMessage } from "./newt/handleGetConfigMessage"; import { MessageHandler } from "./ws"; export const messageHandlers: Record = { - "newt/wg/register": handleRegisterMessage, + "newt/wg/register": handleNewtRegisterMessage, + "olm/wg/register": handleOlmRegisterMessage, "newt/wg/get-config": handleGetConfigMessage, }; diff --git a/server/routers/newt/handleRegisterMessage.ts b/server/routers/newt/handleNewtRegisterMessage.ts similarity index 96% rename from server/routers/newt/handleRegisterMessage.ts rename to server/routers/newt/handleNewtRegisterMessage.ts index 0f086698..8e263034 100644 --- a/server/routers/newt/handleRegisterMessage.ts +++ b/server/routers/newt/handleNewtRegisterMessage.ts @@ -11,8 +11,10 @@ import { eq, and, sql } from "drizzle-orm"; import { addPeer, deletePeer } from "../gerbil/peers"; import logger from "@server/logger"; -export const handleRegisterMessage: MessageHandler = async (context) => { - const { message, newt, sendToClient } = context; +export const handleNewtRegisterMessage: MessageHandler = async (context) => { + const { message, client, sendToClient } = context; + + const newt = client; logger.info("Handling register message!"); diff --git a/server/routers/olm/createOlm.ts b/server/routers/olm/createOlm.ts new file mode 100644 index 00000000..d43c4cc6 --- /dev/null +++ b/server/routers/olm/createOlm.ts @@ -0,0 +1,106 @@ +import { NextFunction, Request, Response } from "express"; +import db from "@server/db"; +import { hash } from "@node-rs/argon2"; +import HttpCode from "@server/types/HttpCode"; +import { z } from "zod"; +import { newts } from "@server/db/schema"; +import createHttpError from "http-errors"; +import response from "@server/lib/response"; +import { SqliteError } from "better-sqlite3"; +import moment from "moment"; +import { generateSessionToken } from "@server/auth/sessions/app"; +import { createNewtSession } from "@server/auth/sessions/newt"; +import { fromError } from "zod-validation-error"; +import { hashPassword } from "@server/auth/password"; + +export const createNewtBodySchema = z.object({}); + +export type CreateNewtBody = z.infer; + +export type CreateNewtResponse = { + token: string; + newtId: string; + secret: string; +}; + +const createNewtSchema = z + .object({ + newtId: z.string(), + secret: z.string() + }) + .strict(); + +export async function createNewt( + req: Request, + res: Response, + next: NextFunction +): Promise { + try { + + const parsedBody = createNewtSchema.safeParse(req.body); + if (!parsedBody.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedBody.error).toString() + ) + ); + } + + const { newtId, secret } = parsedBody.data; + + if (!req.userOrgRoleId) { + return next( + createHttpError(HttpCode.FORBIDDEN, "User does not have a role") + ); + } + + const secretHash = await hashPassword(secret); + + await db.insert(newts).values({ + newtId: newtId, + secretHash, + dateCreated: moment().toISOString(), + }); + + // give the newt their default permissions: + // await db.insert(newtActions).values({ + // newtId: newtId, + // actionId: ActionsEnum.createOrg, + // orgId: null, + // }); + + const token = generateSessionToken(); + await createNewtSession(token, newtId); + + return response(res, { + data: { + newtId, + secret, + token, + }, + success: true, + error: false, + message: "Newt created successfully", + status: HttpCode.OK, + }); + } catch (e) { + if (e instanceof SqliteError && e.code === "SQLITE_CONSTRAINT_UNIQUE") { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + "A newt with that email address already exists" + ) + ); + } else { + console.error(e); + + return next( + createHttpError( + HttpCode.INTERNAL_SERVER_ERROR, + "Failed to create newt" + ) + ); + } + } +} diff --git a/server/routers/olm/getToken.ts b/server/routers/olm/getToken.ts new file mode 100644 index 00000000..e6ae0cd6 --- /dev/null +++ b/server/routers/olm/getToken.ts @@ -0,0 +1,115 @@ +import { generateSessionToken } from "@server/auth/sessions/app"; +import db from "@server/db"; +import { newts } from "@server/db/schema"; +import HttpCode from "@server/types/HttpCode"; +import response from "@server/lib/response"; +import { eq } from "drizzle-orm"; +import { NextFunction, Request, Response } from "express"; +import createHttpError from "http-errors"; +import { z } from "zod"; +import { fromError } from "zod-validation-error"; +import { + createNewtSession, + validateNewtSessionToken +} from "@server/auth/sessions/newt"; +import { verifyPassword } from "@server/auth/password"; +import logger from "@server/logger"; +import config from "@server/lib/config"; + +export const newtGetTokenBodySchema = z.object({ + newtId: z.string(), + secret: z.string(), + token: z.string().optional() +}); + +export type NewtGetTokenBody = z.infer; + +export async function getToken( + req: Request, + res: Response, + next: NextFunction +): Promise { + const parsedBody = newtGetTokenBodySchema.safeParse(req.body); + + if (!parsedBody.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedBody.error).toString() + ) + ); + } + + const { newtId, secret, token } = parsedBody.data; + + try { + if (token) { + const { session, newt } = await validateNewtSessionToken(token); + if (session) { + if (config.getRawConfig().app.log_failed_attempts) { + logger.info( + `Newt session already valid. Newt ID: ${newtId}. IP: ${req.ip}.` + ); + } + return response(res, { + data: null, + success: true, + error: false, + message: "Token session already valid", + status: HttpCode.OK + }); + } + } + + const existingNewtRes = await db + .select() + .from(newts) + .where(eq(newts.newtId, newtId)); + if (!existingNewtRes || !existingNewtRes.length) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + "No newt found with that newtId" + ) + ); + } + + const existingNewt = existingNewtRes[0]; + + const validSecret = await verifyPassword( + secret, + existingNewt.secretHash + ); + if (!validSecret) { + if (config.getRawConfig().app.log_failed_attempts) { + logger.info( + `Newt id or secret is incorrect. Newt: ID ${newtId}. IP: ${req.ip}.` + ); + } + return next( + createHttpError(HttpCode.BAD_REQUEST, "Secret is incorrect") + ); + } + + const resToken = generateSessionToken(); + await createNewtSession(resToken, existingNewt.newtId); + + return response<{ token: string }>(res, { + data: { + token: resToken + }, + success: true, + error: false, + message: "Token created successfully", + status: HttpCode.OK + }); + } catch (e) { + console.error(e); + return next( + createHttpError( + HttpCode.INTERNAL_SERVER_ERROR, + "Failed to authenticate newt" + ) + ); + } +} diff --git a/server/routers/olm/handleGetConfigMessage.ts b/server/routers/olm/handleGetConfigMessage.ts new file mode 100644 index 00000000..6e4f7ebf --- /dev/null +++ b/server/routers/olm/handleGetConfigMessage.ts @@ -0,0 +1,147 @@ +import { z } from "zod"; +import { MessageHandler } from "../ws"; +import logger from "@server/logger"; +import { fromError } from "zod-validation-error"; +import db from "@server/db"; +import { olms, Site, sites } from "@server/db/schema"; +import { eq, isNotNull } from "drizzle-orm"; +import { findNextAvailableCidr } from "@server/lib/ip"; +import config from "@server/lib/config"; + +const inputSchema = z.object({ + publicKey: z.string(), + endpoint: z.string(), + listenPort: z.number() +}); + +type Input = z.infer; + +export const handleGetConfigMessage: MessageHandler = async (context) => { + const { message, newt, sendToClient } = context; + + logger.debug("Handling Newt get config message!"); + + if (!newt) { + logger.warn("Newt not found"); + return; + } + + if (!newt.siteId) { + logger.warn("Newt has no site!"); // TODO: Maybe we create the site here? + return; + } + + const parsed = inputSchema.safeParse(message.data); + if (!parsed.success) { + logger.error( + "handleGetConfigMessage: Invalid input: " + + fromError(parsed.error).toString() + ); + return; + } + + const { publicKey, endpoint, listenPort } = message.data as Input; + + const siteId = newt.siteId; + + const [siteRes] = await db + .select() + .from(sites) + .where(eq(sites.siteId, siteId)); + + if (!siteRes) { + logger.warn("handleGetConfigMessage: Site not found"); + return; + } + + let site: Site | undefined; + if (!site) { + const address = await getNextAvailableSubnet(); + + // create a new exit node + const [updateRes] = await db + .update(sites) + .set({ + publicKey, + endpoint, + address, + listenPort + }) + .where(eq(sites.siteId, siteId)) + .returning(); + + site = updateRes; + + logger.info(`Updated site ${siteId} with new WG Newt info`); + } else { + site = siteRes; + } + + if (!site) { + logger.error("handleGetConfigMessage: Failed to update site"); + return; + } + + const clientsRes = await db + .select() + .from(olms) + .where(eq(olms.siteId, siteId)); + + const peers = await Promise.all( + clientsRes.map(async (client) => { + return { + publicKey: client.pubKey, + allowedIps: "0.0.0.0/0" + }; + }) + ); + + const configResponse = { + listenPort: site.listenPort, // ????? + // ipAddress: exitNode[0].address, + peers + }; + + logger.debug("Sending config: ", configResponse); + + return { + message: { + type: "olm/wg/connect", // what to make the response type? + data: { + config: configResponse + } + }, + broadcast: false, // Send to all clients + excludeSender: false // Include sender in broadcast + }; +}; + +async function getNextAvailableSubnet(): Promise { + const existingAddresses = await db + .select({ + address: sites.address + }) + .from(sites) + .where(isNotNull(sites.address)); + + const addresses = existingAddresses + .map((a) => a.address) + .filter((a) => a) as string[]; + + let subnet = findNextAvailableCidr( + addresses, + config.getRawConfig().wg_site.block_size, + config.getRawConfig().wg_site.subnet_group + ); + if (!subnet) { + throw new Error("No available subnets remaining in space"); + } + + // replace the last octet with 1 + subnet = + subnet.split(".").slice(0, 3).join(".") + + ".1" + + "/" + + subnet.split("/")[1]; + return subnet; +} diff --git a/server/routers/olm/handleOlmRegisterMessage.ts b/server/routers/olm/handleOlmRegisterMessage.ts new file mode 100644 index 00000000..33786f2d --- /dev/null +++ b/server/routers/olm/handleOlmRegisterMessage.ts @@ -0,0 +1,93 @@ +import db from "@server/db"; +import { MessageHandler } from "../ws"; +import { + exitNodes, + resources, + sites, + Target, + targets +} from "@server/db/schema"; +import { eq, and, sql } from "drizzle-orm"; +import { addPeer, deletePeer } from "../gerbil/peers"; +import logger from "@server/logger"; + +export const handleOlmRegisterMessage: MessageHandler = async (context) => { + const { message, client, sendToClient } = context; + + const olm = client; + + logger.info("Handling register message!"); + + if (!olm) { + logger.warn("Olm not found"); + return; + } + + if (!olm.siteId) { + logger.warn("Olm has no site!"); // TODO: Maybe we create the site here? + return; + } + + const siteId = olm.siteId; + + const { publicKey } = message.data; + if (!publicKey) { + logger.warn("Public key not provided"); + return; + } + + const [site] = await db + .select() + .from(sites) + .where(eq(sites.siteId, siteId)) + .limit(1); + + if (!site || !site.exitNodeId) { + logger.warn("Site not found or does not have exit node"); + return; + } + + await db + .update(sites) + .set({ + pubKey: publicKey + }) + .where(eq(sites.siteId, siteId)) + .returning(); + + const [exitNode] = await db + .select() + .from(exitNodes) + .where(eq(exitNodes.exitNodeId, site.exitNodeId)) + .limit(1); + + if (site.pubKey && site.pubKey !== publicKey) { + logger.info("Public key mismatch. Deleting old peer..."); + await deletePeer(site.exitNodeId, site.pubKey); + } + + if (!site.subnet) { + logger.warn("Site has no subnet"); + return; + } + + // add the peer to the exit node + await addPeer(site.exitNodeId, { + publicKey: publicKey, + allowedIps: [site.subnet] + }); + + return { + message: { + type: "olm/wg/connect", + data: { + endpoint: `${exitNode.endpoint}:${exitNode.listenPort}`, + publicKey: exitNode.publicKey, + serverIP: exitNode.address.split("/")[0], + tunnelIP: site.subnet.split("/")[0] + } + }, + broadcast: false, // Send to all olms + excludeSender: false // Include sender in broadcast + }; +}; diff --git a/server/routers/olm/index.ts b/server/routers/olm/index.ts new file mode 100644 index 00000000..7265331b --- /dev/null +++ b/server/routers/olm/index.ts @@ -0,0 +1 @@ +export * from "./pickOlmDefaults"; \ No newline at end of file diff --git a/server/routers/client/pickClientDefaults.ts b/server/routers/olm/pickOlmDefaults.ts similarity index 94% rename from server/routers/client/pickClientDefaults.ts rename to server/routers/olm/pickOlmDefaults.ts index eb765fc2..24ddcace 100644 --- a/server/routers/client/pickClientDefaults.ts +++ b/server/routers/olm/pickOlmDefaults.ts @@ -1,6 +1,6 @@ import { Request, Response, NextFunction } from "express"; import { db } from "@server/db"; -import { clients, sites } from "@server/db/schema"; +import { olms, sites } from "@server/db/schema"; import { eq } from "drizzle-orm"; import response from "@server/lib/response"; import HttpCode from "@server/types/HttpCode"; @@ -30,7 +30,7 @@ export type PickClientDefaultsResponse = { clientSecret: string; }; -export async function pickClientDefaults( +export async function pickOlmDefaults( req: Request, res: Response, next: NextFunction @@ -71,10 +71,10 @@ export async function pickClientDefaults( const clientsQuery = await db .select({ - subnet: clients.subnet + subnet: olms.subnet }) - .from(clients) - .where(eq(clients.siteId, site.siteId)); + .from(olms) + .where(eq(olms.siteId, site.siteId)); let subnets = clientsQuery.map((client) => client.subnet); diff --git a/server/routers/ws.ts b/server/routers/ws.ts index afe422d0..1c24f48e 100644 --- a/server/routers/ws.ts +++ b/server/routers/ws.ts @@ -3,10 +3,11 @@ import { Server as HttpServer } from "http"; import { WebSocket, WebSocketServer } from "ws"; import { IncomingMessage } from "http"; import { Socket } from "net"; -import { Newt, newts, NewtSession } from "@server/db/schema"; +import { Newt, newts, NewtSession, Olm, olms, OlmSession } from "@server/db/schema"; import { eq } from "drizzle-orm"; import db from "@server/db"; import { validateNewtSessionToken } from "@server/auth/sessions/newt"; +import { validateOlmSessionToken } from "@server/auth/sessions/olm"; import { messageHandlers } from "./messageHandlers"; import logger from "@server/logger"; @@ -15,13 +16,17 @@ interface WebSocketRequest extends IncomingMessage { token?: string; } +type ClientType = 'newt' | 'olm'; + interface AuthenticatedWebSocket extends WebSocket { - newt?: Newt; + client?: Newt | Olm; + clientType?: ClientType; } interface TokenPayload { - newt: Newt; - session: NewtSession; + client: Newt | Olm; + session: NewtSession | OlmSession; + clientType: ClientType; } interface WSMessage { @@ -33,15 +38,16 @@ interface HandlerResponse { message: WSMessage; broadcast?: boolean; excludeSender?: boolean; - targetNewtId?: string; + targetClientId?: string; } interface HandlerContext { message: WSMessage; senderWs: WebSocket; - newt: Newt | undefined; - sendToClient: (newtId: string, message: WSMessage) => boolean; - broadcastToAllExcept: (message: WSMessage, excludeNewtId?: string) => void; + client: Newt | Olm | undefined; + clientType: ClientType; + sendToClient: (clientId: string, message: WSMessage) => boolean; + broadcastToAllExcept: (message: WSMessage, excludeClientId?: string) => void; connectedClients: Map; } @@ -54,34 +60,32 @@ const wss: WebSocketServer = new WebSocketServer({ noServer: true }); let connectedClients: Map = new Map(); // Helper functions for client management -const addClient = (newtId: string, ws: AuthenticatedWebSocket): void => { - const existingClients = connectedClients.get(newtId) || []; +const addClient = (clientId: string, ws: AuthenticatedWebSocket, clientType: ClientType): void => { + const existingClients = connectedClients.get(clientId) || []; existingClients.push(ws); - connectedClients.set(newtId, existingClients); - logger.info(`Client added to tracking - Newt ID: ${newtId}, Total connections: ${existingClients.length}`); + connectedClients.set(clientId, existingClients); + logger.info(`Client added to tracking - ${clientType.toUpperCase()} ID: ${clientId}, Total connections: ${existingClients.length}`); }; -const removeClient = (newtId: string, ws: AuthenticatedWebSocket): void => { - const existingClients = connectedClients.get(newtId) || []; +const removeClient = (clientId: string, ws: AuthenticatedWebSocket, clientType: ClientType): void => { + const existingClients = connectedClients.get(clientId) || []; const updatedClients = existingClients.filter(client => client !== ws); - if (updatedClients.length === 0) { - connectedClients.delete(newtId); - logger.info(`All connections removed for Newt ID: ${newtId}`); + connectedClients.delete(clientId); + logger.info(`All connections removed for ${clientType.toUpperCase()} ID: ${clientId}`); } else { - connectedClients.set(newtId, updatedClients); - logger.info(`Connection removed - Newt ID: ${newtId}, Remaining connections: ${updatedClients.length}`); + connectedClients.set(clientId, updatedClients); + logger.info(`Connection removed - ${clientType.toUpperCase()} ID: ${clientId}, Remaining connections: ${updatedClients.length}`); } }; // Helper functions for sending messages -const sendToClient = (newtId: string, message: WSMessage): boolean => { - const clients = connectedClients.get(newtId); +const sendToClient = (clientId: string, message: WSMessage): boolean => { + const clients = connectedClients.get(clientId); if (!clients || clients.length === 0) { - logger.info(`No active connections found for Newt ID: ${newtId}`); + logger.info(`No active connections found for Client ID: ${clientId}`); return false; } - const messageString = JSON.stringify(message); clients.forEach(client => { if (client.readyState === WebSocket.OPEN) { @@ -91,9 +95,9 @@ const sendToClient = (newtId: string, message: WSMessage): boolean => { return true; }; -const broadcastToAllExcept = (message: WSMessage, excludeNewtId?: string): void => { - connectedClients.forEach((clients, newtId) => { - if (newtId !== excludeNewtId) { +const broadcastToAllExcept = (message: WSMessage, excludeClientId?: string): void => { + connectedClients.forEach((clients, clientId) => { + if (clientId !== excludeClientId) { clients.forEach(client => { if (client.readyState === WebSocket.OPEN) { client.send(JSON.stringify(message)); @@ -103,84 +107,88 @@ const broadcastToAllExcept = (message: WSMessage, excludeNewtId?: string): void }); }; -// Token verification middleware (unchanged) -const verifyToken = async (token: string): Promise => { +// Token verification middleware +const verifyToken = async (token: string, clientType: ClientType): Promise => { try { - const { session, newt } = await validateNewtSessionToken(token); - - if (!session || !newt) { - return null; + if (clientType === 'newt') { + const { session, newt } = await validateNewtSessionToken(token); + if (!session || !newt) { + return null; + } + const existingNewt = await db + .select() + .from(newts) + .where(eq(newts.newtId, newt.newtId)); + if (!existingNewt || !existingNewt[0]) { + return null; + } + return { client: existingNewt[0], session, clientType }; + } else { + const { session, olm } = await validateOlmSessionToken(token); + if (!session || !olm) { + return null; + } + const existingOlm = await db + .select() + .from(olms) + .where(eq(olms.olmId, olm.olmId)); + if (!existingOlm || !existingOlm[0]) { + return null; + } + return { client: existingOlm[0], session, clientType }; } - - const existingNewt = await db - .select() - .from(newts) - .where(eq(newts.newtId, newt.newtId)); - - if (!existingNewt || !existingNewt[0]) { - return null; - } - - return { newt: existingNewt[0], session }; } catch (error) { logger.error("Token verification failed:", error); return null; } }; -const setupConnection = (ws: AuthenticatedWebSocket, newt: Newt): void => { +const setupConnection = (ws: AuthenticatedWebSocket, client: Newt | Olm, clientType: ClientType): void => { logger.info("Establishing websocket connection"); - - if (!newt) { - logger.error("Connection attempt without newt"); + if (!client) { + logger.error("Connection attempt without client"); return ws.terminate(); } - ws.newt = newt; + ws.client = client; + ws.clientType = clientType; // Add client to tracking - addClient(newt.newtId, ws); + const clientId = clientType === 'newt' ? (client as Newt).newtId : (client as Olm).olmId; + addClient(clientId, ws, clientType); ws.on("message", async (data) => { try { const message: WSMessage = JSON.parse(data.toString()); - // logger.info(`Message received from Newt ID ${newtId}:`, message); - // Validate message format if (!message.type || typeof message.type !== "string") { throw new Error("Invalid message format: missing or invalid type"); } - // Get the appropriate handler for the message type const handler = messageHandlers[message.type]; if (!handler) { throw new Error(`Unsupported message type: ${message.type}`); } - // Process the message and get response const response = await handler({ message, senderWs: ws, - newt: ws.newt, + client: ws.client, + clientType: ws.clientType!, sendToClient, broadcastToAllExcept, connectedClients }); - // Send response if one was returned if (response) { if (response.broadcast) { - // Broadcast to all clients except sender if specified - broadcastToAllExcept(response.message, response.excludeSender ? newt.newtId : undefined); - } else if (response.targetNewtId) { - // Send to specific client if targetNewtId is provided - sendToClient(response.targetNewtId, response.message); + broadcastToAllExcept(response.message, response.excludeSender ? clientId : undefined); + } else if (response.targetClientId) { + sendToClient(response.targetClientId, response.message); } else { - // Send back to sender ws.send(JSON.stringify(response.message)); } } - } catch (error) { logger.error("Message handling error:", error); ws.send(JSON.stringify({ @@ -194,18 +202,18 @@ const setupConnection = (ws: AuthenticatedWebSocket, newt: Newt): void => { }); ws.on("close", () => { - removeClient(newt.newtId, ws); - logger.info(`Client disconnected - Newt ID: ${newt.newtId}`); + removeClient(clientId, ws, clientType); + logger.info(`Client disconnected - ${clientType.toUpperCase()} ID: ${clientId}`); }); ws.on("error", (error: Error) => { - logger.error(`WebSocket error for Newt ID ${newt.newtId}:`, error); + logger.error(`WebSocket error for ${clientType.toUpperCase()} ID ${clientId}:`, error); }); - logger.info(`WebSocket connection established - Newt ID: ${newt.newtId}`); + logger.info(`WebSocket connection established - ${clientType.toUpperCase()} ID: ${clientId}`); }; -// Router endpoint (unchanged) +// Router endpoint router.get("/ws", (req: Request, res: Response) => { res.status(200).send("WebSocket endpoint"); }); @@ -214,18 +222,22 @@ router.get("/ws", (req: Request, res: Response) => { const handleWSUpgrade = (server: HttpServer): void => { server.on("upgrade", async (request: WebSocketRequest, socket: Socket, head: Buffer) => { try { - const token = request.url?.includes("?") - ? new URLSearchParams(request.url.split("?")[1]).get("token") || "" - : request.headers["sec-websocket-protocol"]; + const url = new URL(request.url || '', `http://${request.headers.host}`); + const token = url.searchParams.get('token') || request.headers["sec-websocket-protocol"] || ''; + let clientType = url.searchParams.get('clientType') as ClientType; - if (!token) { - logger.warn("Unauthorized connection attempt: no token..."); + if (!clientType) { + clientType = "newt"; + } + + if (!token || !clientType || !['newt', 'olm'].includes(clientType)) { + logger.warn("Unauthorized connection attempt: invalid token or client type..."); socket.write("HTTP/1.1 401 Unauthorized\r\n\r\n"); socket.destroy(); return; } - const tokenPayload = await verifyToken(token); + const tokenPayload = await verifyToken(token, clientType); if (!tokenPayload) { logger.warn("Unauthorized connection attempt: invalid token..."); socket.write("HTTP/1.1 401 Unauthorized\r\n\r\n"); @@ -234,7 +246,7 @@ const handleWSUpgrade = (server: HttpServer): void => { } wss.handleUpgrade(request, socket, head, (ws: AuthenticatedWebSocket) => { - setupConnection(ws, tokenPayload.newt); + setupConnection(ws, tokenPayload.client, tokenPayload.clientType); }); } catch (error) { logger.error("WebSocket upgrade error:", error); @@ -250,4 +262,4 @@ export { sendToClient, broadcastToAllExcept, connectedClients -}; +}; \ No newline at end of file From b9de0f8e389ffcb9ec7db969012a24d351fb7b3b Mon Sep 17 00:00:00 2001 From: Owen Date: Fri, 21 Feb 2025 10:55:38 -0500 Subject: [PATCH 004/135] Working on handlers --- .../routers/newt/handleNewtRegisterMessage.ts | 2 +- server/routers/newt/peers.ts | 46 ++++++ server/routers/olm/handleGetConfigMessage.ts | 147 ------------------ .../routers/olm/handleOlmRegisterMessage.ts | 35 ++--- server/routers/olm/index.ts | 3 +- 5 files changed, 62 insertions(+), 171 deletions(-) create mode 100644 server/routers/newt/peers.ts delete mode 100644 server/routers/olm/handleGetConfigMessage.ts diff --git a/server/routers/newt/handleNewtRegisterMessage.ts b/server/routers/newt/handleNewtRegisterMessage.ts index 8e263034..54a62735 100644 --- a/server/routers/newt/handleNewtRegisterMessage.ts +++ b/server/routers/newt/handleNewtRegisterMessage.ts @@ -16,7 +16,7 @@ export const handleNewtRegisterMessage: MessageHandler = async (context) => { const newt = client; - logger.info("Handling register message!"); + logger.info("Handling register newt message!"); if (!newt) { logger.warn("Newt not found"); diff --git a/server/routers/newt/peers.ts b/server/routers/newt/peers.ts new file mode 100644 index 00000000..ee22c052 --- /dev/null +++ b/server/routers/newt/peers.ts @@ -0,0 +1,46 @@ +import db from '@server/db'; +import { newts, sites } from '@server/db/schema'; +import { eq } from 'drizzle-orm'; +import { sendToClient } from '../ws'; + +export async function addPeer(siteId: number, peer: { + publicKey: string; + allowedIps: string[]; +}) { + + const [site] = await db.select().from(sites).where(eq(sites.siteId, siteId)).limit(1); + if (!site) { + throw new Error(`Exit node with ID ${siteId} not found`); + } + + // get the newt on the site + const [newt] = await db.select().from(newts).where(eq(newts.siteId, siteId)).limit(1); + if (!newt) { + throw new Error(`Newt not found for site ${siteId}`); + } + + sendToClient(newt.newtId, { + type: 'add_peer', + data: peer + }); +} + +export async function deletePeer(siteId: number, publicKey: string) { + const [site] = await db.select().from(sites).where(eq(sites.siteId, siteId)).limit(1); + if (!site) { + throw new Error(`Exit node with ID ${siteId} not found`); + } + + // get the newt on the site + const [newt] = await db.select().from(newts).where(eq(newts.siteId, siteId)).limit(1); + if (!newt) { + throw new Error(`Newt not found for site ${siteId}`); + } + + sendToClient(newt.newtId, { + type: 'delete_peer', + data: { + publicKey + } + }); +} \ No newline at end of file diff --git a/server/routers/olm/handleGetConfigMessage.ts b/server/routers/olm/handleGetConfigMessage.ts deleted file mode 100644 index 6e4f7ebf..00000000 --- a/server/routers/olm/handleGetConfigMessage.ts +++ /dev/null @@ -1,147 +0,0 @@ -import { z } from "zod"; -import { MessageHandler } from "../ws"; -import logger from "@server/logger"; -import { fromError } from "zod-validation-error"; -import db from "@server/db"; -import { olms, Site, sites } from "@server/db/schema"; -import { eq, isNotNull } from "drizzle-orm"; -import { findNextAvailableCidr } from "@server/lib/ip"; -import config from "@server/lib/config"; - -const inputSchema = z.object({ - publicKey: z.string(), - endpoint: z.string(), - listenPort: z.number() -}); - -type Input = z.infer; - -export const handleGetConfigMessage: MessageHandler = async (context) => { - const { message, newt, sendToClient } = context; - - logger.debug("Handling Newt get config message!"); - - if (!newt) { - logger.warn("Newt not found"); - return; - } - - if (!newt.siteId) { - logger.warn("Newt has no site!"); // TODO: Maybe we create the site here? - return; - } - - const parsed = inputSchema.safeParse(message.data); - if (!parsed.success) { - logger.error( - "handleGetConfigMessage: Invalid input: " + - fromError(parsed.error).toString() - ); - return; - } - - const { publicKey, endpoint, listenPort } = message.data as Input; - - const siteId = newt.siteId; - - const [siteRes] = await db - .select() - .from(sites) - .where(eq(sites.siteId, siteId)); - - if (!siteRes) { - logger.warn("handleGetConfigMessage: Site not found"); - return; - } - - let site: Site | undefined; - if (!site) { - const address = await getNextAvailableSubnet(); - - // create a new exit node - const [updateRes] = await db - .update(sites) - .set({ - publicKey, - endpoint, - address, - listenPort - }) - .where(eq(sites.siteId, siteId)) - .returning(); - - site = updateRes; - - logger.info(`Updated site ${siteId} with new WG Newt info`); - } else { - site = siteRes; - } - - if (!site) { - logger.error("handleGetConfigMessage: Failed to update site"); - return; - } - - const clientsRes = await db - .select() - .from(olms) - .where(eq(olms.siteId, siteId)); - - const peers = await Promise.all( - clientsRes.map(async (client) => { - return { - publicKey: client.pubKey, - allowedIps: "0.0.0.0/0" - }; - }) - ); - - const configResponse = { - listenPort: site.listenPort, // ????? - // ipAddress: exitNode[0].address, - peers - }; - - logger.debug("Sending config: ", configResponse); - - return { - message: { - type: "olm/wg/connect", // what to make the response type? - data: { - config: configResponse - } - }, - broadcast: false, // Send to all clients - excludeSender: false // Include sender in broadcast - }; -}; - -async function getNextAvailableSubnet(): Promise { - const existingAddresses = await db - .select({ - address: sites.address - }) - .from(sites) - .where(isNotNull(sites.address)); - - const addresses = existingAddresses - .map((a) => a.address) - .filter((a) => a) as string[]; - - let subnet = findNextAvailableCidr( - addresses, - config.getRawConfig().wg_site.block_size, - config.getRawConfig().wg_site.subnet_group - ); - if (!subnet) { - throw new Error("No available subnets remaining in space"); - } - - // replace the last octet with 1 - subnet = - subnet.split(".").slice(0, 3).join(".") + - ".1" + - "/" + - subnet.split("/")[1]; - return subnet; -} diff --git a/server/routers/olm/handleOlmRegisterMessage.ts b/server/routers/olm/handleOlmRegisterMessage.ts index 33786f2d..859f756c 100644 --- a/server/routers/olm/handleOlmRegisterMessage.ts +++ b/server/routers/olm/handleOlmRegisterMessage.ts @@ -1,14 +1,11 @@ import db from "@server/db"; import { MessageHandler } from "../ws"; import { - exitNodes, - resources, + olms, sites, - Target, - targets } from "@server/db/schema"; -import { eq, and, sql } from "drizzle-orm"; -import { addPeer, deletePeer } from "../gerbil/peers"; +import { eq, } from "drizzle-orm"; +import { addPeer, deletePeer } from "../newt/peers"; import logger from "@server/logger"; export const handleOlmRegisterMessage: MessageHandler = async (context) => { @@ -16,7 +13,7 @@ export const handleOlmRegisterMessage: MessageHandler = async (context) => { const olm = client; - logger.info("Handling register message!"); + logger.info("Handling register olm message!"); if (!olm) { logger.warn("Olm not found"); @@ -42,28 +39,22 @@ export const handleOlmRegisterMessage: MessageHandler = async (context) => { .where(eq(sites.siteId, siteId)) .limit(1); - if (!site || !site.exitNodeId) { + if (!site) { logger.warn("Site not found or does not have exit node"); return; } await db - .update(sites) + .update(olms) .set({ pubKey: publicKey }) - .where(eq(sites.siteId, siteId)) + .where(eq(olms.olmId, olm.olmId)) .returning(); - const [exitNode] = await db - .select() - .from(exitNodes) - .where(eq(exitNodes.exitNodeId, site.exitNodeId)) - .limit(1); - - if (site.pubKey && site.pubKey !== publicKey) { + if (olm.pubKey && olm.pubKey !== publicKey) { logger.info("Public key mismatch. Deleting old peer..."); - await deletePeer(site.exitNodeId, site.pubKey); + await deletePeer(site.siteId, site.pubKey); } if (!site.subnet) { @@ -72,7 +63,7 @@ export const handleOlmRegisterMessage: MessageHandler = async (context) => { } // add the peer to the exit node - await addPeer(site.exitNodeId, { + await addPeer(site.siteId, { publicKey: publicKey, allowedIps: [site.subnet] }); @@ -81,9 +72,9 @@ export const handleOlmRegisterMessage: MessageHandler = async (context) => { message: { type: "olm/wg/connect", data: { - endpoint: `${exitNode.endpoint}:${exitNode.listenPort}`, - publicKey: exitNode.publicKey, - serverIP: exitNode.address.split("/")[0], + endpoint: `${site.endpoint}:${site.listenPort}`, + publicKey: site.publicKey, + serverIP: site.address!.split("/")[0], tunnelIP: site.subnet.split("/")[0] } }, diff --git a/server/routers/olm/index.ts b/server/routers/olm/index.ts index 7265331b..4c073152 100644 --- a/server/routers/olm/index.ts +++ b/server/routers/olm/index.ts @@ -1 +1,2 @@ -export * from "./pickOlmDefaults"; \ No newline at end of file +export * from "./pickOlmDefaults"; +export * from "./handleOlmRegisterMessage"; \ No newline at end of file From 346f2db5fbf406b14ab19ffda10710170843d438 Mon Sep 17 00:00:00 2001 From: miloschwartz Date: Fri, 21 Feb 2025 11:57:01 -0500 Subject: [PATCH 005/135] add client olm relationship --- server/db/schema.ts | 46 +++++++++++++++---- server/routers/client/index.ts | 1 + .../pickClientDefaults.ts} | 46 +++++++++++-------- server/routers/external.ts | 4 +- server/routers/olm/index.ts | 3 +- 5 files changed, 69 insertions(+), 31 deletions(-) create mode 100644 server/routers/client/index.ts rename server/routers/{olm/pickOlmDefaults.ts => client/pickClientDefaults.ts} (75%) diff --git a/server/db/schema.ts b/server/db/schema.ts index 9564fdff..931bf6a4 100644 --- a/server/db/schema.ts +++ b/server/db/schema.ts @@ -61,7 +61,9 @@ export const resources = sqliteTable("resources", { .notNull() .default(false), isBaseDomain: integer("isBaseDomain", { mode: "boolean" }), - applyRules: integer("applyRules", { mode: "boolean" }).notNull().default(false) + applyRules: integer("applyRules", { mode: "boolean" }) + .notNull() + .default(false) }); export const targets = sqliteTable("targets", { @@ -114,22 +116,27 @@ export const newts = sqliteTable("newt", { }) }); -export const olms = sqliteTable("olms", { - olmId: text("id").primaryKey(), - secretHash: text("secretHash").notNull(), - dateCreated: text("dateCreated").notNull(), +export const clients = sqliteTable("clients", { + clientId: integer("id").primaryKey({ autoIncrement: true }), siteId: integer("siteId").references(() => sites.siteId, { onDelete: "cascade" }), - - // wgstuff pubKey: text("pubKey"), subnet: text("subnet").notNull(), megabytesIn: integer("bytesIn"), megabytesOut: integer("bytesOut"), lastBandwidthUpdate: text("lastBandwidthUpdate"), - type: text("type").notNull(), // "newt" or "wireguard" - online: integer("online", { mode: "boolean" }).notNull().default(false), + type: text("type").notNull(), // "olm" + online: integer("online", { mode: "boolean" }).notNull().default(false) +}); + +export const olms = sqliteTable("olms", { + olmId: text("id").primaryKey(), + secretHash: text("secretHash").notNull(), + dateCreated: text("dateCreated").notNull(), + clientId: integer("clientId").references(() => clients.clientId, { + onDelete: "cascade" + }) }); export const twoFactorBackupCodes = sqliteTable("twoFactorBackupCodes", { @@ -259,6 +266,24 @@ export const userSites = sqliteTable("userSites", { .references(() => sites.siteId, { onDelete: "cascade" }) }); +export const userClients = sqliteTable("userClients", { + userId: text("userId") + .notNull() + .references(() => users.userId, { onDelete: "cascade" }), + clientId: integer("clientId") + .notNull() + .references(() => clients.clientId, { onDelete: "cascade" }) +}); + +export const roleClients = sqliteTable("roleClients", { + roleId: integer("roleId") + .notNull() + .references(() => roles.roleId, { onDelete: "cascade" }), + clientId: integer("clientId") + .notNull() + .references(() => clients.clientId, { onDelete: "cascade" }) +}); + export const roleResources = sqliteTable("roleResources", { roleId: integer("roleId") .notNull() @@ -451,3 +476,6 @@ export type ResourceAccessToken = InferSelectModel; export type ResourceWhitelist = InferSelectModel; export type VersionMigration = InferSelectModel; export type ResourceRule = InferSelectModel; +export type Client = InferSelectModel; +export type RoleClient = InferSelectModel; +export type UserClient = InferSelectModel; diff --git a/server/routers/client/index.ts b/server/routers/client/index.ts new file mode 100644 index 00000000..5b493724 --- /dev/null +++ b/server/routers/client/index.ts @@ -0,0 +1 @@ +export * from "./pickClientDefaults"; diff --git a/server/routers/olm/pickOlmDefaults.ts b/server/routers/client/pickClientDefaults.ts similarity index 75% rename from server/routers/olm/pickOlmDefaults.ts rename to server/routers/client/pickClientDefaults.ts index 24ddcace..fd048259 100644 --- a/server/routers/olm/pickOlmDefaults.ts +++ b/server/routers/client/pickClientDefaults.ts @@ -1,6 +1,6 @@ import { Request, Response, NextFunction } from "express"; import { db } from "@server/db"; -import { olms, sites } from "@server/db/schema"; +import { clients, olms, sites } from "@server/db/schema"; import { eq } from "drizzle-orm"; import response from "@server/lib/response"; import HttpCode from "@server/types/HttpCode"; @@ -30,7 +30,7 @@ export type PickClientDefaultsResponse = { clientSecret: string; }; -export async function pickOlmDefaults( +export async function pickClientDefaults( req: Request, res: Response, next: NextFunction @@ -58,29 +58,39 @@ export async function pickOlmDefaults( } // make sure all the required fields are present - if ( - !site.address || - !site.publicKey || - !site.listenPort || - !site.endpoint - ) { + + const sitesRequiredFields = z.object({ + address: z.string(), + publicKey: z.string(), + listenPort: z.number(), + endpoint: z.string() + }); + + const parsedSite = sitesRequiredFields.safeParse(site); + if (!parsedSite.success) { return next( - createHttpError(HttpCode.BAD_REQUEST, "Site has no address") + createHttpError( + HttpCode.BAD_REQUEST, + "Unable to pick client defaults because: " + + fromError(parsedSite.error).toString() + ) ); } + const { address, publicKey, listenPort, endpoint } = parsedSite.data; + const clientsQuery = await db .select({ - subnet: olms.subnet + subnet: clients.subnet }) - .from(olms) - .where(eq(olms.siteId, site.siteId)); + .from(clients) + .where(eq(clients.siteId, site.siteId)); let subnets = clientsQuery.map((client) => client.subnet); // exclude the exit node address by replacing after the / with a site block size subnets.push( - site.address.replace( + address.replace( /\/\d+$/, `/${config.getRawConfig().wg_site.block_size}` ) @@ -88,7 +98,7 @@ export async function pickOlmDefaults( const newSubnet = findNextAvailableCidr( subnets, config.getRawConfig().wg_site.block_size, - site.address + address ); if (!newSubnet) { return next( @@ -105,11 +115,11 @@ export async function pickOlmDefaults( return response(res, { data: { siteId: site.siteId, - address: site.address, - publicKey: site.publicKey, + address: address, + publicKey: publicKey, name: site.name, - listenPort: site.listenPort, - endpoint: site.endpoint, + listenPort: listenPort, + endpoint: endpoint, subnet: newSubnet, clientId, clientSecret: secret diff --git a/server/routers/external.ts b/server/routers/external.ts index cb5fbfd7..778bf288 100644 --- a/server/routers/external.ts +++ b/server/routers/external.ts @@ -7,7 +7,7 @@ import * as target from "./target"; import * as user from "./user"; import * as auth from "./auth"; import * as role from "./role"; -import * as olm from "./olm"; +import * as client from "./client"; import * as accessToken from "./accessToken"; import HttpCode from "@server/types/HttpCode"; import { @@ -100,7 +100,7 @@ authenticated.get( "/site/:siteId/pick-client-defaults", verifyOrgAccess, verifyUserHasAction(ActionsEnum.createClient), - olm.pickOlmDefaults + client.pickClientDefaults ); // authenticated.get( diff --git a/server/routers/olm/index.ts b/server/routers/olm/index.ts index 4c073152..d29a2ef1 100644 --- a/server/routers/olm/index.ts +++ b/server/routers/olm/index.ts @@ -1,2 +1 @@ -export * from "./pickOlmDefaults"; -export * from "./handleOlmRegisterMessage"; \ No newline at end of file +export * from "./handleOlmRegisterMessage"; From bec303821b8b05e67c1a9ee87277e787273ba5a5 Mon Sep 17 00:00:00 2001 From: Owen Date: Fri, 21 Feb 2025 12:17:56 -0500 Subject: [PATCH 006/135] Handle types in handlers --- server/routers/newt/handleGetConfigMessage.ts | 5 +-- .../routers/newt/handleNewtRegisterMessage.ts | 4 +-- .../routers/olm/handleOlmRegisterMessage.ts | 34 +++++++++++++------ 3 files changed, 28 insertions(+), 15 deletions(-) diff --git a/server/routers/newt/handleGetConfigMessage.ts b/server/routers/newt/handleGetConfigMessage.ts index 17ac63dd..9bcde107 100644 --- a/server/routers/newt/handleGetConfigMessage.ts +++ b/server/routers/newt/handleGetConfigMessage.ts @@ -3,7 +3,7 @@ import { MessageHandler } from "../ws"; import logger from "@server/logger"; import { fromError } from "zod-validation-error"; import db from "@server/db"; -import { clients, Site, sites } from "@server/db/schema"; +import { clients, Newt, Site, sites } from "@server/db/schema"; import { eq, isNotNull } from "drizzle-orm"; import { findNextAvailableCidr } from "@server/lib/ip"; import config from "@server/lib/config"; @@ -17,7 +17,8 @@ const inputSchema = z.object({ type Input = z.infer; export const handleGetConfigMessage: MessageHandler = async (context) => { - const { message, newt, sendToClient } = context; + const { message, client, sendToClient } = context; + const newt = client as Newt; logger.debug("Handling Newt get config message!"); diff --git a/server/routers/newt/handleNewtRegisterMessage.ts b/server/routers/newt/handleNewtRegisterMessage.ts index 54a62735..21ad4ba6 100644 --- a/server/routers/newt/handleNewtRegisterMessage.ts +++ b/server/routers/newt/handleNewtRegisterMessage.ts @@ -2,6 +2,7 @@ import db from "@server/db"; import { MessageHandler } from "../ws"; import { exitNodes, + Newt, resources, sites, Target, @@ -13,8 +14,7 @@ import logger from "@server/logger"; export const handleNewtRegisterMessage: MessageHandler = async (context) => { const { message, client, sendToClient } = context; - - const newt = client; + const newt = client as Newt; logger.info("Handling register newt message!"); diff --git a/server/routers/olm/handleOlmRegisterMessage.ts b/server/routers/olm/handleOlmRegisterMessage.ts index 859f756c..147de81c 100644 --- a/server/routers/olm/handleOlmRegisterMessage.ts +++ b/server/routers/olm/handleOlmRegisterMessage.ts @@ -1,6 +1,8 @@ import db from "@server/db"; import { MessageHandler } from "../ws"; import { + clients, + Olm, olms, sites, } from "@server/db/schema"; @@ -9,9 +11,8 @@ import { addPeer, deletePeer } from "../newt/peers"; import logger from "@server/logger"; export const handleOlmRegisterMessage: MessageHandler = async (context) => { - const { message, client, sendToClient } = context; - - const olm = client; + const { message, client: c, sendToClient } = context; + const olm = c as Olm; logger.info("Handling register olm message!"); @@ -20,12 +21,12 @@ export const handleOlmRegisterMessage: MessageHandler = async (context) => { return; } - if (!olm.siteId) { + if (!olm.clientId) { logger.warn("Olm has no site!"); // TODO: Maybe we create the site here? return; } - const siteId = olm.siteId; + const clientId = olm.clientId; const { publicKey } = message.data; if (!publicKey) { @@ -33,28 +34,39 @@ export const handleOlmRegisterMessage: MessageHandler = async (context) => { return; } + const [client] = await db + .select() + .from(clients) + .where(eq(clients.clientId, clientId)) + .limit(1); + + if (!client || !client.siteId) { + logger.warn("Site not found or does not have exit node"); + return; + } + const [site] = await db .select() .from(sites) - .where(eq(sites.siteId, siteId)) + .where(eq(sites.siteId, client.siteId)) .limit(1); - if (!site) { + if (!client) { logger.warn("Site not found or does not have exit node"); return; } await db - .update(olms) + .update(clients) .set({ pubKey: publicKey }) - .where(eq(olms.olmId, olm.olmId)) + .where(eq(clients.clientId, olm.clientId)) .returning(); - if (olm.pubKey && olm.pubKey !== publicKey) { + if (client.pubKey && client.pubKey !== publicKey) { logger.info("Public key mismatch. Deleting old peer..."); - await deletePeer(site.siteId, site.pubKey); + await deletePeer(site.siteId, client.pubKey); } if (!site.subnet) { From ef69bf9256cedbfcb0555523215224b15beec6ae Mon Sep 17 00:00:00 2001 From: Owen Date: Fri, 21 Feb 2025 12:34:05 -0500 Subject: [PATCH 007/135] Update ws paths --- server/routers/newt/handleGetConfigMessage.ts | 4 ++-- server/routers/newt/peers.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/server/routers/newt/handleGetConfigMessage.ts b/server/routers/newt/handleGetConfigMessage.ts index 9bcde107..4f03bdd8 100644 --- a/server/routers/newt/handleGetConfigMessage.ts +++ b/server/routers/newt/handleGetConfigMessage.ts @@ -92,7 +92,7 @@ export const handleGetConfigMessage: MessageHandler = async (context) => { clientsRes.map(async (client) => { return { publicKey: client.pubKey, - allowedIps: "0.0.0.0/0" + allowedIps: "0.0.0.0/0" // TODO: We should lock this down more }; }) ); @@ -107,7 +107,7 @@ export const handleGetConfigMessage: MessageHandler = async (context) => { return { message: { - type: "newt/wg/connect", // what to make the response type? + type: "newt/wg/receive-config", // what to make the response type? data: { config: configResponse } diff --git a/server/routers/newt/peers.ts b/server/routers/newt/peers.ts index ee22c052..afc3b5d6 100644 --- a/server/routers/newt/peers.ts +++ b/server/routers/newt/peers.ts @@ -20,7 +20,7 @@ export async function addPeer(siteId: number, peer: { } sendToClient(newt.newtId, { - type: 'add_peer', + type: 'newt/wg/peer/add', data: peer }); } @@ -38,7 +38,7 @@ export async function deletePeer(siteId: number, publicKey: string) { } sendToClient(newt.newtId, { - type: 'delete_peer', + type: 'newt/wg/peer/remove', data: { publicKey } From a57d32d05d04db68d63682fa2df080f6d33e2cd5 Mon Sep 17 00:00:00 2001 From: Owen Date: Fri, 21 Feb 2025 12:41:02 -0500 Subject: [PATCH 008/135] Add receive bandwidth --- server/routers/gerbil/receiveBandwidth.ts | 9 +-- server/routers/messageHandlers.ts | 3 +- .../newt/handleReceiveBandwidthMessage.ts | 68 +++++++++++++++++++ server/routers/newt/index.ts | 3 +- 4 files changed, 77 insertions(+), 6 deletions(-) create mode 100644 server/routers/newt/handleReceiveBandwidthMessage.ts diff --git a/server/routers/gerbil/receiveBandwidth.ts b/server/routers/gerbil/receiveBandwidth.ts index b2063c08..b8839577 100644 --- a/server/routers/gerbil/receiveBandwidth.ts +++ b/server/routers/gerbil/receiveBandwidth.ts @@ -30,12 +30,13 @@ export const receiveBandwidth = async ( const { publicKey, bytesIn, bytesOut } = peer; // Find the site by public key - const site = await trx.query.sites.findFirst({ - where: eq(sites.pubKey, publicKey) - }); + const [site] = await trx + .select() + .from(sites) + .where(eq(sites.pubKey, publicKey)) + .limit(1); if (!site) { - logger.warn(`Site not found for public key: ${publicKey}`); continue; } let online = site.online; diff --git a/server/routers/messageHandlers.ts b/server/routers/messageHandlers.ts index bf8f357c..f23ea0a8 100644 --- a/server/routers/messageHandlers.ts +++ b/server/routers/messageHandlers.ts @@ -1,4 +1,4 @@ -import { handleNewtRegisterMessage } from "./newt"; +import { handleNewtRegisterMessage, handleReceiveBandwidthMessage } from "./newt"; import { handleOlmRegisterMessage } from "./olm"; import { handleGetConfigMessage } from "./newt/handleGetConfigMessage"; import { MessageHandler } from "./ws"; @@ -7,4 +7,5 @@ export const messageHandlers: Record = { "newt/wg/register": handleNewtRegisterMessage, "olm/wg/register": handleOlmRegisterMessage, "newt/wg/get-config": handleGetConfigMessage, + "newt/receive-bandwidth": handleReceiveBandwidthMessage }; diff --git a/server/routers/newt/handleReceiveBandwidthMessage.ts b/server/routers/newt/handleReceiveBandwidthMessage.ts new file mode 100644 index 00000000..1e1642aa --- /dev/null +++ b/server/routers/newt/handleReceiveBandwidthMessage.ts @@ -0,0 +1,68 @@ +import db from "@server/db"; +import { MessageHandler } from "../ws"; +import { clients, Newt } from "@server/db/schema"; +import { eq } from "drizzle-orm"; +import logger from "@server/logger"; + +interface PeerBandwidth { + publicKey: string; + bytesIn: number; + bytesOut: number; +} + +export const handleReceiveBandwidthMessage: MessageHandler = async (context) => { + const { message, client, sendToClient } = context; + const newt = client as Newt; + + const bandwidthData: PeerBandwidth[] = message.data; + + if (!Array.isArray(bandwidthData)) { + throw new Error("Invalid bandwidth data"); + } + + await db.transaction(async (trx) => { + for (const peer of bandwidthData) { + const { publicKey, bytesIn, bytesOut } = peer; + + // Find the site by public key + const [client] = await trx + .select() + .from(clients) + .where(eq(clients.pubKey, publicKey)) + .limit(1); + + if (!client) { + continue; + } + let online = client.online; + + // if the bandwidth for the site is > 0 then set it to online. if it has been less than 0 (no update) for 5 minutes then set it to offline + if (bytesIn > 0 || bytesOut > 0) { + online = true; + } else if (client.lastBandwidthUpdate) { + const lastBandwidthUpdate = new Date( + client.lastBandwidthUpdate + ); + const currentTime = new Date(); + const diff = + currentTime.getTime() - lastBandwidthUpdate.getTime(); + if (diff < 300000) { + online = false; + } + } + + // Update the site's bandwidth usage + await trx + .update(clients) + .set({ + megabytesOut: (client.megabytesIn || 0) + bytesIn, + megabytesIn: (client.megabytesOut || 0) + bytesOut, + lastBandwidthUpdate: new Date().toISOString(), + online + }) + .where(eq(clients.clientId, client.clientId)); + } + }); + + logger.info("Handling register olm message!"); +}; diff --git a/server/routers/newt/index.ts b/server/routers/newt/index.ts index dcc49749..84b9a6e9 100644 --- a/server/routers/newt/index.ts +++ b/server/routers/newt/index.ts @@ -1,3 +1,4 @@ export * from "./createNewt"; export * from "./getToken"; -export * from "./handleRegisterMessage"; \ No newline at end of file +export * from "./handleNewtRegisterMessage"; +export* from "./handleReceiveBandwidthMessage"; \ No newline at end of file From 757d628bc84c729e1ca5e103a714b8fc6b44c5f9 Mon Sep 17 00:00:00 2001 From: Owen Date: Fri, 21 Feb 2025 12:52:24 -0500 Subject: [PATCH 009/135] Handle port correctly --- install/fs/config.yml | 5 ++++ server/lib/config.ts | 2 ++ server/routers/newt/handleGetConfigMessage.ts | 24 ++++++++++++++++++- 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/install/fs/config.yml b/install/fs/config.yml index 8e4411e7..cf9e6464 100644 --- a/install/fs/config.yml +++ b/install/fs/config.yml @@ -32,6 +32,11 @@ gerbil: site_block_size: 30 subnet_group: 100.89.137.0/20 +wg_site: + start_port: 51820 + block_size: 24 + subnet_group: 100.89.137.0/20 + rate_limits: global: window_minutes: 1 diff --git a/server/lib/config.ts b/server/lib/config.ts index fc1c0531..f607fe0d 100644 --- a/server/lib/config.ts +++ b/server/lib/config.ts @@ -12,6 +12,7 @@ import { } from "@server/lib/consts"; import { passwordSchema } from "@server/auth/passwordSchema"; import stoi from "./stoi"; +import { start } from "repl"; const portSchema = z.number().positive().gt(0).lte(65535); const hostnameSchema = z @@ -112,6 +113,7 @@ const configSchema = z.object({ wg_site: z.object({ block_size: z.number().positive().gt(0), subnet_group: z.string(), + start_port: portSchema }), rate_limits: z.object({ global: z.object({ diff --git a/server/routers/newt/handleGetConfigMessage.ts b/server/routers/newt/handleGetConfigMessage.ts index 4f03bdd8..6d8cb8c8 100644 --- a/server/routers/newt/handleGetConfigMessage.ts +++ b/server/routers/newt/handleGetConfigMessage.ts @@ -41,7 +41,7 @@ export const handleGetConfigMessage: MessageHandler = async (context) => { return; } - const { publicKey, endpoint, listenPort } = message.data as Input; + const { publicKey, endpoint } = message.data as Input; const siteId = newt.siteId; @@ -58,6 +58,7 @@ export const handleGetConfigMessage: MessageHandler = async (context) => { let site: Site | undefined; if (!site) { const address = await getNextAvailableSubnet(); + const listenPort = await getNextAvailablePort(); // create a new exit node const [updateRes] = await db @@ -146,3 +147,24 @@ async function getNextAvailableSubnet(): Promise { subnet.split("/")[1]; return subnet; } + +async function getNextAvailablePort(): Promise { + // Get all existing ports from exitNodes table + const existingPorts = await db.select({ + listenPort: sites.listenPort, + }).from(sites); + + // Find the first available port between 1024 and 65535 + let nextPort = config.getRawConfig().wg_site.start_port; + for (const port of existingPorts) { + if (port.listenPort && port.listenPort > nextPort) { + break; + } + nextPort++; + if (nextPort > 65535) { + throw new Error('No available ports remaining in space'); + } + } + + return nextPort; +} From 35ccdd30145b8c9307d81c582563b144fc90eef3 Mon Sep 17 00:00:00 2001 From: miloschwartz Date: Fri, 21 Feb 2025 14:39:10 -0500 Subject: [PATCH 010/135] add createClient for testing --- server/db/schema.ts | 4 + server/routers/client/createClient.ts | 171 ++++++++++++++++++++++++++ server/routers/client/deleteClient.ts | 94 ++++++++++++++ server/routers/client/index.ts | 2 + server/routers/external.ts | 7 ++ server/routers/site/createSite.ts | 2 +- 6 files changed, 279 insertions(+), 1 deletion(-) create mode 100644 server/routers/client/createClient.ts create mode 100644 server/routers/client/deleteClient.ts diff --git a/server/db/schema.ts b/server/db/schema.ts index 931bf6a4..02602e32 100644 --- a/server/db/schema.ts +++ b/server/db/schema.ts @@ -121,6 +121,10 @@ export const clients = sqliteTable("clients", { siteId: integer("siteId").references(() => sites.siteId, { onDelete: "cascade" }), + orgId: text("orgId").references(() => orgs.orgId, { + onDelete: "cascade" + }), + name: text("name").notNull(), pubKey: text("pubKey"), subnet: text("subnet").notNull(), megabytesIn: integer("bytesIn"), diff --git a/server/routers/client/createClient.ts b/server/routers/client/createClient.ts new file mode 100644 index 00000000..85f1735e --- /dev/null +++ b/server/routers/client/createClient.ts @@ -0,0 +1,171 @@ +import { Request, Response, NextFunction } from "express"; +import { z } from "zod"; +import { db } from "@server/db"; +import { + roles, + userSites, + sites, + roleSites, + Site, + Client, + clients, + roleClients, + userClients, + olms +} from "@server/db/schema"; +import response from "@server/lib/response"; +import HttpCode from "@server/types/HttpCode"; +import createHttpError from "http-errors"; +import logger from "@server/logger"; +import { eq, and } from "drizzle-orm"; +import { addPeer } from "../gerbil/peers"; +import { fromError } from "zod-validation-error"; +import { newts } from "@server/db/schema"; +import moment from "moment"; +import { hashPassword } from "@server/auth/password"; + +const createClientParamsSchema = z + .object({ + siteId: z + .string() + .transform((val) => parseInt(val)) + .pipe(z.number()) + }) + .strict(); + +const createClientSchema = z + .object({ + name: z.string().min(1).max(255), + siteId: z.number().int().positive(), + pubKey: z.string(), + subnet: z.string(), + olmId: z.string(), + secret: z.string(), + type: z.enum(["olm"]) + }) + .strict(); + +export type CreateClientBody = z.infer; + +export type CreateClientResponse = Client; + +export async function createClient( + req: Request, + res: Response, + next: NextFunction +): Promise { + try { + const parsedBody = createClientSchema.safeParse(req.body); + if (!parsedBody.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedBody.error).toString() + ) + ); + } + + const { name, type, siteId, pubKey, subnet, olmId, secret } = + parsedBody.data; + + const parsedParams = createClientParamsSchema.safeParse(req.params); + if (!parsedParams.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedParams.error).toString() + ) + ); + } + + const { siteId: paramSiteId } = parsedParams.data; + + if (siteId != paramSiteId) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + "Site ID in body does not match site ID in URL" + ) + ); + } + + if (!req.userOrgRoleId) { + return next( + createHttpError(HttpCode.FORBIDDEN, "User does not have a role") + ); + } + + const [site] = await db + .select() + .from(sites) + .where(eq(sites.siteId, siteId)); + + if (!site) { + return next(createHttpError(HttpCode.NOT_FOUND, "Site not found")); + } + + await db.transaction(async (trx) => { + const adminRole = await trx + .select() + .from(roles) + .where( + and(eq(roles.isAdmin, true), eq(roles.orgId, site.orgId)) + ) + .limit(1); + + if (adminRole.length === 0) { + trx.rollback(); + return next( + createHttpError(HttpCode.NOT_FOUND, `Admin role not found`) + ); + } + + const [newClient] = await trx + .insert(clients) + .values({ + siteId, + orgId: site.orgId, + name, + pubKey, + subnet, + type + }) + .returning(); + + await trx.insert(roleClients).values({ + roleId: adminRole[0].roleId, + clientId: newClient.clientId + }); + + if (req.userOrgRoleId != adminRole[0].roleId) { + // make sure the user can access the site + trx.insert(userClients).values({ + userId: req.user?.userId!, + clientId: newClient.clientId + }); + } + + const secretHash = await hashPassword(secret); + + await trx.insert(olms).values({ + olmId, + secretHash, + clientId: newClient.clientId, + dateCreated: moment().toISOString() + }); + + return response(res, { + data: newClient, + success: true, + error: false, + message: "Site created successfully", + status: HttpCode.CREATED + }); + }); + } catch (error) { + logger.error(error); + return next( + createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred") + ); + } +} diff --git a/server/routers/client/deleteClient.ts b/server/routers/client/deleteClient.ts new file mode 100644 index 00000000..6ee12734 --- /dev/null +++ b/server/routers/client/deleteClient.ts @@ -0,0 +1,94 @@ +import { Request, Response, NextFunction } from "express"; +import { z } from "zod"; +import { db } from "@server/db"; +import { newts, newtSessions, sites } from "@server/db/schema"; +import { eq } from "drizzle-orm"; +import response from "@server/lib/response"; +import HttpCode from "@server/types/HttpCode"; +import createHttpError from "http-errors"; +import logger from "@server/logger"; +import { deletePeer } from "../gerbil/peers"; +import { fromError } from "zod-validation-error"; +import { sendToClient } from "../ws"; + +const deleteClientSchema = z + .object({ + siteId: z.string().transform(Number).pipe(z.number().int().positive()) + }) + .strict(); + +export async function deleteSite( + req: Request, + res: Response, + next: NextFunction +): Promise { + try { + const parsedParams = deleteClientSchema.safeParse(req.params); + if (!parsedParams.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedParams.error).toString() + ) + ); + } + + const { siteId } = parsedParams.data; + + const [site] = await db + .select() + .from(sites) + .where(eq(sites.siteId, siteId)) + .limit(1); + + if (!site) { + return next( + createHttpError( + HttpCode.NOT_FOUND, + `Site with ID ${siteId} not found` + ) + ); + } + + await db.transaction(async (trx) => { + if (site.pubKey) { + if (site.type == "wireguard") { + await deletePeer(site.exitNodeId!, site.pubKey); + } else if (site.type == "newt") { + // get the newt on the site by querying the newt table for siteId + const [deletedNewt] = await trx + .delete(newts) + .where(eq(newts.siteId, siteId)) + .returning(); + if (deletedNewt) { + const payload = { + type: `newt/terminate`, + data: {} + }; + sendToClient(deletedNewt.newtId, payload); + + // delete all of the sessions for the newt + await trx + .delete(newtSessions) + .where(eq(newtSessions.newtId, deletedNewt.newtId)); + } + } + } + + await trx.delete(sites).where(eq(sites.siteId, siteId)); + }); + + return response(res, { + data: null, + success: true, + error: false, + message: "Site deleted successfully", + status: HttpCode.OK + }); + } catch (error) { + logger.error(error); + return next( + createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred") + ); + } +} diff --git a/server/routers/client/index.ts b/server/routers/client/index.ts index 5b493724..98abfe06 100644 --- a/server/routers/client/index.ts +++ b/server/routers/client/index.ts @@ -1 +1,3 @@ export * from "./pickClientDefaults"; +export * from "./createClient"; +export * from "./deleteClient"; diff --git a/server/routers/external.ts b/server/routers/external.ts index 778bf288..d2e680ec 100644 --- a/server/routers/external.ts +++ b/server/routers/external.ts @@ -103,6 +103,13 @@ authenticated.get( client.pickClientDefaults ); +authenticated.put( + "/site/:siteId/client", + verifySiteAccess, + verifyUserHasAction(ActionsEnum.createClient), + client.createClient +); + // authenticated.get( // "/site/:siteId/roles", // verifySiteAccess, diff --git a/server/routers/site/createSite.ts b/server/routers/site/createSite.ts index 9a5e6f46..1b30b5c8 100644 --- a/server/routers/site/createSite.ts +++ b/server/routers/site/createSite.ts @@ -35,7 +35,7 @@ const createSiteSchema = z subnet: z.string().optional(), newtId: z.string().optional(), secret: z.string().optional(), - type: z.string() + type: z.enum(["newt", "wireguard"]) }) .strict(); From 11920ca9976a231fede3eb4670bebcb37e583cf4 Mon Sep 17 00:00:00 2001 From: miloschwartz Date: Fri, 21 Feb 2025 15:26:48 -0500 Subject: [PATCH 011/135] add deleteClient endpoint --- server/auth/actions.ts | 3 +- server/db/schema.ts | 8 +- server/middlewares/index.ts | 1 + server/middlewares/verifyClientAccess.ts | 131 +++++++++++++++++++++++ server/routers/client/deleteClient.ts | 50 ++------- server/routers/external.ts | 13 ++- 6 files changed, 161 insertions(+), 45 deletions(-) create mode 100644 server/middlewares/verifyClientAccess.ts diff --git a/server/auth/actions.ts b/server/auth/actions.ts index 972ff1a7..fef085a1 100644 --- a/server/auth/actions.ts +++ b/server/auth/actions.ts @@ -62,7 +62,8 @@ export enum ActionsEnum { deleteResourceRule = "deleteResourceRule", listResourceRules = "listResourceRules", updateResourceRule = "updateResourceRule", - createClient = "createClient" + createClient = "createClient", + deleteClient = "deleteClient" } export async function checkUserActionPermission( diff --git a/server/db/schema.ts b/server/db/schema.ts index 02602e32..449618f7 100644 --- a/server/db/schema.ts +++ b/server/db/schema.ts @@ -121,9 +121,11 @@ export const clients = sqliteTable("clients", { siteId: integer("siteId").references(() => sites.siteId, { onDelete: "cascade" }), - orgId: text("orgId").references(() => orgs.orgId, { - onDelete: "cascade" - }), + orgId: text("orgId") + .references(() => orgs.orgId, { + onDelete: "cascade" + }) + .notNull(), name: text("name").notNull(), pubKey: text("pubKey"), subnet: text("subnet").notNull(), diff --git a/server/middlewares/index.ts b/server/middlewares/index.ts index 03de18cb..a0f01d08 100644 --- a/server/middlewares/index.ts +++ b/server/middlewares/index.ts @@ -14,3 +14,4 @@ export * from "./verifyAdmin"; export * from "./verifySetResourceUsers"; export * from "./verifyUserInRole"; export * from "./verifyAccessTokenAccess"; +export * from "./verifyClientAccess"; diff --git a/server/middlewares/verifyClientAccess.ts b/server/middlewares/verifyClientAccess.ts new file mode 100644 index 00000000..b024d416 --- /dev/null +++ b/server/middlewares/verifyClientAccess.ts @@ -0,0 +1,131 @@ +import { Request, Response, NextFunction } from "express"; +import { db } from "@server/db"; +import { userOrgs, clients, roleClients, userClients } from "@server/db/schema"; +import { and, eq } from "drizzle-orm"; +import createHttpError from "http-errors"; +import HttpCode from "@server/types/HttpCode"; + +export async function verifyClientAccess( + req: Request, + res: Response, + next: NextFunction +) { + const userId = req.user!.userId; // Assuming you have user information in the request + const clientId = parseInt( + req.params.clientId || req.body.clientId || req.query.clientId + ); + + if (!userId) { + return next( + createHttpError(HttpCode.UNAUTHORIZED, "User not authenticated") + ); + } + + if (isNaN(clientId)) { + return next(createHttpError(HttpCode.BAD_REQUEST, "Invalid client ID")); + } + + try { + // Get the client + const [client] = await db + .select() + .from(clients) + .where(eq(clients.clientId, clientId)) + .limit(1); + + if (!client) { + return next( + createHttpError( + HttpCode.NOT_FOUND, + `Client with ID ${clientId} not found` + ) + ); + } + + if (!client.orgId) { + return next( + createHttpError( + HttpCode.INTERNAL_SERVER_ERROR, + `Client with ID ${clientId} does not have an organization ID` + ) + ); + } + + if (!req.userOrg) { + // Get user's role ID in the organization + const userOrgRole = await db + .select() + .from(userOrgs) + .where( + and( + eq(userOrgs.userId, userId), + eq(userOrgs.orgId, client.orgId) + ) + ) + .limit(1); + req.userOrg = userOrgRole[0]; + } + + if (!req.userOrg) { + return next( + createHttpError( + HttpCode.FORBIDDEN, + "User does not have access to this organization" + ) + ); + } + + const userOrgRoleId = req.userOrg.roleId; + req.userOrgRoleId = userOrgRoleId; + req.userOrgId = client.orgId; + + // Check role-based site access first + const [roleClientAccess] = await db + .select() + .from(roleClients) + .where( + and( + eq(roleClients.clientId, clientId), + eq(roleClients.roleId, userOrgRoleId) + ) + ) + .limit(1); + + if (roleClientAccess) { + // User has access to the site through their role + return next(); + } + + // If role doesn't have access, check user-specific site access + const [userClientAccess] = await db + .select() + .from(userClients) + .where( + and( + eq(userClients.userId, userId), + eq(userClients.clientId, clientId) + ) + ) + .limit(1); + + if (userClientAccess) { + // User has direct access to the site + return next(); + } + + // If we reach here, the user doesn't have access to the site + return next( + createHttpError( + HttpCode.FORBIDDEN, + "User does not have access to this client" + ) + ); + } catch (error) { + return next( + createHttpError( + HttpCode.INTERNAL_SERVER_ERROR, + "Error verifying site access" + ) + ); + } +} diff --git a/server/routers/client/deleteClient.ts b/server/routers/client/deleteClient.ts index 6ee12734..2036a3ce 100644 --- a/server/routers/client/deleteClient.ts +++ b/server/routers/client/deleteClient.ts @@ -1,23 +1,21 @@ import { Request, Response, NextFunction } from "express"; import { z } from "zod"; import { db } from "@server/db"; -import { newts, newtSessions, sites } from "@server/db/schema"; +import { clients, sites } from "@server/db/schema"; import { eq } from "drizzle-orm"; import response from "@server/lib/response"; import HttpCode from "@server/types/HttpCode"; import createHttpError from "http-errors"; import logger from "@server/logger"; -import { deletePeer } from "../gerbil/peers"; import { fromError } from "zod-validation-error"; -import { sendToClient } from "../ws"; const deleteClientSchema = z .object({ - siteId: z.string().transform(Number).pipe(z.number().int().positive()) + clientId: z.string().transform(Number).pipe(z.number().int().positive()) }) .strict(); -export async function deleteSite( +export async function deleteClient( req: Request, res: Response, next: NextFunction @@ -33,56 +31,30 @@ export async function deleteSite( ); } - const { siteId } = parsedParams.data; + const { clientId } = parsedParams.data; - const [site] = await db + const [client] = await db .select() - .from(sites) - .where(eq(sites.siteId, siteId)) + .from(clients) + .where(eq(clients.clientId, clientId)) .limit(1); - if (!site) { + if (!client) { return next( createHttpError( HttpCode.NOT_FOUND, - `Site with ID ${siteId} not found` + `Site with ID ${clientId} not found` ) ); } - await db.transaction(async (trx) => { - if (site.pubKey) { - if (site.type == "wireguard") { - await deletePeer(site.exitNodeId!, site.pubKey); - } else if (site.type == "newt") { - // get the newt on the site by querying the newt table for siteId - const [deletedNewt] = await trx - .delete(newts) - .where(eq(newts.siteId, siteId)) - .returning(); - if (deletedNewt) { - const payload = { - type: `newt/terminate`, - data: {} - }; - sendToClient(deletedNewt.newtId, payload); - - // delete all of the sessions for the newt - await trx - .delete(newtSessions) - .where(eq(newtSessions.newtId, deletedNewt.newtId)); - } - } - } - - await trx.delete(sites).where(eq(sites.siteId, siteId)); - }); + await db.delete(sites).where(eq(sites.siteId, clientId)); return response(res, { data: null, success: true, error: false, - message: "Site deleted successfully", + message: "Client deleted successfully", status: HttpCode.OK }); } catch (error) { diff --git a/server/routers/external.ts b/server/routers/external.ts index d2e680ec..8e064ab7 100644 --- a/server/routers/external.ts +++ b/server/routers/external.ts @@ -22,7 +22,8 @@ import { verifyRoleAccess, verifySetResourceUsers, verifyUserAccess, - getUserOrgs + getUserOrgs, + verifyClientAccess } from "@server/middlewares"; import { verifyUserHasAction } from "../middlewares/verifyUserHasAction"; import { ActionsEnum } from "@server/auth/actions"; @@ -98,7 +99,7 @@ authenticated.get( authenticated.get( "/site/:siteId/pick-client-defaults", - verifyOrgAccess, + verifySiteAccess, verifyUserHasAction(ActionsEnum.createClient), client.pickClientDefaults ); @@ -110,6 +111,14 @@ authenticated.put( client.createClient ); +authenticated.delete( + "/client/:clientId", + verifyClientAccess, + verifyUserHasAction(ActionsEnum.deleteClient), + client.deleteClient +); + + // authenticated.get( // "/site/:siteId/roles", // verifySiteAccess, From 6e1bfdac58cbbe4bac511736aae585da2c51e9b1 Mon Sep 17 00:00:00 2001 From: miloschwartz Date: Fri, 21 Feb 2025 15:45:34 -0500 Subject: [PATCH 012/135] add listClients endpoint --- server/auth/actions.ts | 3 +- server/routers/client/index.ts | 1 + server/routers/client/listClients.ts | 162 +++++++++++++++++++++++++++ server/routers/external.ts | 9 +- 4 files changed, 172 insertions(+), 3 deletions(-) create mode 100644 server/routers/client/listClients.ts diff --git a/server/auth/actions.ts b/server/auth/actions.ts index fef085a1..6331b78b 100644 --- a/server/auth/actions.ts +++ b/server/auth/actions.ts @@ -63,7 +63,8 @@ export enum ActionsEnum { listResourceRules = "listResourceRules", updateResourceRule = "updateResourceRule", createClient = "createClient", - deleteClient = "deleteClient" + deleteClient = "deleteClient", + listClients = "listClients" } export async function checkUserActionPermission( diff --git a/server/routers/client/index.ts b/server/routers/client/index.ts index 98abfe06..686d08e9 100644 --- a/server/routers/client/index.ts +++ b/server/routers/client/index.ts @@ -1,3 +1,4 @@ export * from "./pickClientDefaults"; export * from "./createClient"; export * from "./deleteClient"; +export * from "./listClients"; diff --git a/server/routers/client/listClients.ts b/server/routers/client/listClients.ts new file mode 100644 index 00000000..ad03bbf7 --- /dev/null +++ b/server/routers/client/listClients.ts @@ -0,0 +1,162 @@ +import { db } from "@server/db"; +import { + clients, + orgs, + roleClients, + roleSites, + sites, + userClients, + userSites +} from "@server/db/schema"; +import logger from "@server/logger"; +import HttpCode from "@server/types/HttpCode"; +import response from "@server/lib/response"; +import { and, count, eq, inArray, or, sql } from "drizzle-orm"; +import { NextFunction, Request, Response } from "express"; +import createHttpError from "http-errors"; +import { z } from "zod"; +import { fromError } from "zod-validation-error"; + +const listClientsParamsSchema = z + .object({ + orgId: z.string() + }) + .strict(); + +const listClientsSchema = z.object({ + limit: z + .string() + .optional() + .default("1000") + .transform(Number) + .pipe(z.number().int().positive()), + offset: z + .string() + .optional() + .default("0") + .transform(Number) + .pipe(z.number().int().nonnegative()) +}); + +function queryClients(orgId: string, accessibleClientIds: number[]) { + return db + .select({ + siteId: sites.siteId, + niceId: sites.niceId, + name: sites.name, + pubKey: sites.pubKey, + subnet: sites.subnet, + megabytesIn: sites.megabytesIn, + megabytesOut: sites.megabytesOut, + orgName: orgs.name, + type: sites.type, + online: sites.online + }) + .from(clients) + .leftJoin(orgs, eq(clients.orgId, orgs.orgId)) + .where( + and( + inArray(clients.clientId, accessibleClientIds), + eq(clients.orgId, orgId) + ) + ); +} + +export type ListClientsResponse = { + clients: Awaited>; + pagination: { total: number; limit: number; offset: number }; +}; + +export async function listClients( + req: Request, + res: Response, + next: NextFunction +): Promise { + try { + const parsedQuery = listClientsSchema.safeParse(req.query); + if (!parsedQuery.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedQuery.error) + ) + ); + } + const { limit, offset } = parsedQuery.data; + + const parsedParams = listClientsParamsSchema.safeParse(req.params); + if (!parsedParams.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedParams.error) + ) + ); + } + const { orgId } = parsedParams.data; + + if (orgId && orgId !== req.userOrgId) { + return next( + createHttpError( + HttpCode.FORBIDDEN, + "User does not have access to this organization" + ) + ); + } + + const accessibleClients = await db + .select({ + clientId: sql`COALESCE(${userClients.clientId}, ${roleClients.clientId})` + }) + .from(userClients) + .fullJoin( + roleClients, + eq(userClients.clientId, roleClients.clientId) + ) + .where( + or( + eq(userSites.userId, req.user!.userId), + eq(roleSites.roleId, req.userOrgRoleId!) + ) + ); + + const accessibleSiteIds = accessibleClients.map( + (site) => site.clientId + ); + const baseQuery = queryClients(orgId, accessibleSiteIds); + + let countQuery = db + .select({ count: count() }) + .from(sites) + .where( + and( + inArray(sites.siteId, accessibleSiteIds), + eq(sites.orgId, orgId) + ) + ); + + const clientsList = await baseQuery.limit(limit).offset(offset); + const totalCountResult = await countQuery; + const totalCount = totalCountResult[0].count; + + return response(res, { + data: { + clients: clientsList, + pagination: { + total: totalCount, + limit, + offset + } + }, + success: true, + error: false, + message: "Clients retrieved successfully", + status: HttpCode.OK + }); + } catch (error) { + logger.error(error); + return next( + createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred") + ); + } +} diff --git a/server/routers/external.ts b/server/routers/external.ts index 8e064ab7..bc831bdf 100644 --- a/server/routers/external.ts +++ b/server/routers/external.ts @@ -104,6 +104,13 @@ authenticated.get( client.pickClientDefaults ); +authenticated.get( + "/org/:orgId/clients", + verifyOrgAccess, + verifyUserHasAction(ActionsEnum.listClients), + client.listClients +); + authenticated.put( "/site/:siteId/client", verifySiteAccess, @@ -118,7 +125,6 @@ authenticated.delete( client.deleteClient ); - // authenticated.get( // "/site/:siteId/roles", // verifySiteAccess, @@ -256,7 +262,6 @@ authenticated.delete( target.deleteTarget ); - authenticated.put( "/org/:orgId/role", verifyOrgAccess, From b1f4971f254a4ebe4f06199110c28bf5bf1811b9 Mon Sep 17 00:00:00 2001 From: Owen Date: Fri, 21 Feb 2025 16:12:21 -0500 Subject: [PATCH 013/135] Get new api endpoints working --- install/fs/config.yml | 3 +- server/db/schema.ts | 2 +- server/lib/config.ts | 3 +- server/routers/client/createClient.ts | 8 +-- server/routers/client/pickClientDefaults.ts | 10 ++-- server/routers/external.ts | 6 ++- .../newt/{getToken.ts => getNewtToken.ts} | 2 +- server/routers/newt/handleGetConfigMessage.ts | 10 ++-- .../newt/handleReceiveBandwidthMessage.ts | 9 ++-- server/routers/newt/index.ts | 2 +- .../olm/{getToken.ts => getOlmToken.ts} | 54 ++++++++++--------- server/routers/olm/index.ts | 2 + server/routers/site/pickSiteDefaults.ts | 19 +++++-- 13 files changed, 79 insertions(+), 51 deletions(-) rename server/routers/newt/{getToken.ts => getNewtToken.ts} (98%) rename server/routers/olm/{getToken.ts => getOlmToken.ts} (64%) diff --git a/install/fs/config.yml b/install/fs/config.yml index cf9e6464..620ccbf3 100644 --- a/install/fs/config.yml +++ b/install/fs/config.yml @@ -35,7 +35,8 @@ gerbil: wg_site: start_port: 51820 block_size: 24 - subnet_group: 100.89.137.0/20 + subnet_group: 100.89.138.0/20 + site_block_size: 30 rate_limits: global: diff --git a/server/db/schema.ts b/server/db/schema.ts index 449618f7..6b7dcfda 100644 --- a/server/db/schema.ts +++ b/server/db/schema.ts @@ -173,7 +173,7 @@ export const olmSessions = sqliteTable("clientSession", { sessionId: text("id").primaryKey(), olmId: text("olmId") .notNull() - .references(() => newts.newtId, { onDelete: "cascade" }), + .references(() => olms.olmId, { onDelete: "cascade" }), expiresAt: integer("expiresAt").notNull() }); diff --git a/server/lib/config.ts b/server/lib/config.ts index f607fe0d..336b3ccb 100644 --- a/server/lib/config.ts +++ b/server/lib/config.ts @@ -113,7 +113,8 @@ const configSchema = z.object({ wg_site: z.object({ block_size: z.number().positive().gt(0), subnet_group: z.string(), - start_port: portSchema + start_port: portSchema, + site_block_size: z.number().positive().gt(0) }), rate_limits: z.object({ global: z.object({ diff --git a/server/routers/client/createClient.ts b/server/routers/client/createClient.ts index 85f1735e..ef5be02f 100644 --- a/server/routers/client/createClient.ts +++ b/server/routers/client/createClient.ts @@ -37,10 +37,10 @@ const createClientSchema = z .object({ name: z.string().min(1).max(255), siteId: z.number().int().positive(), - pubKey: z.string(), - subnet: z.string(), - olmId: z.string(), - secret: z.string(), + pubKey: z.string().optional(), + subnet: z.string().optional(), + olmId: z.string().optional(), + secret: z.string().optional(), type: z.enum(["olm"]) }) .strict(); diff --git a/server/routers/client/pickClientDefaults.ts b/server/routers/client/pickClientDefaults.ts index fd048259..c8292374 100644 --- a/server/routers/client/pickClientDefaults.ts +++ b/server/routers/client/pickClientDefaults.ts @@ -14,7 +14,7 @@ import { fromError } from "zod-validation-error"; const getSiteSchema = z .object({ - siteId: z.number().int().positive() + siteId: z.string().transform(Number).pipe(z.number()) }) .strict(); @@ -92,12 +92,16 @@ export async function pickClientDefaults( subnets.push( address.replace( /\/\d+$/, - `/${config.getRawConfig().wg_site.block_size}` + `/${config.getRawConfig().wg_site.site_block_size}` ) ); + logger.debug(`Subnets: ${subnets}`); + logger.debug(`Address: ${address}`); + logger.debug(`Block size: ${config.getRawConfig().wg_site.block_size}`); + logger.debug(`Site block size: ${config.getRawConfig().wg_site.site_block_size}`); const newSubnet = findNextAvailableCidr( subnets, - config.getRawConfig().wg_site.block_size, + config.getRawConfig().wg_site.site_block_size, address ); if (!newSubnet) { diff --git a/server/routers/external.ts b/server/routers/external.ts index bc831bdf..8c5fcb93 100644 --- a/server/routers/external.ts +++ b/server/routers/external.ts @@ -28,7 +28,8 @@ import { import { verifyUserHasAction } from "../middlewares/verifyUserHasAction"; import { ActionsEnum } from "@server/auth/actions"; import { verifyUserIsOrgOwner } from "../middlewares/verifyUserIsOrgOwner"; -import { createNewt, getToken } from "./newt"; +import { createNewt, getNewtToken } from "./newt"; +import { getOlmToken } from "./olm"; import rateLimit from "express-rate-limit"; import createHttpError from "http-errors"; @@ -501,7 +502,8 @@ authRouter.use( authRouter.put("/signup", auth.signup); authRouter.post("/login", auth.login); authRouter.post("/logout", auth.logout); -authRouter.post("/newt/get-token", getToken); +authRouter.post("/newt/get-token", getNewtToken); +authRouter.post("/olm/get-token", getOlmToken); authRouter.post("/2fa/enable", verifySessionUserMiddleware, auth.verifyTotp); authRouter.post( diff --git a/server/routers/newt/getToken.ts b/server/routers/newt/getNewtToken.ts similarity index 98% rename from server/routers/newt/getToken.ts rename to server/routers/newt/getNewtToken.ts index e6ae0cd6..80977f81 100644 --- a/server/routers/newt/getToken.ts +++ b/server/routers/newt/getNewtToken.ts @@ -24,7 +24,7 @@ export const newtGetTokenBodySchema = z.object({ export type NewtGetTokenBody = z.infer; -export async function getToken( +export async function getNewtToken( req: Request, res: Response, next: NextFunction diff --git a/server/routers/newt/handleGetConfigMessage.ts b/server/routers/newt/handleGetConfigMessage.ts index 6d8cb8c8..7058a0c7 100644 --- a/server/routers/newt/handleGetConfigMessage.ts +++ b/server/routers/newt/handleGetConfigMessage.ts @@ -10,8 +10,7 @@ import config from "@server/lib/config"; const inputSchema = z.object({ publicKey: z.string(), - endpoint: z.string(), - listenPort: z.number() + endpoint: z.string() }); type Input = z.infer; @@ -20,6 +19,9 @@ export const handleGetConfigMessage: MessageHandler = async (context) => { const { message, client, sendToClient } = context; const newt = client as Newt; + logger.debug(JSON.stringify(message.data)); + + logger.debug("Handling Newt get config message!"); if (!newt) { @@ -99,8 +101,8 @@ export const handleGetConfigMessage: MessageHandler = async (context) => { ); const configResponse = { - listenPort: site.listenPort, // ????? - // ipAddress: exitNode[0].address, + listenPort: site.listenPort, + ipAddress: site.address, peers }; diff --git a/server/routers/newt/handleReceiveBandwidthMessage.ts b/server/routers/newt/handleReceiveBandwidthMessage.ts index 1e1642aa..a20e2426 100644 --- a/server/routers/newt/handleReceiveBandwidthMessage.ts +++ b/server/routers/newt/handleReceiveBandwidthMessage.ts @@ -12,9 +12,12 @@ interface PeerBandwidth { export const handleReceiveBandwidthMessage: MessageHandler = async (context) => { const { message, client, sendToClient } = context; - const newt = client as Newt; - const bandwidthData: PeerBandwidth[] = message.data; + if (!message.data.bandwidthData) { + logger.warn("No bandwidth data provided"); + } + + const bandwidthData: PeerBandwidth[] = message.data.bandwidthData; if (!Array.isArray(bandwidthData)) { throw new Error("Invalid bandwidth data"); @@ -63,6 +66,4 @@ export const handleReceiveBandwidthMessage: MessageHandler = async (context) => .where(eq(clients.clientId, client.clientId)); } }); - - logger.info("Handling register olm message!"); }; diff --git a/server/routers/newt/index.ts b/server/routers/newt/index.ts index 84b9a6e9..aa72fc6f 100644 --- a/server/routers/newt/index.ts +++ b/server/routers/newt/index.ts @@ -1,4 +1,4 @@ export * from "./createNewt"; -export * from "./getToken"; +export * from "./getNewtToken"; export * from "./handleNewtRegisterMessage"; export* from "./handleReceiveBandwidthMessage"; \ No newline at end of file diff --git a/server/routers/olm/getToken.ts b/server/routers/olm/getOlmToken.ts similarity index 64% rename from server/routers/olm/getToken.ts rename to server/routers/olm/getOlmToken.ts index e6ae0cd6..ce40a35e 100644 --- a/server/routers/olm/getToken.ts +++ b/server/routers/olm/getOlmToken.ts @@ -1,6 +1,6 @@ import { generateSessionToken } from "@server/auth/sessions/app"; import db from "@server/db"; -import { newts } from "@server/db/schema"; +import { olms } from "@server/db/schema"; import HttpCode from "@server/types/HttpCode"; import response from "@server/lib/response"; import { eq } from "drizzle-orm"; @@ -9,27 +9,27 @@ import createHttpError from "http-errors"; import { z } from "zod"; import { fromError } from "zod-validation-error"; import { - createNewtSession, - validateNewtSessionToken -} from "@server/auth/sessions/newt"; + createOlmSession, + validateOlmSessionToken +} from "@server/auth/sessions/olm"; import { verifyPassword } from "@server/auth/password"; import logger from "@server/logger"; import config from "@server/lib/config"; -export const newtGetTokenBodySchema = z.object({ - newtId: z.string(), +export const olmGetTokenBodySchema = z.object({ + olmId: z.string(), secret: z.string(), token: z.string().optional() }); -export type NewtGetTokenBody = z.infer; +export type OlmGetTokenBody = z.infer; -export async function getToken( +export async function getOlmToken( req: Request, res: Response, next: NextFunction ): Promise { - const parsedBody = newtGetTokenBodySchema.safeParse(req.body); + const parsedBody = olmGetTokenBodySchema.safeParse(req.body); if (!parsedBody.success) { return next( @@ -40,15 +40,15 @@ export async function getToken( ); } - const { newtId, secret, token } = parsedBody.data; + const { olmId, secret, token } = parsedBody.data; try { if (token) { - const { session, newt } = await validateNewtSessionToken(token); + const { session, olm } = await validateOlmSessionToken(token); if (session) { if (config.getRawConfig().app.log_failed_attempts) { logger.info( - `Newt session already valid. Newt ID: ${newtId}. IP: ${req.ip}.` + `Olm session already valid. Olm ID: ${olmId}. IP: ${req.ip}.` ); } return response(res, { @@ -61,29 +61,31 @@ export async function getToken( } } - const existingNewtRes = await db + const existingOlmRes = await db .select() - .from(newts) - .where(eq(newts.newtId, newtId)); - if (!existingNewtRes || !existingNewtRes.length) { + .from(olms) + .where(eq(olms.olmId, olmId)); + if (!existingOlmRes || !existingOlmRes.length) { return next( createHttpError( HttpCode.BAD_REQUEST, - "No newt found with that newtId" + "No olm found with that olmId" ) ); } - const existingNewt = existingNewtRes[0]; + logger.debug("Existing olm: ", existingOlmRes); + + const existingOlm = existingOlmRes[0]; const validSecret = await verifyPassword( secret, - existingNewt.secretHash + existingOlm.secretHash ); if (!validSecret) { if (config.getRawConfig().app.log_failed_attempts) { logger.info( - `Newt id or secret is incorrect. Newt: ID ${newtId}. IP: ${req.ip}.` + `Olm id or secret is incorrect. Olm: ID ${olmId}. IP: ${req.ip}.` ); } return next( @@ -91,8 +93,12 @@ export async function getToken( ); } + logger.debug("Creating new olm session token"); + const resToken = generateSessionToken(); - await createNewtSession(resToken, existingNewt.newtId); + await createOlmSession(resToken, existingOlm.olmId); + + logger.debug("Token created successfully"); return response<{ token: string }>(res, { data: { @@ -103,12 +109,12 @@ export async function getToken( message: "Token created successfully", status: HttpCode.OK }); - } catch (e) { - console.error(e); + } catch (error) { + logger.error(error); return next( createHttpError( HttpCode.INTERNAL_SERVER_ERROR, - "Failed to authenticate newt" + "Failed to authenticate olm" ) ); } diff --git a/server/routers/olm/index.ts b/server/routers/olm/index.ts index d29a2ef1..616480cc 100644 --- a/server/routers/olm/index.ts +++ b/server/routers/olm/index.ts @@ -1 +1,3 @@ export * from "./handleOlmRegisterMessage"; +export * from "./getOlmToken"; +export * from "./createOlm"; \ No newline at end of file diff --git a/server/routers/site/pickSiteDefaults.ts b/server/routers/site/pickSiteDefaults.ts index 56d072e0..79c2b324 100644 --- a/server/routers/site/pickSiteDefaults.ts +++ b/server/routers/site/pickSiteDefaults.ts @@ -45,7 +45,7 @@ export async function pickSiteDefaults( // list all of the sites on that exit node const sitesQuery = await db .select({ - subnet: sites.subnet, + subnet: sites.subnet }) .from(sites) .where(eq(sites.exitNodeId, exitNode.exitNodeId)); @@ -53,8 +53,17 @@ export async function pickSiteDefaults( // TODO: we need to lock this subnet for some time so someone else does not take it let subnets = sitesQuery.map((site) => site.subnet); // exclude the exit node address by replacing after the / with a site block size - subnets.push(exitNode.address.replace(/\/\d+$/, `/${config.getRawConfig().gerbil.site_block_size}`)); - const newSubnet = findNextAvailableCidr(subnets, config.getRawConfig().gerbil.site_block_size, exitNode.address); + subnets.push( + exitNode.address.replace( + /\/\d+$/, + `/${config.getRawConfig().gerbil.site_block_size}` + ) + ); + const newSubnet = findNextAvailableCidr( + subnets, + config.getRawConfig().gerbil.site_block_size, + exitNode.address + ); if (!newSubnet) { return next( createHttpError( @@ -77,12 +86,12 @@ export async function pickSiteDefaults( endpoint: exitNode.endpoint, subnet: newSubnet, newtId, - newtSecret: secret, + newtSecret: secret }, success: true, error: false, message: "Organization retrieved successfully", - status: HttpCode.OK, + status: HttpCode.OK }); } catch (error) { logger.error(error); From b21e758eb803b2395a582bd927176939ff2765c6 Mon Sep 17 00:00:00 2001 From: Owen Date: Fri, 21 Feb 2025 16:19:12 -0500 Subject: [PATCH 014/135] Initial olm test working --- server/routers/newt/handleNewtRegisterMessage.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/routers/newt/handleNewtRegisterMessage.ts b/server/routers/newt/handleNewtRegisterMessage.ts index 21ad4ba6..ec60641f 100644 --- a/server/routers/newt/handleNewtRegisterMessage.ts +++ b/server/routers/newt/handleNewtRegisterMessage.ts @@ -9,7 +9,7 @@ import { targets } from "@server/db/schema"; import { eq, and, sql } from "drizzle-orm"; -import { addPeer, deletePeer } from "../gerbil/peers"; +import { addPeer, deletePeer } from "../newt/peers"; import logger from "@server/logger"; export const handleNewtRegisterMessage: MessageHandler = async (context) => { From 098723b88d2793f18a0f3729e2e9cdb18ff386a2 Mon Sep 17 00:00:00 2001 From: miloschwartz Date: Fri, 21 Feb 2025 16:58:30 -0500 Subject: [PATCH 015/135] clients frontend demo first pass --- server/routers/client/createClient.ts | 6 + server/routers/client/listClients.ts | 31 +- server/routers/client/pickClientDefaults.ts | 21 +- server/routers/site/createSite.ts | 2 +- .../settings/clients/ClientsDataTable.tsx | 153 ++++++++ .../[orgId]/settings/clients/ClientsTable.tsx | 271 ++++++++++++++ .../settings/clients/CreateClientsForm.tsx | 336 ++++++++++++++++++ .../settings/clients/CreateClientsModal.tsx | 80 +++++ src/app/[orgId]/settings/clients/page.tsx | 57 +++ src/app/[orgId]/settings/layout.tsx | 7 +- 10 files changed, 940 insertions(+), 24 deletions(-) create mode 100644 src/app/[orgId]/settings/clients/ClientsDataTable.tsx create mode 100644 src/app/[orgId]/settings/clients/ClientsTable.tsx create mode 100644 src/app/[orgId]/settings/clients/CreateClientsForm.tsx create mode 100644 src/app/[orgId]/settings/clients/CreateClientsModal.tsx create mode 100644 src/app/[orgId]/settings/clients/page.tsx diff --git a/server/routers/client/createClient.ts b/server/routers/client/createClient.ts index 85f1735e..03b7826e 100644 --- a/server/routers/client/createClient.ts +++ b/server/routers/client/createClient.ts @@ -104,6 +104,12 @@ export async function createClient( return next(createHttpError(HttpCode.NOT_FOUND, "Site not found")); } + if (site.type !== "newt") { + return next( + createHttpError(HttpCode.BAD_REQUEST, "Site is not a newt site") + ); + } + await db.transaction(async (trx) => { const adminRole = await trx .select() diff --git a/server/routers/client/listClients.ts b/server/routers/client/listClients.ts index ad03bbf7..9273d54a 100644 --- a/server/routers/client/listClients.ts +++ b/server/routers/client/listClients.ts @@ -3,10 +3,8 @@ import { clients, orgs, roleClients, - roleSites, sites, userClients, - userSites } from "@server/db/schema"; import logger from "@server/logger"; import HttpCode from "@server/types/HttpCode"; @@ -41,16 +39,17 @@ const listClientsSchema = z.object({ function queryClients(orgId: string, accessibleClientIds: number[]) { return db .select({ - siteId: sites.siteId, - niceId: sites.niceId, - name: sites.name, - pubKey: sites.pubKey, - subnet: sites.subnet, - megabytesIn: sites.megabytesIn, - megabytesOut: sites.megabytesOut, + clientId: clients.clientId, + orgId: clients.orgId, + siteId: clients.siteId, + name: clients.name, + pubKey: clients.pubKey, + subnet: clients.subnet, + megabytesIn: clients.megabytesIn, + megabytesOut: clients.megabytesOut, orgName: orgs.name, - type: sites.type, - online: sites.online + type: clients.type, + online: clients.online }) .from(clients) .leftJoin(orgs, eq(clients.orgId, orgs.orgId)) @@ -115,22 +114,22 @@ export async function listClients( ) .where( or( - eq(userSites.userId, req.user!.userId), - eq(roleSites.roleId, req.userOrgRoleId!) + eq(userClients.userId, req.user!.userId), + eq(roleClients.roleId, req.userOrgRoleId!) ) ); - const accessibleSiteIds = accessibleClients.map( + const accessibleClientIds = accessibleClients.map( (site) => site.clientId ); - const baseQuery = queryClients(orgId, accessibleSiteIds); + const baseQuery = queryClients(orgId, accessibleClientIds); let countQuery = db .select({ count: count() }) .from(sites) .where( and( - inArray(sites.siteId, accessibleSiteIds), + inArray(sites.siteId, accessibleClientIds), eq(sites.orgId, orgId) ) ); diff --git a/server/routers/client/pickClientDefaults.ts b/server/routers/client/pickClientDefaults.ts index fd048259..32caaa3b 100644 --- a/server/routers/client/pickClientDefaults.ts +++ b/server/routers/client/pickClientDefaults.ts @@ -14,7 +14,7 @@ import { fromError } from "zod-validation-error"; const getSiteSchema = z .object({ - siteId: z.number().int().positive() + siteId: z.string().transform(Number).pipe(z.number()) }) .strict(); @@ -26,8 +26,8 @@ export type PickClientDefaultsResponse = { listenPort: number; endpoint: string; subnet: string; - clientId: string; - clientSecret: string; + olmId: string; + olmSecret: string; }; export async function pickClientDefaults( @@ -57,6 +57,15 @@ export async function pickClientDefaults( return next(createHttpError(HttpCode.NOT_FOUND, "Site not found")); } + if (site.type !== "newt") { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + "Site is not a newt site" + ) + ); + } + // make sure all the required fields are present const sitesRequiredFields = z.object({ @@ -109,7 +118,7 @@ export async function pickClientDefaults( ); } - const clientId = generateId(15); + const olmId = generateId(15); const secret = generateId(48); return response(res, { @@ -121,8 +130,8 @@ export async function pickClientDefaults( listenPort: listenPort, endpoint: endpoint, subnet: newSubnet, - clientId, - clientSecret: secret + olmId: olmId, + olmSecret: secret }, success: true, error: false, diff --git a/server/routers/site/createSite.ts b/server/routers/site/createSite.ts index 1b30b5c8..a1d1876f 100644 --- a/server/routers/site/createSite.ts +++ b/server/routers/site/createSite.ts @@ -35,7 +35,7 @@ const createSiteSchema = z subnet: z.string().optional(), newtId: z.string().optional(), secret: z.string().optional(), - type: z.enum(["newt", "wireguard"]) + type: z.enum(["newt", "wireguard", "local"]) }) .strict(); diff --git a/src/app/[orgId]/settings/clients/ClientsDataTable.tsx b/src/app/[orgId]/settings/clients/ClientsDataTable.tsx new file mode 100644 index 00000000..07bee372 --- /dev/null +++ b/src/app/[orgId]/settings/clients/ClientsDataTable.tsx @@ -0,0 +1,153 @@ +"use client"; + +import { + ColumnDef, + flexRender, + getCoreRowModel, + useReactTable, + getPaginationRowModel, + SortingState, + getSortedRowModel, + ColumnFiltersState, + getFilteredRowModel +} from "@tanstack/react-table"; + +import { + Table, + TableBody, + TableCell, + TableContainer, + TableHead, + TableHeader, + TableRow +} from "@/components/ui/table"; +import { Button } from "@app/components/ui/button"; +import { useState } from "react"; +import { Input } from "@app/components/ui/input"; +import { DataTablePagination } from "@app/components/DataTablePagination"; +import { Plus, Search } from "lucide-react"; + +interface DataTableProps { + columns: ColumnDef[]; + data: TData[]; + addClient?: () => void; +} + +export function ClientsDataTable({ + addClient, + columns, + data +}: DataTableProps) { + const [sorting, setSorting] = useState([]); + const [columnFilters, setColumnFilters] = useState([]); + + const table = useReactTable({ + data, + columns, + getCoreRowModel: getCoreRowModel(), + getPaginationRowModel: getPaginationRowModel(), + onSortingChange: setSorting, + getSortedRowModel: getSortedRowModel(), + onColumnFiltersChange: setColumnFilters, + getFilteredRowModel: getFilteredRowModel(), + initialState: { + pagination: { + pageSize: 20, + pageIndex: 0 + } + }, + state: { + sorting, + columnFilters + } + }); + + return ( +
+
+
+ + table + .getColumn("name") + ?.setFilterValue(event.target.value) + } + className="w-full pl-8" + /> + +
+ +
+ + + + {table.getHeaderGroups().map((headerGroup) => ( + + {headerGroup.headers.map((header) => { + return ( + + {header.isPlaceholder + ? null + : flexRender( + header.column.columnDef + .header, + header.getContext() + )} + + ); + })} + + ))} + + + {table.getRowModel().rows?.length ? ( + table.getRowModel().rows.map((row) => ( + + {row.getVisibleCells().map((cell) => ( + + {flexRender( + cell.column.columnDef.cell, + cell.getContext() + )} + + ))} + + )) + ) : ( + + + No clients. Create one to get started. + + + )} + +
+
+
+ +
+
+ ); +} diff --git a/src/app/[orgId]/settings/clients/ClientsTable.tsx b/src/app/[orgId]/settings/clients/ClientsTable.tsx new file mode 100644 index 00000000..bba06c00 --- /dev/null +++ b/src/app/[orgId]/settings/clients/ClientsTable.tsx @@ -0,0 +1,271 @@ +"use client"; + +import { ColumnDef } from "@tanstack/react-table"; +import { ClientsDataTable } from "./ClientsDataTable"; +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuTrigger +} from "@app/components/ui/dropdown-menu"; +import { Button } from "@app/components/ui/button"; +import { + ArrowRight, + ArrowUpDown, + Check, + MoreHorizontal, + X +} from "lucide-react"; +import Link from "next/link"; +import { useRouter } from "next/navigation"; +import { useState } from "react"; +import ConfirmDeleteDialog from "@app/components/ConfirmDeleteDialog"; +import { toast } from "@app/hooks/useToast"; +import { formatAxiosError } from "@app/lib/api"; +import { createApiClient } from "@app/lib/api"; +import { useEnvContext } from "@app/hooks/useEnvContext"; +import CreateClientFormModal from "./CreateClientsModal"; + +export type ClientRow = { + id: number; + name: string; + mbIn: string; + mbOut: string; + orgId: string; + online: boolean; +}; + +type ClientTableProps = { + clients: ClientRow[]; + orgId: string; +}; + +export default function ClientsTable({ clients, orgId }: ClientTableProps) { + const router = useRouter(); + + const [isCreateModalOpen, setIsCreateModalOpen] = useState(false); + const [isDeleteModalOpen, setIsDeleteModalOpen] = useState(false); + const [selectedClient, setSelectedClient] = useState( + null + ); + const [rows, setRows] = useState(clients); + + const api = createApiClient(useEnvContext()); + + const deleteSite = (clientId: number) => { + api.delete(`/client/${clientId}`) + .catch((e) => { + console.error("Error deleting client", e); + toast({ + variant: "destructive", + title: "Error deleting client", + description: formatAxiosError(e, "Error deleting client") + }); + }) + .then(() => { + router.refresh(); + setIsDeleteModalOpen(false); + + const newRows = rows.filter((row) => row.id !== clientId); + + setRows(newRows); + }); + }; + + const columns: ColumnDef[] = [ + { + id: "dots", + cell: ({ row }) => { + const clientRow = row.original; + const router = useRouter(); + + return ( + + + + + + {/* */} + {/* */} + {/* View settings */} + {/* */} + {/* */} + { + setSelectedClient(clientRow); + setIsDeleteModalOpen(true); + }} + > + Delete + + + + ); + } + }, + { + accessorKey: "name", + header: ({ column }) => { + return ( + + ); + } + }, + { + accessorKey: "online", + header: ({ column }) => { + return ( + + ); + }, + cell: ({ row }) => { + const originalRow = row.original; + if (originalRow.online) { + return ( + +
+ Online +
+ ); + } else { + return ( + +
+ Offline +
+ ); + } + } + }, + { + accessorKey: "mbIn", + header: ({ column }) => { + return ( + + ); + } + }, + { + accessorKey: "mbOut", + header: ({ column }) => { + return ( + + ); + } + } + // { + // id: "actions", + // cell: ({ row }) => { + // const siteRow = row.original; + // return ( + //
+ // + // + // + //
+ // ); + // } + // } + ]; + + return ( + <> + { + setRows([val, ...rows]); + }} + orgId={orgId} + /> + + {selectedClient && ( + { + setIsDeleteModalOpen(val); + setSelectedClient(null); + }} + dialog={ +
+

+ Are you sure you want to remove the client{" "} + + {selectedClient?.name || selectedClient?.id} + {" "} + from the site and organization? +

+ +

+ + Once removed, the client will no longer be + able to connect to the site.{" "} + +

+ +

+ To confirm, please type the name of the client + below. +

+
+ } + buttonText="Confirm Delete Client" + onConfirm={async () => deleteSite(selectedClient!.id)} + string={selectedClient.name} + title="Delete Client" + /> + )} + + { + setIsCreateModalOpen(true); + }} + /> + + ); +} diff --git a/src/app/[orgId]/settings/clients/CreateClientsForm.tsx b/src/app/[orgId]/settings/clients/CreateClientsForm.tsx new file mode 100644 index 00000000..ba5a9838 --- /dev/null +++ b/src/app/[orgId]/settings/clients/CreateClientsForm.tsx @@ -0,0 +1,336 @@ +"use client"; + +import { + Form, + FormControl, + FormDescription, + FormField, + FormItem, + FormLabel, + FormMessage +} from "@app/components/ui/form"; +import { Input } from "@app/components/ui/input"; +import { toast } from "@app/hooks/useToast"; +import { zodResolver } from "@hookform/resolvers/zod"; +import { useEffect, useState } from "react"; +import { useForm } from "react-hook-form"; +import { z } from "zod"; +import CopyTextBox from "@app/components/CopyTextBox"; +import { Checkbox } from "@app/components/ui/checkbox"; +import { formatAxiosError } from "@app/lib/api"; +import { createApiClient } from "@app/lib/api"; +import { useEnvContext } from "@app/hooks/useEnvContext"; +import { AxiosResponse } from "axios"; +import { Collapsible } from "@app/components/ui/collapsible"; +import { ClientRow } from "./ClientsTable"; +import { + CreateClientResponse, + PickClientDefaultsResponse +} from "@server/routers/client"; +import { ListSitesResponse } from "@server/routers/site"; +import { + Popover, + PopoverContent, + PopoverTrigger +} from "@app/components/ui/popover"; +import { Button } from "@app/components/ui/button"; +import { cn } from "@app/lib/cn"; +import { CaretSortIcon, CheckIcon } from "@radix-ui/react-icons"; +import { + Command, + CommandEmpty, + CommandGroup, + CommandInput, + CommandItem, + CommandList +} from "@app/components/ui/command"; + +const createClientFormSchema = z.object({ + name: z + .string() + .min(2, { + message: "Name must be at least 2 characters." + }) + .max(30, { + message: "Name must not be longer than 30 characters." + }), + siteId: z.coerce.number() +}); + +type CreateSiteFormValues = z.infer; + +const defaultValues: Partial = { + name: "" +}; + +type CreateSiteFormProps = { + onCreate?: (client: ClientRow) => void; + setLoading?: (loading: boolean) => void; + setChecked?: (checked: boolean) => void; + orgId: string; +}; + +export default function CreateClientForm({ + onCreate, + setLoading, + setChecked, + orgId +}: CreateSiteFormProps) { + const api = createApiClient(useEnvContext()); + const { env } = useEnvContext(); + + const [sites, setSites] = useState([]); + const [isLoading, setIsLoading] = useState(false); + const [isChecked, setIsChecked] = useState(false); + const [isOpen, setIsOpen] = useState(false); + const [clientDefaults, setClientDefaults] = + useState(null); + const [olmCommand, setOlmCommand] = useState(null); + + const handleCheckboxChange = (checked: boolean) => { + setIsChecked(checked); + }; + + const form = useForm({ + resolver: zodResolver(createClientFormSchema), + defaultValues + }); + + useEffect(() => { + if (!open) return; + + // reset all values + setLoading?.(false); + setIsLoading(false); + form.reset(); + setChecked?.(false); + setClientDefaults(null); + + const fetchSites = async () => { + const res = await api.get>( + `/org/${orgId}/sites/` + ); + setSites(res.data.data.sites); + + if (res.data.data.sites.length > 0) { + form.setValue("siteId", res.data.data.sites[0].siteId); + } + }; + + fetchSites(); + }, [open]); + + useEffect(() => { + const siteId = form.getValues("siteId"); + + if (siteId === undefined || siteId === null) return; + + api.get(`/site/${siteId}/pick-client-defaults`) + .catch((e) => { + toast({ + variant: "destructive", + title: `Error fetching client defaults for site ${siteId}`, + description: formatAxiosError(e) + }); + }) + .then((res) => { + if (res && res.status === 200) { + const data = res.data.data; + setClientDefaults(data); + const olmConfig = `olm --id ${data?.olmId} --secret ${data?.olmSecret} --endpoint ${env.app.dashboardUrl}`; + setOlmCommand(olmConfig); + } + }); + }, [form.watch("siteId")]); + + async function onSubmit(data: CreateSiteFormValues) { + setLoading?.(true); + setIsLoading(true); + + if (!clientDefaults) { + toast({ + variant: "destructive", + title: "Error creating site", + description: "Site defaults not found" + }); + setLoading?.(false); + setIsLoading(false); + return; + } + + const payload = { + name: data.name, + siteId: data.siteId, + orgId, + subnet: clientDefaults.subnet, + secret: clientDefaults.olmSecret, + olmId: clientDefaults.olmId + }; + + const res = await api + .put< + AxiosResponse + >(`/site/${data.siteId}/client`, payload) + .catch((e) => { + toast({ + variant: "destructive", + title: "Error creating client", + description: formatAxiosError(e) + }); + }); + + if (res && res.status === 201) { + const data = res.data.data; + + onCreate?.({ + name: data.name, + id: data.clientId, + mbIn: "0 MB", + mbOut: "0 MB", + orgId: orgId as string, + online: false + }); + } + + setLoading?.(false); + setIsLoading(false); + } + + return ( +
+
+ + ( + + Name + + + + + + )} + /> + + ( + + Client + + + + + + + + + + + + No site found. + + + {sites.map((site) => ( + { + form.setValue( + "siteId", + site.siteId + ); + }} + > + + {site.name} + + ))} + + + + + + + The client will be have connectivity to this + site. + + + + )} + /> + +
+
+ +
+ +
+
+
+ + You will only be able to see the configuration once. + +
+ +
+ + +
+ + +
+ ); +} diff --git a/src/app/[orgId]/settings/clients/CreateClientsModal.tsx b/src/app/[orgId]/settings/clients/CreateClientsModal.tsx new file mode 100644 index 00000000..450e655f --- /dev/null +++ b/src/app/[orgId]/settings/clients/CreateClientsModal.tsx @@ -0,0 +1,80 @@ +"use client"; + +import { Button } from "@app/components/ui/button"; +import { useState } from "react"; +import { + Credenza, + CredenzaBody, + CredenzaClose, + CredenzaContent, + CredenzaDescription, + CredenzaFooter, + CredenzaHeader, + CredenzaTitle +} from "@app/components/Credenza"; +import CreateClientForm from "./CreateClientsForm"; +import { ClientRow } from "./ClientsTable"; + +type CreateClientFormProps = { + open: boolean; + setOpen: (open: boolean) => void; + onCreate?: (client: ClientRow) => void; + orgId: string; +}; + +export default function CreateClientFormModal({ + open, + setOpen, + onCreate, + orgId +}: CreateClientFormProps) { + const [loading, setLoading] = useState(false); + const [isChecked, setIsChecked] = useState(false); + + return ( + <> + { + setOpen(val); + setLoading(false); + }} + > + + + Create Client + + Create a new client to connect to your sites + + + +
+ setLoading(val)} + setChecked={(val) => setIsChecked(val)} + onCreate={onCreate} + orgId={orgId} + /> +
+
+ + + + + + +
+
+ + ); +} diff --git a/src/app/[orgId]/settings/clients/page.tsx b/src/app/[orgId]/settings/clients/page.tsx new file mode 100644 index 00000000..145d99a7 --- /dev/null +++ b/src/app/[orgId]/settings/clients/page.tsx @@ -0,0 +1,57 @@ +import { internal } from "@app/lib/api"; +import { authCookieHeader } from "@app/lib/api/cookies"; +import { AxiosResponse } from "axios"; +import { ClientRow } from "./ClientsTable"; +import SettingsSectionTitle from "@app/components/SettingsSectionTitle"; +import { ListClientsResponse } from "@server/routers/client"; +import ClientsTable from "./ClientsTable"; + +type ClientsPageProps = { + params: Promise<{ orgId: string }>; +}; + +export const dynamic = "force-dynamic"; + +export default async function ClientsPage(props: ClientsPageProps) { + const params = await props.params; + let clients: ListClientsResponse["clients"] = []; + try { + const res = await internal.get>( + `/org/${params.orgId}/clients`, + await authCookieHeader() + ); + clients = res.data.data.clients; + } catch (e) {} + + function formatSize(mb: number): string { + if (mb >= 1024 * 1024) { + return `${(mb / (1024 * 1024)).toFixed(2)} TB`; + } else if (mb >= 1024) { + return `${(mb / 1024).toFixed(2)} GB`; + } else { + return `${mb.toFixed(2)} MB`; + } + } + + const clientRows: ClientRow[] = clients.map((client) => { + return { + name: client.name, + id: client.clientId, + mbIn: formatSize(client.megabytesIn || 0), + mbOut: formatSize(client.megabytesOut || 0), + orgId: params.orgId, + online: client.online + }; + }); + + return ( + <> + + + + + ); +} diff --git a/src/app/[orgId]/settings/layout.tsx b/src/app/[orgId]/settings/layout.tsx index b0b561a2..79727e91 100644 --- a/src/app/[orgId]/settings/layout.tsx +++ b/src/app/[orgId]/settings/layout.tsx @@ -1,6 +1,6 @@ import { Metadata } from "next"; import { TopbarNav } from "@app/components/TopbarNav"; -import { Cog, Combine, Link, Settings, Users, Waypoints } from "lucide-react"; +import { Cog, Combine, Laptop, Link, Settings, Users, Waypoints, Workflow } from "lucide-react"; import { Header } from "@app/components/Header"; import { verifySession } from "@app/lib/auth/verifySession"; import { redirect } from "next/navigation"; @@ -30,6 +30,11 @@ const topNavItems = [ href: "/{orgId}/settings/resources", icon: }, + { + title: "Clients", + href: "/{orgId}/settings/clients", + icon: + }, { title: "Users & Roles", href: "/{orgId}/settings/access", From 3830ad65fc78f6bf48a80f7c1d37ef6676d2ac68 Mon Sep 17 00:00:00 2001 From: miloschwartz Date: Fri, 21 Feb 2025 17:13:23 -0500 Subject: [PATCH 016/135] more frontend for clients --- server/routers/client/pickClientDefaults.ts | 4 +-- server/routers/newt/handleGetConfigMessage.ts | 4 +-- .../settings/clients/CreateClientsForm.tsx | 35 ++++++++++--------- 3 files changed, 22 insertions(+), 21 deletions(-) diff --git a/server/routers/client/pickClientDefaults.ts b/server/routers/client/pickClientDefaults.ts index b2d3d946..858c1bab 100644 --- a/server/routers/client/pickClientDefaults.ts +++ b/server/routers/client/pickClientDefaults.ts @@ -77,11 +77,11 @@ export async function pickClientDefaults( const parsedSite = sitesRequiredFields.safeParse(site); if (!parsedSite.success) { + logger.error("Unable to pick client defaults because: " + fromError(parsedSite.error).toString()); return next( createHttpError( HttpCode.BAD_REQUEST, - "Unable to pick client defaults because: " + - fromError(parsedSite.error).toString() + "Site is not configured to accept client connectivity" ) ); } diff --git a/server/routers/newt/handleGetConfigMessage.ts b/server/routers/newt/handleGetConfigMessage.ts index 7058a0c7..391a2af7 100644 --- a/server/routers/newt/handleGetConfigMessage.ts +++ b/server/routers/newt/handleGetConfigMessage.ts @@ -20,7 +20,7 @@ export const handleGetConfigMessage: MessageHandler = async (context) => { const newt = client as Newt; logger.debug(JSON.stringify(message.data)); - + logger.debug("Handling Newt get config message!"); @@ -60,7 +60,7 @@ export const handleGetConfigMessage: MessageHandler = async (context) => { let site: Site | undefined; if (!site) { const address = await getNextAvailableSubnet(); - const listenPort = await getNextAvailablePort(); + const listenPort = await getNextAvailablePort(); // create a new exit node const [updateRes] = await db diff --git a/src/app/[orgId]/settings/clients/CreateClientsForm.tsx b/src/app/[orgId]/settings/clients/CreateClientsForm.tsx index ba5a9838..e787ed35 100644 --- a/src/app/[orgId]/settings/clients/CreateClientsForm.tsx +++ b/src/app/[orgId]/settings/clients/CreateClientsForm.tsx @@ -110,10 +110,13 @@ export default function CreateClientForm({ const res = await api.get>( `/org/${orgId}/sites/` ); - setSites(res.data.data.sites); + const sites = res.data.data.sites.filter( + (s) => s.type === "newt" && s.subnet + ); + setSites(sites); - if (res.data.data.sites.length > 0) { - form.setValue("siteId", res.data.data.sites[0].siteId); + if (sites.length > 0) { + form.setValue("siteId", sites[0].siteId); } }; @@ -289,32 +292,30 @@ export default function CreateClientForm({ The client will be have connectivity to this - site. + site. The site must be configured to accept + client connections. )} /> -
-
- + {olmCommand && ( +
+
- +
+ + You will only be able to see the configuration + once. +
- - You will only be able to see the configuration once. - -
+ )}
Date: Fri, 21 Feb 2025 17:13:20 -0500 Subject: [PATCH 017/135] Fix up ws messages --- server/routers/newt/handleGetConfigMessage.ts | 4 ++-- server/routers/newt/handleNewtRegisterMessage.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/server/routers/newt/handleGetConfigMessage.ts b/server/routers/newt/handleGetConfigMessage.ts index 391a2af7..7226ce24 100644 --- a/server/routers/newt/handleGetConfigMessage.ts +++ b/server/routers/newt/handleGetConfigMessage.ts @@ -95,7 +95,7 @@ export const handleGetConfigMessage: MessageHandler = async (context) => { clientsRes.map(async (client) => { return { publicKey: client.pubKey, - allowedIps: "0.0.0.0/0" // TODO: We should lock this down more + allowedIps: ["0.0.0.0/0"] // TODO: We should lock this down more }; }) ); @@ -112,7 +112,7 @@ export const handleGetConfigMessage: MessageHandler = async (context) => { message: { type: "newt/wg/receive-config", // what to make the response type? data: { - config: configResponse + ...configResponse } }, broadcast: false, // Send to all clients diff --git a/server/routers/newt/handleNewtRegisterMessage.ts b/server/routers/newt/handleNewtRegisterMessage.ts index ec60641f..21ad4ba6 100644 --- a/server/routers/newt/handleNewtRegisterMessage.ts +++ b/server/routers/newt/handleNewtRegisterMessage.ts @@ -9,7 +9,7 @@ import { targets } from "@server/db/schema"; import { eq, and, sql } from "drizzle-orm"; -import { addPeer, deletePeer } from "../newt/peers"; +import { addPeer, deletePeer } from "../gerbil/peers"; import logger from "@server/logger"; export const handleNewtRegisterMessage: MessageHandler = async (context) => { From 6cf3bf02551f62fb269e4e2405d1f501362ab6d2 Mon Sep 17 00:00:00 2001 From: Owen Date: Fri, 21 Feb 2025 17:22:05 -0500 Subject: [PATCH 018/135] Not optional --- server/routers/client/createClient.ts | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/server/routers/client/createClient.ts b/server/routers/client/createClient.ts index 08d20d37..2ae59117 100644 --- a/server/routers/client/createClient.ts +++ b/server/routers/client/createClient.ts @@ -37,10 +37,9 @@ const createClientSchema = z .object({ name: z.string().min(1).max(255), siteId: z.number().int().positive(), - pubKey: z.string().optional(), - subnet: z.string().optional(), - olmId: z.string().optional(), - secret: z.string().optional(), + subnet: z.string(), + olmId: z.string(), + secret: z.string(), type: z.enum(["olm"]) }) .strict(); @@ -65,7 +64,7 @@ export async function createClient( ); } - const { name, type, siteId, pubKey, subnet, olmId, secret } = + const { name, type, siteId, subnet, olmId, secret } = parsedBody.data; const parsedParams = createClientParamsSchema.safeParse(req.params); @@ -104,12 +103,6 @@ export async function createClient( return next(createHttpError(HttpCode.NOT_FOUND, "Site not found")); } - if (site.type !== "newt") { - return next( - createHttpError(HttpCode.BAD_REQUEST, "Site is not a newt site") - ); - } - await db.transaction(async (trx) => { const adminRole = await trx .select() @@ -132,7 +125,6 @@ export async function createClient( siteId, orgId: site.orgId, name, - pubKey, subnet, type }) From f99efbb1e90110b5dda131a9b4a7a4e082418677 Mon Sep 17 00:00:00 2001 From: Owen Date: Fri, 21 Feb 2025 17:23:22 -0500 Subject: [PATCH 019/135] Update the public key and the endpoint --- server/routers/newt/handleGetConfigMessage.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/server/routers/newt/handleGetConfigMessage.ts b/server/routers/newt/handleGetConfigMessage.ts index 7226ce24..be460ab0 100644 --- a/server/routers/newt/handleGetConfigMessage.ts +++ b/server/routers/newt/handleGetConfigMessage.ts @@ -78,6 +78,16 @@ export const handleGetConfigMessage: MessageHandler = async (context) => { logger.info(`Updated site ${siteId} with new WG Newt info`); } else { + // update the endpoint and the public key + const [siteRes] = await db + .update(sites) + .set({ + publicKey, + endpoint + }) + .where(eq(sites.siteId, siteId)) + .returning(); + site = siteRes; } From 2e36c97d1cb4b61260bd5800a00dbb9ff10d6e43 Mon Sep 17 00:00:00 2001 From: miloschwartz Date: Fri, 21 Feb 2025 17:24:54 -0500 Subject: [PATCH 020/135] set checked --- src/app/[orgId]/settings/clients/CreateClientsForm.tsx | 5 ++++- src/app/[orgId]/settings/clients/CreateClientsModal.tsx | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/app/[orgId]/settings/clients/CreateClientsForm.tsx b/src/app/[orgId]/settings/clients/CreateClientsForm.tsx index e787ed35..f37c2fd4 100644 --- a/src/app/[orgId]/settings/clients/CreateClientsForm.tsx +++ b/src/app/[orgId]/settings/clients/CreateClientsForm.tsx @@ -89,6 +89,9 @@ export default function CreateClientForm({ const handleCheckboxChange = (checked: boolean) => { setIsChecked(checked); + if (setChecked) { + setChecked(checked); + } }; const form = useForm({ @@ -230,7 +233,7 @@ export default function CreateClientForm({ name="siteId" render={({ field }) => ( - Client + Site diff --git a/src/app/[orgId]/settings/clients/CreateClientsModal.tsx b/src/app/[orgId]/settings/clients/CreateClientsModal.tsx index 450e655f..ef6df0f3 100644 --- a/src/app/[orgId]/settings/clients/CreateClientsModal.tsx +++ b/src/app/[orgId]/settings/clients/CreateClientsModal.tsx @@ -44,7 +44,7 @@ export default function CreateClientFormModal({ Create Client - Create a new client to connect to your sites + Create a new client to connect to your site From a4d3a5ad4dab7531622840dfa7dc1302ce206dd0 Mon Sep 17 00:00:00 2001 From: miloschwartz Date: Fri, 21 Feb 2025 17:30:15 -0500 Subject: [PATCH 021/135] don't sent orgId in createClient --- src/app/[orgId]/settings/clients/CreateClientsForm.tsx | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/app/[orgId]/settings/clients/CreateClientsForm.tsx b/src/app/[orgId]/settings/clients/CreateClientsForm.tsx index f37c2fd4..3220f74d 100644 --- a/src/app/[orgId]/settings/clients/CreateClientsForm.tsx +++ b/src/app/[orgId]/settings/clients/CreateClientsForm.tsx @@ -24,6 +24,7 @@ import { AxiosResponse } from "axios"; import { Collapsible } from "@app/components/ui/collapsible"; import { ClientRow } from "./ClientsTable"; import { + CreateClientBody, CreateClientResponse, PickClientDefaultsResponse } from "@server/routers/client"; @@ -167,11 +168,11 @@ export default function CreateClientForm({ const payload = { name: data.name, siteId: data.siteId, - orgId, subnet: clientDefaults.subnet, + olmId: clientDefaults.olmId, secret: clientDefaults.olmSecret, - olmId: clientDefaults.olmId - }; + type: "olm" + } as CreateClientBody; const res = await api .put< From 450b0bf4fa6cef0979d2e7d630f030e7fe5a4a7c Mon Sep 17 00:00:00 2001 From: Owen Date: Fri, 21 Feb 2025 18:16:07 -0500 Subject: [PATCH 022/135] Use the right ip --- server/routers/olm/handleOlmRegisterMessage.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/routers/olm/handleOlmRegisterMessage.ts b/server/routers/olm/handleOlmRegisterMessage.ts index 147de81c..003cc3d6 100644 --- a/server/routers/olm/handleOlmRegisterMessage.ts +++ b/server/routers/olm/handleOlmRegisterMessage.ts @@ -87,7 +87,7 @@ export const handleOlmRegisterMessage: MessageHandler = async (context) => { endpoint: `${site.endpoint}:${site.listenPort}`, publicKey: site.publicKey, serverIP: site.address!.split("/")[0], - tunnelIP: site.subnet.split("/")[0] + tunnelIP: client.subnet.split("/")[0] } }, broadcast: false, // Send to all olms From b9080a1ec185bfae743b94152e82c175c0344a33 Mon Sep 17 00:00:00 2001 From: Owen Date: Fri, 21 Feb 2025 18:51:16 -0500 Subject: [PATCH 023/135] Working! --- install/fs/config.yml | 2 +- server/lib/config.ts | 2 +- server/routers/client/createClient.ts | 2 +- server/routers/client/pickClientDefaults.ts | 11 ++++------- server/routers/newt/handleGetConfigMessage.ts | 8 ++++---- server/routers/olm/handleOlmRegisterMessage.ts | 6 +++--- server/routers/site/pickSiteDefaults.ts | 2 +- 7 files changed, 15 insertions(+), 18 deletions(-) diff --git a/install/fs/config.yml b/install/fs/config.yml index 620ccbf3..bee904c1 100644 --- a/install/fs/config.yml +++ b/install/fs/config.yml @@ -32,7 +32,7 @@ gerbil: site_block_size: 30 subnet_group: 100.89.137.0/20 -wg_site: +newt: start_port: 51820 block_size: 24 subnet_group: 100.89.138.0/20 diff --git a/server/lib/config.ts b/server/lib/config.ts index 336b3ccb..9a115884 100644 --- a/server/lib/config.ts +++ b/server/lib/config.ts @@ -110,7 +110,7 @@ const configSchema = z.object({ block_size: z.number().positive().gt(0), site_block_size: z.number().positive().gt(0) }), - wg_site: z.object({ + newt: z.object({ block_size: z.number().positive().gt(0), subnet_group: z.string(), start_port: portSchema, diff --git a/server/routers/client/createClient.ts b/server/routers/client/createClient.ts index 2ae59117..0dd1e2da 100644 --- a/server/routers/client/createClient.ts +++ b/server/routers/client/createClient.ts @@ -18,7 +18,7 @@ import HttpCode from "@server/types/HttpCode"; import createHttpError from "http-errors"; import logger from "@server/logger"; import { eq, and } from "drizzle-orm"; -import { addPeer } from "../gerbil/peers"; +import { addPeer } from "../newt/peers"; import { fromError } from "zod-validation-error"; import { newts } from "@server/db/schema"; import moment from "moment"; diff --git a/server/routers/client/pickClientDefaults.ts b/server/routers/client/pickClientDefaults.ts index 858c1bab..33802b03 100644 --- a/server/routers/client/pickClientDefaults.ts +++ b/server/routers/client/pickClientDefaults.ts @@ -101,16 +101,13 @@ export async function pickClientDefaults( subnets.push( address.replace( /\/\d+$/, - `/${config.getRawConfig().wg_site.site_block_size}` + `/${config.getRawConfig().newt.site_block_size}` ) ); - logger.debug(`Subnets: ${subnets}`); - logger.debug(`Address: ${address}`); - logger.debug(`Block size: ${config.getRawConfig().wg_site.block_size}`); - logger.debug(`Site block size: ${config.getRawConfig().wg_site.site_block_size}`); + const newSubnet = findNextAvailableCidr( subnets, - config.getRawConfig().wg_site.site_block_size, + config.getRawConfig().newt.site_block_size, address ); if (!newSubnet) { @@ -133,7 +130,7 @@ export async function pickClientDefaults( name: site.name, listenPort: listenPort, endpoint: endpoint, - subnet: newSubnet, + subnet: `${newSubnet.split("/")[0]}/${config.getRawConfig().newt.block_size}`, // we want the block size of the whole subnet olmId: olmId, olmSecret: secret }, diff --git a/server/routers/newt/handleGetConfigMessage.ts b/server/routers/newt/handleGetConfigMessage.ts index be460ab0..ba8b5347 100644 --- a/server/routers/newt/handleGetConfigMessage.ts +++ b/server/routers/newt/handleGetConfigMessage.ts @@ -105,7 +105,7 @@ export const handleGetConfigMessage: MessageHandler = async (context) => { clientsRes.map(async (client) => { return { publicKey: client.pubKey, - allowedIps: ["0.0.0.0/0"] // TODO: We should lock this down more + allowedIps: [client.subnet] }; }) ); @@ -144,8 +144,8 @@ async function getNextAvailableSubnet(): Promise { let subnet = findNextAvailableCidr( addresses, - config.getRawConfig().wg_site.block_size, - config.getRawConfig().wg_site.subnet_group + config.getRawConfig().newt.block_size, + config.getRawConfig().newt.subnet_group ); if (!subnet) { throw new Error("No available subnets remaining in space"); @@ -167,7 +167,7 @@ async function getNextAvailablePort(): Promise { }).from(sites); // Find the first available port between 1024 and 65535 - let nextPort = config.getRawConfig().wg_site.start_port; + let nextPort = config.getRawConfig().newt.start_port; for (const port of existingPorts) { if (port.listenPort && port.listenPort > nextPort) { break; diff --git a/server/routers/olm/handleOlmRegisterMessage.ts b/server/routers/olm/handleOlmRegisterMessage.ts index 003cc3d6..0c82a523 100644 --- a/server/routers/olm/handleOlmRegisterMessage.ts +++ b/server/routers/olm/handleOlmRegisterMessage.ts @@ -73,11 +73,11 @@ export const handleOlmRegisterMessage: MessageHandler = async (context) => { logger.warn("Site has no subnet"); return; } - + // add the peer to the exit node await addPeer(site.siteId, { publicKey: publicKey, - allowedIps: [site.subnet] + allowedIps: [client.subnet] }); return { @@ -87,7 +87,7 @@ export const handleOlmRegisterMessage: MessageHandler = async (context) => { endpoint: `${site.endpoint}:${site.listenPort}`, publicKey: site.publicKey, serverIP: site.address!.split("/")[0], - tunnelIP: client.subnet.split("/")[0] + tunnelIP: client.subnet } }, broadcast: false, // Send to all olms diff --git a/server/routers/site/pickSiteDefaults.ts b/server/routers/site/pickSiteDefaults.ts index 79c2b324..02c204d4 100644 --- a/server/routers/site/pickSiteDefaults.ts +++ b/server/routers/site/pickSiteDefaults.ts @@ -84,7 +84,7 @@ export async function pickSiteDefaults( name: exitNode.name, listenPort: exitNode.listenPort, endpoint: exitNode.endpoint, - subnet: newSubnet, + subnet: `${newSubnet.split("/")[0]}/${config.getRawConfig().gerbil.block_size}`, // we want the block size of the whole subnet newtId, newtSecret: secret }, From fcf6abd36ea0a77136ca07fcded41d166edf773b Mon Sep 17 00:00:00 2001 From: miloschwartz Date: Fri, 21 Feb 2025 18:51:27 -0500 Subject: [PATCH 024/135] delete client instead of site --- server/routers/client/deleteClient.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/routers/client/deleteClient.ts b/server/routers/client/deleteClient.ts index 2036a3ce..9f1fb93e 100644 --- a/server/routers/client/deleteClient.ts +++ b/server/routers/client/deleteClient.ts @@ -43,12 +43,12 @@ export async function deleteClient( return next( createHttpError( HttpCode.NOT_FOUND, - `Site with ID ${clientId} not found` + `Client with ID ${clientId} not found` ) ); } - await db.delete(sites).where(eq(sites.siteId, clientId)); + await db.delete(clients).where(eq(clients.clientId, clientId)); return response(res, { data: null, From 9f54f4d81a38a70d18eef4625391175bfc5f780c Mon Sep 17 00:00:00 2001 From: miloschwartz Date: Fri, 21 Feb 2025 22:20:19 -0500 Subject: [PATCH 025/135] show site link in clients table --- server/db/schema.ts | 8 +++-- server/routers/client/listClients.ts | 5 ++- .../[orgId]/settings/clients/ClientsTable.tsx | 36 +++++++++++++++++-- .../settings/clients/CreateClientsForm.tsx | 4 +++ src/app/[orgId]/settings/clients/page.tsx | 2 ++ 5 files changed, 48 insertions(+), 7 deletions(-) diff --git a/server/db/schema.ts b/server/db/schema.ts index 6b7dcfda..24c705a9 100644 --- a/server/db/schema.ts +++ b/server/db/schema.ts @@ -118,9 +118,11 @@ export const newts = sqliteTable("newt", { export const clients = sqliteTable("clients", { clientId: integer("id").primaryKey({ autoIncrement: true }), - siteId: integer("siteId").references(() => sites.siteId, { - onDelete: "cascade" - }), + siteId: integer("siteId") + .references(() => sites.siteId, { + onDelete: "cascade" + }) + .notNull(), orgId: text("orgId") .references(() => orgs.orgId, { onDelete: "cascade" diff --git a/server/routers/client/listClients.ts b/server/routers/client/listClients.ts index 9273d54a..1bef1488 100644 --- a/server/routers/client/listClients.ts +++ b/server/routers/client/listClients.ts @@ -42,6 +42,7 @@ function queryClients(orgId: string, accessibleClientIds: number[]) { clientId: clients.clientId, orgId: clients.orgId, siteId: clients.siteId, + siteNiceId: sites.niceId, name: clients.name, pubKey: clients.pubKey, subnet: clients.subnet, @@ -49,10 +50,12 @@ function queryClients(orgId: string, accessibleClientIds: number[]) { megabytesOut: clients.megabytesOut, orgName: orgs.name, type: clients.type, - online: clients.online + online: clients.online, + siteName: sites.name }) .from(clients) .leftJoin(orgs, eq(clients.orgId, orgs.orgId)) + .innerJoin(sites, eq(clients.siteId, sites.siteId)) .where( and( inArray(clients.clientId, accessibleClientIds), diff --git a/src/app/[orgId]/settings/clients/ClientsTable.tsx b/src/app/[orgId]/settings/clients/ClientsTable.tsx index bba06c00..c7af53e3 100644 --- a/src/app/[orgId]/settings/clients/ClientsTable.tsx +++ b/src/app/[orgId]/settings/clients/ClientsTable.tsx @@ -12,6 +12,7 @@ import { Button } from "@app/components/ui/button"; import { ArrowRight, ArrowUpDown, + ArrowUpRight, Check, MoreHorizontal, X @@ -28,6 +29,8 @@ import CreateClientFormModal from "./CreateClientsModal"; export type ClientRow = { id: number; + siteId: string; + siteName: string; name: string; mbIn: string; mbOut: string; @@ -125,6 +128,33 @@ export default function ClientsTable({ clients, orgId }: ClientTableProps) { ); } }, + { + accessorKey: "siteName", + header: ({ column }) => { + return ( + + ); + }, + cell: ({ row }) => { + const r = row.original; + return ( + + + + ); + } + }, { accessorKey: "online", header: ({ column }) => { @@ -135,7 +165,7 @@ export default function ClientsTable({ clients, orgId }: ClientTableProps) { column.toggleSorting(column.getIsSorted() === "asc") } > - Online + Connectivity ); @@ -146,14 +176,14 @@ export default function ClientsTable({ clients, orgId }: ClientTableProps) { return (
- Online + Connected
); } else { return (
- Offline + Disconnected
); } diff --git a/src/app/[orgId]/settings/clients/CreateClientsForm.tsx b/src/app/[orgId]/settings/clients/CreateClientsForm.tsx index 3220f74d..09bdb7f9 100644 --- a/src/app/[orgId]/settings/clients/CreateClientsForm.tsx +++ b/src/app/[orgId]/settings/clients/CreateClientsForm.tsx @@ -189,8 +189,12 @@ export default function CreateClientForm({ if (res && res.status === 201) { const data = res.data.data; + const site = sites.find((site) => site.siteId === data.siteId); + onCreate?.({ name: data.name, + siteId: site!.niceId, + siteName: site!.name, id: data.clientId, mbIn: "0 MB", mbOut: "0 MB", diff --git a/src/app/[orgId]/settings/clients/page.tsx b/src/app/[orgId]/settings/clients/page.tsx index 145d99a7..2a422fc9 100644 --- a/src/app/[orgId]/settings/clients/page.tsx +++ b/src/app/[orgId]/settings/clients/page.tsx @@ -36,6 +36,8 @@ export default async function ClientsPage(props: ClientsPageProps) { const clientRows: ClientRow[] = clients.map((client) => { return { name: client.name, + siteName: client.siteName, + siteId: client.siteNiceId, id: client.clientId, mbIn: formatSize(client.megabytesIn || 0), mbOut: formatSize(client.megabytesOut || 0), From a9a9391b3942e0fa427747f6dedf51a611af56bd Mon Sep 17 00:00:00 2001 From: Owen Date: Fri, 21 Feb 2025 22:29:30 -0500 Subject: [PATCH 026/135] Receive new holepunch info --- server/db/schema.ts | 7 +- server/routers/gerbil/updateHolePunch.ts | 91 ++++++++++++++++++++++++ 2 files changed, 96 insertions(+), 2 deletions(-) create mode 100644 server/routers/gerbil/updateHolePunch.ts diff --git a/server/db/schema.ts b/server/db/schema.ts index 24c705a9..7355d0ca 100644 --- a/server/db/schema.ts +++ b/server/db/schema.ts @@ -31,7 +31,8 @@ export const sites = sqliteTable("sites", { address: text("address"), // this is the address of the wireguard interface in gerbil endpoint: text("endpoint"), // this is how to reach gerbil externally - gets put into the wireguard config publicKey: text("pubicKey"), - listenPort: integer("listenPort") + listenPort: integer("listenPort"), + lastHolePunch: integer("lastHolePunch"), }); export const resources = sqliteTable("resources", { @@ -135,7 +136,9 @@ export const clients = sqliteTable("clients", { megabytesOut: integer("bytesOut"), lastBandwidthUpdate: text("lastBandwidthUpdate"), type: text("type").notNull(), // "olm" - online: integer("online", { mode: "boolean" }).notNull().default(false) + online: integer("online", { mode: "boolean" }).notNull().default(false), + endpoint: text("endpoint"), + lastHolePunch: integer("lastHolePunch"), }); export const olms = sqliteTable("olms", { diff --git a/server/routers/gerbil/updateHolePunch.ts b/server/routers/gerbil/updateHolePunch.ts new file mode 100644 index 00000000..50648f13 --- /dev/null +++ b/server/routers/gerbil/updateHolePunch.ts @@ -0,0 +1,91 @@ +import { Request, Response, NextFunction } from "express"; +import { z } from "zod"; +import { clients, newts, olms, sites } from "@server/db/schema"; +import { db } from "@server/db"; +import { eq } from "drizzle-orm"; +import HttpCode from "@server/types/HttpCode"; +import createHttpError from "http-errors"; +import logger from "@server/logger"; +import { fromError } from "zod-validation-error"; + +// Define Zod schema for request validation +const updateHolePunchSchema = z.object({ + olmId: z.string().optional(), + newtId: z.string().optional(), + ip: z.string(), + port: z.number(), + timestamp: z.number() +}); + +export async function updateHolePunch( + req: Request, + res: Response, + next: NextFunction +): Promise { + try { + // Validate request parameters + const parsedParams = updateHolePunchSchema.safeParse(req.body); + if (!parsedParams.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedParams.error).toString() + ) + ); + } + + const { olmId, newtId, ip, port, timestamp } = parsedParams.data; + + if (olmId) { + const [olm] = await db + .select() + .from(olms) + .where(eq(olms.olmId, olmId)); + + if (!olm || !olm.clientId) { + logger.warn(`Olm not found: ${olmId}`); + return next( + createHttpError(HttpCode.NOT_FOUND, "Olm not found") + ); + } + + await db + .update(clients) + .set({ + endpoint: `${ip}:${port}`, + lastHolePunch: timestamp + }) + .where(eq(clients.clientId, olm.clientId)); + } else if (newtId) { + const [newt] = await db + .select() + .from(newts) + .where(eq(newts.newtId, newtId)); + + if (!newt || !newt.siteId) { + logger.warn(`Newt not found: ${newtId}`); + return next( + createHttpError(HttpCode.NOT_FOUND, "New not found") + ); + } + + await db + .update(sites) + .set({ + endpoint: `${ip}:${port}`, + lastHolePunch: timestamp + }) + .where(eq(sites.siteId, newt.siteId)); + } + + return res.status(HttpCode.OK).send({}); + } catch (error) { + logger.error(error); + return next( + createHttpError( + HttpCode.INTERNAL_SERVER_ERROR, + "An error occurred..." + ) + ); + } +} From e4c5be43507eb4c28e8671a4633ecfc1f18aba28 Mon Sep 17 00:00:00 2001 From: Owen Date: Sat, 22 Feb 2025 11:20:56 -0500 Subject: [PATCH 027/135] Initial hp working? --- server/routers/gerbil/index.ts | 1 + server/routers/gerbil/updateHolePunch.ts | 2 + server/routers/internal.ts | 1 + server/routers/newt/handleGetConfigMessage.ts | 38 ++++++++++++------- server/routers/newt/peers.ts | 1 + .../routers/olm/handleOlmRegisterMessage.ts | 22 ++++++++++- 6 files changed, 49 insertions(+), 16 deletions(-) diff --git a/server/routers/gerbil/index.ts b/server/routers/gerbil/index.ts index 82f82c4c..bcf1eb24 100644 --- a/server/routers/gerbil/index.ts +++ b/server/routers/gerbil/index.ts @@ -1,2 +1,3 @@ export * from "./getConfig"; export * from "./receiveBandwidth"; +export * from "./updateHolePunch"; \ No newline at end of file diff --git a/server/routers/gerbil/updateHolePunch.ts b/server/routers/gerbil/updateHolePunch.ts index 50648f13..36002f57 100644 --- a/server/routers/gerbil/updateHolePunch.ts +++ b/server/routers/gerbil/updateHolePunch.ts @@ -35,6 +35,8 @@ export async function updateHolePunch( } const { olmId, newtId, ip, port, timestamp } = parsedParams.data; + + logger.debug(`Got hole punch with ip: ${ip}, port: ${port} for olmId: ${olmId} or newtId: ${newtId}`); if (olmId) { const [olm] = await db diff --git a/server/routers/internal.ts b/server/routers/internal.ts index ead70d13..8392cc6e 100644 --- a/server/routers/internal.ts +++ b/server/routers/internal.ts @@ -34,6 +34,7 @@ internalRouter.use("/gerbil", gerbilRouter); gerbilRouter.post("/get-config", gerbil.getConfig); gerbilRouter.post("/receive-bandwidth", gerbil.receiveBandwidth); +gerbilRouter.post("/update-hole-punch", gerbil.updateHolePunch); // Badger routes const badgerRouter = Router(); diff --git a/server/routers/newt/handleGetConfigMessage.ts b/server/routers/newt/handleGetConfigMessage.ts index ba8b5347..602b02f0 100644 --- a/server/routers/newt/handleGetConfigMessage.ts +++ b/server/routers/newt/handleGetConfigMessage.ts @@ -21,7 +21,6 @@ export const handleGetConfigMessage: MessageHandler = async (context) => { logger.debug(JSON.stringify(message.data)); - logger.debug("Handling Newt get config message!"); if (!newt) { @@ -67,7 +66,7 @@ export const handleGetConfigMessage: MessageHandler = async (context) => { .update(sites) .set({ publicKey, - endpoint, + // endpoint, address, listenPort }) @@ -82,8 +81,8 @@ export const handleGetConfigMessage: MessageHandler = async (context) => { const [siteRes] = await db .update(sites) .set({ - publicKey, - endpoint + publicKey + // endpoint }) .where(eq(sites.siteId, siteId)) .returning(); @@ -101,13 +100,22 @@ export const handleGetConfigMessage: MessageHandler = async (context) => { .from(clients) .where(eq(clients.siteId, siteId)); + const now = new Date().getTime() / 1000; const peers = await Promise.all( - clientsRes.map(async (client) => { - return { - publicKey: client.pubKey, - allowedIps: [client.subnet] - }; - }) + clientsRes + .filter((client) => { + if (client.lastHolePunch && now - client.lastHolePunch > 6) { + logger.warn("Client last hole punch is too old"); + return; + } + }) + .map(async (client) => { + return { + publicKey: client.pubKey, + allowedIps: [client.subnet], + endpoint: client.endpoint + }; + }) ); const configResponse = { @@ -162,9 +170,11 @@ async function getNextAvailableSubnet(): Promise { async function getNextAvailablePort(): Promise { // Get all existing ports from exitNodes table - const existingPorts = await db.select({ - listenPort: sites.listenPort, - }).from(sites); + const existingPorts = await db + .select({ + listenPort: sites.listenPort + }) + .from(sites); // Find the first available port between 1024 and 65535 let nextPort = config.getRawConfig().newt.start_port; @@ -174,7 +184,7 @@ async function getNextAvailablePort(): Promise { } nextPort++; if (nextPort > 65535) { - throw new Error('No available ports remaining in space'); + throw new Error("No available ports remaining in space"); } } diff --git a/server/routers/newt/peers.ts b/server/routers/newt/peers.ts index afc3b5d6..a4bf8ae7 100644 --- a/server/routers/newt/peers.ts +++ b/server/routers/newt/peers.ts @@ -6,6 +6,7 @@ import { sendToClient } from '../ws'; export async function addPeer(siteId: number, peer: { publicKey: string; allowedIps: string[]; + endpoint: string; }) { const [site] = await db.select().from(sites).where(eq(sites.siteId, siteId)).limit(1); diff --git a/server/routers/olm/handleOlmRegisterMessage.ts b/server/routers/olm/handleOlmRegisterMessage.ts index 0c82a523..d7021588 100644 --- a/server/routers/olm/handleOlmRegisterMessage.ts +++ b/server/routers/olm/handleOlmRegisterMessage.ts @@ -56,6 +56,23 @@ export const handleOlmRegisterMessage: MessageHandler = async (context) => { return; } + // make sure we hand endpoints for both the site and the client and the lastHolePunch is not too old + if (!site.endpoint || !client.endpoint) { + logger.warn("Site or client has no endpoint or listen port"); + return; + } + + const now = new Date().getTime() / 1000; + if (site.lastHolePunch && now - site.lastHolePunch > 6) { + logger.warn("Site last hole punch is too old"); + return; + } + + if (client.lastHolePunch && now - client.lastHolePunch > 6) { + logger.warn("Client last hole punch is too old"); + return; + } + await db .update(clients) .set({ @@ -77,14 +94,15 @@ export const handleOlmRegisterMessage: MessageHandler = async (context) => { // add the peer to the exit node await addPeer(site.siteId, { publicKey: publicKey, - allowedIps: [client.subnet] + allowedIps: [client.subnet], + endpoint: client.endpoint }); return { message: { type: "olm/wg/connect", data: { - endpoint: `${site.endpoint}:${site.listenPort}`, + endpoint: site.endpoint, publicKey: site.publicKey, serverIP: site.address!.split("/")[0], tunnelIP: client.subnet From bebe40c8e85b2d0cc75f067ad3fd38ebb8e105c7 Mon Sep 17 00:00:00 2001 From: Owen Date: Sat, 22 Feb 2025 12:53:35 -0500 Subject: [PATCH 028/135] HP works! --- server/routers/newt/handleGetConfigMessage.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/routers/newt/handleGetConfigMessage.ts b/server/routers/newt/handleGetConfigMessage.ts index 602b02f0..aa994db5 100644 --- a/server/routers/newt/handleGetConfigMessage.ts +++ b/server/routers/newt/handleGetConfigMessage.ts @@ -57,7 +57,7 @@ export const handleGetConfigMessage: MessageHandler = async (context) => { } let site: Site | undefined; - if (!site) { + if (!siteRes.address) { const address = await getNextAvailableSubnet(); const listenPort = await getNextAvailablePort(); From afd87d07a37486402d581c9274253ba23e20ec71 Mon Sep 17 00:00:00 2001 From: Owen Date: Sun, 23 Feb 2025 16:49:41 -0500 Subject: [PATCH 029/135] Basic relay working! --- server/routers/gerbil/getAllRelays.ts | 89 +++++++++++++++++++ server/routers/gerbil/getConfig.ts | 4 +- server/routers/gerbil/index.ts | 3 +- server/routers/gerbil/updateHolePunch.ts | 33 +++++-- server/routers/internal.ts | 1 + server/routers/newt/handleGetConfigMessage.ts | 5 +- server/routers/newt/peers.ts | 5 ++ server/routers/olm/getOlmToken.ts | 2 - 8 files changed, 125 insertions(+), 17 deletions(-) create mode 100644 server/routers/gerbil/getAllRelays.ts diff --git a/server/routers/gerbil/getAllRelays.ts b/server/routers/gerbil/getAllRelays.ts new file mode 100644 index 00000000..2284d4eb --- /dev/null +++ b/server/routers/gerbil/getAllRelays.ts @@ -0,0 +1,89 @@ +import { Request, Response, NextFunction } from "express"; +import { z } from "zod"; +import { clients, exitNodes, newts, olms, Site, sites } from "@server/db/schema"; +import { db } from "@server/db"; +import { eq } from "drizzle-orm"; +import HttpCode from "@server/types/HttpCode"; +import createHttpError from "http-errors"; +import logger from "@server/logger"; +import { fromError } from "zod-validation-error"; + +// Define Zod schema for request validation +const getAllRelaysSchema = z.object({ + publicKey: z.string().optional(), +}); + +export async function getAllRelays( + req: Request, + res: Response, + next: NextFunction +): Promise { + try { + // Validate request parameters + const parsedParams = getAllRelaysSchema.safeParse(req.body); + if (!parsedParams.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedParams.error).toString() + ) + ); + } + + const { publicKey } = parsedParams.data; + + if (!publicKey) { + return next(createHttpError(HttpCode.BAD_REQUEST, 'publicKey is required')); + } + + // Fetch exit node + let [exitNode] = await db.select().from(exitNodes).where(eq(exitNodes.publicKey, publicKey)); + if (!exitNode) { + return next(createHttpError(HttpCode.NOT_FOUND, "Exit node not found")); + } + + // Fetch sites for this exit node + const sitesRes = await db.select().from(sites).where(eq(sites.exitNodeId, exitNode.exitNodeId)); + + if (sitesRes.length === 0) { + return next(createHttpError(HttpCode.NOT_FOUND, "No sites found for this exit node")); + } + + // get the clients on each site and map them to the site + const sitesAndClients = await Promise.all(sitesRes.map(async (site) => { + const clientsRes = await db.select().from(clients).where(eq(clients.siteId, site.siteId)); + return { + site, + clients: clientsRes + }; + })); + + let mappings: { [key: string]: { + destinationIp: string; + destinationPort: number; + } } = {}; + + for (const siteAndClients of sitesAndClients) { + const { site, clients } = siteAndClients; + for (const client of clients) { + if (!client.endpoint || !site.endpoint || !site.subnet) { + continue; + } + mappings[client.endpoint] = { + destinationIp: site.subnet.split("/")[0], + destinationPort: parseInt(site.endpoint.split(":")[1]) + }; + } + } + + return res.status(HttpCode.OK).send({ mappings }); + } catch (error) { + logger.error(error); + return next( + createHttpError( + HttpCode.INTERNAL_SERVER_ERROR, + "An error occurred..." + ) + ); + } +} diff --git a/server/routers/gerbil/getConfig.ts b/server/routers/gerbil/getConfig.ts index 95e0df6b..0c50944e 100644 --- a/server/routers/gerbil/getConfig.ts +++ b/server/routers/gerbil/getConfig.ts @@ -79,9 +79,7 @@ export async function getConfig(req: Request, res: Response, next: NextFunction) } // Fetch sites for this exit node - const sitesRes = await db.query.sites.findMany({ - where: eq(sites.exitNodeId, exitNode[0].exitNodeId), - }); + const sitesRes = await db.select().from(sites).where(eq(sites.exitNodeId, exitNode[0].exitNodeId)); const peers = await Promise.all(sitesRes.map(async (site) => { return { diff --git a/server/routers/gerbil/index.ts b/server/routers/gerbil/index.ts index bcf1eb24..4a4f3b60 100644 --- a/server/routers/gerbil/index.ts +++ b/server/routers/gerbil/index.ts @@ -1,3 +1,4 @@ export * from "./getConfig"; export * from "./receiveBandwidth"; -export * from "./updateHolePunch"; \ No newline at end of file +export * from "./updateHolePunch"; +export * from "./getAllRelays"; \ No newline at end of file diff --git a/server/routers/gerbil/updateHolePunch.ts b/server/routers/gerbil/updateHolePunch.ts index 36002f57..68a7282f 100644 --- a/server/routers/gerbil/updateHolePunch.ts +++ b/server/routers/gerbil/updateHolePunch.ts @@ -1,6 +1,6 @@ import { Request, Response, NextFunction } from "express"; import { z } from "zod"; -import { clients, newts, olms, sites } from "@server/db/schema"; +import { clients, newts, olms, Site, sites } from "@server/db/schema"; import { db } from "@server/db"; import { eq } from "drizzle-orm"; import HttpCode from "@server/types/HttpCode"; @@ -36,7 +36,9 @@ export async function updateHolePunch( const { olmId, newtId, ip, port, timestamp } = parsedParams.data; - logger.debug(`Got hole punch with ip: ${ip}, port: ${port} for olmId: ${olmId} or newtId: ${newtId}`); + // logger.debug(`Got hole punch with ip: ${ip}, port: ${port} for olmId: ${olmId} or newtId: ${newtId}`); + + let site: Site | undefined; if (olmId) { const [olm] = await db @@ -51,13 +53,19 @@ export async function updateHolePunch( ); } - await db + const [client] = await db .update(clients) .set({ endpoint: `${ip}:${port}`, lastHolePunch: timestamp }) - .where(eq(clients.clientId, olm.clientId)); + .where(eq(clients.clientId, olm.clientId)) + .returning(); + + [site] = await db + .select() + .from(sites) + .where(eq(sites.siteId, client.siteId)); } else if (newtId) { const [newt] = await db .select() @@ -71,16 +79,27 @@ export async function updateHolePunch( ); } - await db + [site] = await db .update(sites) .set({ endpoint: `${ip}:${port}`, lastHolePunch: timestamp }) - .where(eq(sites.siteId, newt.siteId)); + .where(eq(sites.siteId, newt.siteId)) + .returning(); } - return res.status(HttpCode.OK).send({}); + if (!site || !site.endpoint || !site.subnet) { + logger.warn(`Site not found for olmId: ${olmId} or newtId: ${newtId}`); + return next( + createHttpError(HttpCode.NOT_FOUND, "Site not found") + ); + } + + return res.status(HttpCode.OK).send({ + destinationIp: site.subnet.split("/")[0], + destinationPort: parseInt(site.endpoint.split(":")[1]) + }); } catch (error) { logger.error(error); return next( diff --git a/server/routers/internal.ts b/server/routers/internal.ts index 8392cc6e..6b57e3f6 100644 --- a/server/routers/internal.ts +++ b/server/routers/internal.ts @@ -35,6 +35,7 @@ internalRouter.use("/gerbil", gerbilRouter); gerbilRouter.post("/get-config", gerbil.getConfig); gerbilRouter.post("/receive-bandwidth", gerbil.receiveBandwidth); gerbilRouter.post("/update-hole-punch", gerbil.updateHolePunch); +gerbilRouter.post("/get-all-relays", gerbil.getAllRelays); // Badger routes const badgerRouter = Router(); diff --git a/server/routers/newt/handleGetConfigMessage.ts b/server/routers/newt/handleGetConfigMessage.ts index aa994db5..62934d2f 100644 --- a/server/routers/newt/handleGetConfigMessage.ts +++ b/server/routers/newt/handleGetConfigMessage.ts @@ -10,7 +10,6 @@ import config from "@server/lib/config"; const inputSchema = z.object({ publicKey: z.string(), - endpoint: z.string() }); type Input = z.infer; @@ -42,7 +41,7 @@ export const handleGetConfigMessage: MessageHandler = async (context) => { return; } - const { publicKey, endpoint } = message.data as Input; + const { publicKey } = message.data as Input; const siteId = newt.siteId; @@ -66,7 +65,6 @@ export const handleGetConfigMessage: MessageHandler = async (context) => { .update(sites) .set({ publicKey, - // endpoint, address, listenPort }) @@ -82,7 +80,6 @@ export const handleGetConfigMessage: MessageHandler = async (context) => { .update(sites) .set({ publicKey - // endpoint }) .where(eq(sites.siteId, siteId)) .returning(); diff --git a/server/routers/newt/peers.ts b/server/routers/newt/peers.ts index a4bf8ae7..5808484f 100644 --- a/server/routers/newt/peers.ts +++ b/server/routers/newt/peers.ts @@ -2,6 +2,7 @@ import db from '@server/db'; import { newts, sites } from '@server/db/schema'; import { eq } from 'drizzle-orm'; import { sendToClient } from '../ws'; +import logger from '@server/logger'; export async function addPeer(siteId: number, peer: { publicKey: string; @@ -24,6 +25,8 @@ export async function addPeer(siteId: number, peer: { type: 'newt/wg/peer/add', data: peer }); + + logger.info(`Added peer ${peer.publicKey} to newt ${newt.newtId}`); } export async function deletePeer(siteId: number, publicKey: string) { @@ -44,4 +47,6 @@ export async function deletePeer(siteId: number, publicKey: string) { publicKey } }); + + logger.info(`Deleted peer ${publicKey} from newt ${newt.newtId}`); } \ No newline at end of file diff --git a/server/routers/olm/getOlmToken.ts b/server/routers/olm/getOlmToken.ts index ce40a35e..00ab9358 100644 --- a/server/routers/olm/getOlmToken.ts +++ b/server/routers/olm/getOlmToken.ts @@ -74,8 +74,6 @@ export async function getOlmToken( ); } - logger.debug("Existing olm: ", existingOlmRes); - const existingOlm = existingOlmRes[0]; const validSecret = await verifyPassword( From bacc5e421397e4b1112012c78cf55442dd0d152f Mon Sep 17 00:00:00 2001 From: Owen Date: Sun, 23 Feb 2025 20:18:03 -0500 Subject: [PATCH 030/135] Add relay message --- server/routers/messageHandlers.ts | 5 +- server/routers/olm/handleOlmRelayMessage.ts | 81 +++++++++++++++++++++ server/routers/olm/index.ts | 3 +- 3 files changed, 86 insertions(+), 3 deletions(-) create mode 100644 server/routers/olm/handleOlmRelayMessage.ts diff --git a/server/routers/messageHandlers.ts b/server/routers/messageHandlers.ts index f23ea0a8..759a88ea 100644 --- a/server/routers/messageHandlers.ts +++ b/server/routers/messageHandlers.ts @@ -1,5 +1,5 @@ import { handleNewtRegisterMessage, handleReceiveBandwidthMessage } from "./newt"; -import { handleOlmRegisterMessage } from "./olm"; +import { handleOlmRegisterMessage, handleOlmRelayMessage } from "./olm"; import { handleGetConfigMessage } from "./newt/handleGetConfigMessage"; import { MessageHandler } from "./ws"; @@ -7,5 +7,6 @@ export const messageHandlers: Record = { "newt/wg/register": handleNewtRegisterMessage, "olm/wg/register": handleOlmRegisterMessage, "newt/wg/get-config": handleGetConfigMessage, - "newt/receive-bandwidth": handleReceiveBandwidthMessage + "newt/receive-bandwidth": handleReceiveBandwidthMessage, + "olm/wg/relay": handleOlmRelayMessage }; diff --git a/server/routers/olm/handleOlmRelayMessage.ts b/server/routers/olm/handleOlmRelayMessage.ts new file mode 100644 index 00000000..9cf0ceb2 --- /dev/null +++ b/server/routers/olm/handleOlmRelayMessage.ts @@ -0,0 +1,81 @@ +import db from "@server/db"; +import { MessageHandler } from "../ws"; +import { clients, Olm, olms, sites } from "@server/db/schema"; +import { eq } from "drizzle-orm"; +import { addPeer, deletePeer } from "../newt/peers"; +import logger from "@server/logger"; + +export const handleOlmRelayMessage: MessageHandler = async (context) => { + const { message, client: c, sendToClient } = context; + const olm = c as Olm; + + logger.info("Handling relay olm message!"); + + if (!olm) { + logger.warn("Olm not found"); + return; + } + + if (!olm.clientId) { + logger.warn("Olm has no site!"); // TODO: Maybe we create the site here? + return; + } + + const clientId = olm.clientId; + + const [client] = await db + .select() + .from(clients) + .where(eq(clients.clientId, clientId)) + .limit(1); + + if (!client || !client.siteId) { + logger.warn("Site not found or does not have exit node"); + return; + } + + const [site] = await db + .select() + .from(sites) + .where(eq(sites.siteId, client.siteId)) + .limit(1); + + if (!client) { + logger.warn("Site not found or does not have exit node"); + return; + } + + // make sure we hand endpoints for both the site and the client and the lastHolePunch is not too old + if (!client.pubKey) { + logger.warn("Site or client has no endpoint or listen port"); + return; + } + + if (!site.subnet) { + logger.warn("Site has no subnet"); + return; + } + + await deletePeer(site.siteId, client.pubKey); + + // add the peer to the exit node + await addPeer(site.siteId, { + publicKey: client.pubKey, + allowedIps: [client.subnet], + endpoint: "" + }); + + return { + message: { + type: "olm/wg/connect", + data: { + endpoint: site.endpoint, + publicKey: site.publicKey, + serverIP: site.address!.split("/")[0], + tunnelIP: client.subnet + } + }, + broadcast: false, // Send to all olms + excludeSender: false // Include sender in broadcast + }; +}; diff --git a/server/routers/olm/index.ts b/server/routers/olm/index.ts index 616480cc..b7373961 100644 --- a/server/routers/olm/index.ts +++ b/server/routers/olm/index.ts @@ -1,3 +1,4 @@ export * from "./handleOlmRegisterMessage"; export * from "./getOlmToken"; -export * from "./createOlm"; \ No newline at end of file +export * from "./createOlm"; +export * from "./handleOlmRelayMessage"; \ No newline at end of file From 8ee6a3f134dc7e007d9ca76395ad68db5fb5335f Mon Sep 17 00:00:00 2001 From: Owen Date: Sun, 23 Feb 2025 21:35:13 -0500 Subject: [PATCH 031/135] Add bruno --- bruno/Clients/createClient.bru | 22 ++++++++++++++++++++++ bruno/Clients/pickClientDefaults.bru | 11 +++++++++++ 2 files changed, 33 insertions(+) create mode 100644 bruno/Clients/createClient.bru create mode 100644 bruno/Clients/pickClientDefaults.bru diff --git a/bruno/Clients/createClient.bru b/bruno/Clients/createClient.bru new file mode 100644 index 00000000..7577bb28 --- /dev/null +++ b/bruno/Clients/createClient.bru @@ -0,0 +1,22 @@ +meta { + name: createClient + type: http + seq: 1 +} + +put { + url: http://localhost:3000/api/v1/site/1/client + body: json + auth: none +} + +body:json { + { + "siteId": 1, + "name": "test", + "type": "olm", + "subnet": "100.90.129.4/30", + "olmId": "029yzunhx6nh3y5", + "secret": "l0ymp075y3d4rccb25l6sqpgar52k09etunui970qq5gj7x6" + } +} diff --git a/bruno/Clients/pickClientDefaults.bru b/bruno/Clients/pickClientDefaults.bru new file mode 100644 index 00000000..61509c11 --- /dev/null +++ b/bruno/Clients/pickClientDefaults.bru @@ -0,0 +1,11 @@ +meta { + name: pickClientDefaults + type: http + seq: 2 +} + +get { + url: http://localhost:3000/api/v1/site/1/pick-client-defaults + body: none + auth: none +} From 733e0e07c362a7d43b4b3d261023c6fb82d90140 Mon Sep 17 00:00:00 2001 From: Owen Date: Mon, 24 Feb 2025 20:21:57 -0500 Subject: [PATCH 032/135] Fix subnet issues --- server/routers/client/pickClientDefaults.ts | 3 ++- server/routers/gerbil/getAllRelays.ts | 4 +++- server/routers/olm/handleOlmRegisterMessage.ts | 2 +- server/routers/site/pickSiteDefaults.ts | 3 ++- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/server/routers/client/pickClientDefaults.ts b/server/routers/client/pickClientDefaults.ts index 33802b03..d77ae3bb 100644 --- a/server/routers/client/pickClientDefaults.ts +++ b/server/routers/client/pickClientDefaults.ts @@ -130,7 +130,8 @@ export async function pickClientDefaults( name: site.name, listenPort: listenPort, endpoint: endpoint, - subnet: `${newSubnet.split("/")[0]}/${config.getRawConfig().newt.block_size}`, // we want the block size of the whole subnet + // subnet: `${newSubnet.split("/")[0]}/${config.getRawConfig().newt.block_size}`, // we want the block size of the whole subnet + subnet: newSubnet, olmId: olmId, olmSecret: secret }, diff --git a/server/routers/gerbil/getAllRelays.ts b/server/routers/gerbil/getAllRelays.ts index 2284d4eb..edc702b7 100644 --- a/server/routers/gerbil/getAllRelays.ts +++ b/server/routers/gerbil/getAllRelays.ts @@ -46,7 +46,9 @@ export async function getAllRelays( const sitesRes = await db.select().from(sites).where(eq(sites.exitNodeId, exitNode.exitNodeId)); if (sitesRes.length === 0) { - return next(createHttpError(HttpCode.NOT_FOUND, "No sites found for this exit node")); + return { + mappings: {} + } } // get the clients on each site and map them to the site diff --git a/server/routers/olm/handleOlmRegisterMessage.ts b/server/routers/olm/handleOlmRegisterMessage.ts index d7021588..8ae44b13 100644 --- a/server/routers/olm/handleOlmRegisterMessage.ts +++ b/server/routers/olm/handleOlmRegisterMessage.ts @@ -105,7 +105,7 @@ export const handleOlmRegisterMessage: MessageHandler = async (context) => { endpoint: site.endpoint, publicKey: site.publicKey, serverIP: site.address!.split("/")[0], - tunnelIP: client.subnet + tunnelIP: `${client.subnet.split("/")[0]}/${site.address!.split("/")[1]}`, // put the client ip in the same subnet as the site. TODO: Is this right? Maybe we need th make .subnet work properly! } }, broadcast: false, // Send to all olms diff --git a/server/routers/site/pickSiteDefaults.ts b/server/routers/site/pickSiteDefaults.ts index 02c204d4..e0d8b03c 100644 --- a/server/routers/site/pickSiteDefaults.ts +++ b/server/routers/site/pickSiteDefaults.ts @@ -84,7 +84,8 @@ export async function pickSiteDefaults( name: exitNode.name, listenPort: exitNode.listenPort, endpoint: exitNode.endpoint, - subnet: `${newSubnet.split("/")[0]}/${config.getRawConfig().gerbil.block_size}`, // we want the block size of the whole subnet + // subnet: `${newSubnet.split("/")[0]}/${config.getRawConfig().gerbil.block_size}`, // we want the block size of the whole subnet + subnet: newSubnet, newtId, newtSecret: secret }, From 8e4bccffbffe06ed5eaa730e06d10996df051714 Mon Sep 17 00:00:00 2001 From: Owen Date: Thu, 27 Feb 2025 23:54:24 -0500 Subject: [PATCH 033/135] Dont use bytes out for online Persistant keep alive means that this will always increase --- server/routers/newt/handleReceiveBandwidthMessage.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/routers/newt/handleReceiveBandwidthMessage.ts b/server/routers/newt/handleReceiveBandwidthMessage.ts index a20e2426..80b0c57d 100644 --- a/server/routers/newt/handleReceiveBandwidthMessage.ts +++ b/server/routers/newt/handleReceiveBandwidthMessage.ts @@ -27,7 +27,7 @@ export const handleReceiveBandwidthMessage: MessageHandler = async (context) => for (const peer of bandwidthData) { const { publicKey, bytesIn, bytesOut } = peer; - // Find the site by public key + // Find the client by public key const [client] = await trx .select() .from(clients) @@ -39,8 +39,8 @@ export const handleReceiveBandwidthMessage: MessageHandler = async (context) => } let online = client.online; - // if the bandwidth for the site is > 0 then set it to online. if it has been less than 0 (no update) for 5 minutes then set it to offline - if (bytesIn > 0 || bytesOut > 0) { + // if the bandwidth for the client is > 0 then set it to online. if it has been less than 0 (no update) for 5 minutes then set it to offline + if (bytesIn > 0) { // only track the bytes in because we are always sending bytes out with persistent keep alive online = true; } else if (client.lastBandwidthUpdate) { const lastBandwidthUpdate = new Date( @@ -54,7 +54,7 @@ export const handleReceiveBandwidthMessage: MessageHandler = async (context) => } } - // Update the site's bandwidth usage + // Update the client's bandwidth usage await trx .update(clients) .set({ From 14e61366835f0eaeaf8afbae396b8991f2aeefc0 Mon Sep 17 00:00:00 2001 From: Owen Date: Fri, 14 Mar 2025 09:18:13 -0400 Subject: [PATCH 034/135] Send down gerbil pub so it can encrypt --- .../routers/olm/handleOlmRegisterMessage.ts | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/server/routers/olm/handleOlmRegisterMessage.ts b/server/routers/olm/handleOlmRegisterMessage.ts index 8ae44b13..cf3164be 100644 --- a/server/routers/olm/handleOlmRegisterMessage.ts +++ b/server/routers/olm/handleOlmRegisterMessage.ts @@ -1,12 +1,7 @@ import db from "@server/db"; import { MessageHandler } from "../ws"; -import { - clients, - Olm, - olms, - sites, -} from "@server/db/schema"; -import { eq, } from "drizzle-orm"; +import { clients, Olm, olms, sites } from "@server/db/schema"; +import { eq } from "drizzle-orm"; import { addPeer, deletePeer } from "../newt/peers"; import logger from "@server/logger"; @@ -62,6 +57,16 @@ export const handleOlmRegisterMessage: MessageHandler = async (context) => { return; } + sendToClient(olm.olmId, { + type: "olm/wg/holepunch", + data: { + endpoint: site.endpoint, + publicKey: site.publicKey, + serverIP: site.address!.split("/")[0], + tunnelIP: `${client.subnet.split("/")[0]}/${site.address!.split("/")[1]}` // put the client ip in the same subnet as the site. TODO: Is this right? Maybe we need th make .subnet work properly! + } + }); + const now = new Date().getTime() / 1000; if (site.lastHolePunch && now - site.lastHolePunch > 6) { logger.warn("Site last hole punch is too old"); @@ -90,7 +95,7 @@ export const handleOlmRegisterMessage: MessageHandler = async (context) => { logger.warn("Site has no subnet"); return; } - + // add the peer to the exit node await addPeer(site.siteId, { publicKey: publicKey, @@ -105,7 +110,7 @@ export const handleOlmRegisterMessage: MessageHandler = async (context) => { endpoint: site.endpoint, publicKey: site.publicKey, serverIP: site.address!.split("/")[0], - tunnelIP: `${client.subnet.split("/")[0]}/${site.address!.split("/")[1]}`, // put the client ip in the same subnet as the site. TODO: Is this right? Maybe we need th make .subnet work properly! + tunnelIP: `${client.subnet.split("/")[0]}/${site.address!.split("/")[1]}` // put the client ip in the same subnet as the site. TODO: Is this right? Maybe we need th make .subnet work properly! } }, broadcast: false, // Send to all olms From 779a1c303f5da041ec99af8b6b14a6d723c54140 Mon Sep 17 00:00:00 2001 From: Owen Date: Sat, 15 Mar 2025 21:45:53 -0400 Subject: [PATCH 035/135] Verify holepunch token and send gerbil pub key --- server/routers/gerbil/updateHolePunch.ts | 45 ++++++++++++++++--- .../routers/olm/handleOlmRegisterMessage.ts | 32 ++++++++----- src/app/[orgId]/settings/layout.tsx | 3 +- 3 files changed, 62 insertions(+), 18 deletions(-) diff --git a/server/routers/gerbil/updateHolePunch.ts b/server/routers/gerbil/updateHolePunch.ts index 68a7282f..26608345 100644 --- a/server/routers/gerbil/updateHolePunch.ts +++ b/server/routers/gerbil/updateHolePunch.ts @@ -7,11 +7,14 @@ import HttpCode from "@server/types/HttpCode"; import createHttpError from "http-errors"; import logger from "@server/logger"; import { fromError } from "zod-validation-error"; +import { validateNewtSessionToken } from "@server/auth/sessions/newt"; +import { validateOlmSessionToken } from "@server/auth/sessions/olm"; // Define Zod schema for request validation const updateHolePunchSchema = z.object({ olmId: z.string().optional(), newtId: z.string().optional(), + token: z.string(), ip: z.string(), port: z.number(), timestamp: z.number() @@ -34,13 +37,28 @@ export async function updateHolePunch( ); } - const { olmId, newtId, ip, port, timestamp } = parsedParams.data; - + const { olmId, newtId, ip, port, timestamp, token } = parsedParams.data; + // logger.debug(`Got hole punch with ip: ${ip}, port: ${port} for olmId: ${olmId} or newtId: ${newtId}`); let site: Site | undefined; if (olmId) { + const { session, olm: olmSession } = + await validateOlmSessionToken(token); + if (!session || !olmSession) { + return next( + createHttpError(HttpCode.UNAUTHORIZED, "Unauthorized") + ); + } + + if (olmId !== olmSession.olmId) { + logger.warn(`Olm ID mismatch: ${olmId} !== ${olmSession.olmId}`); + return next( + createHttpError(HttpCode.UNAUTHORIZED, "Unauthorized") + ); + } + const [olm] = await db .select() .from(olms) @@ -66,7 +84,24 @@ export async function updateHolePunch( .select() .from(sites) .where(eq(sites.siteId, client.siteId)); + } else if (newtId) { + const { session, newt: newtSession } = + await validateNewtSessionToken(token); + + if (!session || !newtSession) { + return next( + createHttpError(HttpCode.UNAUTHORIZED, "Unauthorized") + ); + } + + if (newtId !== newtSession.newtId) { + logger.warn(`Newt ID mismatch: ${newtId} !== ${newtSession.newtId}`); + return next( + createHttpError(HttpCode.UNAUTHORIZED, "Unauthorized") + ); + } + const [newt] = await db .select() .from(newts) @@ -90,10 +125,10 @@ export async function updateHolePunch( } if (!site || !site.endpoint || !site.subnet) { - logger.warn(`Site not found for olmId: ${olmId} or newtId: ${newtId}`); - return next( - createHttpError(HttpCode.NOT_FOUND, "Site not found") + logger.warn( + `Site not found for olmId: ${olmId} or newtId: ${newtId}` ); + return next(createHttpError(HttpCode.NOT_FOUND, "Site not found")); } return res.status(HttpCode.OK).send({ diff --git a/server/routers/olm/handleOlmRegisterMessage.ts b/server/routers/olm/handleOlmRegisterMessage.ts index cf3164be..4bf46744 100644 --- a/server/routers/olm/handleOlmRegisterMessage.ts +++ b/server/routers/olm/handleOlmRegisterMessage.ts @@ -1,6 +1,6 @@ import db from "@server/db"; import { MessageHandler } from "../ws"; -import { clients, Olm, olms, sites } from "@server/db/schema"; +import { clients, exitNodes, Olm, olms, sites } from "@server/db/schema"; import { eq } from "drizzle-orm"; import { addPeer, deletePeer } from "../newt/peers"; import logger from "@server/logger"; @@ -46,27 +46,35 @@ export const handleOlmRegisterMessage: MessageHandler = async (context) => { .where(eq(sites.siteId, client.siteId)) .limit(1); - if (!client) { + if (!site) { logger.warn("Site not found or does not have exit node"); return; } + if (!site.exitNodeId) { + logger.warn("Site does not have exit node"); + return; + } + + const [exitNode] = await db + .select() + .from(exitNodes) + .where(eq(exitNodes.exitNodeId, site.exitNodeId)) + .limit(1); + + sendToClient(olm.olmId, { + type: "olm/wg/holepunch", + data: { + serverPubKey: exitNode.publicKey, + } + }); + // make sure we hand endpoints for both the site and the client and the lastHolePunch is not too old if (!site.endpoint || !client.endpoint) { logger.warn("Site or client has no endpoint or listen port"); return; } - sendToClient(olm.olmId, { - type: "olm/wg/holepunch", - data: { - endpoint: site.endpoint, - publicKey: site.publicKey, - serverIP: site.address!.split("/")[0], - tunnelIP: `${client.subnet.split("/")[0]}/${site.address!.split("/")[1]}` // put the client ip in the same subnet as the site. TODO: Is this right? Maybe we need th make .subnet work properly! - } - }); - const now = new Date().getTime() / 1000; if (site.lastHolePunch && now - site.lastHolePunch > 6) { logger.warn("Site last hole punch is too old"); diff --git a/src/app/[orgId]/settings/layout.tsx b/src/app/[orgId]/settings/layout.tsx index 0f9f2ffe..d94da3fb 100644 --- a/src/app/[orgId]/settings/layout.tsx +++ b/src/app/[orgId]/settings/layout.tsx @@ -5,7 +5,8 @@ import { LinkIcon, Settings, Users, - Waypoints + Waypoints, + Workflow } from "lucide-react"; import { Header } from "@app/components/Header"; import { verifySession } from "@app/lib/auth/verifySession"; From 87012c47ea6bd12da7b7ac84e41ed4a670750a01 Mon Sep 17 00:00:00 2001 From: Owen Date: Tue, 25 Mar 2025 22:01:08 -0400 Subject: [PATCH 036/135] Start changes for multi site clients - Org subnet and assign sites and clients out of the same subnet group on each org - Add join table for client on multiple sites - Start to handle websocket endpoints for these multiple connections --- install/config/config.yml | 4 +- server/db/schema.ts | 19 +- server/lib/config.ts | 4 +- server/lib/ip.ts | 59 ++++++ server/routers/client/pickClientDefaults.ts | 38 +--- server/routers/newt/handleGetConfigMessage.ts | 90 ++------- .../routers/olm/handleOlmRegisterMessage.ts | 186 ++++++++++-------- server/routers/org/createOrg.ts | 6 +- 8 files changed, 210 insertions(+), 196 deletions(-) diff --git a/install/config/config.yml b/install/config/config.yml index 043b1421..d972c637 100644 --- a/install/config/config.yml +++ b/install/config/config.yml @@ -38,11 +38,9 @@ gerbil: site_block_size: 30 subnet_group: 100.89.137.0/20 -newt: - start_port: 51820 +orgs: block_size: 24 subnet_group: 100.89.138.0/20 - site_block_size: 30 rate_limits: global: diff --git a/server/db/schema.ts b/server/db/schema.ts index 74e9ecf2..875754f5 100644 --- a/server/db/schema.ts +++ b/server/db/schema.ts @@ -11,7 +11,8 @@ export const domains = sqliteTable("domains", { export const orgs = sqliteTable("orgs", { orgId: text("orgId").primaryKey(), - name: text("name").notNull() + name: text("name").notNull(), + subnet: text("subnet").notNull(), }); export const orgDomains = sqliteTable("orgDomains", { @@ -47,7 +48,6 @@ export const sites = sqliteTable("sites", { address: text("address"), // this is the address of the wireguard interface in gerbil endpoint: text("endpoint"), // this is how to reach gerbil externally - gets put into the wireguard config publicKey: text("pubicKey"), - listenPort: integer("listenPort"), lastHolePunch: integer("lastHolePunch"), }); @@ -138,11 +138,6 @@ export const newts = sqliteTable("newt", { export const clients = sqliteTable("clients", { clientId: integer("id").primaryKey({ autoIncrement: true }), - siteId: integer("siteId") - .references(() => sites.siteId, { - onDelete: "cascade" - }) - .notNull(), orgId: text("orgId") .references(() => orgs.orgId, { onDelete: "cascade" @@ -160,6 +155,15 @@ export const clients = sqliteTable("clients", { lastHolePunch: integer("lastHolePunch"), }); +export const clientSites = sqliteTable("clientSites", { + clientId: integer("clientId") + .notNull() + .references(() => clients.clientId, { onDelete: "cascade" }), + siteId: integer("siteId") + .notNull() + .references(() => sites.siteId, { onDelete: "cascade" }), +}); + export const olms = sqliteTable("olms", { olmId: text("id").primaryKey(), secretHash: text("secretHash").notNull(), @@ -516,6 +520,7 @@ export type ResourceWhitelist = InferSelectModel; export type VersionMigration = InferSelectModel; export type ResourceRule = InferSelectModel; export type Client = InferSelectModel; +export type ClientSite = InferSelectModel; export type RoleClient = InferSelectModel; export type UserClient = InferSelectModel; export type Domain = InferSelectModel; diff --git a/server/lib/config.ts b/server/lib/config.ts index b41be6ec..9df6a55f 100644 --- a/server/lib/config.ts +++ b/server/lib/config.ts @@ -105,11 +105,9 @@ const configSchema = z.object({ block_size: z.number().positive().gt(0), site_block_size: z.number().positive().gt(0) }), - newt: z.object({ + orgs: z.object({ block_size: z.number().positive().gt(0), subnet_group: z.string(), - start_port: portSchema, - site_block_size: z.number().positive().gt(0) }), rate_limits: z.object({ global: z.object({ diff --git a/server/lib/ip.ts b/server/lib/ip.ts index 86fe1169..a3a78027 100644 --- a/server/lib/ip.ts +++ b/server/lib/ip.ts @@ -1,3 +1,8 @@ +import db from "@server/db"; +import { clients, orgs, sites } from "@server/db/schema"; +import { and, eq, isNotNull } from "drizzle-orm"; +import config from "@server/lib/config"; + interface IPRange { start: bigint; end: bigint; @@ -204,4 +209,58 @@ export function isIpInCidr(ip: string, cidr: string): boolean { const ipBigInt = ipToBigInt(ip); const range = cidrToRange(cidr); return ipBigInt >= range.start && ipBigInt <= range.end; +} + +export async function getNextAvailableClientSubnet(orgId: string): Promise { + const existingAddressesSites = await db + .select({ + address: sites.address + }) + .from(sites) + .where(and(isNotNull(sites.address), eq(sites.orgId, orgId))); + + const existingAddressesClients = await db + .select({ + address: clients.subnet + }) + .from(clients) + .where(and(isNotNull(clients.subnet), eq(clients.orgId, orgId))); + + const addresses = [ + ...existingAddressesSites.map((site) => site.address), + ...existingAddressesClients.map((client) => client.address) + ].filter((address) => address !== null) as string[]; + + let subnet = findNextAvailableCidr( + addresses, + 32, + config.getRawConfig().orgs.subnet_group + ); // pick the sites address in the org + if (!subnet) { + throw new Error("No available subnets remaining in space"); + } + + return subnet; +} + +export async function getNextAvailableOrgSubnet(): Promise { + const existingAddresses = await db + .select({ + subnet: orgs.subnet + }) + .from(orgs) + .where(isNotNull(orgs.subnet)); + + const addresses = existingAddresses.map((org) => org.subnet); + + let subnet = findNextAvailableCidr( + addresses, + config.getRawConfig().orgs.block_size, + config.getRawConfig().orgs.subnet_group + ); + if (!subnet) { + throw new Error("No available subnets remaining in space"); + } + + return subnet; } \ No newline at end of file diff --git a/server/routers/client/pickClientDefaults.ts b/server/routers/client/pickClientDefaults.ts index d77ae3bb..5e87759d 100644 --- a/server/routers/client/pickClientDefaults.ts +++ b/server/routers/client/pickClientDefaults.ts @@ -6,7 +6,7 @@ import response from "@server/lib/response"; import HttpCode from "@server/types/HttpCode"; import createHttpError from "http-errors"; import logger from "@server/logger"; -import { findNextAvailableCidr } from "@server/lib/ip"; +import { findNextAvailableCidr, getNextAvailableClientSubnet } from "@server/lib/ip"; import { generateId } from "@server/auth/sessions/app"; import config from "@server/lib/config"; import { z } from "zod"; @@ -88,37 +88,8 @@ export async function pickClientDefaults( const { address, publicKey, listenPort, endpoint } = parsedSite.data; - const clientsQuery = await db - .select({ - subnet: clients.subnet - }) - .from(clients) - .where(eq(clients.siteId, site.siteId)); - - let subnets = clientsQuery.map((client) => client.subnet); - - // exclude the exit node address by replacing after the / with a site block size - subnets.push( - address.replace( - /\/\d+$/, - `/${config.getRawConfig().newt.site_block_size}` - ) - ); - - const newSubnet = findNextAvailableCidr( - subnets, - config.getRawConfig().newt.site_block_size, - address - ); - if (!newSubnet) { - return next( - createHttpError( - HttpCode.INTERNAL_SERVER_ERROR, - "No available subnets" - ) - ); - } - + const newSubnet = await getNextAvailableClientSubnet(site.orgId); + const olmId = generateId(15); const secret = generateId(48); @@ -130,8 +101,7 @@ export async function pickClientDefaults( name: site.name, listenPort: listenPort, endpoint: endpoint, - // subnet: `${newSubnet.split("/")[0]}/${config.getRawConfig().newt.block_size}`, // we want the block size of the whole subnet - subnet: newSubnet, + subnet: `${newSubnet.split("/")[0]}/${config.getRawConfig().orgs.block_size}`, // we want the block size of the whole org olmId: olmId, olmSecret: secret }, diff --git a/server/routers/newt/handleGetConfigMessage.ts b/server/routers/newt/handleGetConfigMessage.ts index 62934d2f..00b5ee64 100644 --- a/server/routers/newt/handleGetConfigMessage.ts +++ b/server/routers/newt/handleGetConfigMessage.ts @@ -3,13 +3,12 @@ import { MessageHandler } from "../ws"; import logger from "@server/logger"; import { fromError } from "zod-validation-error"; import db from "@server/db"; -import { clients, Newt, Site, sites } from "@server/db/schema"; -import { eq, isNotNull } from "drizzle-orm"; -import { findNextAvailableCidr } from "@server/lib/ip"; -import config from "@server/lib/config"; +import { clients, clientSites, Newt, Site, sites } from "@server/db/schema"; +import { eq } from "drizzle-orm"; +import { getNextAvailableClientSubnet } from "@server/lib/ip"; const inputSchema = z.object({ - publicKey: z.string(), + publicKey: z.string() }); type Input = z.infer; @@ -57,16 +56,15 @@ export const handleGetConfigMessage: MessageHandler = async (context) => { let site: Site | undefined; if (!siteRes.address) { - const address = await getNextAvailableSubnet(); - const listenPort = await getNextAvailablePort(); + let address = await getNextAvailableClientSubnet(siteRes.orgId); + address = address.split("/")[0]; // get the first part of the CIDR // create a new exit node const [updateRes] = await db .update(sites) .set({ publicKey, - address, - listenPort + address }) .where(eq(sites.siteId, siteId)) .returning(); @@ -95,28 +93,33 @@ export const handleGetConfigMessage: MessageHandler = async (context) => { const clientsRes = await db .select() .from(clients) - .where(eq(clients.siteId, siteId)); + .innerJoin(clientSites, eq(clients.clientId, clientSites.clientId)) + .where(eq(clientSites.siteId, siteId)); const now = new Date().getTime() / 1000; const peers = await Promise.all( clientsRes .filter((client) => { - if (client.lastHolePunch && now - client.lastHolePunch > 6) { + // This filter wasn't returning anything - fixed to properly filter clients + if ( + !client.clients.lastHolePunch || + now - client.clients.lastHolePunch > 6 + ) { logger.warn("Client last hole punch is too old"); - return; + return false; } + return true; }) .map(async (client) => { return { - publicKey: client.pubKey, - allowedIps: [client.subnet], - endpoint: client.endpoint + publicKey: client.clients.pubKey, + allowedIps: [client.clients.subnet], + endpoint: client.clients.endpoint }; }) ); const configResponse = { - listenPort: site.listenPort, ipAddress: site.address, peers }; @@ -133,57 +136,4 @@ export const handleGetConfigMessage: MessageHandler = async (context) => { broadcast: false, // Send to all clients excludeSender: false // Include sender in broadcast }; -}; - -async function getNextAvailableSubnet(): Promise { - const existingAddresses = await db - .select({ - address: sites.address - }) - .from(sites) - .where(isNotNull(sites.address)); - - const addresses = existingAddresses - .map((a) => a.address) - .filter((a) => a) as string[]; - - let subnet = findNextAvailableCidr( - addresses, - config.getRawConfig().newt.block_size, - config.getRawConfig().newt.subnet_group - ); - if (!subnet) { - throw new Error("No available subnets remaining in space"); - } - - // replace the last octet with 1 - subnet = - subnet.split(".").slice(0, 3).join(".") + - ".1" + - "/" + - subnet.split("/")[1]; - return subnet; -} - -async function getNextAvailablePort(): Promise { - // Get all existing ports from exitNodes table - const existingPorts = await db - .select({ - listenPort: sites.listenPort - }) - .from(sites); - - // Find the first available port between 1024 and 65535 - let nextPort = config.getRawConfig().newt.start_port; - for (const port of existingPorts) { - if (port.listenPort && port.listenPort > nextPort) { - break; - } - nextPort++; - if (nextPort > 65535) { - throw new Error("No available ports remaining in space"); - } - } - - return nextPort; -} +}; \ No newline at end of file diff --git a/server/routers/olm/handleOlmRegisterMessage.ts b/server/routers/olm/handleOlmRegisterMessage.ts index 4bf46744..ac658b78 100644 --- a/server/routers/olm/handleOlmRegisterMessage.ts +++ b/server/routers/olm/handleOlmRegisterMessage.ts @@ -1,91 +1,53 @@ import db from "@server/db"; import { MessageHandler } from "../ws"; -import { clients, exitNodes, Olm, olms, sites } from "@server/db/schema"; -import { eq } from "drizzle-orm"; +import { clients, clientSites, exitNodes, Olm, olms, sites } from "@server/db/schema"; +import { eq, inArray } from "drizzle-orm"; import { addPeer, deletePeer } from "../newt/peers"; import logger from "@server/logger"; export const handleOlmRegisterMessage: MessageHandler = async (context) => { const { message, client: c, sendToClient } = context; const olm = c as Olm; - logger.info("Handling register olm message!"); - if (!olm) { logger.warn("Olm not found"); return; } - if (!olm.clientId) { - logger.warn("Olm has no site!"); // TODO: Maybe we create the site here? + logger.warn("Olm has no client ID!"); return; } - const clientId = olm.clientId; - const { publicKey } = message.data; if (!publicKey) { logger.warn("Public key not provided"); return; } - + + // Get the client const [client] = await db .select() .from(clients) .where(eq(clients.clientId, clientId)) .limit(1); - - if (!client || !client.siteId) { - logger.warn("Site not found or does not have exit node"); + + if (!client) { + logger.warn("Client not found"); return; } - - const [site] = await db + + // Get all site associations for this client + const clientSiteAssociations = await db .select() - .from(sites) - .where(eq(sites.siteId, client.siteId)) - .limit(1); - - if (!site) { - logger.warn("Site not found or does not have exit node"); + .from(clientSites) + .where(eq(clientSites.clientId, clientId)); + + if (clientSiteAssociations.length === 0) { + logger.warn("Client is not associated with any sites"); return; } - - if (!site.exitNodeId) { - logger.warn("Site does not have exit node"); - return; - } - - const [exitNode] = await db - .select() - .from(exitNodes) - .where(eq(exitNodes.exitNodeId, site.exitNodeId)) - .limit(1); - - sendToClient(olm.olmId, { - type: "olm/wg/holepunch", - data: { - serverPubKey: exitNode.publicKey, - } - }); - - // make sure we hand endpoints for both the site and the client and the lastHolePunch is not too old - if (!site.endpoint || !client.endpoint) { - logger.warn("Site or client has no endpoint or listen port"); - return; - } - - const now = new Date().getTime() / 1000; - if (site.lastHolePunch && now - site.lastHolePunch > 6) { - logger.warn("Site last hole punch is too old"); - return; - } - - if (client.lastHolePunch && now - client.lastHolePunch > 6) { - logger.warn("Client last hole punch is too old"); - return; - } - + + // Update the client's public key await db .update(clients) .set({ @@ -93,35 +55,103 @@ export const handleOlmRegisterMessage: MessageHandler = async (context) => { }) .where(eq(clients.clientId, olm.clientId)) .returning(); - - if (client.pubKey && client.pubKey !== publicKey) { - logger.info("Public key mismatch. Deleting old peer..."); - await deletePeer(site.siteId, client.pubKey); + + // Check if public key changed and handle old peer deletion later + const pubKeyChanged = client.pubKey && client.pubKey !== publicKey; + + // Get all sites data + const siteIds = clientSiteAssociations.map(cs => cs.siteId); + const sitesData = await db + .select() + .from(sites) + .where(inArray(sites.siteId, siteIds)); + + // Prepare an array to store site configurations + const siteConfigurations = []; + const now = new Date().getTime() / 1000; + + // Process each site + for (const site of sitesData) { + if (!site.exitNodeId) { + logger.warn(`Site ${site.siteId} does not have exit node, skipping`); + continue; + } + + // Get the exit node for this site + const [exitNode] = await db + .select() + .from(exitNodes) + .where(eq(exitNodes.exitNodeId, site.exitNodeId)) + .limit(1); + + // Validate endpoint and hole punch status + if (!site.endpoint) { + logger.warn(`Site ${site.siteId} has no endpoint, skipping`); + continue; + } + + if (site.lastHolePunch && now - site.lastHolePunch > 6) { + logger.warn(`Site ${site.siteId} last hole punch is too old, skipping`); + continue; + } + + if (client.lastHolePunch && now - client.lastHolePunch > 6) { + logger.warn("Client last hole punch is too old, skipping all sites"); + break; + } + + // If public key changed, delete old peer from this site + if (pubKeyChanged) { + logger.info(`Public key mismatch. Deleting old peer from site ${site.siteId}...`); + await deletePeer(site.siteId, client.pubKey); + } + + if (!site.subnet) { + logger.warn(`Site ${site.siteId} has no subnet, skipping`); + continue; + } + + // Add the peer to the exit node for this site + await addPeer(site.siteId, { + publicKey: publicKey, + allowedIps: [client.subnet], + endpoint: client.endpoint + }); + + // Add site configuration to the array + siteConfigurations.push({ + siteId: site.siteId, + endpoint: site.endpoint, + publicKey: site.publicKey, + serverIP: site.address, + }); + + // Send holepunch message for each site + sendToClient(olm.olmId, { + type: "olm/wg/holepunch", + data: { + serverPubKey: exitNode.publicKey, + siteId: site.siteId + } + }); } - - if (!site.subnet) { - logger.warn("Site has no subnet"); + + // If we have no valid site configurations, don't send a connect message + if (siteConfigurations.length === 0) { + logger.warn("No valid site configurations found"); return; } - - // add the peer to the exit node - await addPeer(site.siteId, { - publicKey: publicKey, - allowedIps: [client.subnet], - endpoint: client.endpoint - }); - + + // Return connect message with all site configurations return { message: { type: "olm/wg/connect", data: { - endpoint: site.endpoint, - publicKey: site.publicKey, - serverIP: site.address!.split("/")[0], - tunnelIP: `${client.subnet.split("/")[0]}/${site.address!.split("/")[1]}` // put the client ip in the same subnet as the site. TODO: Is this right? Maybe we need th make .subnet work properly! + sites: siteConfigurations, + tunnelIP: client.subnet, } }, - broadcast: false, // Send to all olms - excludeSender: false // Include sender in broadcast + broadcast: false, + excludeSender: false }; -}; +}; \ No newline at end of file diff --git a/server/routers/org/createOrg.ts b/server/routers/org/createOrg.ts index 381ce20e..fef5e2ac 100644 --- a/server/routers/org/createOrg.ts +++ b/server/routers/org/createOrg.ts @@ -19,6 +19,7 @@ import { createAdminRole } from "@server/setup/ensureActions"; import config from "@server/lib/config"; import { fromError } from "zod-validation-error"; import { defaultRoleAllowedActions } from "../role"; +import { getNextAvailableOrgSubnet } from "@server/lib/ip"; const createOrgSchema = z .object({ @@ -88,6 +89,8 @@ export async function createOrg( let error = ""; let org: Org | null = null; + const subnet = await getNextAvailableOrgSubnet(); + await db.transaction(async (trx) => { const allDomains = await trx .select() @@ -98,7 +101,8 @@ export async function createOrg( .insert(orgs) .values({ orgId, - name + name, + subnet, }) .returning(); From 926ec831e2cb6a0a6684ed78e4aab33c6a4bab17 Mon Sep 17 00:00:00 2001 From: Owen Date: Wed, 26 Mar 2025 21:23:26 -0400 Subject: [PATCH 037/135] Finish conversion of olm reg to multi site --- server/db/schema.ts | 3 + .../routers/olm/handleOlmRegisterMessage.ts | 123 +++++++++--------- 2 files changed, 68 insertions(+), 58 deletions(-) diff --git a/server/db/schema.ts b/server/db/schema.ts index 875754f5..002fc442 100644 --- a/server/db/schema.ts +++ b/server/db/schema.ts @@ -143,6 +143,9 @@ export const clients = sqliteTable("clients", { onDelete: "cascade" }) .notNull(), + exitNodeId: integer("exitNode").references(() => exitNodes.exitNodeId, { + onDelete: "set null" + }), name: text("name").notNull(), pubKey: text("pubKey"), subnet: text("subnet").notNull(), diff --git a/server/routers/olm/handleOlmRegisterMessage.ts b/server/routers/olm/handleOlmRegisterMessage.ts index ac658b78..88c5ba4c 100644 --- a/server/routers/olm/handleOlmRegisterMessage.ts +++ b/server/routers/olm/handleOlmRegisterMessage.ts @@ -1,6 +1,13 @@ import db from "@server/db"; import { MessageHandler } from "../ws"; -import { clients, clientSites, exitNodes, Olm, olms, sites } from "@server/db/schema"; +import { + clients, + clientSites, + exitNodes, + Olm, + olms, + sites +} from "@server/db/schema"; import { eq, inArray } from "drizzle-orm"; import { addPeer, deletePeer } from "../newt/peers"; import logger from "@server/logger"; @@ -23,30 +30,36 @@ export const handleOlmRegisterMessage: MessageHandler = async (context) => { logger.warn("Public key not provided"); return; } - + // Get the client const [client] = await db .select() .from(clients) .where(eq(clients.clientId, clientId)) .limit(1); - + if (!client) { logger.warn("Client not found"); return; } + + if (client.exitNodeId) { + // Get the exit node for this site + const [exitNode] = await db + .select() + .from(exitNodes) + .where(eq(exitNodes.exitNodeId, client.exitNodeId)) + .limit(1); - // Get all site associations for this client - const clientSiteAssociations = await db - .select() - .from(clientSites) - .where(eq(clientSites.clientId, clientId)); - - if (clientSiteAssociations.length === 0) { - logger.warn("Client is not associated with any sites"); - return; + // Send holepunch message for each site + sendToClient(olm.olmId, { + type: "olm/wg/holepunch", + data: { + serverPubKey: exitNode.publicKey + } + }); } - + // Update the client's public key await db .update(clients) @@ -55,103 +68,97 @@ export const handleOlmRegisterMessage: MessageHandler = async (context) => { }) .where(eq(clients.clientId, olm.clientId)) .returning(); - + // Check if public key changed and handle old peer deletion later const pubKeyChanged = client.pubKey && client.pubKey !== publicKey; - + // Get all sites data - const siteIds = clientSiteAssociations.map(cs => cs.siteId); const sitesData = await db .select() .from(sites) - .where(inArray(sites.siteId, siteIds)); - + .innerJoin(clientSites, eq(sites.siteId, clientSites.siteId)) + .where(eq(clientSites.clientId, client.clientId)); + // Prepare an array to store site configurations const siteConfigurations = []; const now = new Date().getTime() / 1000; - + // Process each site - for (const site of sitesData) { + for (const { sites: site } of sitesData) { if (!site.exitNodeId) { - logger.warn(`Site ${site.siteId} does not have exit node, skipping`); + logger.warn( + `Site ${site.siteId} does not have exit node, skipping` + ); continue; } - - // Get the exit node for this site - const [exitNode] = await db - .select() - .from(exitNodes) - .where(eq(exitNodes.exitNodeId, site.exitNodeId)) - .limit(1); - + // Validate endpoint and hole punch status if (!site.endpoint) { logger.warn(`Site ${site.siteId} has no endpoint, skipping`); continue; } - + if (site.lastHolePunch && now - site.lastHolePunch > 6) { - logger.warn(`Site ${site.siteId} last hole punch is too old, skipping`); + logger.warn( + `Site ${site.siteId} last hole punch is too old, skipping` + ); continue; } - + if (client.lastHolePunch && now - client.lastHolePunch > 6) { - logger.warn("Client last hole punch is too old, skipping all sites"); + logger.warn( + "Client last hole punch is too old, skipping all sites" + ); break; } - + // If public key changed, delete old peer from this site if (pubKeyChanged) { - logger.info(`Public key mismatch. Deleting old peer from site ${site.siteId}...`); - await deletePeer(site.siteId, client.pubKey); + logger.info( + `Public key mismatch. Deleting old peer from site ${site.siteId}...` + ); + await deletePeer(site.siteId, client.pubKey!); } - + if (!site.subnet) { logger.warn(`Site ${site.siteId} has no subnet, skipping`); continue; } - + // Add the peer to the exit node for this site - await addPeer(site.siteId, { - publicKey: publicKey, - allowedIps: [client.subnet], - endpoint: client.endpoint - }); - + if (client.endpoint) { + await addPeer(site.siteId, { + publicKey: publicKey, + allowedIps: [client.subnet], + endpoint: client.endpoint + }); + } + // Add site configuration to the array siteConfigurations.push({ siteId: site.siteId, endpoint: site.endpoint, publicKey: site.publicKey, - serverIP: site.address, - }); - - // Send holepunch message for each site - sendToClient(olm.olmId, { - type: "olm/wg/holepunch", - data: { - serverPubKey: exitNode.publicKey, - siteId: site.siteId - } + serverIP: site.address }); } - + // If we have no valid site configurations, don't send a connect message if (siteConfigurations.length === 0) { logger.warn("No valid site configurations found"); return; } - + // Return connect message with all site configurations return { message: { type: "olm/wg/connect", data: { sites: siteConfigurations, - tunnelIP: client.subnet, + tunnelIP: client.subnet } }, broadcast: false, excludeSender: false }; -}; \ No newline at end of file +}; From dac49f7fdc2d0796c261c3c2f78f2496c934b93b Mon Sep 17 00:00:00 2001 From: Owen Date: Wed, 26 Mar 2025 21:39:12 -0400 Subject: [PATCH 038/135] Converting more multi site --- server/routers/client/createClient.ts | 3 --- server/routers/client/pickClientDefaults.ts | 18 ++++++++++++++++-- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/server/routers/client/createClient.ts b/server/routers/client/createClient.ts index 0dd1e2da..19b41b66 100644 --- a/server/routers/client/createClient.ts +++ b/server/routers/client/createClient.ts @@ -18,9 +18,7 @@ import HttpCode from "@server/types/HttpCode"; import createHttpError from "http-errors"; import logger from "@server/logger"; import { eq, and } from "drizzle-orm"; -import { addPeer } from "../newt/peers"; import { fromError } from "zod-validation-error"; -import { newts } from "@server/db/schema"; import moment from "moment"; import { hashPassword } from "@server/auth/password"; @@ -122,7 +120,6 @@ export async function createClient( const [newClient] = await trx .insert(clients) .values({ - siteId, orgId: site.orgId, name, subnet, diff --git a/server/routers/client/pickClientDefaults.ts b/server/routers/client/pickClientDefaults.ts index 5e87759d..b56a4bf4 100644 --- a/server/routers/client/pickClientDefaults.ts +++ b/server/routers/client/pickClientDefaults.ts @@ -1,6 +1,6 @@ import { Request, Response, NextFunction } from "express"; import { db } from "@server/db"; -import { clients, olms, sites } from "@server/db/schema"; +import { clients, exitNodes, olms, sites } from "@server/db/schema"; import { eq } from "drizzle-orm"; import response from "@server/lib/response"; import HttpCode from "@server/types/HttpCode"; @@ -19,6 +19,7 @@ const getSiteSchema = z .strict(); export type PickClientDefaultsResponse = { + exitNodeId: number; siteId: number; address: string; publicKey: string; @@ -66,8 +67,20 @@ export async function pickClientDefaults( ); } - // make sure all the required fields are present + // TODO: more intelligent way to pick the exit node + // make sure there is an exit node by counting the exit nodes table + const nodes = await db.select().from(exitNodes); + if (nodes.length === 0) { + return next( + createHttpError(HttpCode.NOT_FOUND, "No exit nodes available") + ); + } + + // get the first exit node + const exitNode = nodes[0]; + + // make sure all the required fields are present const sitesRequiredFields = z.object({ address: z.string(), publicKey: z.string(), @@ -95,6 +108,7 @@ export async function pickClientDefaults( return response(res, { data: { + exitNodeId: exitNode.exitNodeId, siteId: site.siteId, address: address, publicKey: publicKey, From 15eb6663942c8a59053d8c879cb3827ae1cea377 Mon Sep 17 00:00:00 2001 From: Owen Date: Wed, 26 Mar 2025 21:41:20 -0400 Subject: [PATCH 039/135] Remove olm/wg/relay message --- server/routers/messageHandlers.ts | 5 +- server/routers/olm/handleOlmRelayMessage.ts | 81 --------------------- server/routers/olm/index.ts | 3 +- 3 files changed, 3 insertions(+), 86 deletions(-) delete mode 100644 server/routers/olm/handleOlmRelayMessage.ts diff --git a/server/routers/messageHandlers.ts b/server/routers/messageHandlers.ts index 759a88ea..08f5e7ce 100644 --- a/server/routers/messageHandlers.ts +++ b/server/routers/messageHandlers.ts @@ -1,5 +1,5 @@ import { handleNewtRegisterMessage, handleReceiveBandwidthMessage } from "./newt"; -import { handleOlmRegisterMessage, handleOlmRelayMessage } from "./olm"; +import { handleOlmRegisterMessage } from "./olm"; import { handleGetConfigMessage } from "./newt/handleGetConfigMessage"; import { MessageHandler } from "./ws"; @@ -7,6 +7,5 @@ export const messageHandlers: Record = { "newt/wg/register": handleNewtRegisterMessage, "olm/wg/register": handleOlmRegisterMessage, "newt/wg/get-config": handleGetConfigMessage, - "newt/receive-bandwidth": handleReceiveBandwidthMessage, - "olm/wg/relay": handleOlmRelayMessage + "newt/receive-bandwidth": handleReceiveBandwidthMessage }; diff --git a/server/routers/olm/handleOlmRelayMessage.ts b/server/routers/olm/handleOlmRelayMessage.ts deleted file mode 100644 index 9cf0ceb2..00000000 --- a/server/routers/olm/handleOlmRelayMessage.ts +++ /dev/null @@ -1,81 +0,0 @@ -import db from "@server/db"; -import { MessageHandler } from "../ws"; -import { clients, Olm, olms, sites } from "@server/db/schema"; -import { eq } from "drizzle-orm"; -import { addPeer, deletePeer } from "../newt/peers"; -import logger from "@server/logger"; - -export const handleOlmRelayMessage: MessageHandler = async (context) => { - const { message, client: c, sendToClient } = context; - const olm = c as Olm; - - logger.info("Handling relay olm message!"); - - if (!olm) { - logger.warn("Olm not found"); - return; - } - - if (!olm.clientId) { - logger.warn("Olm has no site!"); // TODO: Maybe we create the site here? - return; - } - - const clientId = olm.clientId; - - const [client] = await db - .select() - .from(clients) - .where(eq(clients.clientId, clientId)) - .limit(1); - - if (!client || !client.siteId) { - logger.warn("Site not found or does not have exit node"); - return; - } - - const [site] = await db - .select() - .from(sites) - .where(eq(sites.siteId, client.siteId)) - .limit(1); - - if (!client) { - logger.warn("Site not found or does not have exit node"); - return; - } - - // make sure we hand endpoints for both the site and the client and the lastHolePunch is not too old - if (!client.pubKey) { - logger.warn("Site or client has no endpoint or listen port"); - return; - } - - if (!site.subnet) { - logger.warn("Site has no subnet"); - return; - } - - await deletePeer(site.siteId, client.pubKey); - - // add the peer to the exit node - await addPeer(site.siteId, { - publicKey: client.pubKey, - allowedIps: [client.subnet], - endpoint: "" - }); - - return { - message: { - type: "olm/wg/connect", - data: { - endpoint: site.endpoint, - publicKey: site.publicKey, - serverIP: site.address!.split("/")[0], - tunnelIP: client.subnet - } - }, - broadcast: false, // Send to all olms - excludeSender: false // Include sender in broadcast - }; -}; diff --git a/server/routers/olm/index.ts b/server/routers/olm/index.ts index b7373961..616480cc 100644 --- a/server/routers/olm/index.ts +++ b/server/routers/olm/index.ts @@ -1,4 +1,3 @@ export * from "./handleOlmRegisterMessage"; export * from "./getOlmToken"; -export * from "./createOlm"; -export * from "./handleOlmRelayMessage"; \ No newline at end of file +export * from "./createOlm"; \ No newline at end of file From 619cfef1c7dc0e90330f64da7f195110041c1aec Mon Sep 17 00:00:00 2001 From: Owen Date: Thu, 27 Mar 2025 14:27:41 -0400 Subject: [PATCH 040/135] Working on multi client --- server/routers/client/listClients.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/server/routers/client/listClients.ts b/server/routers/client/listClients.ts index 1bef1488..c7977ea4 100644 --- a/server/routers/client/listClients.ts +++ b/server/routers/client/listClients.ts @@ -41,7 +41,6 @@ function queryClients(orgId: string, accessibleClientIds: number[]) { .select({ clientId: clients.clientId, orgId: clients.orgId, - siteId: clients.siteId, siteNiceId: sites.niceId, name: clients.name, pubKey: clients.pubKey, @@ -55,7 +54,6 @@ function queryClients(orgId: string, accessibleClientIds: number[]) { }) .from(clients) .leftJoin(orgs, eq(clients.orgId, orgs.orgId)) - .innerJoin(sites, eq(clients.siteId, sites.siteId)) .where( and( inArray(clients.clientId, accessibleClientIds), From 4b6985718a1f7afeb622a842872f80097c3f2972 Mon Sep 17 00:00:00 2001 From: Owen Date: Thu, 27 Mar 2025 22:13:39 -0400 Subject: [PATCH 041/135] Fix ip picking from subnet in exclusion --- server/lib/ip.test.ts | 15 ++++++++++++++- server/lib/ip.ts | 32 +++++++++++++++++++------------- 2 files changed, 33 insertions(+), 14 deletions(-) diff --git a/server/lib/ip.test.ts b/server/lib/ip.test.ts index 2c2dd057..67a2faaa 100644 --- a/server/lib/ip.test.ts +++ b/server/lib/ip.test.ts @@ -4,7 +4,14 @@ import { assertEquals } from "@test/assert"; // Test cases function testFindNextAvailableCidr() { console.log("Running findNextAvailableCidr tests..."); - + + // Test 0: Basic IPv4 allocation with a subnet in the wrong range + { + const existing = ["100.90.130.1/30", "100.90.128.4/30"]; + const result = findNextAvailableCidr(existing, 30, "100.90.130.1/24"); + assertEquals(result, "100.90.130.4/30", "Basic IPv4 allocation failed"); + } + // Test 1: Basic IPv4 allocation { const existing = ["10.0.0.0/16", "10.1.0.0/16"]; @@ -26,6 +33,12 @@ function testFindNextAvailableCidr() { assertEquals(result, null, "No available space test failed"); } + // Test 4: Empty existing + { + const existing: string[] = []; + const result = findNextAvailableCidr(existing, 30, "10.0.0.0/8"); + assertEquals(result, "10.0.0.0/30", "Empty existing test failed"); + } // // Test 4: IPv6 allocation // { // const existing = ["2001:db8::/32", "2001:db8:1::/32"]; diff --git a/server/lib/ip.ts b/server/lib/ip.ts index 86fe1169..a4f05126 100644 --- a/server/lib/ip.ts +++ b/server/lib/ip.ts @@ -132,7 +132,6 @@ export function findNextAvailableCidr( blockSize: number, startCidr?: string ): string | null { - if (!startCidr && existingCidrs.length === 0) { return null; } @@ -150,40 +149,47 @@ export function findNextAvailableCidr( existingCidrs.some(cidr => detectIpVersion(cidr.split('/')[0]) !== version)) { throw new Error('All CIDRs must be of the same IP version'); } - + + // Extract the network part from startCidr to ensure we stay in the right subnet + const startCidrRange = cidrToRange(startCidr); + // Convert existing CIDRs to ranges and sort them const existingRanges = existingCidrs .map(cidr => cidrToRange(cidr)) .sort((a, b) => (a.start < b.start ? -1 : 1)); - + // Calculate block size const maxPrefix = version === 4 ? 32 : 128; const blockSizeBigInt = BigInt(1) << BigInt(maxPrefix - blockSize); - + // Start from the beginning of the given CIDR - let current = cidrToRange(startCidr).start; - const maxIp = cidrToRange(startCidr).end; - + let current = startCidrRange.start; + const maxIp = startCidrRange.end; + // Iterate through existing ranges for (let i = 0; i <= existingRanges.length; i++) { const nextRange = existingRanges[i]; + // Align current to block size const alignedCurrent = current + ((blockSizeBigInt - (current % blockSizeBigInt)) % blockSizeBigInt); - + // Check if we've gone beyond the maximum allowed IP if (alignedCurrent + blockSizeBigInt - BigInt(1) > maxIp) { return null; } - + // If we're at the end of existing ranges or found a gap if (!nextRange || alignedCurrent + blockSizeBigInt - BigInt(1) < nextRange.start) { return `${bigIntToIp(alignedCurrent, version)}/${blockSize}`; } - - // Move current pointer to after the current range - current = nextRange.end + BigInt(1); + + // If next range overlaps with our search space, move past it + if (nextRange.end >= startCidrRange.start && nextRange.start <= maxIp) { + // Move current pointer to after the current range + current = nextRange.end + BigInt(1); + } } - + return null; } From a665e3aae9aa65f4e77492a89397468b5326f7a0 Mon Sep 17 00:00:00 2001 From: Owen Date: Thu, 27 Mar 2025 22:13:57 -0400 Subject: [PATCH 042/135] Fix issues with relaying and holepunching --- server/routers/client/pickClientDefaults.ts | 2 +- server/routers/gerbil/updateHolePunch.ts | 2 +- server/routers/newt/handleGetConfigMessage.ts | 10 +++++----- server/routers/olm/handleOlmRelayMessage.ts | 9 ++------- src/app/[orgId]/settings/sites/create/page.tsx | 2 +- 5 files changed, 10 insertions(+), 15 deletions(-) diff --git a/server/routers/client/pickClientDefaults.ts b/server/routers/client/pickClientDefaults.ts index d77ae3bb..b51004f3 100644 --- a/server/routers/client/pickClientDefaults.ts +++ b/server/routers/client/pickClientDefaults.ts @@ -97,7 +97,7 @@ export async function pickClientDefaults( let subnets = clientsQuery.map((client) => client.subnet); - // exclude the exit node address by replacing after the / with a site block size + // exclude the newt address by replacing after the / with a site block size subnets.push( address.replace( /\/\d+$/, diff --git a/server/routers/gerbil/updateHolePunch.ts b/server/routers/gerbil/updateHolePunch.ts index 26608345..e2ba01c6 100644 --- a/server/routers/gerbil/updateHolePunch.ts +++ b/server/routers/gerbil/updateHolePunch.ts @@ -133,7 +133,7 @@ export async function updateHolePunch( return res.status(HttpCode.OK).send({ destinationIp: site.subnet.split("/")[0], - destinationPort: parseInt(site.endpoint.split(":")[1]) + destinationPort: site.listenPort }); } catch (error) { logger.error(error); diff --git a/server/routers/newt/handleGetConfigMessage.ts b/server/routers/newt/handleGetConfigMessage.ts index 62934d2f..3c1d7bd6 100644 --- a/server/routers/newt/handleGetConfigMessage.ts +++ b/server/routers/newt/handleGetConfigMessage.ts @@ -10,6 +10,7 @@ import config from "@server/lib/config"; const inputSchema = z.object({ publicKey: z.string(), + port: z.number().int().positive(), }); type Input = z.infer; @@ -41,7 +42,7 @@ export const handleGetConfigMessage: MessageHandler = async (context) => { return; } - const { publicKey } = message.data as Input; + const { publicKey, port } = message.data as Input; const siteId = newt.siteId; @@ -58,7 +59,6 @@ export const handleGetConfigMessage: MessageHandler = async (context) => { let site: Site | undefined; if (!siteRes.address) { const address = await getNextAvailableSubnet(); - const listenPort = await getNextAvailablePort(); // create a new exit node const [updateRes] = await db @@ -66,7 +66,7 @@ export const handleGetConfigMessage: MessageHandler = async (context) => { .set({ publicKey, address, - listenPort + listenPort: port, }) .where(eq(sites.siteId, siteId)) .returning(); @@ -79,7 +79,8 @@ export const handleGetConfigMessage: MessageHandler = async (context) => { const [siteRes] = await db .update(sites) .set({ - publicKey + publicKey, + listenPort: port, }) .where(eq(sites.siteId, siteId)) .returning(); @@ -116,7 +117,6 @@ export const handleGetConfigMessage: MessageHandler = async (context) => { ); const configResponse = { - listenPort: site.listenPort, ipAddress: site.address, peers }; diff --git a/server/routers/olm/handleOlmRelayMessage.ts b/server/routers/olm/handleOlmRelayMessage.ts index 9cf0ceb2..ef42e05a 100644 --- a/server/routers/olm/handleOlmRelayMessage.ts +++ b/server/routers/olm/handleOlmRelayMessage.ts @@ -67,13 +67,8 @@ export const handleOlmRelayMessage: MessageHandler = async (context) => { return { message: { - type: "olm/wg/connect", - data: { - endpoint: site.endpoint, - publicKey: site.publicKey, - serverIP: site.address!.split("/")[0], - tunnelIP: client.subnet - } + type: "olm/wg/relay-success", + data: {} }, broadcast: false, // Send to all olms excludeSender: false // Include sender in broadcast diff --git a/src/app/[orgId]/settings/sites/create/page.tsx b/src/app/[orgId]/settings/sites/create/page.tsx index a8002705..acafd5bb 100644 --- a/src/app/[orgId]/settings/sites/create/page.tsx +++ b/src/app/[orgId]/settings/sites/create/page.tsx @@ -324,7 +324,7 @@ PersistentKeepalive = 5`; let payload: CreateSiteBody = { name: data.name, - type: data.method + type: data.method as any, }; if (data.method == "wireguard") { From 81c7954e0c1b14eba394c66cae3d97f883c65363 Mon Sep 17 00:00:00 2001 From: Owen Date: Mon, 31 Mar 2025 15:19:37 -0400 Subject: [PATCH 043/135] Remove unused argon --- server/routers/site/createSite.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/server/routers/site/createSite.ts b/server/routers/site/createSite.ts index a1d1876f..f79149cc 100644 --- a/server/routers/site/createSite.ts +++ b/server/routers/site/createSite.ts @@ -10,7 +10,6 @@ import { eq, and } from "drizzle-orm"; import { getUniqueSiteName } from "@server/db/names"; import { addPeer } from "../gerbil/peers"; import { fromError } from "zod-validation-error"; -import { hash } from "@node-rs/argon2"; import { newts } from "@server/db/schema"; import moment from "moment"; import { hashPassword } from "@server/auth/password"; From cd059728fd0bff01f1089d41dd9a54d1daa9706f Mon Sep 17 00:00:00 2001 From: Owen Date: Mon, 31 Mar 2025 15:25:30 -0400 Subject: [PATCH 044/135] Put back handle relay message --- server/routers/messageHandlers.ts | 5 +++-- server/routers/olm/index.ts | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/server/routers/messageHandlers.ts b/server/routers/messageHandlers.ts index 08f5e7ce..759a88ea 100644 --- a/server/routers/messageHandlers.ts +++ b/server/routers/messageHandlers.ts @@ -1,5 +1,5 @@ import { handleNewtRegisterMessage, handleReceiveBandwidthMessage } from "./newt"; -import { handleOlmRegisterMessage } from "./olm"; +import { handleOlmRegisterMessage, handleOlmRelayMessage } from "./olm"; import { handleGetConfigMessage } from "./newt/handleGetConfigMessage"; import { MessageHandler } from "./ws"; @@ -7,5 +7,6 @@ export const messageHandlers: Record = { "newt/wg/register": handleNewtRegisterMessage, "olm/wg/register": handleOlmRegisterMessage, "newt/wg/get-config": handleGetConfigMessage, - "newt/receive-bandwidth": handleReceiveBandwidthMessage + "newt/receive-bandwidth": handleReceiveBandwidthMessage, + "olm/wg/relay": handleOlmRelayMessage }; diff --git a/server/routers/olm/index.ts b/server/routers/olm/index.ts index 616480cc..b7373961 100644 --- a/server/routers/olm/index.ts +++ b/server/routers/olm/index.ts @@ -1,3 +1,4 @@ export * from "./handleOlmRegisterMessage"; export * from "./getOlmToken"; -export * from "./createOlm"; \ No newline at end of file +export * from "./createOlm"; +export * from "./handleOlmRelayMessage"; \ No newline at end of file From 8fbd8a905f0c2c2402b34817350944e742ca902e Mon Sep 17 00:00:00 2001 From: Owen Date: Mon, 31 Mar 2025 15:34:18 -0400 Subject: [PATCH 045/135] We do not need to send any message back --- server/routers/olm/handleOlmRelayMessage.ts | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/server/routers/olm/handleOlmRelayMessage.ts b/server/routers/olm/handleOlmRelayMessage.ts index ef42e05a..f0a226ba 100644 --- a/server/routers/olm/handleOlmRelayMessage.ts +++ b/server/routers/olm/handleOlmRelayMessage.ts @@ -65,12 +65,5 @@ export const handleOlmRelayMessage: MessageHandler = async (context) => { endpoint: "" }); - return { - message: { - type: "olm/wg/relay-success", - data: {} - }, - broadcast: false, // Send to all olms - excludeSender: false // Include sender in broadcast - }; + return }; From 1baa02de89178df5706aa139d4f885d46a5bce20 Mon Sep 17 00:00:00 2001 From: Owen Date: Mon, 31 Mar 2025 15:45:51 -0400 Subject: [PATCH 046/135] Fix relay ws message --- server/routers/newt/peers.ts | 26 +++++++++++++ server/routers/olm/handleOlmRelayMessage.ts | 42 +++++++++------------ 2 files changed, 43 insertions(+), 25 deletions(-) diff --git a/server/routers/newt/peers.ts b/server/routers/newt/peers.ts index 5808484f..99aacf0d 100644 --- a/server/routers/newt/peers.ts +++ b/server/routers/newt/peers.ts @@ -49,4 +49,30 @@ export async function deletePeer(siteId: number, publicKey: string) { }); logger.info(`Deleted peer ${publicKey} from newt ${newt.newtId}`); +} + +export async function updatePeer(siteId: number, publicKey: string, peer: { + allowedIps?: string[]; + endpoint?: string; +}) { + const [site] = await db.select().from(sites).where(eq(sites.siteId, siteId)).limit(1); + if (!site) { + throw new Error(`Exit node with ID ${siteId} not found`); + } + + // get the newt on the site + const [newt] = await db.select().from(newts).where(eq(newts.siteId, siteId)).limit(1); + if (!newt) { + throw new Error(`Newt not found for site ${siteId}`); + } + + sendToClient(newt.newtId, { + type: 'newt/wg/peer/update', + data: { + publicKey, + ...peer + } + }); + + logger.info(`Updated peer ${publicKey} on newt ${newt.newtId}`); } \ No newline at end of file diff --git a/server/routers/olm/handleOlmRelayMessage.ts b/server/routers/olm/handleOlmRelayMessage.ts index f0a226ba..bef707e3 100644 --- a/server/routers/olm/handleOlmRelayMessage.ts +++ b/server/routers/olm/handleOlmRelayMessage.ts @@ -1,8 +1,8 @@ import db from "@server/db"; import { MessageHandler } from "../ws"; -import { clients, Olm, olms, sites } from "@server/db/schema"; +import { clients, clientSites, Olm, olms, sites } from "@server/db/schema"; import { eq } from "drizzle-orm"; -import { addPeer, deletePeer } from "../newt/peers"; +import { updatePeer } from "../newt/peers"; import logger from "@server/logger"; export const handleOlmRelayMessage: MessageHandler = async (context) => { @@ -29,17 +29,6 @@ export const handleOlmRelayMessage: MessageHandler = async (context) => { .where(eq(clients.clientId, clientId)) .limit(1); - if (!client || !client.siteId) { - logger.warn("Site not found or does not have exit node"); - return; - } - - const [site] = await db - .select() - .from(sites) - .where(eq(sites.siteId, client.siteId)) - .limit(1); - if (!client) { logger.warn("Site not found or does not have exit node"); return; @@ -51,19 +40,22 @@ export const handleOlmRelayMessage: MessageHandler = async (context) => { return; } - if (!site.subnet) { - logger.warn("Site has no subnet"); - return; + const sitesData = await db + .select() + .from(sites) + .innerJoin(clientSites, eq(sites.siteId, clientSites.siteId)) + .where(eq(clientSites.clientId, client.clientId)); + + let jobs: Array> = []; + for (const site of sitesData) { + // update the peer on the exit node + const job = updatePeer(site.sites.siteId, client.pubKey, { + endpoint: "" // this removes the endpoint + }); + jobs.push(job); } - await deletePeer(site.siteId, client.pubKey); + await Promise.all(jobs); - // add the peer to the exit node - await addPeer(site.siteId, { - publicKey: client.pubKey, - allowedIps: [client.subnet], - endpoint: "" - }); - - return + return; }; From 56e1684e2eabad9e05c8d234d0c7582c1f81a462 Mon Sep 17 00:00:00 2001 From: Owen Date: Mon, 31 Mar 2025 16:21:01 -0400 Subject: [PATCH 047/135] Update api endpoints for new association --- server/auth/actions.ts | 1 + server/routers/client/createClient.ts | 55 ++++----- server/routers/client/deleteClient.ts | 16 ++- server/routers/client/index.ts | 1 + server/routers/client/pickClientDefaults.ts | 95 --------------- server/routers/client/updateClient.ts | 124 ++++++++++++++++++++ server/routers/external.ts | 13 +- 7 files changed, 172 insertions(+), 133 deletions(-) create mode 100644 server/routers/client/updateClient.ts diff --git a/server/auth/actions.ts b/server/auth/actions.ts index 26242b4a..815fa137 100644 --- a/server/auth/actions.ts +++ b/server/auth/actions.ts @@ -64,6 +64,7 @@ export enum ActionsEnum { updateResourceRule = "updateResourceRule", createClient = "createClient", deleteClient = "deleteClient", + updateClient = "updateClient", listClients = "listClients", listOrgDomains = "listOrgDomains", } diff --git a/server/routers/client/createClient.ts b/server/routers/client/createClient.ts index 19b41b66..8bc2af49 100644 --- a/server/routers/client/createClient.ts +++ b/server/routers/client/createClient.ts @@ -3,15 +3,12 @@ import { z } from "zod"; import { db } from "@server/db"; import { roles, - userSites, - sites, - roleSites, - Site, Client, clients, roleClients, userClients, - olms + olms, + clientSites } from "@server/db/schema"; import response from "@server/lib/response"; import HttpCode from "@server/types/HttpCode"; @@ -21,21 +18,19 @@ import { eq, and } from "drizzle-orm"; import { fromError } from "zod-validation-error"; import moment from "moment"; import { hashPassword } from "@server/auth/password"; +import { getNextAvailableClientSubnet } from "@server/lib/ip"; +import config from "@server/lib/config"; const createClientParamsSchema = z .object({ - siteId: z - .string() - .transform((val) => parseInt(val)) - .pipe(z.number()) + orgId: z.string() }) .strict(); const createClientSchema = z .object({ name: z.string().min(1).max(255), - siteId: z.number().int().positive(), - subnet: z.string(), + siteIds: z.array(z.string().transform(Number).pipe(z.number())), olmId: z.string(), secret: z.string(), type: z.enum(["olm"]) @@ -62,7 +57,7 @@ export async function createClient( ); } - const { name, type, siteId, subnet, olmId, secret } = + const { name, type, siteIds, olmId, secret } = parsedBody.data; const parsedParams = createClientParamsSchema.safeParse(req.params); @@ -75,16 +70,7 @@ export async function createClient( ); } - const { siteId: paramSiteId } = parsedParams.data; - - if (siteId != paramSiteId) { - return next( - createHttpError( - HttpCode.BAD_REQUEST, - "Site ID in body does not match site ID in URL" - ) - ); - } + const { orgId } = parsedParams.data; if (!req.userOrgRoleId) { return next( @@ -92,21 +78,16 @@ export async function createClient( ); } - const [site] = await db - .select() - .from(sites) - .where(eq(sites.siteId, siteId)); + const newSubnet = await getNextAvailableClientSubnet(orgId); - if (!site) { - return next(createHttpError(HttpCode.NOT_FOUND, "Site not found")); - } + const subnet = `${newSubnet.split("/")[0]}/${config.getRawConfig().orgs.block_size}` // we want the block size of the whole org await db.transaction(async (trx) => { const adminRole = await trx .select() .from(roles) .where( - and(eq(roles.isAdmin, true), eq(roles.orgId, site.orgId)) + and(eq(roles.isAdmin, true), eq(roles.orgId, orgId)) ) .limit(1); @@ -120,7 +101,7 @@ export async function createClient( const [newClient] = await trx .insert(clients) .values({ - orgId: site.orgId, + orgId, name, subnet, type @@ -140,6 +121,16 @@ export async function createClient( }); } + // Create site to client associations + if (siteIds && siteIds.length > 0) { + await trx.insert(clientSites).values( + siteIds.map(siteId => ({ + clientId: newClient.clientId, + siteId + })) + ); + } + const secretHash = await hashPassword(secret); await trx.insert(olms).values({ @@ -163,4 +154,4 @@ export async function createClient( createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred") ); } -} +} \ No newline at end of file diff --git a/server/routers/client/deleteClient.ts b/server/routers/client/deleteClient.ts index 9f1fb93e..f99f6fee 100644 --- a/server/routers/client/deleteClient.ts +++ b/server/routers/client/deleteClient.ts @@ -1,7 +1,7 @@ import { Request, Response, NextFunction } from "express"; import { z } from "zod"; import { db } from "@server/db"; -import { clients, sites } from "@server/db/schema"; +import { clients, clientSites } from "@server/db/schema"; import { eq } from "drizzle-orm"; import response from "@server/lib/response"; import HttpCode from "@server/types/HttpCode"; @@ -48,7 +48,17 @@ export async function deleteClient( ); } - await db.delete(clients).where(eq(clients.clientId, clientId)); + await db.transaction(async (trx) => { + // Delete the client-site associations first + await trx + .delete(clientSites) + .where(eq(clientSites.clientId, clientId)); + + // Then delete the client itself + await trx + .delete(clients) + .where(eq(clients.clientId, clientId)); + }); return response(res, { data: null, @@ -63,4 +73,4 @@ export async function deleteClient( createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred") ); } -} +} \ No newline at end of file diff --git a/server/routers/client/index.ts b/server/routers/client/index.ts index 686d08e9..144957ed 100644 --- a/server/routers/client/index.ts +++ b/server/routers/client/index.ts @@ -2,3 +2,4 @@ export * from "./pickClientDefaults"; export * from "./createClient"; export * from "./deleteClient"; export * from "./listClients"; +export * from "./updateClient"; \ No newline at end of file diff --git a/server/routers/client/pickClientDefaults.ts b/server/routers/client/pickClientDefaults.ts index b56a4bf4..231bc409 100644 --- a/server/routers/client/pickClientDefaults.ts +++ b/server/routers/client/pickClientDefaults.ts @@ -1,32 +1,11 @@ import { Request, Response, NextFunction } from "express"; -import { db } from "@server/db"; -import { clients, exitNodes, olms, sites } from "@server/db/schema"; -import { eq } from "drizzle-orm"; import response from "@server/lib/response"; import HttpCode from "@server/types/HttpCode"; import createHttpError from "http-errors"; import logger from "@server/logger"; -import { findNextAvailableCidr, getNextAvailableClientSubnet } from "@server/lib/ip"; import { generateId } from "@server/auth/sessions/app"; -import config from "@server/lib/config"; -import { z } from "zod"; -import { fromError } from "zod-validation-error"; - -const getSiteSchema = z - .object({ - siteId: z.string().transform(Number).pipe(z.number()) - }) - .strict(); export type PickClientDefaultsResponse = { - exitNodeId: number; - siteId: number; - address: string; - publicKey: string; - name: string; - listenPort: number; - endpoint: string; - subnet: string; olmId: string; olmSecret: string; }; @@ -37,85 +16,11 @@ export async function pickClientDefaults( next: NextFunction ): Promise { try { - const parsedParams = getSiteSchema.safeParse(req.params); - if (!parsedParams.success) { - return next( - createHttpError( - HttpCode.BAD_REQUEST, - fromError(parsedParams.error).toString() - ) - ); - } - - const { siteId } = parsedParams.data; - - const [site] = await db - .select() - .from(sites) - .where(eq(sites.siteId, siteId)); - - if (!site) { - return next(createHttpError(HttpCode.NOT_FOUND, "Site not found")); - } - - if (site.type !== "newt") { - return next( - createHttpError( - HttpCode.BAD_REQUEST, - "Site is not a newt site" - ) - ); - } - - // TODO: more intelligent way to pick the exit node - - // make sure there is an exit node by counting the exit nodes table - const nodes = await db.select().from(exitNodes); - if (nodes.length === 0) { - return next( - createHttpError(HttpCode.NOT_FOUND, "No exit nodes available") - ); - } - - // get the first exit node - const exitNode = nodes[0]; - - // make sure all the required fields are present - const sitesRequiredFields = z.object({ - address: z.string(), - publicKey: z.string(), - listenPort: z.number(), - endpoint: z.string() - }); - - const parsedSite = sitesRequiredFields.safeParse(site); - if (!parsedSite.success) { - logger.error("Unable to pick client defaults because: " + fromError(parsedSite.error).toString()); - return next( - createHttpError( - HttpCode.BAD_REQUEST, - "Site is not configured to accept client connectivity" - ) - ); - } - - const { address, publicKey, listenPort, endpoint } = parsedSite.data; - - const newSubnet = await getNextAvailableClientSubnet(site.orgId); - const olmId = generateId(15); const secret = generateId(48); return response(res, { data: { - exitNodeId: exitNode.exitNodeId, - siteId: site.siteId, - address: address, - publicKey: publicKey, - name: site.name, - listenPort: listenPort, - endpoint: endpoint, - subnet: `${newSubnet.split("/")[0]}/${config.getRawConfig().orgs.block_size}`, // we want the block size of the whole org olmId: olmId, olmSecret: secret }, diff --git a/server/routers/client/updateClient.ts b/server/routers/client/updateClient.ts new file mode 100644 index 00000000..9bdd4295 --- /dev/null +++ b/server/routers/client/updateClient.ts @@ -0,0 +1,124 @@ +import { Request, Response, NextFunction } from "express"; +import { z } from "zod"; +import { db } from "@server/db"; +import { + clients, + clientSites +} from "@server/db/schema"; +import response from "@server/lib/response"; +import HttpCode from "@server/types/HttpCode"; +import createHttpError from "http-errors"; +import logger from "@server/logger"; +import { eq, and } from "drizzle-orm"; +import { fromError } from "zod-validation-error"; + +const updateClientParamsSchema = z + .object({ + clientId: z.string().transform(Number).pipe(z.number().int().positive()) + }) + .strict(); + +const updateClientSchema = z + .object({ + name: z.string().min(1).max(255).optional(), + siteIds: z.array(z.string().transform(Number).pipe(z.number())).optional() + }) + .strict(); + +export type UpdateClientBody = z.infer; + +export async function updateClient( + req: Request, + res: Response, + next: NextFunction +): Promise { + try { + const parsedBody = updateClientSchema.safeParse(req.body); + if (!parsedBody.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedBody.error).toString() + ) + ); + } + + const { name, siteIds } = parsedBody.data; + + const parsedParams = updateClientParamsSchema.safeParse(req.params); + if (!parsedParams.success) { + return next( + createHttpError( + HttpCode.BAD_REQUEST, + fromError(parsedParams.error).toString() + ) + ); + } + + const { clientId } = parsedParams.data; + + // Fetch the client to make sure it exists and the user has access to it + const [client] = await db + .select() + .from(clients) + .where(eq(clients.clientId, clientId)) + .limit(1); + + if (!client) { + return next( + createHttpError( + HttpCode.NOT_FOUND, + `Client with ID ${clientId} not found` + ) + ); + } + + await db.transaction(async (trx) => { + // Update client name if provided + if (name) { + await trx + .update(clients) + .set({ name }) + .where(eq(clients.clientId, clientId)); + } + + // Update site associations if provided + if (siteIds) { + // Delete existing site associations + await trx + .delete(clientSites) + .where(eq(clientSites.clientId, clientId)); + + // Create new site associations + if (siteIds.length > 0) { + await trx.insert(clientSites).values( + siteIds.map(siteId => ({ + clientId, + siteId + })) + ); + } + } + + // Fetch the updated client + const [updatedClient] = await trx + .select() + .from(clients) + .where(eq(clients.clientId, clientId)) + .limit(1); + + return response(res, { + data: updatedClient, + success: true, + error: false, + message: "Client updated successfully", + status: HttpCode.OK + }); + }); + } catch (error) { + logger.error(error); + return next( + createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred") + ); + } +} \ No newline at end of file diff --git a/server/routers/external.ts b/server/routers/external.ts index 6b01aacc..5001daf5 100644 --- a/server/routers/external.ts +++ b/server/routers/external.ts @@ -102,7 +102,7 @@ authenticated.get( ); authenticated.get( - "/site/:siteId/pick-client-defaults", + "/pick-client-defaults", verifySiteAccess, verifyUserHasAction(ActionsEnum.createClient), client.pickClientDefaults @@ -116,8 +116,8 @@ authenticated.get( ); authenticated.put( - "/site/:siteId/client", - verifySiteAccess, + "/org/:orgId/client", + verifyOrgAccess, verifyUserHasAction(ActionsEnum.createClient), client.createClient ); @@ -129,6 +129,13 @@ authenticated.delete( client.deleteClient ); +authenticated.post( + "/client/:clientId", + verifyClientAccess, // this will check if the user has access to the client + verifyUserHasAction(ActionsEnum.updateClient), // this will check if the user has permission to update the client + client.updateClient +); + // authenticated.get( // "/site/:siteId/roles", // verifySiteAccess, From bcd80e19d49fbf12e741e2e89ebc20939f908156 Mon Sep 17 00:00:00 2001 From: Owen Date: Mon, 31 Mar 2025 16:23:02 -0400 Subject: [PATCH 048/135] Update list clients --- server/routers/client/listClients.ts | 68 +++++++++++++++++++++++----- 1 file changed, 57 insertions(+), 11 deletions(-) diff --git a/server/routers/client/listClients.ts b/server/routers/client/listClients.ts index c7977ea4..f9cf5295 100644 --- a/server/routers/client/listClients.ts +++ b/server/routers/client/listClients.ts @@ -5,6 +5,7 @@ import { roleClients, sites, userClients, + clientSites } from "@server/db/schema"; import logger from "@server/logger"; import HttpCode from "@server/types/HttpCode"; @@ -41,7 +42,6 @@ function queryClients(orgId: string, accessibleClientIds: number[]) { .select({ clientId: clients.clientId, orgId: clients.orgId, - siteNiceId: sites.niceId, name: clients.name, pubKey: clients.pubKey, subnet: clients.subnet, @@ -49,8 +49,7 @@ function queryClients(orgId: string, accessibleClientIds: number[]) { megabytesOut: clients.megabytesOut, orgName: orgs.name, type: clients.type, - online: clients.online, - siteName: sites.name + online: clients.online }) .from(clients) .leftJoin(orgs, eq(clients.orgId, orgs.orgId)) @@ -62,8 +61,27 @@ function queryClients(orgId: string, accessibleClientIds: number[]) { ); } +async function getSiteAssociations(clientIds: number[]) { + if (clientIds.length === 0) return []; + + return db + .select({ + clientId: clientSites.clientId, + siteId: clientSites.siteId, + siteName: sites.name, + siteNiceId: sites.niceId + }) + .from(clientSites) + .leftJoin(sites, eq(clientSites.siteId, sites.siteId)) + .where(inArray(clientSites.clientId, clientIds)); +} + export type ListClientsResponse = { - clients: Awaited>; + clients: Array>[0] & { sites: Array<{ + siteId: number; + siteName: string | null; + siteNiceId: string | null; + }> }>; pagination: { total: number; limit: number; offset: number }; }; @@ -121,17 +139,18 @@ export async function listClients( ); const accessibleClientIds = accessibleClients.map( - (site) => site.clientId + (client) => client.clientId ); const baseQuery = queryClients(orgId, accessibleClientIds); - let countQuery = db + // Get client count + const countQuery = db .select({ count: count() }) - .from(sites) + .from(clients) .where( and( - inArray(sites.siteId, accessibleClientIds), - eq(sites.orgId, orgId) + inArray(clients.clientId, accessibleClientIds), + eq(clients.orgId, orgId) ) ); @@ -139,9 +158,36 @@ export async function listClients( const totalCountResult = await countQuery; const totalCount = totalCountResult[0].count; + // Get associated sites for all clients + const clientIds = clientsList.map(client => client.clientId); + const siteAssociations = await getSiteAssociations(clientIds); + + // Group site associations by client ID + const sitesByClient = siteAssociations.reduce((acc, association) => { + if (!acc[association.clientId]) { + acc[association.clientId] = []; + } + acc[association.clientId].push({ + siteId: association.siteId, + siteName: association.siteName, + siteNiceId: association.siteNiceId + }); + return acc; + }, {} as Record>); + + // Merge clients with their site associations + const clientsWithSites = clientsList.map(client => ({ + ...client, + sites: sitesByClient[client.clientId] || [] + })); + return response(res, { data: { - clients: clientsList, + clients: clientsWithSites, pagination: { total: totalCount, limit, @@ -159,4 +205,4 @@ export async function listClients( createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred") ); } -} +} \ No newline at end of file From 875fa215c5dc27cd2ca28ed768cb3dfe1fe58c38 Mon Sep 17 00:00:00 2001 From: Owen Date: Mon, 31 Mar 2025 16:27:40 -0400 Subject: [PATCH 049/135] Update create client form --- package-lock.json | 73 ++++++++ package.json | 1 + .../settings/clients/CreateClientsForm.tsx | 170 ++++++++++-------- .../settings/clients/CreateClientsModal.tsx | 2 +- src/components/ui/scroll-area.tsx | 48 +++++ 5 files changed, 222 insertions(+), 72 deletions(-) create mode 100644 src/components/ui/scroll-area.tsx diff --git a/package-lock.json b/package-lock.json index c9e0a334..81fc0c38 100644 --- a/package-lock.json +++ b/package-lock.json @@ -22,6 +22,7 @@ "@radix-ui/react-label": "2.1.1", "@radix-ui/react-popover": "1.1.4", "@radix-ui/react-radio-group": "1.2.2", + "@radix-ui/react-scroll-area": "1.2.3", "@radix-ui/react-select": "2.1.4", "@radix-ui/react-separator": "1.1.1", "@radix-ui/react-slot": "1.1.1", @@ -3536,6 +3537,78 @@ } } }, + "node_modules/@radix-ui/react-scroll-area": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/@radix-ui/react-scroll-area/-/react-scroll-area-1.2.3.tgz", + "integrity": "sha512-l7+NNBfBYYJa9tNqVcP2AGvxdE3lmE6kFTBXdvHgUaZuy+4wGCL1Cl2AfaR7RKyimj7lZURGLwFO59k4eBnDJQ==", + "license": "MIT", + "dependencies": { + "@radix-ui/number": "1.1.0", + "@radix-ui/primitive": "1.1.1", + "@radix-ui/react-compose-refs": "1.1.1", + "@radix-ui/react-context": "1.1.1", + "@radix-ui/react-direction": "1.1.0", + "@radix-ui/react-presence": "1.1.2", + "@radix-ui/react-primitive": "2.0.2", + "@radix-ui/react-use-callback-ref": "1.1.0", + "@radix-ui/react-use-layout-effect": "1.1.0" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-scroll-area/node_modules/@radix-ui/react-primitive": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/@radix-ui/react-primitive/-/react-primitive-2.0.2.tgz", + "integrity": "sha512-Ec/0d38EIuvDF+GZjcMU/Ze6MxntVJYO/fRlCPhCaVUyPY9WTalHJw54tp9sXeJo3tlShWpy41vQRgLRGOuz+w==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-slot": "1.1.2" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-scroll-area/node_modules/@radix-ui/react-slot": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@radix-ui/react-slot/-/react-slot-1.1.2.tgz", + "integrity": "sha512-YAKxaiGsSQJ38VzKH86/BPRC4rh+b1Jpa+JneA5LRE7skmLPNAyeG8kPJj/oo4STLvlrs8vkf/iYyc3A5stYCQ==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-compose-refs": "1.1.1" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, "node_modules/@radix-ui/react-select": { "version": "2.1.4", "resolved": "https://registry.npmjs.org/@radix-ui/react-select/-/react-select-2.1.4.tgz", diff --git a/package.json b/package.json index 08cb73aa..b791d980 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,7 @@ "@radix-ui/react-label": "2.1.1", "@radix-ui/react-popover": "1.1.4", "@radix-ui/react-radio-group": "1.2.2", + "@radix-ui/react-scroll-area": "1.2.3", "@radix-ui/react-select": "2.1.4", "@radix-ui/react-separator": "1.1.1", "@radix-ui/react-slot": "1.1.1", diff --git a/src/app/[orgId]/settings/clients/CreateClientsForm.tsx b/src/app/[orgId]/settings/clients/CreateClientsForm.tsx index 09bdb7f9..77275471 100644 --- a/src/app/[orgId]/settings/clients/CreateClientsForm.tsx +++ b/src/app/[orgId]/settings/clients/CreateClientsForm.tsx @@ -21,7 +21,6 @@ import { formatAxiosError } from "@app/lib/api"; import { createApiClient } from "@app/lib/api"; import { useEnvContext } from "@app/hooks/useEnvContext"; import { AxiosResponse } from "axios"; -import { Collapsible } from "@app/components/ui/collapsible"; import { ClientRow } from "./ClientsTable"; import { CreateClientBody, @@ -45,6 +44,9 @@ import { CommandItem, CommandList } from "@app/components/ui/command"; +import { ScrollArea } from "@app/components/ui/scroll-area"; +import { Badge } from "@app/components/ui/badge"; +import { X } from "lucide-react"; const createClientFormSchema = z.object({ name: z @@ -55,16 +57,19 @@ const createClientFormSchema = z.object({ .max(30, { message: "Name must not be longer than 30 characters." }), - siteId: z.coerce.number() + siteIds: z.array(z.number()).min(1, { + message: "Select at least one site." + }) }); -type CreateSiteFormValues = z.infer; +type CreateClientFormValues = z.infer; -const defaultValues: Partial = { - name: "" +const defaultValues: Partial = { + name: "", + siteIds: [] }; -type CreateSiteFormProps = { +type CreateClientFormProps = { onCreate?: (client: ClientRow) => void; setLoading?: (loading: boolean) => void; setChecked?: (checked: boolean) => void; @@ -76,7 +81,7 @@ export default function CreateClientForm({ setLoading, setChecked, orgId -}: CreateSiteFormProps) { +}: CreateClientFormProps) { const api = createApiClient(useEnvContext()); const { env } = useEnvContext(); @@ -87,6 +92,7 @@ export default function CreateClientForm({ const [clientDefaults, setClientDefaults] = useState(null); const [olmCommand, setOlmCommand] = useState(null); + const [selectedSites, setSelectedSites] = useState>([]); const handleCheckboxChange = (checked: boolean) => { setIsChecked(checked); @@ -95,11 +101,16 @@ export default function CreateClientForm({ } }; - const form = useForm({ + const form = useForm({ resolver: zodResolver(createClientFormSchema), defaultValues }); + useEffect(() => { + // Update form value when selectedSites changes + form.setValue('siteIds', selectedSites.map(site => site.id)); + }, [selectedSites, form]); + useEffect(() => { if (!open) return; @@ -109,6 +120,7 @@ export default function CreateClientForm({ form.reset(); setChecked?.(false); setClientDefaults(null); + setSelectedSites([]); const fetchSites = async () => { const res = await api.get>( @@ -118,25 +130,19 @@ export default function CreateClientForm({ (s) => s.type === "newt" && s.subnet ); setSites(sites); - - if (sites.length > 0) { - form.setValue("siteId", sites[0].siteId); - } }; fetchSites(); }, [open]); useEffect(() => { - const siteId = form.getValues("siteId"); + if (selectedSites.length === 0) return; - if (siteId === undefined || siteId === null) return; - - api.get(`/site/${siteId}/pick-client-defaults`) + api.get(`/pick-client-defaults`) .catch((e) => { toast({ variant: "destructive", - title: `Error fetching client defaults for site ${siteId}`, + title: `Error fetching client defaults`, description: formatAxiosError(e) }); }) @@ -148,17 +154,27 @@ export default function CreateClientForm({ setOlmCommand(olmConfig); } }); - }, [form.watch("siteId")]); + }, [selectedSites]); - async function onSubmit(data: CreateSiteFormValues) { + const addSite = (siteId: number, siteName: string) => { + if (!selectedSites.some(site => site.id === siteId)) { + setSelectedSites([...selectedSites, { id: siteId, name: siteName }]); + } + }; + + const removeSite = (siteId: number) => { + setSelectedSites(selectedSites.filter(site => site.id !== siteId)); + }; + + async function onSubmit(data: CreateClientFormValues) { setLoading?.(true); setIsLoading(true); if (!clientDefaults) { toast({ variant: "destructive", - title: "Error creating site", - description: "Site defaults not found" + title: "Error creating client", + description: "Client defaults not found" }); setLoading?.(false); setIsLoading(false); @@ -167,17 +183,17 @@ export default function CreateClientForm({ const payload = { name: data.name, - siteId: data.siteId, - subnet: clientDefaults.subnet, + siteIds: data.siteIds, olmId: clientDefaults.olmId, secret: clientDefaults.olmSecret, type: "olm" } as CreateClientBody; const res = await api - .put< - AxiosResponse - >(`/site/${data.siteId}/client`, payload) + .put>( + `/org/${orgId}/client`, + payload + ) .catch((e) => { toast({ variant: "destructive", @@ -189,12 +205,14 @@ export default function CreateClientForm({ if (res && res.status === 201) { const data = res.data.data; - const site = sites.find((site) => site.siteId === data.siteId); + // For now we'll just use the first site for display purposes + // The actual client will be associated with all selected sites + const firstSite = sites.find((site) => site.siteId === selectedSites[0]?.id); onCreate?.({ name: data.name, - siteId: site!.niceId, - siteName: site!.name, + siteId: firstSite?.niceId || "", + siteName: firstSite?.name || "", id: data.clientId, mbIn: "0 MB", mbOut: "0 MB", @@ -213,7 +231,7 @@ export default function CreateClientForm({
( + name="siteIds" + render={() => ( - Site + Sites @@ -247,61 +265,71 @@ export default function CreateClientForm({ role="combobox" className={cn( "justify-between", - !field.value && + selectedSites.length === 0 && "text-muted-foreground" )} > - {field.value - ? sites.find( - (site) => - site.siteId === - field.value - )?.name - : "Select site"} + {selectedSites.length > 0 + ? `${selectedSites.length} site${selectedSites.length !== 1 ? 's' : ''} selected` + : "Select sites"} - + - + - No site found. + No sites found. - {sites.map((site) => ( - { - form.setValue( - "siteId", - site.siteId - ); - }} - > - - {site.name} - - ))} + + {sites.map((site) => ( + { + addSite(site.siteId, site.name); + }} + > + s.id === site.siteId) + ? "opacity-100" + : "opacity-0" + )} + /> + {site.name} + + ))} + + + {selectedSites.length > 0 && ( +
+ {selectedSites.map(site => ( + + {site.name} + + + ))} +
+ )} + - The client will be have connectivity to this - site. The site must be configured to accept - client connections. + The client will have connectivity to the selected sites. The sites must be configured to accept client connections.
@@ -342,4 +370,4 @@ export default function CreateClientForm({
); -} +} \ No newline at end of file diff --git a/src/app/[orgId]/settings/clients/CreateClientsModal.tsx b/src/app/[orgId]/settings/clients/CreateClientsModal.tsx index ef6df0f3..450e655f 100644 --- a/src/app/[orgId]/settings/clients/CreateClientsModal.tsx +++ b/src/app/[orgId]/settings/clients/CreateClientsModal.tsx @@ -44,7 +44,7 @@ export default function CreateClientFormModal({ Create Client - Create a new client to connect to your site + Create a new client to connect to your sites diff --git a/src/components/ui/scroll-area.tsx b/src/components/ui/scroll-area.tsx new file mode 100644 index 00000000..0b4a48d8 --- /dev/null +++ b/src/components/ui/scroll-area.tsx @@ -0,0 +1,48 @@ +"use client" + +import * as React from "react" +import * as ScrollAreaPrimitive from "@radix-ui/react-scroll-area" + +import { cn } from "@/lib/utils" + +const ScrollArea = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef +>(({ className, children, ...props }, ref) => ( + + + {children} + + + + +)) +ScrollArea.displayName = ScrollAreaPrimitive.Root.displayName + +const ScrollBar = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef +>(({ className, orientation = "vertical", ...props }, ref) => ( + + + +)) +ScrollBar.displayName = ScrollAreaPrimitive.ScrollAreaScrollbar.displayName + +export { ScrollArea, ScrollBar } From 96d6ad8142c0b3588242b3093b2c2da23550a16a Mon Sep 17 00:00:00 2001 From: Owen Date: Tue, 1 Apr 2025 10:13:20 -0400 Subject: [PATCH 050/135] Create clients working again --- server/routers/client/createClient.ts | 2 +- server/routers/external.ts | 4 +- .../[orgId]/settings/clients/ClientsTable.tsx | 56 ++++---- .../settings/clients/CreateClientsForm.tsx | 121 ++++++++++-------- .../settings/clients/CreateClientsModal.tsx | 2 +- src/components/ui/scroll-area.tsx | 2 +- 6 files changed, 101 insertions(+), 86 deletions(-) diff --git a/server/routers/client/createClient.ts b/server/routers/client/createClient.ts index 8bc2af49..a86fcff7 100644 --- a/server/routers/client/createClient.ts +++ b/server/routers/client/createClient.ts @@ -30,7 +30,7 @@ const createClientParamsSchema = z const createClientSchema = z .object({ name: z.string().min(1).max(255), - siteIds: z.array(z.string().transform(Number).pipe(z.number())), + siteIds: z.array(z.number().int().positive()), olmId: z.string(), secret: z.string(), type: z.enum(["olm"]) diff --git a/server/routers/external.ts b/server/routers/external.ts index 5001daf5..5601b63a 100644 --- a/server/routers/external.ts +++ b/server/routers/external.ts @@ -102,8 +102,8 @@ authenticated.get( ); authenticated.get( - "/pick-client-defaults", - verifySiteAccess, + "/org/:orgId/pick-client-defaults", + verifyOrgAccess, verifyUserHasAction(ActionsEnum.createClient), client.pickClientDefaults ); diff --git a/src/app/[orgId]/settings/clients/ClientsTable.tsx b/src/app/[orgId]/settings/clients/ClientsTable.tsx index c7af53e3..7b829efb 100644 --- a/src/app/[orgId]/settings/clients/ClientsTable.tsx +++ b/src/app/[orgId]/settings/clients/ClientsTable.tsx @@ -29,8 +29,6 @@ import CreateClientFormModal from "./CreateClientsModal"; export type ClientRow = { id: number; - siteId: string; - siteName: string; name: string; mbIn: string; mbOut: string; @@ -128,33 +126,33 @@ export default function ClientsTable({ clients, orgId }: ClientTableProps) { ); } }, - { - accessorKey: "siteName", - header: ({ column }) => { - return ( - - ); - }, - cell: ({ row }) => { - const r = row.original; - return ( - - - - ); - } - }, + // { + // accessorKey: "siteName", + // header: ({ column }) => { + // return ( + // + // ); + // }, + // cell: ({ row }) => { + // const r = row.original; + // return ( + // + // + // + // ); + // } + // }, { accessorKey: "online", header: ({ column }) => { diff --git a/src/app/[orgId]/settings/clients/CreateClientsForm.tsx b/src/app/[orgId]/settings/clients/CreateClientsForm.tsx index 77275471..e3183c49 100644 --- a/src/app/[orgId]/settings/clients/CreateClientsForm.tsx +++ b/src/app/[orgId]/settings/clients/CreateClientsForm.tsx @@ -57,8 +57,8 @@ const createClientFormSchema = z.object({ .max(30, { message: "Name must not be longer than 30 characters." }), - siteIds: z.array(z.number()).min(1, { - message: "Select at least one site." + siteIds: z.array(z.number()).min(1, { + message: "Select at least one site." }) }); @@ -88,11 +88,12 @@ export default function CreateClientForm({ const [sites, setSites] = useState([]); const [isLoading, setIsLoading] = useState(false); const [isChecked, setIsChecked] = useState(false); - const [isOpen, setIsOpen] = useState(false); const [clientDefaults, setClientDefaults] = useState(null); const [olmCommand, setOlmCommand] = useState(null); - const [selectedSites, setSelectedSites] = useState>([]); + const [selectedSites, setSelectedSites] = useState< + Array<{ id: number; name: string }> + >([]); const handleCheckboxChange = (checked: boolean) => { setIsChecked(checked); @@ -108,7 +109,10 @@ export default function CreateClientForm({ useEffect(() => { // Update form value when selectedSites changes - form.setValue('siteIds', selectedSites.map(site => site.id)); + form.setValue( + "siteIds", + selectedSites.map((site) => site.id) + ); }, [selectedSites, form]); useEffect(() => { @@ -132,38 +136,39 @@ export default function CreateClientForm({ setSites(sites); }; + const fetchDefaults = async () => { + api.get(`/org/${orgId}/pick-client-defaults`) + .catch((e) => { + toast({ + variant: "destructive", + title: `Error fetching client defaults`, + description: formatAxiosError(e) + }); + }) + .then((res) => { + if (res && res.status === 200) { + const data = res.data.data; + setClientDefaults(data); + const olmConfig = `olm --id ${data?.olmId} --secret ${data?.olmSecret} --endpoint ${env.app.dashboardUrl}`; + setOlmCommand(olmConfig); + } + }); + }; fetchSites(); + fetchDefaults(); }, [open]); - useEffect(() => { - if (selectedSites.length === 0) return; - - api.get(`/pick-client-defaults`) - .catch((e) => { - toast({ - variant: "destructive", - title: `Error fetching client defaults`, - description: formatAxiosError(e) - }); - }) - .then((res) => { - if (res && res.status === 200) { - const data = res.data.data; - setClientDefaults(data); - const olmConfig = `olm --id ${data?.olmId} --secret ${data?.olmSecret} --endpoint ${env.app.dashboardUrl}`; - setOlmCommand(olmConfig); - } - }); - }, [selectedSites]); - const addSite = (siteId: number, siteName: string) => { - if (!selectedSites.some(site => site.id === siteId)) { - setSelectedSites([...selectedSites, { id: siteId, name: siteName }]); + if (!selectedSites.some((site) => site.id === siteId)) { + setSelectedSites([ + ...selectedSites, + { id: siteId, name: siteName } + ]); } }; const removeSite = (siteId: number) => { - setSelectedSites(selectedSites.filter(site => site.id !== siteId)); + setSelectedSites(selectedSites.filter((site) => site.id !== siteId)); }; async function onSubmit(data: CreateClientFormValues) { @@ -190,10 +195,9 @@ export default function CreateClientForm({ } as CreateClientBody; const res = await api - .put>( - `/org/${orgId}/client`, - payload - ) + .put< + AxiosResponse + >(`/org/${orgId}/client`, payload) .catch((e) => { toast({ variant: "destructive", @@ -205,14 +209,8 @@ export default function CreateClientForm({ if (res && res.status === 201) { const data = res.data.data; - // For now we'll just use the first site for display purposes - // The actual client will be associated with all selected sites - const firstSite = sites.find((site) => site.siteId === selectedSites[0]?.id); - onCreate?.({ name: data.name, - siteId: firstSite?.niceId || "", - siteName: firstSite?.name || "", id: data.clientId, mbIn: "0 MB", mbOut: "0 MB", @@ -265,12 +263,13 @@ export default function CreateClientForm({ role="combobox" className={cn( "justify-between", - selectedSites.length === 0 && + selectedSites.length === + 0 && "text-muted-foreground" )} > {selectedSites.length > 0 - ? `${selectedSites.length} site${selectedSites.length !== 1 ? 's' : ''} selected` + ? `${selectedSites.length} site${selectedSites.length !== 1 ? "s" : ""} selected` : "Select sites"} @@ -288,15 +287,26 @@ export default function CreateClientForm({ {sites.map((site) => ( { - addSite(site.siteId, site.name); + addSite( + site.siteId, + site.name + ); }} > s.id === site.siteId) + selectedSites.some( + ( + s + ) => + s.id === + site.siteId + ) ? "opacity-100" : "opacity-0" )} @@ -310,15 +320,20 @@ export default function CreateClientForm({ - + {selectedSites.length > 0 && (
- {selectedSites.map(site => ( - + {selectedSites.map((site) => ( + {site.name} -
)} - + - The client will have connectivity to the selected sites. The sites must be configured to accept client connections. + The client will have connectivity to the + selected sites. The sites must be configured + to accept client connections. @@ -370,4 +387,4 @@ export default function CreateClientForm({
); -} \ No newline at end of file +} diff --git a/src/app/[orgId]/settings/clients/CreateClientsModal.tsx b/src/app/[orgId]/settings/clients/CreateClientsModal.tsx index 450e655f..a8921cb1 100644 --- a/src/app/[orgId]/settings/clients/CreateClientsModal.tsx +++ b/src/app/[orgId]/settings/clients/CreateClientsModal.tsx @@ -60,7 +60,7 @@ export default function CreateClientFormModal({