Merge branch 'dev' into user-management-and-resources

This commit is contained in:
Adrian Astles 2025-07-18 22:21:55 +08:00 committed by GitHub
commit a140f27d04
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
291 changed files with 21408 additions and 7510 deletions

View file

@ -8,8 +8,31 @@ export async function copyInConfig() {
const endpoint = config.getRawConfig().gerbil.base_endpoint;
const listenPort = config.getRawConfig().gerbil.start_port;
if (!config.getRawConfig().flags?.disable_config_managed_domains) {
await copyInDomains();
}
const exitNodeName = config.getRawConfig().gerbil.exit_node_name;
if (exitNodeName) {
await db
.update(exitNodes)
.set({ endpoint, listenPort })
.where(eq(exitNodes.name, exitNodeName));
} else {
await db
.update(exitNodes)
.set({ endpoint })
.where(ne(exitNodes.endpoint, endpoint));
await db
.update(exitNodes)
.set({ listenPort })
.where(ne(exitNodes.listenPort, listenPort));
}
}
async function copyInDomains() {
await db.transaction(async (trx) => {
const rawDomains = config.getRawConfig().domains;
const rawDomains = config.getRawConfig().domains!; // always defined if disable flag is not set
const configDomains = Object.entries(rawDomains).map(
([key, value]) => ({
@ -40,13 +63,19 @@ export async function copyInConfig() {
if (existingDomainKeys.has(domainId)) {
await trx
.update(domains)
.set({ baseDomain })
.set({ baseDomain, verified: true, type: "wildcard" })
.where(eq(domains.domainId, domainId))
.execute();
} else {
await trx
.insert(domains)
.values({ domainId, baseDomain, configManaged: true })
.values({
domainId,
baseDomain,
configManaged: true,
type: "wildcard",
verified: true
})
.execute();
}
}
@ -92,7 +121,7 @@ export async function copyInConfig() {
}
let fullDomain = "";
if (resource.isBaseDomain) {
if (!resource.subdomain) {
fullDomain = domain.baseDomain;
} else {
fullDomain = `${resource.subdomain}.${domain.baseDomain}`;
@ -104,15 +133,4 @@ export async function copyInConfig() {
.where(eq(resources.resourceId, resource.resourceId));
}
});
// TODO: eventually each exit node could have a different endpoint
await db
.update(exitNodes)
.set({ endpoint })
.where(ne(exitNodes.endpoint, endpoint));
// TODO: eventually each exit node could have a different port
await db
.update(exitNodes)
.set({ listenPort })
.where(ne(exitNodes.listenPort, listenPort));
}

View file

@ -1,15 +1,9 @@
import { ensureActions } from "./ensureActions";
import { copyInConfig } from "./copyInConfig";
import logger from "@server/logger";
import { clearStaleData } from "./clearStaleData";
export async function runSetupFunctions() {
try {
await copyInConfig(); // copy in the config to the db as needed
await ensureActions(); // make sure all of the actions are in the db and the roles
await clearStaleData();
} catch (error) {
logger.error("Error running setup functions:", error);
process.exit(1);
}
await copyInConfig(); // copy in the config to the db as needed
await ensureActions(); // make sure all of the actions are in the db and the roles
await clearStaleData();
}

View file

@ -30,6 +30,10 @@ async function run() {
}
export async function runMigrations() {
if (process.env.DISABLE_MIGRATIONS) {
console.log("Migrations are disabled. Skipping...");
return;
}
try {
const appVersion = APP_VERSION;

View file

@ -23,7 +23,6 @@ import m19 from "./scriptsSqlite/1.3.0";
import m20 from "./scriptsSqlite/1.5.0";
import m21 from "./scriptsSqlite/1.6.0";
import m22 from "./scriptsSqlite/1.7.0";
import m23 from "./scriptsSqlite/1.8.0";
// THIS CANNOT IMPORT ANYTHING FROM THE SERVER
// EXCEPT FOR THE DATABASE AND THE SCHEMA
@ -47,7 +46,6 @@ const migrations = [
{ version: "1.5.0", run: m20 },
{ version: "1.6.0", run: m21 },
{ version: "1.7.0", run: m22 },
{ version: "1.8.0", run: m23 }
// Add new migrations here as they are created
] as const;
@ -80,24 +78,24 @@ function backupDb() {
}
export async function runMigrations() {
if (process.env.DISABLE_MIGRATIONS) {
console.log("Migrations are disabled. Skipping...");
return;
}
try {
const appVersion = APP_VERSION;
// Check if the database file exists and has tables
const hasTables = await db.select().from(versionMigrations).limit(1).catch(() => false);
if (hasTables) {
if (exists) {
await executeScripts();
} else {
console.log("Running initial migrations...");
console.log("Running migrations...");
try {
migrate(db, {
migrationsFolder: path.join(APP_PATH, "server", "migrations")
migrationsFolder: path.join(__DIRNAME, "init") // put here during the docker build
});
console.log("Initial migrations completed successfully.");
console.log("Migrations completed successfully.");
} catch (error) {
console.error("Error running initial migrations:", error);
throw error;
console.error("Error running migrations:", error);
}
await db

View file

@ -1,21 +1,163 @@
import { db } from "@server/db/pg";
import { db } from "@server/db/pg/driver";
import { sql } from "drizzle-orm";
const version = "1.7.0";
export default async function migration() {
console.log(`Running PostgreSQL setup script ${version}...`);
console.log(`Running setup script ${version}...`);
try {
// Add passwordResetTokenExpiryHours column to orgs table with default value of 1
await db.execute(`
ALTER TABLE orgs ADD COLUMN passwordResetTokenExpiryHours INTEGER NOT NULL DEFAULT 1;
await db.execute(sql`
BEGIN;
CREATE TABLE "clientSites" (
"clientId" integer NOT NULL,
"siteId" integer NOT NULL,
"isRelayed" boolean DEFAULT false NOT NULL
);
CREATE TABLE "clients" (
"id" serial PRIMARY KEY NOT NULL,
"orgId" varchar NOT NULL,
"exitNode" integer,
"name" varchar NOT NULL,
"pubKey" varchar,
"subnet" varchar NOT NULL,
"bytesIn" integer,
"bytesOut" integer,
"lastBandwidthUpdate" varchar,
"lastPing" varchar,
"type" varchar NOT NULL,
"online" boolean DEFAULT false NOT NULL,
"endpoint" varchar,
"lastHolePunch" integer,
"maxConnections" integer
);
CREATE TABLE "clientSession" (
"id" varchar PRIMARY KEY NOT NULL,
"olmId" varchar NOT NULL,
"expiresAt" integer NOT NULL
);
CREATE TABLE "olms" (
"id" varchar PRIMARY KEY NOT NULL,
"secretHash" varchar NOT NULL,
"dateCreated" varchar NOT NULL,
"clientId" integer
);
CREATE TABLE "roleClients" (
"roleId" integer NOT NULL,
"clientId" integer NOT NULL
);
CREATE TABLE "webauthnCredentials" (
"credentialId" varchar PRIMARY KEY NOT NULL,
"userId" varchar NOT NULL,
"publicKey" varchar NOT NULL,
"signCount" integer NOT NULL,
"transports" varchar,
"name" varchar,
"lastUsed" varchar NOT NULL,
"dateCreated" varchar NOT NULL,
"securityKeyName" varchar
);
CREATE TABLE "userClients" (
"userId" varchar NOT NULL,
"clientId" integer NOT NULL
);
CREATE TABLE "webauthnChallenge" (
"sessionId" varchar PRIMARY KEY NOT NULL,
"challenge" varchar NOT NULL,
"securityKeyName" varchar,
"userId" varchar,
"expiresAt" bigint NOT NULL
);
ALTER TABLE "limits" DISABLE ROW LEVEL SECURITY;
DROP TABLE "limits" CASCADE;
ALTER TABLE "sites" ALTER COLUMN "subnet" DROP NOT NULL;
ALTER TABLE "sites" ALTER COLUMN "bytesIn" SET DEFAULT 0;
ALTER TABLE "sites" ALTER COLUMN "bytesOut" SET DEFAULT 0;
ALTER TABLE "domains" ADD COLUMN "type" varchar;
ALTER TABLE "domains" ADD COLUMN "verified" boolean DEFAULT false NOT NULL;
ALTER TABLE "domains" ADD COLUMN "failed" boolean DEFAULT false NOT NULL;
ALTER TABLE "domains" ADD COLUMN "tries" integer DEFAULT 0 NOT NULL;
ALTER TABLE "exitNodes" ADD COLUMN "maxConnections" integer;
ALTER TABLE "newt" ADD COLUMN "version" varchar;
ALTER TABLE "orgs" ADD COLUMN "subnet" varchar;
ALTER TABLE "sites" ADD COLUMN "address" varchar;
ALTER TABLE "sites" ADD COLUMN "endpoint" varchar;
ALTER TABLE "sites" ADD COLUMN "publicKey" varchar;
ALTER TABLE "sites" ADD COLUMN "lastHolePunch" bigint;
ALTER TABLE "sites" ADD COLUMN "listenPort" integer;
ALTER TABLE "user" ADD COLUMN "twoFactorSetupRequested" boolean DEFAULT false;
ALTER TABLE "clientSites" ADD CONSTRAINT "clientSites_clientId_clients_id_fk" FOREIGN KEY ("clientId") REFERENCES "public"."clients"("id") ON DELETE cascade ON UPDATE no action;
ALTER TABLE "clientSites" ADD CONSTRAINT "clientSites_siteId_sites_siteId_fk" FOREIGN KEY ("siteId") REFERENCES "public"."sites"("siteId") ON DELETE cascade ON UPDATE no action;
ALTER TABLE "clients" ADD CONSTRAINT "clients_orgId_orgs_orgId_fk" FOREIGN KEY ("orgId") REFERENCES "public"."orgs"("orgId") ON DELETE cascade ON UPDATE no action;
ALTER TABLE "clients" ADD CONSTRAINT "clients_exitNode_exitNodes_exitNodeId_fk" FOREIGN KEY ("exitNode") REFERENCES "public"."exitNodes"("exitNodeId") ON DELETE set null ON UPDATE no action;
ALTER TABLE "clientSession" ADD CONSTRAINT "clientSession_olmId_olms_id_fk" FOREIGN KEY ("olmId") REFERENCES "public"."olms"("id") ON DELETE cascade ON UPDATE no action;
ALTER TABLE "olms" ADD CONSTRAINT "olms_clientId_clients_id_fk" FOREIGN KEY ("clientId") REFERENCES "public"."clients"("id") ON DELETE cascade ON UPDATE no action;
ALTER TABLE "roleClients" ADD CONSTRAINT "roleClients_roleId_roles_roleId_fk" FOREIGN KEY ("roleId") REFERENCES "public"."roles"("roleId") ON DELETE cascade ON UPDATE no action;
ALTER TABLE "roleClients" ADD CONSTRAINT "roleClients_clientId_clients_id_fk" FOREIGN KEY ("clientId") REFERENCES "public"."clients"("id") ON DELETE cascade ON UPDATE no action;
ALTER TABLE "webauthnCredentials" ADD CONSTRAINT "webauthnCredentials_userId_user_id_fk" FOREIGN KEY ("userId") REFERENCES "public"."user"("id") ON DELETE cascade ON UPDATE no action;
ALTER TABLE "userClients" ADD CONSTRAINT "userClients_userId_user_id_fk" FOREIGN KEY ("userId") REFERENCES "public"."user"("id") ON DELETE cascade ON UPDATE no action;
ALTER TABLE "userClients" ADD CONSTRAINT "userClients_clientId_clients_id_fk" FOREIGN KEY ("clientId") REFERENCES "public"."clients"("id") ON DELETE cascade ON UPDATE no action;
ALTER TABLE "webauthnChallenge" ADD CONSTRAINT "webauthnChallenge_userId_user_id_fk" FOREIGN KEY ("userId") REFERENCES "public"."user"("id") ON DELETE cascade ON UPDATE no action;
ALTER TABLE "resources" DROP COLUMN "isBaseDomain";
COMMIT;
`);
console.log(`Added passwordResetTokenExpiryHours column to orgs table`);
console.log(`Migrated database schema`);
} catch (e) {
console.log("Error adding passwordResetTokenExpiryHours column to orgs table:");
console.log("Unable to migrate database schema");
console.log(e);
throw e;
}
console.log(`${version} PostgreSQL migration complete`);
}
try {
await db.execute(sql`BEGIN`);
// Update all existing orgs to have the default subnet
await db.execute(sql`UPDATE "orgs" SET "subnet" = '100.90.128.0/24'`);
// Get all orgs and their sites to assign sequential IP addresses
const orgsQuery = await db.execute(sql`SELECT "orgId" FROM "orgs"`);
const orgs = orgsQuery.rows as { orgId: string }[];
for (const org of orgs) {
const sitesQuery = await db.execute(sql`
SELECT "siteId" FROM "sites"
WHERE "orgId" = ${org.orgId}
ORDER BY "siteId"
`);
const sites = sitesQuery.rows as { siteId: number }[];
let ipIndex = 1;
for (const site of sites) {
const address = `100.90.128.${ipIndex}/24`;
await db.execute(sql`
UPDATE "sites" SET "address" = ${address}
WHERE "siteId" = ${site.siteId}
`);
ipIndex++;
}
}
await db.execute(sql`COMMIT`);
console.log(`Updated org subnets and site addresses`);
} catch (e) {
await db.execute(sql`ROLLBACK`);
console.log("Unable to update org subnets");
console.log(e);
throw e;
}
console.log(`${version} migration complete`);
}

View file

@ -1,31 +0,0 @@
import { db } from "../../db/sqlite";
import { sql } from "drizzle-orm";
const version = "1.4.0";
export default async function migration() {
console.log(`Running setup script ${version}...`);
try {
db.transaction((trx) => {
trx.run(sql`CREATE TABLE 'securityKey' (
'credentialId' text PRIMARY KEY NOT NULL,
'userId' text NOT NULL,
'publicKey' text NOT NULL,
'signCount' integer NOT NULL,
'transports' text,
'name' text,
'lastUsed' text NOT NULL,
'dateCreated' text NOT NULL,
FOREIGN KEY ('userId') REFERENCES 'user'('id') ON DELETE CASCADE
);`);
});
console.log(`Migrated database schema`);
} catch (e) {
console.log("Unable to migrate database schema");
throw e;
}
console.log(`${version} migration complete`);
}

View file

@ -12,6 +12,7 @@ export default async function migration() {
try {
db.pragma("foreign_keys = OFF");
db.transaction(() => {
// Add passwordResetTokenExpiryHours column to orgs table with default value of 1
db.exec(`
@ -29,26 +30,173 @@ export default async function migration() {
db.pragma("foreign_keys = OFF");
db.transaction(() => {
db.exec(`
CREATE TABLE IF NOT EXISTS securityKey (
credentialId TEXT PRIMARY KEY,
userId TEXT NOT NULL,
publicKey TEXT NOT NULL,
signCount INTEGER NOT NULL,
transports TEXT,
name TEXT,
lastUsed TEXT NOT NULL,
dateCreated TEXT NOT NULL,
FOREIGN KEY (userId) REFERENCES user(id) ON DELETE CASCADE
CREATE TABLE 'clientSites' (
'clientId' integer NOT NULL,
'siteId' integer NOT NULL,
'isRelayed' integer DEFAULT 0 NOT NULL,
FOREIGN KEY ('clientId') REFERENCES 'clients'('id') ON UPDATE no action ON DELETE cascade,
FOREIGN KEY ('siteId') REFERENCES 'sites'('siteId') ON UPDATE no action ON DELETE cascade
);
CREATE TABLE 'clients' (
'id' integer PRIMARY KEY AUTOINCREMENT NOT NULL,
'orgId' text NOT NULL,
'exitNode' integer,
'name' text NOT NULL,
'pubKey' text,
'subnet' text NOT NULL,
'bytesIn' integer,
'bytesOut' integer,
'lastBandwidthUpdate' text,
'lastPing' text,
'type' text NOT NULL,
'online' integer DEFAULT 0 NOT NULL,
'endpoint' text,
'lastHolePunch' integer,
FOREIGN KEY ('orgId') REFERENCES 'orgs'('orgId') ON UPDATE no action ON DELETE cascade,
FOREIGN KEY ('exitNode') REFERENCES 'exitNodes'('exitNodeId') ON UPDATE no action ON DELETE set null
);
CREATE TABLE 'clientSession' (
'id' text PRIMARY KEY NOT NULL,
'olmId' text NOT NULL,
'expiresAt' integer NOT NULL,
FOREIGN KEY ('olmId') REFERENCES 'olms'('id') ON UPDATE no action ON DELETE cascade
);
CREATE TABLE 'olms' (
'id' text PRIMARY KEY NOT NULL,
'secretHash' text NOT NULL,
'dateCreated' text NOT NULL,
'clientId' integer,
FOREIGN KEY ('clientId') REFERENCES 'clients'('id') ON UPDATE no action ON DELETE cascade
);
CREATE TABLE 'roleClients' (
'roleId' integer NOT NULL,
'clientId' integer NOT NULL,
FOREIGN KEY ('roleId') REFERENCES 'roles'('roleId') ON UPDATE no action ON DELETE cascade,
FOREIGN KEY ('clientId') REFERENCES 'clients'('id') ON UPDATE no action ON DELETE cascade
);
CREATE TABLE 'webauthnCredentials' (
'credentialId' text PRIMARY KEY NOT NULL,
'userId' text NOT NULL,
'publicKey' text NOT NULL,
'signCount' integer NOT NULL,
'transports' text,
'name' text,
'lastUsed' text NOT NULL,
'dateCreated' text NOT NULL,
FOREIGN KEY ('userId') REFERENCES 'user'('id') ON UPDATE no action ON DELETE cascade
);
CREATE TABLE 'userClients' (
'userId' text NOT NULL,
'clientId' integer NOT NULL,
FOREIGN KEY ('userId') REFERENCES 'user'('id') ON UPDATE no action ON DELETE cascade,
FOREIGN KEY ('clientId') REFERENCES 'clients'('id') ON UPDATE no action ON DELETE cascade
);
CREATE TABLE 'userDomains' (
'userId' text NOT NULL,
'domainId' text NOT NULL,
FOREIGN KEY ('userId') REFERENCES 'user'('id') ON UPDATE no action ON DELETE cascade,
FOREIGN KEY ('domainId') REFERENCES 'domains'('domainId') ON UPDATE no action ON DELETE cascade
);
CREATE TABLE 'webauthnChallenge' (
'sessionId' text PRIMARY KEY NOT NULL,
'challenge' text NOT NULL,
'securityKeyName' text,
'userId' text,
'expiresAt' integer NOT NULL,
FOREIGN KEY ('userId') REFERENCES 'user'('id') ON UPDATE no action ON DELETE cascade
);
`);
})(); // executes the transaction immediately
db.exec(`
CREATE TABLE '__new_sites' (
'siteId' integer PRIMARY KEY AUTOINCREMENT NOT NULL,
'orgId' text NOT NULL,
'niceId' text NOT NULL,
'exitNode' integer,
'name' text NOT NULL,
'pubKey' text,
'subnet' text,
'bytesIn' integer DEFAULT 0,
'bytesOut' integer DEFAULT 0,
'lastBandwidthUpdate' text,
'type' text NOT NULL,
'online' integer DEFAULT 0 NOT NULL,
'address' text,
'endpoint' text,
'publicKey' text,
'lastHolePunch' integer,
'listenPort' integer,
'dockerSocketEnabled' integer DEFAULT 1 NOT NULL,
FOREIGN KEY ('orgId') REFERENCES 'orgs'('orgId') ON UPDATE no action ON DELETE cascade,
FOREIGN KEY ('exitNode') REFERENCES 'exitNodes'('exitNodeId') ON UPDATE no action ON DELETE set null
);
INSERT INTO '__new_sites' (
'siteId', 'orgId', 'niceId', 'exitNode', 'name', 'pubKey', 'subnet', 'bytesIn', 'bytesOut', 'lastBandwidthUpdate', 'type', 'online', 'address', 'endpoint', 'publicKey', 'lastHolePunch', 'listenPort', 'dockerSocketEnabled'
)
SELECT siteId, orgId, niceId, exitNode, name, pubKey, subnet, bytesIn, bytesOut, lastBandwidthUpdate, type, online, NULL, NULL, NULL, NULL, NULL, dockerSocketEnabled
FROM sites;
DROP TABLE 'sites';
ALTER TABLE '__new_sites' RENAME TO 'sites';
`);
db.exec(`
ALTER TABLE 'domains' ADD 'type' text;
ALTER TABLE 'domains' ADD 'verified' integer DEFAULT 0 NOT NULL;
ALTER TABLE 'domains' ADD 'failed' integer DEFAULT 0 NOT NULL;
ALTER TABLE 'domains' ADD 'tries' integer DEFAULT 0 NOT NULL;
ALTER TABLE 'exitNodes' ADD 'maxConnections' integer;
ALTER TABLE 'newt' ADD 'version' text;
ALTER TABLE 'orgs' ADD 'subnet' text;
ALTER TABLE 'user' ADD 'twoFactorSetupRequested' integer DEFAULT 0;
ALTER TABLE 'resources' DROP COLUMN 'isBaseDomain';
`);
})();
db.pragma("foreign_keys = ON");
console.log(`Created securityKey table`);
console.log(`Migrated database schema`);
} catch (e) {
console.error("Unable to create securityKey table");
console.error(e);
console.log("Unable to migrate database schema");
throw e;
}
db.transaction(() => {
// Update all existing orgs to have the default subnet
db.exec(`UPDATE 'orgs' SET 'subnet' = '100.90.128.0/24'`);
// Get all orgs and their sites to assign sequential IP addresses
const orgs = db.prepare(`SELECT orgId FROM 'orgs'`).all() as {
orgId: string;
}[];
for (const org of orgs) {
const sites = db
.prepare(
`SELECT siteId FROM 'sites' WHERE orgId = ? ORDER BY siteId`
)
.all(org.orgId) as { siteId: number }[];
let ipIndex = 1;
for (const site of sites) {
const address = `100.90.128.${ipIndex}/24`;
db.prepare(
`UPDATE 'sites' SET 'address' = ? WHERE siteId = ?`
).run(address, site.siteId);
ipIndex++;
}
}
})();
console.log(`${version} migration complete`);
}
}

View file

@ -1,38 +0,0 @@
import { APP_PATH } from "@server/lib/consts";
import Database from "better-sqlite3";
import path from "path";
const version = "1.8.0";
export default async function migration() {
console.log(`Running setup script ${version}...`);
const location = path.join(APP_PATH, "db", "db.sqlite");
const db = new Database(location);
try {
db.pragma("foreign_keys = OFF");
db.transaction(() => {
db.exec(`
CREATE TABLE IF NOT EXISTS securityKeyChallenge (
sessionId TEXT PRIMARY KEY,
challenge TEXT NOT NULL,
securityKeyName TEXT,
userId TEXT,
expiresAt INTEGER NOT NULL,
FOREIGN KEY (userId) REFERENCES user(id) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS idx_securityKeyChallenge_expiresAt ON securityKeyChallenge(expiresAt);
`);
})(); // executes the transaction immediately
db.pragma("foreign_keys = ON");
console.log(`Created securityKeyChallenge table`);
} catch (e) {
console.error("Unable to create securityKeyChallenge table");
console.error(e);
throw e;
}
console.log(`${version} migration complete`);
}