diff --git a/newt b/newt new file mode 100755 index 00000000..3805c736 Binary files /dev/null and b/newt differ diff --git a/server/db/index.ts b/server/db/index.ts index 6cf40fec..ba26d1dc 100644 --- a/server/db/index.ts +++ b/server/db/index.ts @@ -1,19 +1,30 @@ -import { drizzle } from "drizzle-orm/better-sqlite3"; +import { drizzle as DrizzleSqlite } from "drizzle-orm/better-sqlite3"; +import { drizzle as DrizzlePostgres } from "drizzle-orm/node-postgres"; import Database from "better-sqlite3"; import * as schema from "@server/db/schemas"; import path from "path"; import fs from "fs/promises"; import { APP_PATH } from "@server/lib/consts"; import { existsSync, mkdirSync } from "fs"; +import { readConfigFile } from "@server/lib/readConfigFile"; export const location = path.join(APP_PATH, "db", "db.sqlite"); export const exists = await checkFileExists(location); bootstrapVolume(); -const sqlite = new Database(location); -export const db = drizzle(sqlite, { schema }); +function createDb() { + const config = readConfigFile(); + if (config.database.type === "postgres") { + return DrizzlePostgres(config.database!.postgres!.connection_string!); + } else { + const sqlite = new Database(location); + return DrizzleSqlite(sqlite, { schema }); + } +} + +export const db = createDb(); export default db; async function checkFileExists(filePath: string): Promise { diff --git a/server/db/migratePostgres.ts b/server/db/migratePostgres.ts new file mode 100644 index 00000000..614b2c66 --- /dev/null +++ b/server/db/migratePostgres.ts @@ -0,0 +1,20 @@ +import { migrate } from "drizzle-orm/node-postgres/migrator"; +import db from "@server/db"; +import path from "path"; + +const migrationsFolder = path.join("server/migrations"); + +const runMigrations = async () => { + console.log("Running migrations..."); + try { + migrate(db as any, { + migrationsFolder: migrationsFolder + }); + console.log("Migrations completed successfully."); + } catch (error) { + console.error("Error running migrations:", error); + process.exit(1); + } +}; + +runMigrations(); diff --git a/server/db/migrate.ts b/server/db/migrateSqlite.ts similarity index 94% rename from server/db/migrate.ts rename to server/db/migrateSqlite.ts index d39f4ae9..7e43cb4f 100644 --- a/server/db/migrate.ts +++ b/server/db/migrateSqlite.ts @@ -7,7 +7,7 @@ const migrationsFolder = path.join("server/migrations"); const runMigrations = async () => { console.log("Running migrations..."); try { - migrate(db, { + migrate(db as any, { migrationsFolder: migrationsFolder, }); console.log("Migrations completed successfully."); diff --git a/server/lib/config.ts b/server/lib/config.ts index 935522ed..678724a4 100644 --- a/server/lib/config.ts +++ b/server/lib/config.ts @@ -1,225 +1,10 @@ -import fs from "fs"; -import yaml from "js-yaml"; import { z } from "zod"; -import { fromError } from "zod-validation-error"; -import { - __DIRNAME, - APP_VERSION, - configFilePath1, - configFilePath2 -} from "@server/lib/consts"; -import { passwordSchema } from "@server/auth/passwordSchema"; -import stoi from "./stoi"; +import { __DIRNAME, APP_VERSION } from "@server/lib/consts"; import db from "@server/db"; import { SupporterKey, supporterKey } from "@server/db/schemas"; import { eq } from "drizzle-orm"; import { license } from "@server/license/license"; - -const portSchema = z.number().positive().gt(0).lte(65535); - -const getEnvOrYaml = (envVar: string) => (valFromYaml: any) => { - return process.env[envVar] ?? valFromYaml; -}; - -const configSchema = z.object({ - app: z.object({ - dashboard_url: z - .string() - .url() - .optional() - .pipe(z.string().url()) - .transform((url) => url.toLowerCase()), - log_level: z - .enum(["debug", "info", "warn", "error"]) - .optional() - .default("info"), - save_logs: z.boolean().optional().default(false), - log_failed_attempts: z.boolean().optional().default(false) - }), - domains: z - .record( - z.string(), - z.object({ - base_domain: z - .string() - .nonempty("base_domain must not be empty") - .transform((url) => url.toLowerCase()), - cert_resolver: z.string().optional().default("letsencrypt"), - prefer_wildcard_cert: z.boolean().optional().default(false) - }) - ) - .refine( - (domains) => { - const keys = Object.keys(domains); - - if (keys.length === 0) { - return false; - } - - return true; - }, - { - message: "At least one domain must be defined" - } - ), - server: z.object({ - integration_port: portSchema - .optional() - .default(3003) - .transform(stoi) - .pipe(portSchema.optional()), - external_port: portSchema - .optional() - .default(3000) - .transform(stoi) - .pipe(portSchema), - internal_port: portSchema - .optional() - .default(3001) - .transform(stoi) - .pipe(portSchema), - next_port: portSchema - .optional() - .default(3002) - .transform(stoi) - .pipe(portSchema), - internal_hostname: z - .string() - .optional() - .default("pangolin") - .transform((url) => url.toLowerCase()), - session_cookie_name: z.string().optional().default("p_session_token"), - resource_access_token_param: z.string().optional().default("p_token"), - resource_access_token_headers: z - .object({ - id: z.string().optional().default("P-Access-Token-Id"), - token: z.string().optional().default("P-Access-Token") - }) - .optional() - .default({}), - resource_session_request_param: z - .string() - .optional() - .default("resource_session_request_param"), - dashboard_session_length_hours: z - .number() - .positive() - .gt(0) - .optional() - .default(720), - resource_session_length_hours: z - .number() - .positive() - .gt(0) - .optional() - .default(720), - cors: z - .object({ - origins: z.array(z.string()).optional(), - methods: z.array(z.string()).optional(), - allowed_headers: z.array(z.string()).optional(), - credentials: z.boolean().optional() - }) - .optional(), - trust_proxy: z.boolean().optional().default(true), - secret: z - .string() - .optional() - .transform(getEnvOrYaml("SERVER_SECRET")) - .pipe(z.string().min(8)) - }), - traefik: z - .object({ - http_entrypoint: z.string().optional().default("web"), - https_entrypoint: z.string().optional().default("websecure"), - additional_middlewares: z.array(z.string()).optional() - }) - .optional() - .default({}), - gerbil: z - .object({ - start_port: portSchema - .optional() - .default(51820) - .transform(stoi) - .pipe(portSchema), - base_endpoint: z - .string() - .optional() - .pipe(z.string()) - .transform((url) => url.toLowerCase()), - use_subdomain: z.boolean().optional().default(false), - subnet_group: z.string().optional().default("100.89.137.0/20"), - block_size: z.number().positive().gt(0).optional().default(24), - site_block_size: z.number().positive().gt(0).optional().default(30) - }) - .optional() - .default({}), - rate_limits: z - .object({ - global: z - .object({ - window_minutes: z - .number() - .positive() - .gt(0) - .optional() - .default(1), - max_requests: z - .number() - .positive() - .gt(0) - .optional() - .default(500) - }) - .optional() - .default({}), - auth: z - .object({ - window_minutes: z.number().positive().gt(0), - max_requests: z.number().positive().gt(0) - }) - .optional() - }) - .optional() - .default({}), - email: z - .object({ - smtp_host: z.string().optional(), - smtp_port: portSchema.optional(), - smtp_user: z.string().optional(), - smtp_pass: z.string().optional(), - smtp_secure: z.boolean().optional(), - smtp_tls_reject_unauthorized: z.boolean().optional(), - no_reply: z.string().email().optional() - }) - .optional(), - users: z.object({ - server_admin: z.object({ - email: z - .string() - .email() - .optional() - .transform(getEnvOrYaml("USERS_SERVERADMIN_EMAIL")) - .pipe(z.string().email()) - .transform((v) => v.toLowerCase()), - password: passwordSchema - .optional() - .transform(getEnvOrYaml("USERS_SERVERADMIN_PASSWORD")) - .pipe(passwordSchema) - }) - }), - flags: z - .object({ - require_email_verification: z.boolean().optional(), - disable_signup_without_invite: z.boolean().optional(), - disable_user_create_org: z.boolean().optional(), - allow_raw_resources: z.boolean().optional(), - allow_base_domain_resources: z.boolean().optional(), - allow_local_sites: z.boolean().optional() - }) - .optional() -}); +import { configSchema, readConfigFile } from "./readConfigFile"; export class Config { private rawConfig!: z.infer; @@ -231,96 +16,57 @@ export class Config { isDev: boolean = process.env.ENVIRONMENT !== "prod"; constructor() { - this.loadConfig(); + this.load(); } - public loadConfig() { - const loadConfig = (configPath: string) => { - try { - const yamlContent = fs.readFileSync(configPath, "utf8"); - const config = yaml.load(yamlContent); - return config; - } catch (error) { - if (error instanceof Error) { - throw new Error( - `Error loading configuration file: ${error.message}` - ); - } - throw error; - } - }; - - let environment: any; - if (fs.existsSync(configFilePath1)) { - environment = loadConfig(configFilePath1); - } else if (fs.existsSync(configFilePath2)) { - environment = loadConfig(configFilePath2); - } - - if (process.env.APP_BASE_DOMAIN) { - console.log( - "You're using deprecated environment variables. Transition to the configuration file. https://docs.fossorial.io/" - ); - } - - if (!environment) { - throw new Error( - "No configuration file found. Please create one. https://docs.fossorial.io/" - ); - } - - const parsedConfig = configSchema.safeParse(environment); - - if (!parsedConfig.success) { - const errors = fromError(parsedConfig.error); - throw new Error(`Invalid configuration file: ${errors}`); - } + public load() { + const parsedConfig = readConfigFile(); process.env.APP_VERSION = APP_VERSION; - process.env.NEXT_PORT = parsedConfig.data.server.next_port.toString(); + process.env.NEXT_PORT = parsedConfig.server.next_port.toString(); process.env.SERVER_EXTERNAL_PORT = - parsedConfig.data.server.external_port.toString(); + parsedConfig.server.external_port.toString(); process.env.SERVER_INTERNAL_PORT = - parsedConfig.data.server.internal_port.toString(); - process.env.FLAGS_EMAIL_VERIFICATION_REQUIRED = parsedConfig.data.flags + parsedConfig.server.internal_port.toString(); + process.env.FLAGS_EMAIL_VERIFICATION_REQUIRED = parsedConfig.flags ?.require_email_verification ? "true" : "false"; - process.env.FLAGS_ALLOW_RAW_RESOURCES = parsedConfig.data.flags + process.env.FLAGS_ALLOW_RAW_RESOURCES = parsedConfig.flags ?.allow_raw_resources ? "true" : "false"; process.env.SESSION_COOKIE_NAME = - parsedConfig.data.server.session_cookie_name; - process.env.EMAIL_ENABLED = parsedConfig.data.email ? "true" : "false"; - process.env.DISABLE_SIGNUP_WITHOUT_INVITE = parsedConfig.data.flags + parsedConfig.server.session_cookie_name; + process.env.EMAIL_ENABLED = parsedConfig.email ? "true" : "false"; + process.env.DISABLE_SIGNUP_WITHOUT_INVITE = parsedConfig.flags ?.disable_signup_without_invite ? "true" : "false"; - process.env.DISABLE_USER_CREATE_ORG = parsedConfig.data.flags + process.env.DISABLE_USER_CREATE_ORG = parsedConfig.flags ?.disable_user_create_org ? "true" : "false"; process.env.RESOURCE_ACCESS_TOKEN_PARAM = - parsedConfig.data.server.resource_access_token_param; + parsedConfig.server.resource_access_token_param; process.env.RESOURCE_ACCESS_TOKEN_HEADERS_ID = - parsedConfig.data.server.resource_access_token_headers.id; + parsedConfig.server.resource_access_token_headers.id; process.env.RESOURCE_ACCESS_TOKEN_HEADERS_TOKEN = - parsedConfig.data.server.resource_access_token_headers.token; + parsedConfig.server.resource_access_token_headers.token; process.env.RESOURCE_SESSION_REQUEST_PARAM = - parsedConfig.data.server.resource_session_request_param; - process.env.FLAGS_ALLOW_BASE_DOMAIN_RESOURCES = parsedConfig.data.flags + parsedConfig.server.resource_session_request_param; + process.env.FLAGS_ALLOW_BASE_DOMAIN_RESOURCES = parsedConfig.flags ?.allow_base_domain_resources ? "true" : "false"; - process.env.DASHBOARD_URL = parsedConfig.data.app.dashboard_url; + process.env.DASHBOARD_URL = parsedConfig.app.dashboard_url; - license.setServerSecret(parsedConfig.data.server.secret); + license.setServerSecret(parsedConfig.server.secret); this.checkKeyStatus(); - this.rawConfig = parsedConfig.data; + this.rawConfig = parsedConfig; } private async checkKeyStatus() { diff --git a/server/lib/readConfigFile.ts b/server/lib/readConfigFile.ts new file mode 100644 index 00000000..63aa2ee2 --- /dev/null +++ b/server/lib/readConfigFile.ts @@ -0,0 +1,281 @@ +import fs from "fs"; +import yaml from "js-yaml"; +import { configFilePath1, configFilePath2 } from "./consts"; +import { z } from "zod"; +import stoi from "./stoi"; +import { passwordSchema } from "@server/auth/passwordSchema"; +import { fromError } from "zod-validation-error"; + +const portSchema = z.number().positive().gt(0).lte(65535); + +const getEnvOrYaml = (envVar: string) => (valFromYaml: any) => { + return process.env[envVar] ?? valFromYaml; +}; + +export const configSchema = z.object({ + app: z.object({ + dashboard_url: z + .string() + .url() + .optional() + .pipe(z.string().url()) + .transform((url) => url.toLowerCase()), + log_level: z + .enum(["debug", "info", "warn", "error"]) + .optional() + .default("info"), + save_logs: z.boolean().optional().default(false), + log_failed_attempts: z.boolean().optional().default(false) + }), + domains: z + .record( + z.string(), + z.object({ + base_domain: z + .string() + .nonempty("base_domain must not be empty") + .transform((url) => url.toLowerCase()), + cert_resolver: z.string().optional().default("letsencrypt"), + prefer_wildcard_cert: z.boolean().optional().default(false) + }) + ) + .refine( + (domains) => { + const keys = Object.keys(domains); + + if (keys.length === 0) { + return false; + } + + return true; + }, + { + message: "At least one domain must be defined" + } + ), + server: z.object({ + integration_port: portSchema + .optional() + .default(3003) + .transform(stoi) + .pipe(portSchema.optional()), + external_port: portSchema + .optional() + .default(3000) + .transform(stoi) + .pipe(portSchema), + internal_port: portSchema + .optional() + .default(3001) + .transform(stoi) + .pipe(portSchema), + next_port: portSchema + .optional() + .default(3002) + .transform(stoi) + .pipe(portSchema), + internal_hostname: z + .string() + .optional() + .default("pangolin") + .transform((url) => url.toLowerCase()), + session_cookie_name: z.string().optional().default("p_session_token"), + resource_access_token_param: z.string().optional().default("p_token"), + resource_access_token_headers: z + .object({ + id: z.string().optional().default("P-Access-Token-Id"), + token: z.string().optional().default("P-Access-Token") + }) + .optional() + .default({}), + resource_session_request_param: z + .string() + .optional() + .default("resource_session_request_param"), + dashboard_session_length_hours: z + .number() + .positive() + .gt(0) + .optional() + .default(720), + resource_session_length_hours: z + .number() + .positive() + .gt(0) + .optional() + .default(720), + cors: z + .object({ + origins: z.array(z.string()).optional(), + methods: z.array(z.string()).optional(), + allowed_headers: z.array(z.string()).optional(), + credentials: z.boolean().optional() + }) + .optional(), + trust_proxy: z.boolean().optional().default(true), + secret: z + .string() + .optional() + .transform(getEnvOrYaml("SERVER_SECRET")) + .pipe(z.string().min(8)) + }), + database: z + .object({ + type: z.enum(["sqlite", "postgres"]).optional().default("sqlite"), + postgres: z + .object({ + connection_string: z.string() + }) + .optional() + }) + .refine( + (data) => { + if (data.type === "postgres" && !data.postgres) { + return false; + } + return true; + }, + { + message: + "Postgres config required" + } + ) + .optional() + .default({}), + traefik: z + .object({ + http_entrypoint: z.string().optional().default("web"), + https_entrypoint: z.string().optional().default("websecure"), + additional_middlewares: z.array(z.string()).optional() + }) + .optional() + .default({}), + gerbil: z + .object({ + start_port: portSchema + .optional() + .default(51820) + .transform(stoi) + .pipe(portSchema), + base_endpoint: z + .string() + .optional() + .pipe(z.string()) + .transform((url) => url.toLowerCase()), + use_subdomain: z.boolean().optional().default(false), + subnet_group: z.string().optional().default("100.89.137.0/20"), + block_size: z.number().positive().gt(0).optional().default(24), + site_block_size: z.number().positive().gt(0).optional().default(30) + }) + .optional() + .default({}), + rate_limits: z + .object({ + global: z + .object({ + window_minutes: z + .number() + .positive() + .gt(0) + .optional() + .default(1), + max_requests: z + .number() + .positive() + .gt(0) + .optional() + .default(500) + }) + .optional() + .default({}), + auth: z + .object({ + window_minutes: z.number().positive().gt(0), + max_requests: z.number().positive().gt(0) + }) + .optional() + }) + .optional() + .default({}), + email: z + .object({ + smtp_host: z.string().optional(), + smtp_port: portSchema.optional(), + smtp_user: z.string().optional(), + smtp_pass: z.string().optional(), + smtp_secure: z.boolean().optional(), + smtp_tls_reject_unauthorized: z.boolean().optional(), + no_reply: z.string().email().optional() + }) + .optional(), + users: z.object({ + server_admin: z.object({ + email: z + .string() + .email() + .optional() + .transform(getEnvOrYaml("USERS_SERVERADMIN_EMAIL")) + .pipe(z.string().email()) + .transform((v) => v.toLowerCase()), + password: passwordSchema + .optional() + .transform(getEnvOrYaml("USERS_SERVERADMIN_PASSWORD")) + .pipe(passwordSchema) + }) + }), + flags: z + .object({ + require_email_verification: z.boolean().optional(), + disable_signup_without_invite: z.boolean().optional(), + disable_user_create_org: z.boolean().optional(), + allow_raw_resources: z.boolean().optional(), + allow_base_domain_resources: z.boolean().optional(), + allow_local_sites: z.boolean().optional() + }) + .optional() +}); + +export function readConfigFile() { + const loadConfig = (configPath: string) => { + try { + const yamlContent = fs.readFileSync(configPath, "utf8"); + const config = yaml.load(yamlContent); + return config; + } catch (error) { + if (error instanceof Error) { + throw new Error( + `Error loading configuration file: ${error.message}` + ); + } + throw error; + } + }; + + let environment: any; + if (fs.existsSync(configFilePath1)) { + environment = loadConfig(configFilePath1); + } else if (fs.existsSync(configFilePath2)) { + environment = loadConfig(configFilePath2); + } + + if (process.env.APP_BASE_DOMAIN) { + console.log( + "You're using deprecated environment variables. Transition to the configuration file. https://docs.fossorial.io/" + ); + } + + if (!environment) { + throw new Error( + "No configuration file found. Please create one. https://docs.fossorial.io/" + ); + } + + const parsedConfig = configSchema.safeParse(environment); + + if (!parsedConfig.success) { + const errors = fromError(parsedConfig.error); + throw new Error(`Invalid configuration file: ${errors}`); + } + + return parsedConfig.data; +}