diff --git a/package-lock.json b/package-lock.json index e0d69052..b0dd35e1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -75,6 +75,7 @@ "npm": "^11.5.2", "oslo": "1.2.1", "pg": "^8.16.2", + "posthog-node": "^5.7.0", "qrcode.react": "4.2.0", "react": "19.1.1", "react-dom": "19.1.1", @@ -2050,6 +2051,7 @@ "version": "4.0.1", "resolved": "https://registry.npmjs.org/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz", "integrity": "sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w==", + "dev": true, "license": "ISC", "dependencies": { "minipass": "^7.0.4" @@ -2983,6 +2985,12 @@ "tslib": "^2.8.1" } }, + "node_modules/@posthog/core": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@posthog/core/-/core-1.0.0.tgz", + "integrity": "sha512-gquQld+duT9DdzLIFoHZkUMW0DZOTSLCtSjuuC/zKFz65Qecbz9p37DHBJMkw0dCuB8Mgh2GtH8Ag3PznJrP3g==", + "license": "MIT" + }, "node_modules/@radix-ui/number": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/@radix-ui/number/-/number-1.1.1.tgz", @@ -6258,6 +6266,7 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz", "integrity": "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==", + "dev": true, "license": "BlueOak-1.0.0", "engines": { "node": ">=18" @@ -8798,6 +8807,7 @@ "version": "4.2.11", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "dev": true, "license": "ISC" }, "node_modules/graphemer": { @@ -9560,6 +9570,7 @@ "version": "3.1.1", "resolved": "https://registry.npmjs.org/isexe/-/isexe-3.1.1.tgz", "integrity": "sha512-LpB/54B+/2J5hqQ7imZHfdU31OlgQqx7ZicVlkm9kzg9/w8GKLEcFfJl/t7DCEDueOyBAD6zCCwTO6Fzs0NoEQ==", + "dev": true, "license": "ISC", "engines": { "node": ">=16" @@ -10405,6 +10416,7 @@ "version": "3.0.2", "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.0.2.tgz", "integrity": "sha512-oG62iEk+CYt5Xj2YqI5Xi9xWUeZhDI8jjQmC5oThVH5JGCTgIjr7ciJDzC7MBzYd//WvR1OTmP5Q38Q8ShQtVA==", + "dev": true, "license": "MIT", "dependencies": { "minipass": "^7.1.2" @@ -10417,6 +10429,7 @@ "version": "3.0.1", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-3.0.1.tgz", "integrity": "sha512-+NsyUUAZDmo6YVHzL/stxSu3t9YS1iljliy3BSDrXJ/dkn1KYdmtZODGGjLcc9XLgVVpH4KshHB8XmZgMhaBXg==", + "dev": true, "license": "MIT", "bin": { "mkdirp": "dist/cjs/src/bin.js" @@ -14183,6 +14196,18 @@ "node": ">=0.10.0" } }, + "node_modules/posthog-node": { + "version": "5.7.0", + "resolved": "https://registry.npmjs.org/posthog-node/-/posthog-node-5.7.0.tgz", + "integrity": "sha512-6J1AIZWtbr2lEbZOO2AzO/h1FPJjUZM4KWcdaL2UQw7FY8J7VNaH3NiaRockASFmglpID7zEY25gV/YwCtuXjg==", + "license": "MIT", + "dependencies": { + "@posthog/core": "1.0.0" + }, + "engines": { + "node": ">=20" + } + }, "node_modules/prebuild-install": { "version": "7.1.3", "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.3.tgz", @@ -15914,6 +15939,7 @@ "version": "7.4.3", "resolved": "https://registry.npmjs.org/tar/-/tar-7.4.3.tgz", "integrity": "sha512-5S7Va8hKfV7W5U6g3aYxXmlPoZVAwUMy9AOKyF2fVuZa2UD3qZjg578OrLRt8PcNN1PleVaL/5/yYATNL0ICUw==", + "dev": true, "license": "ISC", "dependencies": { "@isaacs/fs-minipass": "^4.0.0", @@ -16552,6 +16578,7 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/which/-/which-4.0.0.tgz", "integrity": "sha512-GlaYyEb07DPxYCKhKzplCWBJtvxZcZMrL+4UkrTSJHHPyZU4mYYTv3qaOe77H7EODLSSopAUFAc6W8U4yqvscg==", + "dev": true, "license": "ISC", "dependencies": { "isexe": "^3.1.1" @@ -16857,6 +16884,7 @@ "version": "5.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-5.0.0.tgz", "integrity": "sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==", + "dev": true, "license": "BlueOak-1.0.0", "engines": { "node": ">=18" diff --git a/package.json b/package.json index 937f76fa..7b3464a8 100644 --- a/package.json +++ b/package.json @@ -52,9 +52,9 @@ "@radix-ui/react-tooltip": "^1.2.7", "@react-email/components": "0.5.0", "@react-email/render": "^1.2.0", + "@react-email/tailwind": "1.2.2", "@simplewebauthn/browser": "^13.1.0", "@simplewebauthn/server": "^9.0.3", - "@react-email/tailwind": "1.2.2", "@tailwindcss/forms": "^0.5.10", "@tanstack/react-table": "8.21.3", "arctic": "^3.7.0", @@ -93,6 +93,7 @@ "npm": "^11.5.2", "oslo": "1.2.1", "pg": "^8.16.2", + "posthog-node": "^5.7.0", "qrcode.react": "4.2.0", "react": "19.1.1", "react-dom": "19.1.1", @@ -109,9 +110,9 @@ "winston": "3.17.0", "winston-daily-rotate-file": "5.0.0", "ws": "8.18.3", + "yargs": "18.0.0", "zod": "3.25.76", - "zod-validation-error": "3.5.2", - "yargs": "18.0.0" + "zod-validation-error": "3.5.2" }, "devDependencies": { "@dotenvx/dotenvx": "1.48.4", diff --git a/server/db/sqlite/schema.ts b/server/db/sqlite/schema.ts index db1d8090..5bd81d6a 100644 --- a/server/db/sqlite/schema.ts +++ b/server/db/sqlite/schema.ts @@ -690,3 +690,4 @@ export type ApiKeyAction = InferSelectModel; export type ApiKeyOrg = InferSelectModel; export type OrgDomains = InferSelectModel; export type SetupToken = InferSelectModel; +export type HostMeta = InferSelectModel; diff --git a/server/index.ts b/server/index.ts index d3f90281..8724cb41 100644 --- a/server/index.ts +++ b/server/index.ts @@ -8,11 +8,17 @@ import { createInternalServer } from "./internalServer"; import { ApiKey, ApiKeyOrg, Session, User, UserOrg } from "@server/db"; import { createIntegrationApiServer } from "./integrationApiServer"; import config from "@server/lib/config"; +import { setHostMeta } from "@server/lib/hostMeta"; +import { initTelemetryClient } from "./lib/telemetry.js"; async function startServers() { + await setHostMeta(); + await config.initServer(); await runSetupFunctions(); + initTelemetryClient(); + // Start all servers const apiServer = createApiServer(); const internalServer = createInternalServer(); diff --git a/server/setup/setHostMeta.ts b/server/lib/hostMeta.ts similarity index 57% rename from server/setup/setHostMeta.ts rename to server/lib/hostMeta.ts index 2223d11b..2f2c7ed7 100644 --- a/server/setup/setHostMeta.ts +++ b/server/lib/hostMeta.ts @@ -1,7 +1,9 @@ -import { db } from "@server/db"; +import { db, HostMeta } from "@server/db"; import { hostMeta } from "@server/db"; import { v4 as uuidv4 } from "uuid"; +let gotHostMeta: HostMeta | undefined; + export async function setHostMeta() { const [existing] = await db.select().from(hostMeta).limit(1); @@ -15,3 +17,12 @@ export async function setHostMeta() { .insert(hostMeta) .values({ hostMetaId: id, createdAt: new Date().getTime() }); } + +export async function getHostMeta() { + if (gotHostMeta) { + return gotHostMeta; + } + const [meta] = await db.select().from(hostMeta).limit(1); + gotHostMeta = meta; + return meta; +} diff --git a/server/lib/readConfigFile.ts b/server/lib/readConfigFile.ts index 1bc119fa..c349e50e 100644 --- a/server/lib/readConfigFile.ts +++ b/server/lib/readConfigFile.ts @@ -3,7 +3,6 @@ import yaml from "js-yaml"; import { configFilePath1, configFilePath2 } from "./consts"; import { z } from "zod"; import stoi from "./stoi"; -import { build } from "@server/build"; const portSchema = z.number().positive().gt(0).lte(65535); @@ -25,7 +24,13 @@ export const configSchema = z .optional() .default("info"), save_logs: z.boolean().optional().default(false), - log_failed_attempts: z.boolean().optional().default(false) + log_failed_attempts: z.boolean().optional().default(false), + telmetry: z + .object({ + anonymous_usage: z.boolean().optional().default(true) + }) + .optional() + .default({}) }), domains: z .record( @@ -213,7 +218,10 @@ export const configSchema = z smtp_host: z.string().optional(), smtp_port: portSchema.optional(), smtp_user: z.string().optional(), - smtp_pass: z.string().optional().transform(getEnvOrYaml("EMAIL_SMTP_PASS")), + smtp_pass: z + .string() + .optional() + .transform(getEnvOrYaml("EMAIL_SMTP_PASS")), smtp_secure: z.boolean().optional(), smtp_tls_reject_unauthorized: z.boolean().optional(), no_reply: z.string().email().optional() @@ -229,7 +237,7 @@ export const configSchema = z disable_local_sites: z.boolean().optional(), disable_basic_wireguard_sites: z.boolean().optional(), disable_config_managed_domains: z.boolean().optional(), - enable_clients: z.boolean().optional().default(true), + enable_clients: z.boolean().optional().default(true) }) .optional(), dns: z diff --git a/server/lib/telemetry.ts b/server/lib/telemetry.ts new file mode 100644 index 00000000..8475fb34 --- /dev/null +++ b/server/lib/telemetry.ts @@ -0,0 +1,295 @@ +import { PostHog } from "posthog-node"; +import config from "./config"; +import { getHostMeta } from "./hostMeta"; +import logger from "@server/logger"; +import { apiKeys, db, roles } from "@server/db"; +import { sites, users, orgs, resources, clients, idp } from "@server/db"; +import { eq, count, notInArray } from "drizzle-orm"; +import { APP_VERSION } from "./consts"; +import crypto from "crypto"; +import { UserType } from "@server/types/UserTypes"; + +class TelemetryClient { + private client: PostHog | null = null; + private enabled: boolean; + private intervalId: NodeJS.Timeout | null = null; + + constructor() { + const enabled = config.getRawConfig().app.telmetry.anonymous_usage; + this.enabled = enabled; + const dev = process.env.ENVIRONMENT !== "prod"; + + if (this.enabled && !dev) { + this.client = new PostHog( + "phc_QYuATSSZt6onzssWcYJbXLzQwnunIpdGGDTYhzK3VjX", + { + host: "https://digpangolin.com/relay-O7yI" + } + ); + + process.on("exit", () => { + this.client?.shutdown(); + }); + + this.sendStartupEvents().catch((err) => { + logger.error("Failed to send startup telemetry:", err); + }); + + this.startAnalyticsInterval(); + + logger.info( + "Pangolin now gathers anonymous usage data to help us better understand how the software is used and guide future improvements and feature development. You can find more details, including instructions for opting out of this anonymous data collection, at: https://docs.digpangolin.com/telemetry" + ); + } else if (!this.enabled && !dev) { + logger.info( + "Analytics usage statistics collection is disabled. If you enable this, you can help us make Pangolin better for everyone. Learn more at: https://docs.digpangolin.com/telemetry" + ); + } + } + + private startAnalyticsInterval() { + this.intervalId = setInterval( + () => { + this.collectAndSendAnalytics().catch((err) => { + logger.error("Failed to collect analytics:", err); + }); + }, + 6 * 60 * 60 * 1000 + ); + + this.collectAndSendAnalytics().catch((err) => { + logger.error("Failed to collect initial analytics:", err); + }); + } + + private anon(value: string): string { + return crypto + .createHash("sha256") + .update(value.toLowerCase()) + .digest("hex"); + } + + private async getSystemStats() { + try { + const [sitesCount] = await db + .select({ count: count() }) + .from(sites); + const [usersCount] = await db + .select({ count: count() }) + .from(users); + const [usersInternalCount] = await db + .select({ count: count() }) + .from(users) + .where(eq(users.type, UserType.Internal)); + const [usersOidcCount] = await db + .select({ count: count() }) + .from(users) + .where(eq(users.type, UserType.OIDC)); + const [orgsCount] = await db.select({ count: count() }).from(orgs); + const [resourcesCount] = await db + .select({ count: count() }) + .from(resources); + const [clientsCount] = await db + .select({ count: count() }) + .from(clients); + const [idpCount] = await db.select({ count: count() }).from(idp); + const [onlineSitesCount] = await db + .select({ count: count() }) + .from(sites) + .where(eq(sites.online, true)); + const [numApiKeys] = await db + .select({ count: count() }) + .from(apiKeys); + const [customRoles] = await db + .select({ count: count() }) + .from(roles) + .where(notInArray(roles.name, ["Admin", "Member"])); + + const adminUsers = await db + .select({ email: users.email }) + .from(users) + .where(eq(users.serverAdmin, true)); + + const resourceDetails = await db + .select({ + name: resources.name, + sso: resources.sso, + protocol: resources.protocol, + http: resources.http + }) + .from(resources); + + const siteDetails = await db + .select({ + siteName: sites.name, + megabytesIn: sites.megabytesIn, + megabytesOut: sites.megabytesOut, + type: sites.type, + online: sites.online + }) + .from(sites); + + const supporterKey = config.getSupporterData(); + + return { + numSites: sitesCount.count, + numUsers: usersCount.count, + numUsersInternal: usersInternalCount.count, + numUsersOidc: usersOidcCount.count, + numOrganizations: orgsCount.count, + numResources: resourcesCount.count, + numClients: clientsCount.count, + numIdentityProviders: idpCount.count, + numSitesOnline: onlineSitesCount.count, + resources: resourceDetails, + adminUsers: adminUsers.map((u) => u.email), + sites: siteDetails, + appVersion: APP_VERSION, + numApiKeys: numApiKeys.count, + numCustomRoles: customRoles.count, + supporterStatus: { + valid: supporterKey?.valid || false, + tier: supporterKey?.tier || "None", + githubUsername: supporterKey?.githubUsername || null + } + }; + } catch (error) { + logger.error("Failed to collect system stats:", error); + throw error; + } + } + + private async sendStartupEvents() { + if (!this.enabled || !this.client) return; + + const hostMeta = await getHostMeta(); + if (!hostMeta) return; + + const stats = await this.getSystemStats(); + + this.client.capture({ + distinctId: hostMeta.hostMetaId, + event: "supporter_status", + properties: { + valid: stats.supporterStatus.valid, + tier: stats.supporterStatus.tier, + github_username: stats.supporterStatus.githubUsername + ? this.anon(stats.supporterStatus.githubUsername) + : "None" + } + }); + + this.client.capture({ + distinctId: hostMeta.hostMetaId, + event: "host_startup", + properties: { + host_id: hostMeta.hostMetaId, + app_version: stats.appVersion, + install_timestamp: hostMeta.createdAt + } + }); + + for (const email of stats.adminUsers) { + // There should only be on admin user, but just in case + if (email) { + this.client.capture({ + distinctId: this.anon(email), + event: "admin_user", + properties: { + host_id: hostMeta.hostMetaId, + app_version: stats.appVersion, + hashed_email: this.anon(email) + } + }); + } + } + } + + private async collectAndSendAnalytics() { + if (!this.enabled || !this.client) return; + + try { + const hostMeta = await getHostMeta(); + if (!hostMeta) { + logger.warn( + "Telemetry: Host meta not found, skipping analytics" + ); + return; + } + + const stats = await this.getSystemStats(); + + this.client.capture({ + distinctId: hostMeta.hostMetaId, + event: "system_analytics", + properties: { + app_version: stats.appVersion, + num_sites: stats.numSites, + num_users: stats.numUsers, + num_users_internal: stats.numUsersInternal, + num_users_oidc: stats.numUsersOidc, + num_organizations: stats.numOrganizations, + num_resources: stats.numResources, + num_clients: stats.numClients, + num_identity_providers: stats.numIdentityProviders, + num_sites_online: stats.numSitesOnline, + resources: stats.resources.map((r) => ({ + name: this.anon(r.name), + sso_enabled: r.sso, + protocol: r.protocol, + http_enabled: r.http + })), + sites: stats.sites.map((s) => ({ + site_name: this.anon(s.siteName), + megabytes_in: s.megabytesIn, + megabytes_out: s.megabytesOut, + type: s.type, + online: s.online + })), + num_api_keys: stats.numApiKeys, + num_custom_roles: stats.numCustomRoles + } + }); + } catch (error) { + logger.error("Failed to send analytics:", error); + } + } + + async sendTelemetry(eventName: string, properties: Record) { + if (!this.enabled || !this.client) return; + + const hostMeta = await getHostMeta(); + if (!hostMeta) { + logger.warn("Telemetry: Host meta not found, skipping telemetry"); + return; + } + + this.client.groupIdentify({ + groupType: "host_id", + groupKey: hostMeta.hostMetaId, + properties + }); + } + + shutdown() { + if (this.intervalId) { + clearInterval(this.intervalId); + this.intervalId = null; + } + + if (this.enabled && this.client) { + this.client.shutdown(); + } + } +} + +let telemetryClient!: TelemetryClient; + +export function initTelemetryClient() { + if (!telemetryClient) { + telemetryClient = new TelemetryClient(); + } + return telemetryClient; +} + +export default telemetryClient; diff --git a/server/license/license.ts b/server/license/license.ts index 0adc54fd..aeb628df 100644 --- a/server/license/license.ts +++ b/server/license/license.ts @@ -5,7 +5,7 @@ import NodeCache from "node-cache"; import { validateJWT } from "./licenseJwt"; import { count, eq } from "drizzle-orm"; import moment from "moment"; -import { setHostMeta } from "@server/setup/setHostMeta"; +import { setHostMeta } from "@server/lib/hostMeta"; import { encrypt, decrypt } from "@server/lib/crypto"; const keyTypes = ["HOST", "SITES"] as const; diff --git a/server/logger.ts b/server/logger.ts index cd12d735..15dd6e3f 100644 --- a/server/logger.ts +++ b/server/logger.ts @@ -3,6 +3,7 @@ import config from "@server/lib/config"; import * as winston from "winston"; import path from "path"; import { APP_PATH } from "./lib/consts"; +import telemetryClient from "./lib/telemetry"; const hformat = winston.format.printf( ({ level, label, message, timestamp, stack, ...metadata }) => { diff --git a/server/setup/migrationsPg.ts b/server/setup/migrationsPg.ts index 6b3f20b9..fd9a7c21 100644 --- a/server/setup/migrationsPg.ts +++ b/server/setup/migrationsPg.ts @@ -18,7 +18,7 @@ const migrations = [ { version: "1.6.0", run: m1 }, { version: "1.7.0", run: m2 }, { version: "1.8.0", run: m3 }, - { version: "1.9.0", run: m4 } + // { version: "1.9.0", run: m4 } // Add new migrations here as they are created ] as { version: string; diff --git a/server/setup/migrationsSqlite.ts b/server/setup/migrationsSqlite.ts index 5b0850c8..5411261f 100644 --- a/server/setup/migrationsSqlite.ts +++ b/server/setup/migrationsSqlite.ts @@ -50,7 +50,7 @@ const migrations = [ { version: "1.6.0", run: m21 }, { version: "1.7.0", run: m22 }, { version: "1.8.0", run: m23 }, - { version: "1.9.0", run: m24 }, + // { version: "1.9.0", run: m24 }, // Add new migrations here as they are created ] as const;