add statistics

This commit is contained in:
miloschwartz 2025-08-14 17:06:07 -07:00
parent 74d2527af5
commit 67ba225003
No known key found for this signature in database
11 changed files with 362 additions and 11 deletions

28
package-lock.json generated
View file

@ -75,6 +75,7 @@
"npm": "^11.5.2",
"oslo": "1.2.1",
"pg": "^8.16.2",
"posthog-node": "^5.7.0",
"qrcode.react": "4.2.0",
"react": "19.1.1",
"react-dom": "19.1.1",
@ -2050,6 +2051,7 @@
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz",
"integrity": "sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w==",
"dev": true,
"license": "ISC",
"dependencies": {
"minipass": "^7.0.4"
@ -2983,6 +2985,12 @@
"tslib": "^2.8.1"
}
},
"node_modules/@posthog/core": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/@posthog/core/-/core-1.0.0.tgz",
"integrity": "sha512-gquQld+duT9DdzLIFoHZkUMW0DZOTSLCtSjuuC/zKFz65Qecbz9p37DHBJMkw0dCuB8Mgh2GtH8Ag3PznJrP3g==",
"license": "MIT"
},
"node_modules/@radix-ui/number": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/@radix-ui/number/-/number-1.1.1.tgz",
@ -6258,6 +6266,7 @@
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz",
"integrity": "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==",
"dev": true,
"license": "BlueOak-1.0.0",
"engines": {
"node": ">=18"
@ -8798,6 +8807,7 @@
"version": "4.2.11",
"resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz",
"integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==",
"dev": true,
"license": "ISC"
},
"node_modules/graphemer": {
@ -9560,6 +9570,7 @@
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/isexe/-/isexe-3.1.1.tgz",
"integrity": "sha512-LpB/54B+/2J5hqQ7imZHfdU31OlgQqx7ZicVlkm9kzg9/w8GKLEcFfJl/t7DCEDueOyBAD6zCCwTO6Fzs0NoEQ==",
"dev": true,
"license": "ISC",
"engines": {
"node": ">=16"
@ -10405,6 +10416,7 @@
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.0.2.tgz",
"integrity": "sha512-oG62iEk+CYt5Xj2YqI5Xi9xWUeZhDI8jjQmC5oThVH5JGCTgIjr7ciJDzC7MBzYd//WvR1OTmP5Q38Q8ShQtVA==",
"dev": true,
"license": "MIT",
"dependencies": {
"minipass": "^7.1.2"
@ -10417,6 +10429,7 @@
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-3.0.1.tgz",
"integrity": "sha512-+NsyUUAZDmo6YVHzL/stxSu3t9YS1iljliy3BSDrXJ/dkn1KYdmtZODGGjLcc9XLgVVpH4KshHB8XmZgMhaBXg==",
"dev": true,
"license": "MIT",
"bin": {
"mkdirp": "dist/cjs/src/bin.js"
@ -14183,6 +14196,18 @@
"node": ">=0.10.0"
}
},
"node_modules/posthog-node": {
"version": "5.7.0",
"resolved": "https://registry.npmjs.org/posthog-node/-/posthog-node-5.7.0.tgz",
"integrity": "sha512-6J1AIZWtbr2lEbZOO2AzO/h1FPJjUZM4KWcdaL2UQw7FY8J7VNaH3NiaRockASFmglpID7zEY25gV/YwCtuXjg==",
"license": "MIT",
"dependencies": {
"@posthog/core": "1.0.0"
},
"engines": {
"node": ">=20"
}
},
"node_modules/prebuild-install": {
"version": "7.1.3",
"resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.3.tgz",
@ -15914,6 +15939,7 @@
"version": "7.4.3",
"resolved": "https://registry.npmjs.org/tar/-/tar-7.4.3.tgz",
"integrity": "sha512-5S7Va8hKfV7W5U6g3aYxXmlPoZVAwUMy9AOKyF2fVuZa2UD3qZjg578OrLRt8PcNN1PleVaL/5/yYATNL0ICUw==",
"dev": true,
"license": "ISC",
"dependencies": {
"@isaacs/fs-minipass": "^4.0.0",
@ -16552,6 +16578,7 @@
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/which/-/which-4.0.0.tgz",
"integrity": "sha512-GlaYyEb07DPxYCKhKzplCWBJtvxZcZMrL+4UkrTSJHHPyZU4mYYTv3qaOe77H7EODLSSopAUFAc6W8U4yqvscg==",
"dev": true,
"license": "ISC",
"dependencies": {
"isexe": "^3.1.1"
@ -16857,6 +16884,7 @@
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/yallist/-/yallist-5.0.0.tgz",
"integrity": "sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==",
"dev": true,
"license": "BlueOak-1.0.0",
"engines": {
"node": ">=18"

View file

@ -52,9 +52,9 @@
"@radix-ui/react-tooltip": "^1.2.7",
"@react-email/components": "0.5.0",
"@react-email/render": "^1.2.0",
"@react-email/tailwind": "1.2.2",
"@simplewebauthn/browser": "^13.1.0",
"@simplewebauthn/server": "^9.0.3",
"@react-email/tailwind": "1.2.2",
"@tailwindcss/forms": "^0.5.10",
"@tanstack/react-table": "8.21.3",
"arctic": "^3.7.0",
@ -93,6 +93,7 @@
"npm": "^11.5.2",
"oslo": "1.2.1",
"pg": "^8.16.2",
"posthog-node": "^5.7.0",
"qrcode.react": "4.2.0",
"react": "19.1.1",
"react-dom": "19.1.1",
@ -109,9 +110,9 @@
"winston": "3.17.0",
"winston-daily-rotate-file": "5.0.0",
"ws": "8.18.3",
"yargs": "18.0.0",
"zod": "3.25.76",
"zod-validation-error": "3.5.2",
"yargs": "18.0.0"
"zod-validation-error": "3.5.2"
},
"devDependencies": {
"@dotenvx/dotenvx": "1.48.4",

View file

@ -690,3 +690,4 @@ export type ApiKeyAction = InferSelectModel<typeof apiKeyActions>;
export type ApiKeyOrg = InferSelectModel<typeof apiKeyOrg>;
export type OrgDomains = InferSelectModel<typeof orgDomains>;
export type SetupToken = InferSelectModel<typeof setupTokens>;
export type HostMeta = InferSelectModel<typeof hostMeta>;

View file

@ -8,11 +8,17 @@ import { createInternalServer } from "./internalServer";
import { ApiKey, ApiKeyOrg, Session, User, UserOrg } from "@server/db";
import { createIntegrationApiServer } from "./integrationApiServer";
import config from "@server/lib/config";
import { setHostMeta } from "@server/lib/hostMeta";
import { initTelemetryClient } from "./lib/telemetry.js";
async function startServers() {
await setHostMeta();
await config.initServer();
await runSetupFunctions();
initTelemetryClient();
// Start all servers
const apiServer = createApiServer();
const internalServer = createInternalServer();

View file

@ -1,7 +1,9 @@
import { db } from "@server/db";
import { db, HostMeta } from "@server/db";
import { hostMeta } from "@server/db";
import { v4 as uuidv4 } from "uuid";
let gotHostMeta: HostMeta | undefined;
export async function setHostMeta() {
const [existing] = await db.select().from(hostMeta).limit(1);
@ -15,3 +17,12 @@ export async function setHostMeta() {
.insert(hostMeta)
.values({ hostMetaId: id, createdAt: new Date().getTime() });
}
export async function getHostMeta() {
if (gotHostMeta) {
return gotHostMeta;
}
const [meta] = await db.select().from(hostMeta).limit(1);
gotHostMeta = meta;
return meta;
}

View file

@ -3,7 +3,6 @@ import yaml from "js-yaml";
import { configFilePath1, configFilePath2 } from "./consts";
import { z } from "zod";
import stoi from "./stoi";
import { build } from "@server/build";
const portSchema = z.number().positive().gt(0).lte(65535);
@ -25,7 +24,13 @@ export const configSchema = z
.optional()
.default("info"),
save_logs: z.boolean().optional().default(false),
log_failed_attempts: z.boolean().optional().default(false)
log_failed_attempts: z.boolean().optional().default(false),
telmetry: z
.object({
anonymous_usage: z.boolean().optional().default(true)
})
.optional()
.default({})
}),
domains: z
.record(
@ -213,7 +218,10 @@ export const configSchema = z
smtp_host: z.string().optional(),
smtp_port: portSchema.optional(),
smtp_user: z.string().optional(),
smtp_pass: z.string().optional().transform(getEnvOrYaml("EMAIL_SMTP_PASS")),
smtp_pass: z
.string()
.optional()
.transform(getEnvOrYaml("EMAIL_SMTP_PASS")),
smtp_secure: z.boolean().optional(),
smtp_tls_reject_unauthorized: z.boolean().optional(),
no_reply: z.string().email().optional()
@ -229,7 +237,7 @@ export const configSchema = z
disable_local_sites: z.boolean().optional(),
disable_basic_wireguard_sites: z.boolean().optional(),
disable_config_managed_domains: z.boolean().optional(),
enable_clients: z.boolean().optional().default(true),
enable_clients: z.boolean().optional().default(true)
})
.optional(),
dns: z

295
server/lib/telemetry.ts Normal file
View file

@ -0,0 +1,295 @@
import { PostHog } from "posthog-node";
import config from "./config";
import { getHostMeta } from "./hostMeta";
import logger from "@server/logger";
import { apiKeys, db, roles } from "@server/db";
import { sites, users, orgs, resources, clients, idp } from "@server/db";
import { eq, count, notInArray } from "drizzle-orm";
import { APP_VERSION } from "./consts";
import crypto from "crypto";
import { UserType } from "@server/types/UserTypes";
class TelemetryClient {
private client: PostHog | null = null;
private enabled: boolean;
private intervalId: NodeJS.Timeout | null = null;
constructor() {
const enabled = config.getRawConfig().app.telmetry.anonymous_usage;
this.enabled = enabled;
const dev = process.env.ENVIRONMENT !== "prod";
if (this.enabled && !dev) {
this.client = new PostHog(
"phc_QYuATSSZt6onzssWcYJbXLzQwnunIpdGGDTYhzK3VjX",
{
host: "https://digpangolin.com/relay-O7yI"
}
);
process.on("exit", () => {
this.client?.shutdown();
});
this.sendStartupEvents().catch((err) => {
logger.error("Failed to send startup telemetry:", err);
});
this.startAnalyticsInterval();
logger.info(
"Pangolin now gathers anonymous usage data to help us better understand how the software is used and guide future improvements and feature development. You can find more details, including instructions for opting out of this anonymous data collection, at: https://docs.digpangolin.com/telemetry"
);
} else if (!this.enabled && !dev) {
logger.info(
"Analytics usage statistics collection is disabled. If you enable this, you can help us make Pangolin better for everyone. Learn more at: https://docs.digpangolin.com/telemetry"
);
}
}
private startAnalyticsInterval() {
this.intervalId = setInterval(
() => {
this.collectAndSendAnalytics().catch((err) => {
logger.error("Failed to collect analytics:", err);
});
},
6 * 60 * 60 * 1000
);
this.collectAndSendAnalytics().catch((err) => {
logger.error("Failed to collect initial analytics:", err);
});
}
private anon(value: string): string {
return crypto
.createHash("sha256")
.update(value.toLowerCase())
.digest("hex");
}
private async getSystemStats() {
try {
const [sitesCount] = await db
.select({ count: count() })
.from(sites);
const [usersCount] = await db
.select({ count: count() })
.from(users);
const [usersInternalCount] = await db
.select({ count: count() })
.from(users)
.where(eq(users.type, UserType.Internal));
const [usersOidcCount] = await db
.select({ count: count() })
.from(users)
.where(eq(users.type, UserType.OIDC));
const [orgsCount] = await db.select({ count: count() }).from(orgs);
const [resourcesCount] = await db
.select({ count: count() })
.from(resources);
const [clientsCount] = await db
.select({ count: count() })
.from(clients);
const [idpCount] = await db.select({ count: count() }).from(idp);
const [onlineSitesCount] = await db
.select({ count: count() })
.from(sites)
.where(eq(sites.online, true));
const [numApiKeys] = await db
.select({ count: count() })
.from(apiKeys);
const [customRoles] = await db
.select({ count: count() })
.from(roles)
.where(notInArray(roles.name, ["Admin", "Member"]));
const adminUsers = await db
.select({ email: users.email })
.from(users)
.where(eq(users.serverAdmin, true));
const resourceDetails = await db
.select({
name: resources.name,
sso: resources.sso,
protocol: resources.protocol,
http: resources.http
})
.from(resources);
const siteDetails = await db
.select({
siteName: sites.name,
megabytesIn: sites.megabytesIn,
megabytesOut: sites.megabytesOut,
type: sites.type,
online: sites.online
})
.from(sites);
const supporterKey = config.getSupporterData();
return {
numSites: sitesCount.count,
numUsers: usersCount.count,
numUsersInternal: usersInternalCount.count,
numUsersOidc: usersOidcCount.count,
numOrganizations: orgsCount.count,
numResources: resourcesCount.count,
numClients: clientsCount.count,
numIdentityProviders: idpCount.count,
numSitesOnline: onlineSitesCount.count,
resources: resourceDetails,
adminUsers: adminUsers.map((u) => u.email),
sites: siteDetails,
appVersion: APP_VERSION,
numApiKeys: numApiKeys.count,
numCustomRoles: customRoles.count,
supporterStatus: {
valid: supporterKey?.valid || false,
tier: supporterKey?.tier || "None",
githubUsername: supporterKey?.githubUsername || null
}
};
} catch (error) {
logger.error("Failed to collect system stats:", error);
throw error;
}
}
private async sendStartupEvents() {
if (!this.enabled || !this.client) return;
const hostMeta = await getHostMeta();
if (!hostMeta) return;
const stats = await this.getSystemStats();
this.client.capture({
distinctId: hostMeta.hostMetaId,
event: "supporter_status",
properties: {
valid: stats.supporterStatus.valid,
tier: stats.supporterStatus.tier,
github_username: stats.supporterStatus.githubUsername
? this.anon(stats.supporterStatus.githubUsername)
: "None"
}
});
this.client.capture({
distinctId: hostMeta.hostMetaId,
event: "host_startup",
properties: {
host_id: hostMeta.hostMetaId,
app_version: stats.appVersion,
install_timestamp: hostMeta.createdAt
}
});
for (const email of stats.adminUsers) {
// There should only be on admin user, but just in case
if (email) {
this.client.capture({
distinctId: this.anon(email),
event: "admin_user",
properties: {
host_id: hostMeta.hostMetaId,
app_version: stats.appVersion,
hashed_email: this.anon(email)
}
});
}
}
}
private async collectAndSendAnalytics() {
if (!this.enabled || !this.client) return;
try {
const hostMeta = await getHostMeta();
if (!hostMeta) {
logger.warn(
"Telemetry: Host meta not found, skipping analytics"
);
return;
}
const stats = await this.getSystemStats();
this.client.capture({
distinctId: hostMeta.hostMetaId,
event: "system_analytics",
properties: {
app_version: stats.appVersion,
num_sites: stats.numSites,
num_users: stats.numUsers,
num_users_internal: stats.numUsersInternal,
num_users_oidc: stats.numUsersOidc,
num_organizations: stats.numOrganizations,
num_resources: stats.numResources,
num_clients: stats.numClients,
num_identity_providers: stats.numIdentityProviders,
num_sites_online: stats.numSitesOnline,
resources: stats.resources.map((r) => ({
name: this.anon(r.name),
sso_enabled: r.sso,
protocol: r.protocol,
http_enabled: r.http
})),
sites: stats.sites.map((s) => ({
site_name: this.anon(s.siteName),
megabytes_in: s.megabytesIn,
megabytes_out: s.megabytesOut,
type: s.type,
online: s.online
})),
num_api_keys: stats.numApiKeys,
num_custom_roles: stats.numCustomRoles
}
});
} catch (error) {
logger.error("Failed to send analytics:", error);
}
}
async sendTelemetry(eventName: string, properties: Record<string, any>) {
if (!this.enabled || !this.client) return;
const hostMeta = await getHostMeta();
if (!hostMeta) {
logger.warn("Telemetry: Host meta not found, skipping telemetry");
return;
}
this.client.groupIdentify({
groupType: "host_id",
groupKey: hostMeta.hostMetaId,
properties
});
}
shutdown() {
if (this.intervalId) {
clearInterval(this.intervalId);
this.intervalId = null;
}
if (this.enabled && this.client) {
this.client.shutdown();
}
}
}
let telemetryClient!: TelemetryClient;
export function initTelemetryClient() {
if (!telemetryClient) {
telemetryClient = new TelemetryClient();
}
return telemetryClient;
}
export default telemetryClient;

View file

@ -5,7 +5,7 @@ import NodeCache from "node-cache";
import { validateJWT } from "./licenseJwt";
import { count, eq } from "drizzle-orm";
import moment from "moment";
import { setHostMeta } from "@server/setup/setHostMeta";
import { setHostMeta } from "@server/lib/hostMeta";
import { encrypt, decrypt } from "@server/lib/crypto";
const keyTypes = ["HOST", "SITES"] as const;

View file

@ -3,6 +3,7 @@ import config from "@server/lib/config";
import * as winston from "winston";
import path from "path";
import { APP_PATH } from "./lib/consts";
import telemetryClient from "./lib/telemetry";
const hformat = winston.format.printf(
({ level, label, message, timestamp, stack, ...metadata }) => {

View file

@ -18,7 +18,7 @@ const migrations = [
{ version: "1.6.0", run: m1 },
{ version: "1.7.0", run: m2 },
{ version: "1.8.0", run: m3 },
{ version: "1.9.0", run: m4 }
// { version: "1.9.0", run: m4 }
// Add new migrations here as they are created
] as {
version: string;

View file

@ -50,7 +50,7 @@ const migrations = [
{ version: "1.6.0", run: m21 },
{ version: "1.7.0", run: m22 },
{ version: "1.8.0", run: m23 },
{ version: "1.9.0", run: m24 },
// { version: "1.9.0", run: m24 },
// Add new migrations here as they are created
] as const;