From d22c7826fe63ab58e44988597f3df260192507c8 Mon Sep 17 00:00:00 2001 From: Owen Date: Sun, 9 Feb 2025 16:14:29 -0500 Subject: [PATCH 1/9] Add config files --- install/captcha.html | 338 ++++++++++++++++ install/crowdsec/install.go | 376 ++++++++++++++++++ install/fs/crowdsec/acquis.yaml | 18 + install/fs/crowdsec/config.yaml | 12 + .../fs/crowdsec/local_api_credentials.yaml | 2 + install/fs/crowdsec/profiles.yaml | 25 ++ 6 files changed, 771 insertions(+) create mode 100644 install/captcha.html create mode 100644 install/crowdsec/install.go create mode 100644 install/fs/crowdsec/acquis.yaml create mode 100644 install/fs/crowdsec/config.yaml create mode 100644 install/fs/crowdsec/local_api_credentials.yaml create mode 100644 install/fs/crowdsec/profiles.yaml diff --git a/install/captcha.html b/install/captcha.html new file mode 100644 index 00000000..a40d37a3 --- /dev/null +++ b/install/captcha.html @@ -0,0 +1,338 @@ + + + + + CrowdSec Captcha + + + + + + + +
+
+
+ +

CrowdSec Captcha

+
+
+
+
+
+
+

This security check has been powered by

+ + + + + + + + + + + + + + + + + + + + + CrowdSec + +
+
+
+ + + \ No newline at end of file diff --git a/install/crowdsec/install.go b/install/crowdsec/install.go new file mode 100644 index 00000000..84f3089b --- /dev/null +++ b/install/crowdsec/install.go @@ -0,0 +1,376 @@ +package crowdsec + +import ( + "bytes" + "embed" + "encoding/json" + "fmt" + "io" + "os" + "os/exec" + "strings" + "time" +) + +//go:embed fs/* +var configFiles embed.FS + +// Config holds all configuration values +type Config struct { + DomainName string + EnrollmentKey string + TurnstileSiteKey string + TurnstileSecretKey string + GID string + CrowdsecIP string + TraefikBouncerKey string + PangolinIP string +} + +// DockerContainer represents a Docker container +type DockerContainer struct { + NetworkSettings struct { + Networks map[string]struct { + IPAddress string `json:"IPAddress"` + } `json:"Networks"` + } `json:"NetworkSettings"` +} + +func main() { + if err := run(); err != nil { + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + os.Exit(1) + } +} + +func run() error { + // Create configuration + config := &Config{} + + // Run installation steps + if err := backupConfig(); err != nil { + return fmt.Errorf("backup failed: %v", err) + } + + if err := createPangolinNetwork(); err != nil { + return fmt.Errorf("network creation failed: %v", err) + } + + if err := modifyDockerCompose(); err != nil { + return fmt.Errorf("docker-compose modification failed: %v", err) + } + + if err := createConfigFiles(*config); err != nil { + return fmt.Errorf("config file creation failed: %v", err) + } + + if err := retrieveIPs(config); err != nil { + return fmt.Errorf("IP retrieval failed: %v", err) + } + + if err := retrieveBouncerKey(config); err != nil { + return fmt.Errorf("bouncer key retrieval failed: %v", err) + } + + if err := replacePlaceholders(config); err != nil { + return fmt.Errorf("placeholder replacement failed: %v", err) + } + + if err := deployStack(); err != nil { + return fmt.Errorf("deployment failed: %v", err) + } + + if err := verifyDeployment(); err != nil { + return fmt.Errorf("verification failed: %v", err) + } + + printInstructions() + return nil +} + +func backupConfig() error { + // Backup docker-compose.yml + if _, err := os.Stat("docker-compose.yml"); err == nil { + if err := copyFile("docker-compose.yml", "docker-compose.yml.backup"); err != nil { + return fmt.Errorf("failed to backup docker-compose.yml: %v", err) + } + } + + // Backup config directory + if _, err := os.Stat("config"); err == nil { + cmd := exec.Command("tar", "-czvf", "config.tar.gz", "config") + if err := cmd.Run(); err != nil { + return fmt.Errorf("failed to backup config directory: %v", err) + } + } + + return nil +} + +func createPangolinNetwork() error { + // Check if network exists + cmd := exec.Command("docker", "network", "inspect", "pangolin") + if err := cmd.Run(); err == nil { + fmt.Println("pangolin network already exists") + return nil + } + + // Create network + cmd = exec.Command("docker", "network", "create", "pangolin") + if err := cmd.Run(); err != nil { + return fmt.Errorf("failed to create pangolin network: %v", err) + } + + return nil +} + +func modifyDockerCompose() error { + // Read existing docker-compose.yml + content, err := os.ReadFile("docker-compose.yml") + if err != nil { + return fmt.Errorf("failed to read docker-compose.yml: %v", err) + } + + // Verify required services exist + requiredServices := []string{"services:", "pangolin:", "gerbil:", "traefik:"} + for _, service := range requiredServices { + if !bytes.Contains(content, []byte(service)) { + return fmt.Errorf("required service %s not found in docker-compose.yml", service) + } + } + + // Add crowdsec service + modified := addCrowdsecService(string(content)) + + // Write modified content + if err := os.WriteFile("docker-compose.yml", []byte(modified), 0644); err != nil { + return fmt.Errorf("failed to write modified docker-compose.yml: %v", err) + } + + return nil +} + +func retrieveIPs(config *Config) error { + // Start required containers + cmd := exec.Command("docker", "compose", "up", "-d", "pangolin", "crowdsec") + if err := cmd.Run(); err != nil { + return fmt.Errorf("failed to start containers: %v", err) + } + defer exec.Command("docker", "compose", "down").Run() + + // Wait for containers to start + time.Sleep(10 * time.Second) + + // Get Pangolin IP + pangolinIP, err := getContainerIP("pangolin") + if err != nil { + return fmt.Errorf("failed to get pangolin IP: %v", err) + } + config.PangolinIP = pangolinIP + + // Get CrowdSec IP + crowdsecIP, err := getContainerIP("crowdsec") + if err != nil { + return fmt.Errorf("failed to get crowdsec IP: %v", err) + } + config.CrowdsecIP = crowdsecIP + + return nil +} + +func retrieveBouncerKey(config *Config) error { + // Start crowdsec container + cmd := exec.Command("docker", "compose", "up", "-d", "crowdsec") + if err := cmd.Run(); err != nil { + return fmt.Errorf("failed to start crowdsec: %v", err) + } + defer exec.Command("docker", "compose", "down").Run() + + // Wait for container to start + time.Sleep(10 * time.Second) + + // Get bouncer key + output, err := exec.Command("docker", "exec", "crowdsec", "cscli", "bouncers", "add", "traefik-bouncer").Output() + if err != nil { + return fmt.Errorf("failed to get bouncer key: %v", err) + } + + // Parse key from output + lines := strings.Split(string(output), "\n") + for _, line := range lines { + if strings.Contains(line, "key:") { + config.TraefikBouncerKey = strings.TrimSpace(strings.Split(line, ":")[1]) + break + } + } + + return nil +} + +func replacePlaceholders(config *Config) error { + // Get user input + fmt.Print("Enter your Domain Name (e.g., pangolin.example.com): ") + fmt.Scanln(&config.DomainName) + + fmt.Print("Enter your CrowdSec Enrollment Key: ") + fmt.Scanln(&config.EnrollmentKey) + + fmt.Print("Enter your Cloudflare Turnstile Site Key: ") + fmt.Scanln(&config.TurnstileSiteKey) + + fmt.Print("Enter your Cloudflare Turnstile Secret Key: ") + fmt.Scanln(&config.TurnstileSecretKey) + + fmt.Print("Enter your GID (or leave empty for default 1000): ") + gid := "" + fmt.Scanln(&gid) + if gid == "" { + config.GID = "1000" + } else { + config.GID = gid + } + + return nil +} + +func deployStack() error { + cmd := exec.Command("docker", "compose", "up", "-d") + if err := cmd.Run(); err != nil { + return fmt.Errorf("failed to deploy stack: %v", err) + } + + fmt.Println("Stack deployed. Waiting 2 minutes for services to initialize...") + time.Sleep(2 * time.Minute) + return nil +} + +func verifyDeployment() error { + resp, err := exec.Command("curl", "-s", "http://localhost:6060/metrics").Output() + if err != nil { + return fmt.Errorf("failed to get metrics: %v", err) + } + + if !bytes.Contains(resp, []byte("appsec")) { + return fmt.Errorf("appsec metrics not found in response") + } + + return nil +} + +func printInstructions() { + fmt.Println(` +--- Testing Instructions --- +1. Test Captcha Implementation: + docker exec crowdsec cscli decisions add --ip YOUR_IP --type captcha -d 1h + (Replace YOUR_IP with your actual IP address) + +2. Verify decisions: + docker exec -it crowdsec cscli decisions list + +3. Test security by accessing DOMAIN_NAME/.env (should return 403) + (Replace DOMAIN_NAME with the domain you entered) + +--- Troubleshooting --- +1. If encountering 403 errors: + - Check Traefik logs: docker compose logs traefik -f + - Verify CrowdSec logs: docker compose logs crowdsec + +2. For plugin errors: + - Verify http notifications are commented out in profiles.yaml + - Restart services: docker compose restart traefik crowdsec + +3. For Captcha issues: + - Ensure Turnstile is configured in non-interactive mode + - Verify captcha.html configuration + - Check container network connectivity + +Useful Commands: +- View Traefik logs: docker compose logs traefik -f +- View CrowdSec logs: docker compose logs crowdsec +- List decisions: docker exec -it crowdsec cscli decisions list +- Check metrics: curl http://localhost:6060/metrics | grep appsec +`) +} + +// Helper functions + +func copyFile(src, dst string) error { + source, err := os.Open(src) + if err != nil { + return err + } + defer source.Close() + + destination, err := os.Create(dst) + if err != nil { + return err + } + defer destination.Close() + + _, err = io.Copy(destination, source) + return err +} + +func getContainerIP(containerName string) (string, error) { + output, err := exec.Command("docker", "inspect", containerName).Output() + if err != nil { + return "", err + } + + var containers []DockerContainer + if err := json.Unmarshal(output, &containers); err != nil { + return "", err + } + + if len(containers) == 0 { + return "", fmt.Errorf("no container found") + } + + for _, network := range containers[0].NetworkSettings.Networks { + return network.IPAddress, nil + } + + return "", fmt.Errorf("no IP address found") +} + +func addCrowdsecService(content string) string { + // Implementation of adding crowdsec service to docker-compose.yml + // This would involve string manipulation or template rendering + // The actual implementation would depend on how you want to structure the docker-compose modifications + return content + ` + crowdsec: + image: crowdsecurity/crowdsec:latest + container_name: crowdsec + environment: + GID: "${GID-1000}" + COLLECTIONS: crowdsecurity/traefik crowdsecurity/appsec-virtual-patching crowdsecurity/appsec-generic-rules + ENROLL_INSTANCE_NAME: "pangolin-crowdsec" + PARSERS: crowdsecurity/whitelists + ENROLL_KEY: ${ENROLLMENT_KEY} + ACQUIRE_FILES: "/var/log/traefik/*.log" + ENROLL_TAGS: docker + networks: + - pangolin + healthcheck: + test: ["CMD", "cscli", "capi", "status"] + depends_on: + - gerbil + labels: + - "traefik.enable=false" + volumes: + - ./config/crowdsec:/etc/crowdsec + - ./config/crowdsec/db:/var/lib/crowdsec/data + - ./config/crowdsec_logs/auth.log:/var/log/auth.log:ro + - ./config/crowdsec_logs/syslog:/var/log/syslog:ro + - ./config/crowdsec_logs:/var/log + - ./config/traefik/logs:/var/log/traefik + ports: + - 9090:9090 + - 6060:6060 + expose: + - 9090 + - 6060 + - 7422 + restart: unless-stopped + command: -t` +} diff --git a/install/fs/crowdsec/acquis.yaml b/install/fs/crowdsec/acquis.yaml new file mode 100644 index 00000000..74d8fd1c --- /dev/null +++ b/install/fs/crowdsec/acquis.yaml @@ -0,0 +1,18 @@ +filenames: + - /var/log/auth.log + - /var/log/syslog +labels: + type: syslog +--- +poll_without_inotify: false +filenames: + - /var/log/traefik/*.log +labels: + type: traefik +--- +listen_addr: 0.0.0.0:7422 +appsec_config: crowdsecurity/appsec-default +name: myAppSecComponent +source: appsec +labels: + type: appsec \ No newline at end of file diff --git a/install/fs/crowdsec/config.yaml b/install/fs/crowdsec/config.yaml new file mode 100644 index 00000000..0acf4635 --- /dev/null +++ b/install/fs/crowdsec/config.yaml @@ -0,0 +1,12 @@ +api: + client: + insecure_skip_verify: false + credentials_path: /etc/crowdsec/local_api_credentials.yaml + server: + log_level: info + listen_uri: 0.0.0.0:9090 + profiles_path: /etc/crowdsec/profiles.yaml + trusted_ips: + - 0.0.0.0/0 + - 127.0.0.1 + - ::1 \ No newline at end of file diff --git a/install/fs/crowdsec/local_api_credentials.yaml b/install/fs/crowdsec/local_api_credentials.yaml new file mode 100644 index 00000000..8776e4fd --- /dev/null +++ b/install/fs/crowdsec/local_api_credentials.yaml @@ -0,0 +1,2 @@ +url: http://0.0.0.0:9090 +login: localhost \ No newline at end of file diff --git a/install/fs/crowdsec/profiles.yaml b/install/fs/crowdsec/profiles.yaml new file mode 100644 index 00000000..3796b47f --- /dev/null +++ b/install/fs/crowdsec/profiles.yaml @@ -0,0 +1,25 @@ +name: captcha_remediation +filters: + - Alert.Remediation == true && Alert.GetScope() == "Ip" && Alert.GetScenario() contains "http" +decisions: + - type: captcha + duration: 4h +on_success: break + +--- +name: default_ip_remediation +filters: + - Alert.Remediation == true && Alert.GetScope() == "Ip" +decisions: + - type: ban + duration: 4h +on_success: break + +--- +name: default_range_remediation +filters: + - Alert.Remediation == true && Alert.GetScope() == "Range" +decisions: + - type: ban + duration: 4h +on_success: break \ No newline at end of file From 81c4199e87fdd066495101f85e47bcb6f49a1058 Mon Sep 17 00:00:00 2001 From: Owen Date: Wed, 12 Feb 2025 21:56:13 -0500 Subject: [PATCH 2/9] Complete bash migration --- install/captcha.html | 338 ------------------ install/{crowdsec/install.go => crowdsec.go} | 271 ++++++-------- install/{fs => }/crowdsec/acquis.yaml | 0 install/{fs => }/crowdsec/config.yaml | 0 install/crowdsec/dynamic_config.yml | 108 ++++++ .../crowdsec/local_api_credentials.yaml | 0 install/{fs => }/crowdsec/profiles.yaml | 0 install/crowdsec/traefik_config.yml | 87 +++++ install/main.go | 1 + 9 files changed, 301 insertions(+), 504 deletions(-) delete mode 100644 install/captcha.html rename install/{crowdsec/install.go => crowdsec.go} (55%) rename install/{fs => }/crowdsec/acquis.yaml (100%) rename install/{fs => }/crowdsec/config.yaml (100%) create mode 100644 install/crowdsec/dynamic_config.yml rename install/{fs => }/crowdsec/local_api_credentials.yaml (100%) rename install/{fs => }/crowdsec/profiles.yaml (100%) create mode 100644 install/crowdsec/traefik_config.yml diff --git a/install/captcha.html b/install/captcha.html deleted file mode 100644 index a40d37a3..00000000 --- a/install/captcha.html +++ /dev/null @@ -1,338 +0,0 @@ - - - - - CrowdSec Captcha - - - - - - - -
-
-
- -

CrowdSec Captcha

-
-
-
-
-
-
-

This security check has been powered by

- - - - - - - - - - - - - - - - - - - - - CrowdSec - -
-
-
- - - \ No newline at end of file diff --git a/install/crowdsec/install.go b/install/crowdsec.go similarity index 55% rename from install/crowdsec/install.go rename to install/crowdsec.go index 84f3089b..187fea88 100644 --- a/install/crowdsec/install.go +++ b/install/crowdsec.go @@ -1,31 +1,21 @@ -package crowdsec +package main import ( "bytes" "embed" - "encoding/json" "fmt" + "html/template" "io" + "io/fs" "os" "os/exec" + "path/filepath" "strings" "time" ) -//go:embed fs/* -var configFiles embed.FS - -// Config holds all configuration values -type Config struct { - DomainName string - EnrollmentKey string - TurnstileSiteKey string - TurnstileSecretKey string - GID string - CrowdsecIP string - TraefikBouncerKey string - PangolinIP string -} +//go:embed crowdsec/* +var configCrowdsecFiles embed.FS // DockerContainer represents a Docker container type DockerContainer struct { @@ -36,14 +26,7 @@ type DockerContainer struct { } `json:"NetworkSettings"` } -func main() { - if err := run(); err != nil { - fmt.Fprintf(os.Stderr, "Error: %v\n", err) - os.Exit(1) - } -} - -func run() error { +func installCrowdsec() error { // Create configuration config := &Config{} @@ -52,30 +35,21 @@ func run() error { return fmt.Errorf("backup failed: %v", err) } - if err := createPangolinNetwork(); err != nil { - return fmt.Errorf("network creation failed: %v", err) - } - if err := modifyDockerCompose(); err != nil { return fmt.Errorf("docker-compose modification failed: %v", err) } - if err := createConfigFiles(*config); err != nil { + if err := createCrowdsecFiles(*config); err != nil { return fmt.Errorf("config file creation failed: %v", err) } - if err := retrieveIPs(config); err != nil { - return fmt.Errorf("IP retrieval failed: %v", err) - } + moveFile("config/crowdsec/traefik_config.yaml", "config/traefik/traefik_config.yaml") + moveFile("config/crowdsec/dynamic.yaml", "config/traefik/dynamic.yaml") if err := retrieveBouncerKey(config); err != nil { return fmt.Errorf("bouncer key retrieval failed: %v", err) } - if err := replacePlaceholders(config); err != nil { - return fmt.Errorf("placeholder replacement failed: %v", err) - } - if err := deployStack(); err != nil { return fmt.Errorf("deployment failed: %v", err) } @@ -84,7 +58,6 @@ func run() error { return fmt.Errorf("verification failed: %v", err) } - printInstructions() return nil } @@ -107,23 +80,6 @@ func backupConfig() error { return nil } -func createPangolinNetwork() error { - // Check if network exists - cmd := exec.Command("docker", "network", "inspect", "pangolin") - if err := cmd.Run(); err == nil { - fmt.Println("pangolin network already exists") - return nil - } - - // Create network - cmd = exec.Command("docker", "network", "create", "pangolin") - if err := cmd.Run(); err != nil { - return fmt.Errorf("failed to create pangolin network: %v", err) - } - - return nil -} - func modifyDockerCompose() error { // Read existing docker-compose.yml content, err := os.ReadFile("docker-compose.yml") @@ -150,34 +106,6 @@ func modifyDockerCompose() error { return nil } -func retrieveIPs(config *Config) error { - // Start required containers - cmd := exec.Command("docker", "compose", "up", "-d", "pangolin", "crowdsec") - if err := cmd.Run(); err != nil { - return fmt.Errorf("failed to start containers: %v", err) - } - defer exec.Command("docker", "compose", "down").Run() - - // Wait for containers to start - time.Sleep(10 * time.Second) - - // Get Pangolin IP - pangolinIP, err := getContainerIP("pangolin") - if err != nil { - return fmt.Errorf("failed to get pangolin IP: %v", err) - } - config.PangolinIP = pangolinIP - - // Get CrowdSec IP - crowdsecIP, err := getContainerIP("crowdsec") - if err != nil { - return fmt.Errorf("failed to get crowdsec IP: %v", err) - } - config.CrowdsecIP = crowdsecIP - - return nil -} - func retrieveBouncerKey(config *Config) error { // Start crowdsec container cmd := exec.Command("docker", "compose", "up", "-d", "crowdsec") @@ -207,32 +135,6 @@ func retrieveBouncerKey(config *Config) error { return nil } -func replacePlaceholders(config *Config) error { - // Get user input - fmt.Print("Enter your Domain Name (e.g., pangolin.example.com): ") - fmt.Scanln(&config.DomainName) - - fmt.Print("Enter your CrowdSec Enrollment Key: ") - fmt.Scanln(&config.EnrollmentKey) - - fmt.Print("Enter your Cloudflare Turnstile Site Key: ") - fmt.Scanln(&config.TurnstileSiteKey) - - fmt.Print("Enter your Cloudflare Turnstile Secret Key: ") - fmt.Scanln(&config.TurnstileSecretKey) - - fmt.Print("Enter your GID (or leave empty for default 1000): ") - gid := "" - fmt.Scanln(&gid) - if gid == "" { - config.GID = "1000" - } else { - config.GID = gid - } - - return nil -} - func deployStack() error { cmd := exec.Command("docker", "compose", "up", "-d") if err := cmd.Run(); err != nil { @@ -257,43 +159,6 @@ func verifyDeployment() error { return nil } -func printInstructions() { - fmt.Println(` ---- Testing Instructions --- -1. Test Captcha Implementation: - docker exec crowdsec cscli decisions add --ip YOUR_IP --type captcha -d 1h - (Replace YOUR_IP with your actual IP address) - -2. Verify decisions: - docker exec -it crowdsec cscli decisions list - -3. Test security by accessing DOMAIN_NAME/.env (should return 403) - (Replace DOMAIN_NAME with the domain you entered) - ---- Troubleshooting --- -1. If encountering 403 errors: - - Check Traefik logs: docker compose logs traefik -f - - Verify CrowdSec logs: docker compose logs crowdsec - -2. For plugin errors: - - Verify http notifications are commented out in profiles.yaml - - Restart services: docker compose restart traefik crowdsec - -3. For Captcha issues: - - Ensure Turnstile is configured in non-interactive mode - - Verify captcha.html configuration - - Check container network connectivity - -Useful Commands: -- View Traefik logs: docker compose logs traefik -f -- View CrowdSec logs: docker compose logs crowdsec -- List decisions: docker exec -it crowdsec cscli decisions list -- Check metrics: curl http://localhost:6060/metrics | grep appsec -`) -} - -// Helper functions - func copyFile(src, dst string) error { source, err := os.Open(src) if err != nil { @@ -311,26 +176,12 @@ func copyFile(src, dst string) error { return err } -func getContainerIP(containerName string) (string, error) { - output, err := exec.Command("docker", "inspect", containerName).Output() - if err != nil { - return "", err +func moveFile(src, dst string) error { + if err := copyFile(src, dst); err != nil { + return err } - var containers []DockerContainer - if err := json.Unmarshal(output, &containers); err != nil { - return "", err - } - - if len(containers) == 0 { - return "", fmt.Errorf("no container found") - } - - for _, network := range containers[0].NetworkSettings.Networks { - return network.IPAddress, nil - } - - return "", fmt.Errorf("no IP address found") + return os.Remove(src) } func addCrowdsecService(content string) string { @@ -346,11 +197,8 @@ func addCrowdsecService(content string) string { COLLECTIONS: crowdsecurity/traefik crowdsecurity/appsec-virtual-patching crowdsecurity/appsec-generic-rules ENROLL_INSTANCE_NAME: "pangolin-crowdsec" PARSERS: crowdsecurity/whitelists - ENROLL_KEY: ${ENROLLMENT_KEY} ACQUIRE_FILES: "/var/log/traefik/*.log" ENROLL_TAGS: docker - networks: - - pangolin healthcheck: test: ["CMD", "cscli", "capi", "status"] depends_on: @@ -374,3 +222,94 @@ func addCrowdsecService(content string) string { restart: unless-stopped command: -t` } + +func createCrowdsecFiles(config Config) error { + // Walk through all embedded files + err := fs.WalkDir(configCrowdsecFiles, "crowdsec", func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + + // Skip the root fs directory itself + if path == "fs" { + return nil + } + + // Get the relative path by removing the "fs/" prefix + relPath := strings.TrimPrefix(path, "fs/") + + // skip .DS_Store + if strings.Contains(relPath, ".DS_Store") { + return nil + } + + // Create the full output path under "config/" + outPath := filepath.Join("config", relPath) + + if d.IsDir() { + // Create directory + if err := os.MkdirAll(outPath, 0755); err != nil { + return fmt.Errorf("failed to create directory %s: %v", outPath, err) + } + return nil + } + + // Read the template file + content, err := configFiles.ReadFile(path) + if err != nil { + return fmt.Errorf("failed to read %s: %v", path, err) + } + + // Parse template + tmpl, err := template.New(d.Name()).Parse(string(content)) + if err != nil { + return fmt.Errorf("failed to parse template %s: %v", path, err) + } + + // Ensure parent directory exists + if err := os.MkdirAll(filepath.Dir(outPath), 0755); err != nil { + return fmt.Errorf("failed to create parent directory for %s: %v", outPath, err) + } + + // Create output file + outFile, err := os.Create(outPath) + if err != nil { + return fmt.Errorf("failed to create %s: %v", outPath, err) + } + defer outFile.Close() + + // Execute template + if err := tmpl.Execute(outFile, config); err != nil { + return fmt.Errorf("failed to execute template %s: %v", path, err) + } + + return nil + }) + + if err != nil { + return fmt.Errorf("error walking config files: %v", err) + } + + // get the current directory + dir, err := os.Getwd() + if err != nil { + return fmt.Errorf("failed to get current directory: %v", err) + } + + sourcePath := filepath.Join(dir, "config/docker-compose.yml") + destPath := filepath.Join(dir, "docker-compose.yml") + + // Check if source file exists + if _, err := os.Stat(sourcePath); err != nil { + return fmt.Errorf("source docker-compose.yml not found: %v", err) + } + + // Try to move the file + err = os.Rename(sourcePath, destPath) + if err != nil { + return fmt.Errorf("failed to move docker-compose.yml from %s to %s: %v", + sourcePath, destPath, err) + } + + return nil +} diff --git a/install/fs/crowdsec/acquis.yaml b/install/crowdsec/acquis.yaml similarity index 100% rename from install/fs/crowdsec/acquis.yaml rename to install/crowdsec/acquis.yaml diff --git a/install/fs/crowdsec/config.yaml b/install/crowdsec/config.yaml similarity index 100% rename from install/fs/crowdsec/config.yaml rename to install/crowdsec/config.yaml diff --git a/install/crowdsec/dynamic_config.yml b/install/crowdsec/dynamic_config.yml new file mode 100644 index 00000000..9175b143 --- /dev/null +++ b/install/crowdsec/dynamic_config.yml @@ -0,0 +1,108 @@ +http: + middlewares: + redirect-to-https: + redirectScheme: + scheme: https + default-whitelist: # Whitelist middleware for internal IPs + ipWhiteList: # Internal IP addresses + sourceRange: # Internal IP addresses + - "10.0.0.0/8" # Internal IP addresses + - "192.168.0.0/16" # Internal IP addresses + - "172.16.0.0/12" # Internal IP addresses + # Basic security headers + security-headers: + headers: + customResponseHeaders: # Custom response headers + Server: "" # Remove server header + X-Powered-By: "" # Remove powered by header + X-Forwarded-Proto: "https" # Set forwarded proto to https + sslProxyHeaders: # SSL proxy headers + X-Forwarded-Proto: "https" # Set forwarded proto to https + hostsProxyHeaders: # Hosts proxy headers + - "X-Forwarded-Host" # Set forwarded host + contentTypeNosniff: true # Prevent MIME sniffing + customFrameOptionsValue: "SAMEORIGIN" # Set frame options + referrerPolicy: "strict-origin-when-cross-origin" # Set referrer policy + forceSTSHeader: true # Force STS header + stsIncludeSubdomains: true # Include subdomains + stsSeconds: 63072000 # STS seconds + stsPreload: true # Preload STS + # CrowdSec configuration with proper IP forwarding + crowdsec: + plugin: + crowdsec: + enabled: true # Enable CrowdSec plugin + logLevel: INFO # Log level + updateIntervalSeconds: 15 # Update interval + updateMaxFailure: 0 # Update max failure + defaultDecisionSeconds: 15 # Default decision seconds + httpTimeoutSeconds: 10 # HTTP timeout + crowdsecMode: live # CrowdSec mode + crowdsecAppsecEnabled: true # Enable AppSec + crowdsecAppsecHost: crowdsec:7422 # CrowdSec IP address which you noted down later + crowdsecAppsecFailureBlock: true # Block on failure + crowdsecAppsecUnreachableBlock: true # Block on unreachable + crowdsecLapiKey: "{{.TraefikBouncerKey}}" # CrowdSec API key which you noted down later + crowdsecLapiHost: crowdsec:9090 # CrowdSec + crowdsecLapiScheme: http # CrowdSec API scheme + forwardedHeadersTrustedIPs: # Forwarded headers trusted IPs + - "0.0.0.0/0" # All IP addresses are trusted for forwarded headers (CHANGE MADE HERE) + clientTrustedIPs: # Client trusted IPs (CHANGE MADE HERE) + - "10.0.0.0/8" # Internal LAN IP addresses + - "172.16.0.0/12" # Internal LAN IP addresses + - "192.168.0.0/16" # Internal LAN IP addresses + - "100.89.137.0/20" # Internal LAN IP addresses + + routers: + # HTTP to HTTPS redirect router + main-app-router-redirect: + rule: "Host(`{{.DomainName}}`)" # Dynamic Domain Name + service: next-service + entryPoints: + - web + middlewares: + - redirect-to-https + + # Next.js router (handles everything except API and WebSocket paths) + next-router: + rule: "Host(`{{.DomainName}}`) && !PathPrefix(`/api/v1`)" # Dynamic Domain Name + service: next-service + entryPoints: + - websecure + middlewares: + - security-headers # Add security headers middleware + tls: + certResolver: letsencrypt + + # API router (handles /api/v1 paths) + api-router: + rule: "Host(`{{.DomainName}}`) && PathPrefix(`/api/v1`)" # Dynamic Domain Name + service: api-service + entryPoints: + - websecure + middlewares: + - security-headers # Add security headers middleware + tls: + certResolver: letsencrypt + + # WebSocket router + ws-router: + rule: "Host(`{{.DomainName}}`)" # Dynamic Domain Name + service: api-service + entryPoints: + - websecure + middlewares: + - security-headers # Add security headers middleware + tls: + certResolver: letsencrypt + + services: + next-service: + loadBalancer: + servers: + - url: "http://pangolin:3002" # Next.js server + + api-service: + loadBalancer: + servers: + - url: "http://pangolin:3000" # API/WebSocket server \ No newline at end of file diff --git a/install/fs/crowdsec/local_api_credentials.yaml b/install/crowdsec/local_api_credentials.yaml similarity index 100% rename from install/fs/crowdsec/local_api_credentials.yaml rename to install/crowdsec/local_api_credentials.yaml diff --git a/install/fs/crowdsec/profiles.yaml b/install/crowdsec/profiles.yaml similarity index 100% rename from install/fs/crowdsec/profiles.yaml rename to install/crowdsec/profiles.yaml diff --git a/install/crowdsec/traefik_config.yml b/install/crowdsec/traefik_config.yml new file mode 100644 index 00000000..2ac9125c --- /dev/null +++ b/install/crowdsec/traefik_config.yml @@ -0,0 +1,87 @@ +api: + insecure: true + dashboard: true + +providers: + http: + endpoint: "http://pangolin:3001/api/v1/traefik-config" + pollInterval: "5s" + file: + filename: "/etc/traefik/dynamic_config.yml" + +experimental: + plugins: + badger: + moduleName: "github.com/fosrl/badger" + version: "{{.BadgerVersion}}" + crowdsec: # CrowdSec plugin configuration added + moduleName: "github.com/maxlerebourg/crowdsec-bouncer-traefik-plugin" + version: "v1.3.5" + +log: + level: "INFO" + format: "json" # Log format changed to json for better parsing + +accessLog: # We enable access logs as json + filePath: "/var/log/traefik/access.log" + format: json + filters: + statusCodes: + - "200-299" # Success codes + - "400-499" # Client errors + - "500-599" # Server errors + retryAttempts: true + minDuration: "100ms" # Increased to focus on slower requests + bufferingSize: 100 # Add buffering for better performance + fields: + defaultMode: drop # Start with dropping all fields + names: + ClientAddr: keep # Keep client address for IP tracking + ClientHost: keep # Keep client host for IP tracking + RequestMethod: keep # Keep request method for tracking + RequestPath: keep # Keep request path for tracking + RequestProtocol: keep # Keep request protocol for tracking + DownstreamStatus: keep # Keep downstream status for tracking + DownstreamContentSize: keep # Keep downstream content size for tracking + Duration: keep # Keep request duration for tracking + ServiceName: keep # Keep service name for tracking + StartUTC: keep # Keep start time for tracking + TLSVersion: keep # Keep TLS version for tracking + TLSCipher: keep # Keep TLS cipher for tracking + RetryAttempts: keep # Keep retry attempts for tracking + headers: + defaultMode: drop # Start with dropping all headers + names: + User-Agent: keep # Keep user agent for tracking + X-Real-Ip: keep # Keep real IP for tracking + X-Forwarded-For: keep # Keep forwarded IP for tracking + X-Forwarded-Proto: keep # Keep forwarded protocol for tracking + Content-Type: keep # Keep content type for tracking + Authorization: redact # Redact sensitive information + Cookie: redact # Redact sensitive information + +certificatesResolvers: + letsencrypt: + acme: + httpChallenge: + entryPoint: web + email: "{{.LetsEncryptEmail}}" + storage: "/letsencrypt/acme.json" + caServer: "https://acme-v02.api.letsencrypt.org/directory" + +entryPoints: + web: + address: ":80" + websecure: + address: ":443" + transport: + respondingTimeouts: + readTimeout: "30m" + http: + tls: + certResolver: "letsencrypt" + middlewares: # CHANGE MADE HERE (BOUNCER ENABLED) !!! + - crowdsec@file + +serversTransport: + insecureSkipVerify: true \ No newline at end of file diff --git a/install/main.go b/install/main.go index 4f2deb3a..d5bf5e1e 100644 --- a/install/main.go +++ b/install/main.go @@ -45,6 +45,7 @@ type Config struct { EmailSMTPPass string EmailNoReply string InstallGerbil bool + TraefikBouncerKey string } func main() { From 60449afca5f3186e31c5d8210e116c9a49de884b Mon Sep 17 00:00:00 2001 From: Owen Date: Thu, 13 Feb 2025 22:16:52 -0500 Subject: [PATCH 3/9] Reorg; create crowdsec folder properly now --- install/config.go | 216 ++++++++++++++ install/{fs => config}/config.yml | 0 install/{ => config}/crowdsec/acquis.yaml | 0 install/{ => config}/crowdsec/config.yaml | 0 .../{ => config}/crowdsec/dynamic_config.yml | 8 +- .../crowdsec/local_api_credentials.yaml | 0 install/{ => config}/crowdsec/profiles.yaml | 0 .../{ => config}/crowdsec/traefik_config.yml | 0 install/{fs => config}/docker-compose.yml | 5 +- .../{fs => config}/traefik/dynamic_config.yml | 0 .../{fs => config}/traefik/traefik_config.yml | 0 install/crowdsec.go | 267 ++---------------- install/go.mod | 1 + install/go.sum | 3 + install/main.go | 134 ++++++--- 15 files changed, 349 insertions(+), 285 deletions(-) create mode 100644 install/config.go rename install/{fs => config}/config.yml (100%) rename install/{ => config}/crowdsec/acquis.yaml (100%) rename install/{ => config}/crowdsec/config.yaml (100%) rename install/{ => config}/crowdsec/dynamic_config.yml (92%) rename install/{ => config}/crowdsec/local_api_credentials.yaml (100%) rename install/{ => config}/crowdsec/profiles.yaml (100%) rename install/{ => config}/crowdsec/traefik_config.yml (100%) rename install/{fs => config}/docker-compose.yml (97%) rename install/{fs => config}/traefik/dynamic_config.yml (100%) rename install/{fs => config}/traefik/traefik_config.yml (100%) diff --git a/install/config.go b/install/config.go new file mode 100644 index 00000000..fa10fa53 --- /dev/null +++ b/install/config.go @@ -0,0 +1,216 @@ +package main + +import ( + "bytes" + "fmt" + "os" + + "gopkg.in/yaml.v3" +) + +// TraefikConfig represents the structure of the main Traefik configuration +type TraefikConfig struct { + Experimental struct { + Plugins struct { + Badger struct { + Version string `yaml:"version"` + } `yaml:"badger"` + } `yaml:"plugins"` + } `yaml:"experimental"` + CertificatesResolvers struct { + LetsEncrypt struct { + Acme struct { + Email string `yaml:"email"` + } `yaml:"acme"` + } `yaml:"letsencrypt"` + } `yaml:"certificatesResolvers"` +} + +// DynamicConfig represents the structure of the dynamic configuration +type DynamicConfig struct { + HTTP struct { + Routers map[string]struct { + Rule string `yaml:"rule"` + } `yaml:"routers"` + } `yaml:"http"` +} + +// ConfigValues holds the extracted configuration values +type ConfigValues struct { + DashboardDomain string + LetsEncryptEmail string + BadgerVersion string +} + +// ReadTraefikConfig reads and extracts values from Traefik configuration files +func ReadTraefikConfig(mainConfigPath, dynamicConfigPath string) (*ConfigValues, error) { + // Read main config file + mainConfigData, err := os.ReadFile(mainConfigPath) + if err != nil { + return nil, fmt.Errorf("error reading main config file: %w", err) + } + + var mainConfig TraefikConfig + if err := yaml.Unmarshal(mainConfigData, &mainConfig); err != nil { + return nil, fmt.Errorf("error parsing main config file: %w", err) + } + + // Read dynamic config file + dynamicConfigData, err := os.ReadFile(dynamicConfigPath) + if err != nil { + return nil, fmt.Errorf("error reading dynamic config file: %w", err) + } + + var dynamicConfig DynamicConfig + if err := yaml.Unmarshal(dynamicConfigData, &dynamicConfig); err != nil { + return nil, fmt.Errorf("error parsing dynamic config file: %w", err) + } + + // Extract values + values := &ConfigValues{ + BadgerVersion: mainConfig.Experimental.Plugins.Badger.Version, + LetsEncryptEmail: mainConfig.CertificatesResolvers.LetsEncrypt.Acme.Email, + } + + // Extract DashboardDomain from router rules + // Look for it in the main router rules + for _, router := range dynamicConfig.HTTP.Routers { + if router.Rule != "" { + // Extract domain from Host(`mydomain.com`) + if domain := extractDomainFromRule(router.Rule); domain != "" { + values.DashboardDomain = domain + break + } + } + } + + return values, nil +} + +// extractDomainFromRule extracts the domain from a router rule +func extractDomainFromRule(rule string) string { + // Look for the Host(`mydomain.com`) pattern + if start := findPattern(rule, "Host(`"); start != -1 { + end := findPattern(rule[start:], "`)") + if end != -1 { + return rule[start+6 : start+end] + } + } + return "" +} + +// findPattern finds the start of a pattern in a string +func findPattern(s, pattern string) int { + return bytes.Index([]byte(s), []byte(pattern)) +} + +type Volume string +type Port string +type Expose string + +type HealthCheck struct { + Test []string `yaml:"test,omitempty"` + Interval string `yaml:"interval,omitempty"` + Timeout string `yaml:"timeout,omitempty"` + Retries int `yaml:"retries,omitempty"` +} + +type DependsOnCondition struct { + Condition string `yaml:"condition,omitempty"` +} + +type Service struct { + Image string `yaml:"image,omitempty"` + ContainerName string `yaml:"container_name,omitempty"` + Environment map[string]string `yaml:"environment,omitempty"` + HealthCheck *HealthCheck `yaml:"healthcheck,omitempty"` + DependsOn map[string]DependsOnCondition `yaml:"depends_on,omitempty"` + Labels []string `yaml:"labels,omitempty"` + Volumes []Volume `yaml:"volumes,omitempty"` + Ports []Port `yaml:"ports,omitempty"` + Expose []Expose `yaml:"expose,omitempty"` + Restart string `yaml:"restart,omitempty"` + Command interface{} `yaml:"command,omitempty"` + NetworkMode string `yaml:"network_mode,omitempty"` + CapAdd []string `yaml:"cap_add,omitempty"` +} + +type Network struct { + Driver string `yaml:"driver,omitempty"` + Name string `yaml:"name,omitempty"` +} + +type DockerConfig struct { + Version string `yaml:"version,omitempty"` + Services map[string]Service `yaml:"services"` + Networks map[string]Network `yaml:"networks,omitempty"` +} + +func AddCrowdSecService(configPath string) error { + // Read existing config + data, err := os.ReadFile(configPath) + if err != nil { + return err + } + + // Parse existing config + var config DockerConfig + if err := yaml.Unmarshal(data, &config); err != nil { + return err + } + + // Create CrowdSec service + crowdsecService := Service{ + Image: "crowdsecurity/crowdsec:latest", + ContainerName: "crowdsec", + Environment: map[string]string{ + "GID": "${GID-1000}", + "COLLECTIONS": "crowdsecurity/traefik crowdsecurity/appsec-virtual-patching crowdsecurity/appsec-generic-rules", + "ENROLL_INSTANCE_NAME": "pangolin-crowdsec", + "PARSERS": "crowdsecurity/whitelists", + "ACQUIRE_FILES": "/var/log/traefik/*.log", + "ENROLL_TAGS": "docker", + }, + HealthCheck: &HealthCheck{ + Test: []string{"CMD", "cscli", "capi", "status"}, + }, + DependsOn: map[string]DependsOnCondition{ + "gerbil": {}, + }, + Labels: []string{"traefik.enable=false"}, + Volumes: []Volume{ + "./config/crowdsec:/etc/crowdsec", + "./config/crowdsec/db:/var/lib/crowdsec/data", + "./config/crowdsec_logs/auth.log:/var/log/auth.log:ro", + "./config/crowdsec_logs/syslog:/var/log/syslog:ro", + "./config/crowdsec_logs:/var/log", + "./config/traefik/logs:/var/log/traefik", + }, + Ports: []Port{ + "9090:9090", + "6060:6060", + }, + Expose: []Expose{ + "9090", + "6060", + "7422", + }, + Restart: "unless-stopped", + Command: "-t", + } + + // Add CrowdSec service to config + if config.Services == nil { + config.Services = make(map[string]Service) + } + config.Services["crowdsec"] = crowdsecService + + // Marshal config with better formatting + yamlData, err := yaml.Marshal(&config) + if err != nil { + return err + } + + // Write config back to file + return os.WriteFile(configPath, yamlData, 0644) +} diff --git a/install/fs/config.yml b/install/config/config.yml similarity index 100% rename from install/fs/config.yml rename to install/config/config.yml diff --git a/install/crowdsec/acquis.yaml b/install/config/crowdsec/acquis.yaml similarity index 100% rename from install/crowdsec/acquis.yaml rename to install/config/crowdsec/acquis.yaml diff --git a/install/crowdsec/config.yaml b/install/config/crowdsec/config.yaml similarity index 100% rename from install/crowdsec/config.yaml rename to install/config/crowdsec/config.yaml diff --git a/install/crowdsec/dynamic_config.yml b/install/config/crowdsec/dynamic_config.yml similarity index 92% rename from install/crowdsec/dynamic_config.yml rename to install/config/crowdsec/dynamic_config.yml index 9175b143..d2556971 100644 --- a/install/crowdsec/dynamic_config.yml +++ b/install/config/crowdsec/dynamic_config.yml @@ -56,7 +56,7 @@ http: routers: # HTTP to HTTPS redirect router main-app-router-redirect: - rule: "Host(`{{.DomainName}}`)" # Dynamic Domain Name + rule: "Host(`{{.DashboardDomain}}`)" # Dynamic Domain Name service: next-service entryPoints: - web @@ -65,7 +65,7 @@ http: # Next.js router (handles everything except API and WebSocket paths) next-router: - rule: "Host(`{{.DomainName}}`) && !PathPrefix(`/api/v1`)" # Dynamic Domain Name + rule: "Host(`{{.DashboardDomain}}`) && !PathPrefix(`/api/v1`)" # Dynamic Domain Name service: next-service entryPoints: - websecure @@ -76,7 +76,7 @@ http: # API router (handles /api/v1 paths) api-router: - rule: "Host(`{{.DomainName}}`) && PathPrefix(`/api/v1`)" # Dynamic Domain Name + rule: "Host(`{{.DashboardDomain}}`) && PathPrefix(`/api/v1`)" # Dynamic Domain Name service: api-service entryPoints: - websecure @@ -87,7 +87,7 @@ http: # WebSocket router ws-router: - rule: "Host(`{{.DomainName}}`)" # Dynamic Domain Name + rule: "Host(`{{.DashboardDomain}}`)" # Dynamic Domain Name service: api-service entryPoints: - websecure diff --git a/install/crowdsec/local_api_credentials.yaml b/install/config/crowdsec/local_api_credentials.yaml similarity index 100% rename from install/crowdsec/local_api_credentials.yaml rename to install/config/crowdsec/local_api_credentials.yaml diff --git a/install/crowdsec/profiles.yaml b/install/config/crowdsec/profiles.yaml similarity index 100% rename from install/crowdsec/profiles.yaml rename to install/config/crowdsec/profiles.yaml diff --git a/install/crowdsec/traefik_config.yml b/install/config/crowdsec/traefik_config.yml similarity index 100% rename from install/crowdsec/traefik_config.yml rename to install/config/crowdsec/traefik_config.yml diff --git a/install/fs/docker-compose.yml b/install/config/docker-compose.yml similarity index 97% rename from install/fs/docker-compose.yml rename to install/config/docker-compose.yml index ea673eb0..42604ab4 100644 --- a/install/fs/docker-compose.yml +++ b/install/config/docker-compose.yml @@ -10,7 +10,6 @@ services: interval: "3s" timeout: "3s" retries: 5 - {{if .InstallGerbil}} gerbil: image: fosrl/gerbil:{{.GerbilVersion}} @@ -34,15 +33,13 @@ services: - 443:443 # Port for traefik because of the network_mode - 80:80 # Port for traefik because of the network_mode {{end}} - traefik: image: traefik:v3.3.3 container_name: traefik restart: unless-stopped {{if .InstallGerbil}} network_mode: service:gerbil # Ports appear on the gerbil service -{{end}} -{{if not .InstallGerbil}} +{{end}}{{if not .InstallGerbil}} ports: - 443:443 - 80:80 diff --git a/install/fs/traefik/dynamic_config.yml b/install/config/traefik/dynamic_config.yml similarity index 100% rename from install/fs/traefik/dynamic_config.yml rename to install/config/traefik/dynamic_config.yml diff --git a/install/fs/traefik/traefik_config.yml b/install/config/traefik/traefik_config.yml similarity index 100% rename from install/fs/traefik/traefik_config.yml rename to install/config/traefik/traefik_config.yml diff --git a/install/crowdsec.go b/install/crowdsec.go index 187fea88..02e9c71b 100644 --- a/install/crowdsec.go +++ b/install/crowdsec.go @@ -2,45 +2,26 @@ package main import ( "bytes" - "embed" "fmt" - "html/template" - "io" - "io/fs" "os" "os/exec" - "path/filepath" "strings" "time" ) -//go:embed crowdsec/* -var configCrowdsecFiles embed.FS - -// DockerContainer represents a Docker container -type DockerContainer struct { - NetworkSettings struct { - Networks map[string]struct { - IPAddress string `json:"IPAddress"` - } `json:"Networks"` - } `json:"NetworkSettings"` -} - -func installCrowdsec() error { - // Create configuration - config := &Config{} - +func installCrowdsec(config Config) error { // Run installation steps if err := backupConfig(); err != nil { return fmt.Errorf("backup failed: %v", err) } - if err := modifyDockerCompose(); err != nil { - return fmt.Errorf("docker-compose modification failed: %v", err) + if err := AddCrowdSecService("docker-compose.yml"); err != nil { + return fmt.Errorf("crowdsec service addition failed: %v", err) } - if err := createCrowdsecFiles(*config); err != nil { - return fmt.Errorf("config file creation failed: %v", err) + if err := createConfigFiles(config); err != nil { + fmt.Printf("Error creating config files: %v\n", err) + os.Exit(1) } moveFile("config/crowdsec/traefik_config.yaml", "config/traefik/traefik_config.yaml") @@ -50,14 +31,6 @@ func installCrowdsec() error { return fmt.Errorf("bouncer key retrieval failed: %v", err) } - if err := deployStack(); err != nil { - return fmt.Errorf("deployment failed: %v", err) - } - - if err := verifyDeployment(); err != nil { - return fmt.Errorf("verification failed: %v", err) - } - return nil } @@ -80,33 +53,7 @@ func backupConfig() error { return nil } -func modifyDockerCompose() error { - // Read existing docker-compose.yml - content, err := os.ReadFile("docker-compose.yml") - if err != nil { - return fmt.Errorf("failed to read docker-compose.yml: %v", err) - } - - // Verify required services exist - requiredServices := []string{"services:", "pangolin:", "gerbil:", "traefik:"} - for _, service := range requiredServices { - if !bytes.Contains(content, []byte(service)) { - return fmt.Errorf("required service %s not found in docker-compose.yml", service) - } - } - - // Add crowdsec service - modified := addCrowdsecService(string(content)) - - // Write modified content - if err := os.WriteFile("docker-compose.yml", []byte(modified), 0644); err != nil { - return fmt.Errorf("failed to write modified docker-compose.yml: %v", err) - } - - return nil -} - -func retrieveBouncerKey(config *Config) error { +func retrieveBouncerKey(config Config) error { // Start crowdsec container cmd := exec.Command("docker", "compose", "up", "-d", "crowdsec") if err := cmd.Run(); err != nil { @@ -114,8 +61,24 @@ func retrieveBouncerKey(config *Config) error { } defer exec.Command("docker", "compose", "down").Run() - // Wait for container to start - time.Sleep(10 * time.Second) + // verify that the container is running if not keep waiting for 10 more seconds then return an error + count := 0 + for { + cmd := exec.Command("docker", "inspect", "-f", "{{.State.Running}}", "crowdsec") + output, err := cmd.Output() + if err != nil { + return fmt.Errorf("failed to inspect crowdsec container: %v", err) + } + if strings.TrimSpace(string(output)) == "true" { + break + } + time.Sleep(10 * time.Second) + count++ + + if count > 4 { + return fmt.Errorf("crowdsec container is not running") + } + } // Get bouncer key output, err := exec.Command("docker", "exec", "crowdsec", "cscli", "bouncers", "add", "traefik-bouncer").Output() @@ -135,181 +98,13 @@ func retrieveBouncerKey(config *Config) error { return nil } -func deployStack() error { - cmd := exec.Command("docker", "compose", "up", "-d") - if err := cmd.Run(); err != nil { - return fmt.Errorf("failed to deploy stack: %v", err) - } - - fmt.Println("Stack deployed. Waiting 2 minutes for services to initialize...") - time.Sleep(2 * time.Minute) - return nil -} - -func verifyDeployment() error { - resp, err := exec.Command("curl", "-s", "http://localhost:6060/metrics").Output() +func checkIsCrowdsecInstalledInCompose() bool { + // Read docker-compose.yml + content, err := os.ReadFile("docker-compose.yml") if err != nil { - return fmt.Errorf("failed to get metrics: %v", err) + return false } - if !bytes.Contains(resp, []byte("appsec")) { - return fmt.Errorf("appsec metrics not found in response") - } - - return nil -} - -func copyFile(src, dst string) error { - source, err := os.Open(src) - if err != nil { - return err - } - defer source.Close() - - destination, err := os.Create(dst) - if err != nil { - return err - } - defer destination.Close() - - _, err = io.Copy(destination, source) - return err -} - -func moveFile(src, dst string) error { - if err := copyFile(src, dst); err != nil { - return err - } - - return os.Remove(src) -} - -func addCrowdsecService(content string) string { - // Implementation of adding crowdsec service to docker-compose.yml - // This would involve string manipulation or template rendering - // The actual implementation would depend on how you want to structure the docker-compose modifications - return content + ` - crowdsec: - image: crowdsecurity/crowdsec:latest - container_name: crowdsec - environment: - GID: "${GID-1000}" - COLLECTIONS: crowdsecurity/traefik crowdsecurity/appsec-virtual-patching crowdsecurity/appsec-generic-rules - ENROLL_INSTANCE_NAME: "pangolin-crowdsec" - PARSERS: crowdsecurity/whitelists - ACQUIRE_FILES: "/var/log/traefik/*.log" - ENROLL_TAGS: docker - healthcheck: - test: ["CMD", "cscli", "capi", "status"] - depends_on: - - gerbil - labels: - - "traefik.enable=false" - volumes: - - ./config/crowdsec:/etc/crowdsec - - ./config/crowdsec/db:/var/lib/crowdsec/data - - ./config/crowdsec_logs/auth.log:/var/log/auth.log:ro - - ./config/crowdsec_logs/syslog:/var/log/syslog:ro - - ./config/crowdsec_logs:/var/log - - ./config/traefik/logs:/var/log/traefik - ports: - - 9090:9090 - - 6060:6060 - expose: - - 9090 - - 6060 - - 7422 - restart: unless-stopped - command: -t` -} - -func createCrowdsecFiles(config Config) error { - // Walk through all embedded files - err := fs.WalkDir(configCrowdsecFiles, "crowdsec", func(path string, d fs.DirEntry, err error) error { - if err != nil { - return err - } - - // Skip the root fs directory itself - if path == "fs" { - return nil - } - - // Get the relative path by removing the "fs/" prefix - relPath := strings.TrimPrefix(path, "fs/") - - // skip .DS_Store - if strings.Contains(relPath, ".DS_Store") { - return nil - } - - // Create the full output path under "config/" - outPath := filepath.Join("config", relPath) - - if d.IsDir() { - // Create directory - if err := os.MkdirAll(outPath, 0755); err != nil { - return fmt.Errorf("failed to create directory %s: %v", outPath, err) - } - return nil - } - - // Read the template file - content, err := configFiles.ReadFile(path) - if err != nil { - return fmt.Errorf("failed to read %s: %v", path, err) - } - - // Parse template - tmpl, err := template.New(d.Name()).Parse(string(content)) - if err != nil { - return fmt.Errorf("failed to parse template %s: %v", path, err) - } - - // Ensure parent directory exists - if err := os.MkdirAll(filepath.Dir(outPath), 0755); err != nil { - return fmt.Errorf("failed to create parent directory for %s: %v", outPath, err) - } - - // Create output file - outFile, err := os.Create(outPath) - if err != nil { - return fmt.Errorf("failed to create %s: %v", outPath, err) - } - defer outFile.Close() - - // Execute template - if err := tmpl.Execute(outFile, config); err != nil { - return fmt.Errorf("failed to execute template %s: %v", path, err) - } - - return nil - }) - - if err != nil { - return fmt.Errorf("error walking config files: %v", err) - } - - // get the current directory - dir, err := os.Getwd() - if err != nil { - return fmt.Errorf("failed to get current directory: %v", err) - } - - sourcePath := filepath.Join(dir, "config/docker-compose.yml") - destPath := filepath.Join(dir, "docker-compose.yml") - - // Check if source file exists - if _, err := os.Stat(sourcePath); err != nil { - return fmt.Errorf("source docker-compose.yml not found: %v", err) - } - - // Try to move the file - err = os.Rename(sourcePath, destPath) - if err != nil { - return fmt.Errorf("failed to move docker-compose.yml from %s to %s: %v", - sourcePath, destPath, err) - } - - return nil + // Check for crowdsec service + return bytes.Contains(content, []byte("crowdsec:")) } diff --git a/install/go.mod b/install/go.mod index 85cf49e4..536ac2dd 100644 --- a/install/go.mod +++ b/install/go.mod @@ -5,4 +5,5 @@ go 1.23.0 require ( golang.org/x/sys v0.29.0 // indirect golang.org/x/term v0.28.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/install/go.sum b/install/go.sum index f05f63b4..3316e039 100644 --- a/install/go.sum +++ b/install/go.sum @@ -2,3 +2,6 @@ golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.28.0 h1:/Ts8HFuMR2E6IP/jlo7QVLZHggjKQbhu/7H0LJFr3Gg= golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/install/main.go b/install/main.go index d5bf5e1e..4f7c7df6 100644 --- a/install/main.go +++ b/install/main.go @@ -6,6 +6,7 @@ import ( "fmt" "io/fs" "os" + "io" "os/exec" "path/filepath" "runtime" @@ -24,7 +25,7 @@ func loadVersions(config *Config) { config.BadgerVersion = "replaceme" } -//go:embed fs/* +//go:embed config/* var configFiles embed.FS type Config struct { @@ -46,6 +47,7 @@ type Config struct { EmailNoReply string InstallGerbil bool TraefikBouncerKey string + DoCrowdsecInstall bool } func main() { @@ -57,9 +59,12 @@ func main() { os.Exit(1) } + var config Config + config.DoCrowdsecInstall = false + // check if there is already a config file if _, err := os.Stat("config/config.yml"); err != nil { - config := collectUserInput(reader) + config = collectUserInput(reader) loadVersions(&config) @@ -68,18 +73,53 @@ func main() { os.Exit(1) } + moveFile("config/docker-compose.yml", "docker-compose.yml") + if !isDockerInstalled() && runtime.GOOS == "linux" { if readBool(reader, "Docker is not installed. Would you like to install it?", true) { installDocker() } } + + fmt.Println("\n=== Starting installation ===") + + if isDockerInstalled() { + if readBool(reader, "Would you like to install and start the containers?", true) { + pullAndStartContainers() + } + } } else { - fmt.Println("Config file already exists... skipping configuration") + fmt.Println("Looks like you already installed, so I am going to do the setup...") } - if isDockerInstalled() { - if readBool(reader, "Would you like to install and start the containers?", true) { - pullAndStartContainers() + if !checkIsCrowdsecInstalledInCompose() { + fmt.Println("\n=== Crowdsec Install ===") + // check if crowdsec is installed + if readBool(reader, "Would you like to install Crowdsec?", true) { + + if config.DashboardDomain == "" { + traefikConfig, err := ReadTraefikConfig("config/traefik/traefik_config.yml", "config/traefik/dynamic_config.yml") + if err != nil { + fmt.Printf("Error reading config: %v\n", err) + return + } + config.DashboardDomain = traefikConfig.DashboardDomain + config.LetsEncryptEmail = traefikConfig.LetsEncryptEmail + config.BadgerVersion = traefikConfig.BadgerVersion + + // print the values and check if they are right + fmt.Println("Detected values:") + fmt.Printf("Dashboard Domain: %s\n", config.DashboardDomain) + fmt.Printf("Let's Encrypt Email: %s\n", config.LetsEncryptEmail) + fmt.Printf("Badger Version: %s\n", config.BadgerVersion) + + if !readBool(reader, "Are these values correct?", true) { + config = collectUserInput(reader) + } + } + + config.DoCrowdsecInstall = true + installCrowdsec(config) } } @@ -137,6 +177,11 @@ func readInt(reader *bufio.Reader, prompt string, defaultValue int) int { return value } +func isDockerFilePresent() bool { + _, err := os.Stat("docker-compose.yml") + return !os.IsNotExist(err) +} + func collectUserInput(reader *bufio.Reader) Config { config := Config{} @@ -262,31 +307,33 @@ func createConfigFiles(config Config) error { os.MkdirAll("config/logs", 0755) // Walk through all embedded files - err := fs.WalkDir(configFiles, "fs", func(path string, d fs.DirEntry, err error) error { + err := fs.WalkDir(configFiles, "config", func(path string, d fs.DirEntry, err error) error { if err != nil { return err } // Skip the root fs directory itself - if path == "fs" { + if path == "config" { return nil } - // Get the relative path by removing the "fs/" prefix - relPath := strings.TrimPrefix(path, "fs/") + if !config.DoCrowdsecInstall && strings.Contains(path, "crowdsec") { + return nil + } + + if config.DoCrowdsecInstall && !strings.Contains(path, "crowdsec") { + return nil + } // skip .DS_Store - if strings.Contains(relPath, ".DS_Store") { + if strings.Contains(path, ".DS_Store") { return nil } - // Create the full output path under "config/" - outPath := filepath.Join("config", relPath) - if d.IsDir() { // Create directory - if err := os.MkdirAll(outPath, 0755); err != nil { - return fmt.Errorf("failed to create directory %s: %v", outPath, err) + if err := os.MkdirAll(path, 0755); err != nil { + return fmt.Errorf("failed to create directory %s: %v", path, err) } return nil } @@ -304,14 +351,14 @@ func createConfigFiles(config Config) error { } // Ensure parent directory exists - if err := os.MkdirAll(filepath.Dir(outPath), 0755); err != nil { - return fmt.Errorf("failed to create parent directory for %s: %v", outPath, err) + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + return fmt.Errorf("failed to create parent directory for %s: %v", path, err) } // Create output file - outFile, err := os.Create(outPath) + outFile, err := os.Create(path) if err != nil { - return fmt.Errorf("failed to create %s: %v", outPath, err) + return fmt.Errorf("failed to create %s: %v", path, err) } defer outFile.Close() @@ -327,30 +374,10 @@ func createConfigFiles(config Config) error { return fmt.Errorf("error walking config files: %v", err) } - // get the current directory - dir, err := os.Getwd() - if err != nil { - return fmt.Errorf("failed to get current directory: %v", err) - } - - sourcePath := filepath.Join(dir, "config/docker-compose.yml") - destPath := filepath.Join(dir, "docker-compose.yml") - - // Check if source file exists - if _, err := os.Stat(sourcePath); err != nil { - return fmt.Errorf("source docker-compose.yml not found: %v", err) - } - - // Try to move the file - err = os.Rename(sourcePath, destPath) - if err != nil { - return fmt.Errorf("failed to move docker-compose.yml from %s to %s: %v", - sourcePath, destPath, err) - } - return nil } + func installDocker() error { // Detect Linux distribution cmd := exec.Command("cat", "/etc/os-release") @@ -491,3 +518,28 @@ func pullAndStartContainers() error { return nil } + +func copyFile(src, dst string) error { + source, err := os.Open(src) + if err != nil { + return err + } + defer source.Close() + + destination, err := os.Create(dst) + if err != nil { + return err + } + defer destination.Close() + + _, err = io.Copy(destination, source) + return err +} + +func moveFile(src, dst string) error { + if err := copyFile(src, dst); err != nil { + return err + } + + return os.Remove(src) +} \ No newline at end of file From dabd4a055c2c63996c585018ce3e7d696a3d7868 Mon Sep 17 00:00:00 2001 From: Owen Date: Sat, 15 Feb 2025 22:00:31 -0500 Subject: [PATCH 4/9] Creating structure correctly --- install/config.go | 2 +- install/crowdsec.go | 4 ++-- install/input.txt | 12 ++++++++++++ install/main.go | 38 ++++++++++++++++++++------------------ 4 files changed, 35 insertions(+), 21 deletions(-) create mode 100644 install/input.txt diff --git a/install/config.go b/install/config.go index fa10fa53..49577048 100644 --- a/install/config.go +++ b/install/config.go @@ -164,7 +164,7 @@ func AddCrowdSecService(configPath string) error { Image: "crowdsecurity/crowdsec:latest", ContainerName: "crowdsec", Environment: map[string]string{ - "GID": "${GID-1000}", + "GID": "1000", "COLLECTIONS": "crowdsecurity/traefik crowdsecurity/appsec-virtual-patching crowdsecurity/appsec-generic-rules", "ENROLL_INSTANCE_NAME": "pangolin-crowdsec", "PARSERS": "crowdsecurity/whitelists", diff --git a/install/crowdsec.go b/install/crowdsec.go index 02e9c71b..1d9bef65 100644 --- a/install/crowdsec.go +++ b/install/crowdsec.go @@ -24,8 +24,8 @@ func installCrowdsec(config Config) error { os.Exit(1) } - moveFile("config/crowdsec/traefik_config.yaml", "config/traefik/traefik_config.yaml") - moveFile("config/crowdsec/dynamic.yaml", "config/traefik/dynamic.yaml") + // moveFile("config/crowdsec/traefik_config.yml", "config/traefik/traefik_config.yml") + moveFile("config/crowdsec/dynamic.yml", "config/traefik/dynamic.yml") if err := retrieveBouncerKey(config); err != nil { return fmt.Errorf("bouncer key retrieval failed: %v", err) diff --git a/install/input.txt b/install/input.txt new file mode 100644 index 00000000..9bca8081 --- /dev/null +++ b/install/input.txt @@ -0,0 +1,12 @@ +example.com +pangolin.example.com +admin@example.com +yes +admin@example.com +Password123! +Password123! +yes +no +no +no +yes diff --git a/install/main.go b/install/main.go index 4f7c7df6..e5ad98b9 100644 --- a/install/main.go +++ b/install/main.go @@ -140,22 +140,24 @@ func readString(reader *bufio.Reader, prompt string, defaultValue string) string return input } -func readPassword(prompt string) string { - fmt.Print(prompt + ": ") - - // Read password without echo - password, err := term.ReadPassword(int(syscall.Stdin)) - fmt.Println() // Add a newline since ReadPassword doesn't add one - - if err != nil { - return "" - } - - input := strings.TrimSpace(string(password)) - if input == "" { - return readPassword(prompt) - } - return input +func readPassword(prompt string, reader *bufio.Reader) string { + if term.IsTerminal(int(syscall.Stdin)) { + fmt.Print(prompt + ": ") + // Read password without echo if we're in a terminal + password, err := term.ReadPassword(int(syscall.Stdin)) + fmt.Println() // Add a newline since ReadPassword doesn't add one + if err != nil { + return "" + } + input := strings.TrimSpace(string(password)) + if input == "" { + return readPassword(prompt, reader) + } + return input + } else { + // Fallback to reading from stdin if not in a terminal + return readString(reader, prompt, "") + } } func readBool(reader *bufio.Reader, prompt string, defaultValue bool) bool { @@ -196,8 +198,8 @@ func collectUserInput(reader *bufio.Reader) Config { fmt.Println("\n=== Admin User Configuration ===") config.AdminUserEmail = readString(reader, "Enter admin user email", "admin@"+config.BaseDomain) for { - pass1 := readPassword("Create admin user password") - pass2 := readPassword("Confirm admin user password") + pass1 := readPassword("Create admin user password", reader) + pass2 := readPassword("Confirm admin user password", reader) if pass1 != pass2 { fmt.Println("Passwords do not match") From d3d523b2b8f1de0d25d176be134515a0614b0721 Mon Sep 17 00:00:00 2001 From: Owen Date: Sun, 16 Feb 2025 11:26:45 -0500 Subject: [PATCH 5/9] Refactor docker copy and keep entrypoints --- install/config.go | 268 +++++++++++++-------- install/config/crowdsec/docker-compose.yml | 35 +++ install/config/crowdsec/traefik_config.yml | 2 +- install/crowdsec.go | 35 ++- 4 files changed, 230 insertions(+), 110 deletions(-) create mode 100644 install/config/crowdsec/docker-compose.yml diff --git a/install/config.go b/install/config.go index 49577048..1ebfbecb 100644 --- a/install/config.go +++ b/install/config.go @@ -104,113 +104,175 @@ func findPattern(s, pattern string) int { return bytes.Index([]byte(s), []byte(pattern)) } -type Volume string -type Port string -type Expose string - -type HealthCheck struct { - Test []string `yaml:"test,omitempty"` - Interval string `yaml:"interval,omitempty"` - Timeout string `yaml:"timeout,omitempty"` - Retries int `yaml:"retries,omitempty"` -} - -type DependsOnCondition struct { - Condition string `yaml:"condition,omitempty"` -} - -type Service struct { - Image string `yaml:"image,omitempty"` - ContainerName string `yaml:"container_name,omitempty"` - Environment map[string]string `yaml:"environment,omitempty"` - HealthCheck *HealthCheck `yaml:"healthcheck,omitempty"` - DependsOn map[string]DependsOnCondition `yaml:"depends_on,omitempty"` - Labels []string `yaml:"labels,omitempty"` - Volumes []Volume `yaml:"volumes,omitempty"` - Ports []Port `yaml:"ports,omitempty"` - Expose []Expose `yaml:"expose,omitempty"` - Restart string `yaml:"restart,omitempty"` - Command interface{} `yaml:"command,omitempty"` - NetworkMode string `yaml:"network_mode,omitempty"` - CapAdd []string `yaml:"cap_add,omitempty"` -} - -type Network struct { - Driver string `yaml:"driver,omitempty"` - Name string `yaml:"name,omitempty"` -} - -type DockerConfig struct { - Version string `yaml:"version,omitempty"` - Services map[string]Service `yaml:"services"` - Networks map[string]Network `yaml:"networks,omitempty"` -} - -func AddCrowdSecService(configPath string) error { - // Read existing config - data, err := os.ReadFile(configPath) +func copyEntryPoints(sourceFile, destFile string) error { + // Read source file + sourceData, err := os.ReadFile(sourceFile) if err != nil { - return err + return fmt.Errorf("error reading source file: %w", err) } - // Parse existing config - var config DockerConfig - if err := yaml.Unmarshal(data, &config); err != nil { - return err - } - - // Create CrowdSec service - crowdsecService := Service{ - Image: "crowdsecurity/crowdsec:latest", - ContainerName: "crowdsec", - Environment: map[string]string{ - "GID": "1000", - "COLLECTIONS": "crowdsecurity/traefik crowdsecurity/appsec-virtual-patching crowdsecurity/appsec-generic-rules", - "ENROLL_INSTANCE_NAME": "pangolin-crowdsec", - "PARSERS": "crowdsecurity/whitelists", - "ACQUIRE_FILES": "/var/log/traefik/*.log", - "ENROLL_TAGS": "docker", - }, - HealthCheck: &HealthCheck{ - Test: []string{"CMD", "cscli", "capi", "status"}, - }, - DependsOn: map[string]DependsOnCondition{ - "gerbil": {}, - }, - Labels: []string{"traefik.enable=false"}, - Volumes: []Volume{ - "./config/crowdsec:/etc/crowdsec", - "./config/crowdsec/db:/var/lib/crowdsec/data", - "./config/crowdsec_logs/auth.log:/var/log/auth.log:ro", - "./config/crowdsec_logs/syslog:/var/log/syslog:ro", - "./config/crowdsec_logs:/var/log", - "./config/traefik/logs:/var/log/traefik", - }, - Ports: []Port{ - "9090:9090", - "6060:6060", - }, - Expose: []Expose{ - "9090", - "6060", - "7422", - }, - Restart: "unless-stopped", - Command: "-t", - } - - // Add CrowdSec service to config - if config.Services == nil { - config.Services = make(map[string]Service) - } - config.Services["crowdsec"] = crowdsecService - - // Marshal config with better formatting - yamlData, err := yaml.Marshal(&config) + // Read destination file + destData, err := os.ReadFile(destFile) if err != nil { - return err + return fmt.Errorf("error reading destination file: %w", err) } - // Write config back to file - return os.WriteFile(configPath, yamlData, 0644) + // Parse source YAML + var sourceYAML map[string]interface{} + if err := yaml.Unmarshal(sourceData, &sourceYAML); err != nil { + return fmt.Errorf("error parsing source YAML: %w", err) + } + + // Parse destination YAML + var destYAML map[string]interface{} + if err := yaml.Unmarshal(destData, &destYAML); err != nil { + return fmt.Errorf("error parsing destination YAML: %w", err) + } + + // Get entryPoints section from source + entryPoints, ok := sourceYAML["entryPoints"] + if !ok { + return fmt.Errorf("entryPoints section not found in source file") + } + + // Update entryPoints in destination + destYAML["entryPoints"] = entryPoints + + // Marshal updated destination YAML + updatedData, err := yaml.Marshal(destYAML) + if err != nil { + return fmt.Errorf("error marshaling updated YAML: %w", err) + } + + // Write updated YAML back to destination file + if err := os.WriteFile(destFile, updatedData, 0644); err != nil { + return fmt.Errorf("error writing to destination file: %w", err) + } + + return nil +} + +func copyWebsecureEntryPoint(sourceFile, destFile string) error { + // Read source file + sourceData, err := os.ReadFile(sourceFile) + if err != nil { + return fmt.Errorf("error reading source file: %w", err) + } + + // Read destination file + destData, err := os.ReadFile(destFile) + if err != nil { + return fmt.Errorf("error reading destination file: %w", err) + } + + // Parse source YAML + var sourceYAML map[string]interface{} + if err := yaml.Unmarshal(sourceData, &sourceYAML); err != nil { + return fmt.Errorf("error parsing source YAML: %w", err) + } + + // Parse destination YAML + var destYAML map[string]interface{} + if err := yaml.Unmarshal(destData, &destYAML); err != nil { + return fmt.Errorf("error parsing destination YAML: %w", err) + } + + // Get entryPoints section from source + entryPoints, ok := sourceYAML["entryPoints"].(map[string]interface{}) + if !ok { + return fmt.Errorf("entryPoints section not found in source file or has invalid format") + } + + // Get websecure configuration + websecure, ok := entryPoints["websecure"] + if !ok { + return fmt.Errorf("websecure entrypoint not found in source file") + } + + // Get or create entryPoints section in destination + destEntryPoints, ok := destYAML["entryPoints"].(map[string]interface{}) + if !ok { + // If entryPoints section doesn't exist, create it + destEntryPoints = make(map[string]interface{}) + destYAML["entryPoints"] = destEntryPoints + } + + // Update websecure in destination + destEntryPoints["websecure"] = websecure + + // Marshal updated destination YAML + updatedData, err := yaml.Marshal(destYAML) + if err != nil { + return fmt.Errorf("error marshaling updated YAML: %w", err) + } + + // Write updated YAML back to destination file + if err := os.WriteFile(destFile, updatedData, 0644); err != nil { + return fmt.Errorf("error writing to destination file: %w", err) + } + + return nil +} + +func copyDockerService(sourceFile, destFile, serviceName string) error { + // Read source file + sourceData, err := os.ReadFile(sourceFile) + if err != nil { + return fmt.Errorf("error reading source file: %w", err) + } + + // Read destination file + destData, err := os.ReadFile(destFile) + if err != nil { + return fmt.Errorf("error reading destination file: %w", err) + } + + // Parse source Docker Compose YAML + var sourceCompose map[string]interface{} + if err := yaml.Unmarshal(sourceData, &sourceCompose); err != nil { + return fmt.Errorf("error parsing source Docker Compose file: %w", err) + } + + // Parse destination Docker Compose YAML + var destCompose map[string]interface{} + if err := yaml.Unmarshal(destData, &destCompose); err != nil { + return fmt.Errorf("error parsing destination Docker Compose file: %w", err) + } + + // Get services section from source + sourceServices, ok := sourceCompose["services"].(map[string]interface{}) + if !ok { + return fmt.Errorf("services section not found in source file or has invalid format") + } + + // Get the specific service configuration + serviceConfig, ok := sourceServices[serviceName] + if !ok { + return fmt.Errorf("service '%s' not found in source file", serviceName) + } + + // Get or create services section in destination + destServices, ok := destCompose["services"].(map[string]interface{}) + if !ok { + // If services section doesn't exist, create it + destServices = make(map[string]interface{}) + destCompose["services"] = destServices + } + + // Update service in destination + destServices[serviceName] = serviceConfig + + // Marshal updated destination YAML + // Use yaml.v3 encoder to preserve formatting and comments + updatedData, err := yaml.Marshal(destCompose) + if err != nil { + return fmt.Errorf("error marshaling updated Docker Compose file: %w", err) + } + + // Write updated YAML back to destination file + if err := os.WriteFile(destFile, updatedData, 0644); err != nil { + return fmt.Errorf("error writing to destination file: %w", err) + } + + return nil } diff --git a/install/config/crowdsec/docker-compose.yml b/install/config/crowdsec/docker-compose.yml new file mode 100644 index 00000000..982b3335 --- /dev/null +++ b/install/config/crowdsec/docker-compose.yml @@ -0,0 +1,35 @@ +services: + crowdsec: + image: crowdsecurity/crowdsec:latest + container_name: crowdsec + environment: + GID: "1000" + COLLECTIONS: crowdsecurity/traefik crowdsecurity/appsec-virtual-patching crowdsecurity/appsec-generic-rules + ENROLL_INSTANCE_NAME: "pangolin-crowdsec" + PARSERS: crowdsecurity/whitelists + ACQUIRE_FILES: "/var/log/traefik/*.log" + ENROLL_TAGS: docker + healthcheck: + test: ["CMD", "cscli", "capi", "status"] + depends_on: + - gerbil # Wait for gerbil to be healthy + labels: + - "traefik.enable=false" # Disable traefik for crowdsec + volumes: + # crowdsec container data + - ./config/crowdsec:/etc/crowdsec # crowdsec config + - ./config/crowdsec/db:/var/lib/crowdsec/data # crowdsec db + # log bind mounts into crowdsec + - ./config/crowdsec_logs/auth.log:/var/log/auth.log:ro # auth.log + - ./config/crowdsec_logs/syslog:/var/log/syslog:ro # syslog + - ./config/crowdsec_logs:/var/log # crowdsec logs + - ./config/traefik/logs:/var/log/traefik # traefik logs + ports: + - 9090:9090 # port mapping for local firewall bouncers + - 6060:6060 # metrics endpoint for prometheus + expose: + - 9090 # http api for bouncers + - 6060 # metrics endpoint for prometheus + - 7422 # appsec waf endpoint + restart: unless-stopped + command: -t # Add test config flag to verify configuration \ No newline at end of file diff --git a/install/config/crowdsec/traefik_config.yml b/install/config/crowdsec/traefik_config.yml index 2ac9125c..59356ea7 100644 --- a/install/config/crowdsec/traefik_config.yml +++ b/install/config/crowdsec/traefik_config.yml @@ -80,7 +80,7 @@ entryPoints: http: tls: certResolver: "letsencrypt" - middlewares: # CHANGE MADE HERE (BOUNCER ENABLED) !!! + middlewares: - crowdsec@file serversTransport: diff --git a/install/crowdsec.go b/install/crowdsec.go index 1d9bef65..2b77985f 100644 --- a/install/crowdsec.go +++ b/install/crowdsec.go @@ -15,17 +15,40 @@ func installCrowdsec(config Config) error { return fmt.Errorf("backup failed: %v", err) } - if err := AddCrowdSecService("docker-compose.yml"); err != nil { - return fmt.Errorf("crowdsec service addition failed: %v", err) - } - if err := createConfigFiles(config); err != nil { fmt.Printf("Error creating config files: %v\n", err) os.Exit(1) } - // moveFile("config/crowdsec/traefik_config.yml", "config/traefik/traefik_config.yml") - moveFile("config/crowdsec/dynamic.yml", "config/traefik/dynamic.yml") + if err := copyDockerService("config/crowdsec/docker-compose.yml", "docker-compose.yml", "crowdsec"); err != nil { + fmt.Printf("Error copying docker service: %v\n", err) + os.Exit(1) + } + + if err := copyWebsecureEntryPoint("config/crowdsec/traefik_config.yml", "config/traefik/traefik_config.yml"); err != nil { + fmt.Printf("Error copying entry points: %v\n", err) + os.Exit(1) + } + + if err := copyEntryPoints("config/traefik/traefik_config.yml", "config/crowdsec/traefik_config.yml"); err != nil { + fmt.Printf("Error copying entry points: %v\n", err) + os.Exit(1) + } + + if err := moveFile("config/crowdsec/traefik_config.yml", "config/traefik/traefik_config.yml"); err != nil { + fmt.Printf("Error moving file: %v\n", err) + os.Exit(1) + } + + if err := moveFile("config/crowdsec/dynamic_config.yml", "config/traefik/dynamic_config.yml"); err != nil { + fmt.Printf("Error moving file: %v\n", err) + os.Exit(1) + } + + if err := os.Remove("config/crowdsec/docker-compose.yml"); err != nil { + fmt.Printf("Error removing file: %v\n", err) + os.Exit(1) + } if err := retrieveBouncerKey(config); err != nil { return fmt.Errorf("bouncer key retrieval failed: %v", err) From e6c42e9610feb7e656cc6c951fe84a0f3605c27d Mon Sep 17 00:00:00 2001 From: Owen Date: Sun, 16 Feb 2025 11:31:01 -0500 Subject: [PATCH 6/9] Indent 2 --- install/config.go | 43 ++++++++++++++++++++++++++++++++++++++++--- install/crowdsec.go | 19 ------------------- 2 files changed, 40 insertions(+), 22 deletions(-) diff --git a/install/config.go b/install/config.go index 1ebfbecb..b666b53e 100644 --- a/install/config.go +++ b/install/config.go @@ -4,6 +4,7 @@ import ( "bytes" "fmt" "os" + "os/exec" "gopkg.in/yaml.v3" ) @@ -139,7 +140,8 @@ func copyEntryPoints(sourceFile, destFile string) error { destYAML["entryPoints"] = entryPoints // Marshal updated destination YAML - updatedData, err := yaml.Marshal(destYAML) + // updatedData, err := yaml.Marshal(destYAML) + updatedData, err := MarshalYAMLWithIndent(destYAML, 2) if err != nil { return fmt.Errorf("error marshaling updated YAML: %w", err) } @@ -201,7 +203,8 @@ func copyWebsecureEntryPoint(sourceFile, destFile string) error { destEntryPoints["websecure"] = websecure // Marshal updated destination YAML - updatedData, err := yaml.Marshal(destYAML) + // updatedData, err := yaml.Marshal(destYAML) + updatedData, err := MarshalYAMLWithIndent(destYAML, 2) if err != nil { return fmt.Errorf("error marshaling updated YAML: %w", err) } @@ -264,7 +267,8 @@ func copyDockerService(sourceFile, destFile, serviceName string) error { // Marshal updated destination YAML // Use yaml.v3 encoder to preserve formatting and comments - updatedData, err := yaml.Marshal(destCompose) + // updatedData, err := yaml.Marshal(destCompose) + updatedData, err := MarshalYAMLWithIndent(destCompose, 2) if err != nil { return fmt.Errorf("error marshaling updated Docker Compose file: %w", err) } @@ -276,3 +280,36 @@ func copyDockerService(sourceFile, destFile, serviceName string) error { return nil } + +func backupConfig() error { + // Backup docker-compose.yml + if _, err := os.Stat("docker-compose.yml"); err == nil { + if err := copyFile("docker-compose.yml", "docker-compose.yml.backup"); err != nil { + return fmt.Errorf("failed to backup docker-compose.yml: %v", err) + } + } + + // Backup config directory + if _, err := os.Stat("config"); err == nil { + cmd := exec.Command("tar", "-czvf", "config.tar.gz", "config") + if err := cmd.Run(); err != nil { + return fmt.Errorf("failed to backup config directory: %v", err) + } + } + + return nil +} + +func MarshalYAMLWithIndent(data interface{}, indent int) ([]byte, error) { + buffer := new(bytes.Buffer) + encoder := yaml.NewEncoder(buffer) + encoder.SetIndent(indent) + + err := encoder.Encode(data) + if err != nil { + return nil, err + } + + defer encoder.Close() + return buffer.Bytes(), nil +} diff --git a/install/crowdsec.go b/install/crowdsec.go index 2b77985f..5b777e80 100644 --- a/install/crowdsec.go +++ b/install/crowdsec.go @@ -57,25 +57,6 @@ func installCrowdsec(config Config) error { return nil } -func backupConfig() error { - // Backup docker-compose.yml - if _, err := os.Stat("docker-compose.yml"); err == nil { - if err := copyFile("docker-compose.yml", "docker-compose.yml.backup"); err != nil { - return fmt.Errorf("failed to backup docker-compose.yml: %v", err) - } - } - - // Backup config directory - if _, err := os.Stat("config"); err == nil { - cmd := exec.Command("tar", "-czvf", "config.tar.gz", "config") - if err := cmd.Run(); err != nil { - return fmt.Errorf("failed to backup config directory: %v", err) - } - } - - return nil -} - func retrieveBouncerKey(config Config) error { // Start crowdsec container cmd := exec.Command("docker", "compose", "up", "-d", "crowdsec") From fd11fb81d6eb839c0ac401afd9877405e1824d83 Mon Sep 17 00:00:00 2001 From: Owen Date: Tue, 18 Feb 2025 21:41:23 -0500 Subject: [PATCH 7/9] Remove some config --- install/config.go | 20 +++++ install/config/crowdsec/config.yaml | 12 --- install/config/crowdsec/dynamic_config.yml | 4 +- .../crowdsec/local_api_credentials.yaml | 2 - install/crowdsec.go | 26 +++++- install/main.go | 79 +++++++++++++++++-- 6 files changed, 121 insertions(+), 22 deletions(-) delete mode 100644 install/config/crowdsec/config.yaml delete mode 100644 install/config/crowdsec/local_api_credentials.yaml diff --git a/install/config.go b/install/config.go index b666b53e..f87bb1af 100644 --- a/install/config.go +++ b/install/config.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "os/exec" + "strings" "gopkg.in/yaml.v3" ) @@ -313,3 +314,22 @@ func MarshalYAMLWithIndent(data interface{}, indent int) ([]byte, error) { defer encoder.Close() return buffer.Bytes(), nil } + +func replaceInFile(filepath, oldStr, newStr string) error { + // Read the file content + content, err := os.ReadFile(filepath) + if err != nil { + return fmt.Errorf("error reading file: %v", err) + } + + // Replace the string + newContent := strings.Replace(string(content), oldStr, newStr, -1) + + // Write the modified content back to the file + err = os.WriteFile(filepath, []byte(newContent), 0644) + if err != nil { + return fmt.Errorf("error writing file: %v", err) + } + + return nil +} diff --git a/install/config/crowdsec/config.yaml b/install/config/crowdsec/config.yaml deleted file mode 100644 index 0acf4635..00000000 --- a/install/config/crowdsec/config.yaml +++ /dev/null @@ -1,12 +0,0 @@ -api: - client: - insecure_skip_verify: false - credentials_path: /etc/crowdsec/local_api_credentials.yaml - server: - log_level: info - listen_uri: 0.0.0.0:9090 - profiles_path: /etc/crowdsec/profiles.yaml - trusted_ips: - - 0.0.0.0/0 - - 127.0.0.1 - - ::1 \ No newline at end of file diff --git a/install/config/crowdsec/dynamic_config.yml b/install/config/crowdsec/dynamic_config.yml index d2556971..a3d32dbd 100644 --- a/install/config/crowdsec/dynamic_config.yml +++ b/install/config/crowdsec/dynamic_config.yml @@ -42,8 +42,8 @@ http: crowdsecAppsecHost: crowdsec:7422 # CrowdSec IP address which you noted down later crowdsecAppsecFailureBlock: true # Block on failure crowdsecAppsecUnreachableBlock: true # Block on unreachable - crowdsecLapiKey: "{{.TraefikBouncerKey}}" # CrowdSec API key which you noted down later - crowdsecLapiHost: crowdsec:9090 # CrowdSec + crowdsecLapiKey: "PUT_YOUR_BOUNCER_KEY_HERE_OR_IT_WILL_NOT_WORK" # CrowdSec API key which you noted down later + crowdsecLapiHost: crowdsec:8080 # CrowdSec crowdsecLapiScheme: http # CrowdSec API scheme forwardedHeadersTrustedIPs: # Forwarded headers trusted IPs - "0.0.0.0/0" # All IP addresses are trusted for forwarded headers (CHANGE MADE HERE) diff --git a/install/config/crowdsec/local_api_credentials.yaml b/install/config/crowdsec/local_api_credentials.yaml deleted file mode 100644 index 8776e4fd..00000000 --- a/install/config/crowdsec/local_api_credentials.yaml +++ /dev/null @@ -1,2 +0,0 @@ -url: http://0.0.0.0:9090 -login: localhost \ No newline at end of file diff --git a/install/crowdsec.go b/install/crowdsec.go index 5b777e80..6d25a633 100644 --- a/install/crowdsec.go +++ b/install/crowdsec.go @@ -10,6 +10,11 @@ import ( ) func installCrowdsec(config Config) error { + + if err := stopContainers(); err != nil { + return fmt.Errorf("failed to stop containers: %v", err) + } + // Run installation steps if err := backupConfig(); err != nil { return fmt.Errorf("backup failed: %v", err) @@ -20,6 +25,10 @@ func installCrowdsec(config Config) error { os.Exit(1) } + os.MkdirAll("config/crowdsec/db", 0755) + os.MkdirAll("config/crowdsec_logs/syslog", 0755) + os.MkdirAll("config/traefik/logs", 0755) + if err := copyDockerService("config/crowdsec/docker-compose.yml", "docker-compose.yml", "crowdsec"); err != nil { fmt.Printf("Error copying docker service: %v\n", err) os.Exit(1) @@ -54,16 +63,22 @@ func installCrowdsec(config Config) error { return fmt.Errorf("bouncer key retrieval failed: %v", err) } + // if err := startContainers(); err != nil { + // return fmt.Errorf("failed to start containers: %v", err) + // } + return nil } func retrieveBouncerKey(config Config) error { + + fmt.Println("Retrieving bouncer key. Please be patient...") + // Start crowdsec container cmd := exec.Command("docker", "compose", "up", "-d", "crowdsec") if err := cmd.Run(); err != nil { return fmt.Errorf("failed to start crowdsec: %v", err) } - defer exec.Command("docker", "compose", "down").Run() // verify that the container is running if not keep waiting for 10 more seconds then return an error count := 0 @@ -95,10 +110,19 @@ func retrieveBouncerKey(config Config) error { for _, line := range lines { if strings.Contains(line, "key:") { config.TraefikBouncerKey = strings.TrimSpace(strings.Split(line, ":")[1]) + fmt.Println("Bouncer key:", config.TraefikBouncerKey) break } } + // Stop crowdsec container + cmd = exec.Command("docker", "compose", "down") + if err := cmd.Run(); err != nil { + return fmt.Errorf("failed to stop crowdsec: %v", err) + } + + fmt.Println("Bouncer key retrieved successfully.") + return nil } diff --git a/install/main.go b/install/main.go index e5ad98b9..bca19100 100644 --- a/install/main.go +++ b/install/main.go @@ -179,11 +179,6 @@ func readInt(reader *bufio.Reader, prompt string, defaultValue int) int { return value } -func isDockerFilePresent() bool { - _, err := os.Stat("docker-compose.yml") - return !os.IsNotExist(err) -} - func collectUserInput(reader *bufio.Reader) Config { config := Config{} @@ -521,6 +516,80 @@ func pullAndStartContainers() error { return nil } +// bring containers down +func stopContainers() error { + fmt.Println("Stopping containers...") + + // Check which docker compose command is available + var useNewStyle bool + checkCmd := exec.Command("docker", "compose", "version") + if err := checkCmd.Run(); err == nil { + useNewStyle = true + } else { + // Check if docker-compose (old style) is available + checkCmd = exec.Command("docker-compose", "version") + if err := checkCmd.Run(); err != nil { + return fmt.Errorf("neither 'docker compose' nor 'docker-compose' command is available: %v", err) + } + } + + // Helper function to execute docker compose commands + executeCommand := func(args ...string) error { + var cmd *exec.Cmd + if useNewStyle { + cmd = exec.Command("docker", append([]string{"compose"}, args...)...) + } else { + cmd = exec.Command("docker-compose", args...) + } + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd.Run() + } + + if err := executeCommand("-f", "docker-compose.yml", "down"); err != nil { + return fmt.Errorf("failed to stop containers: %v", err) + } + + return nil +} + +// just start containers +func startContainers() error { + fmt.Println("Starting containers...") + + // Check which docker compose command is available + var useNewStyle bool + checkCmd := exec.Command("docker", "compose", "version") + if err := checkCmd.Run(); err == nil { + useNewStyle = true + } else { + // Check if docker-compose (old style) is available + checkCmd = exec.Command("docker-compose", "version") + if err := checkCmd.Run(); err != nil { + return fmt.Errorf("neither 'docker compose' nor 'docker-compose' command is available: %v", err) + } + } + + // Helper function to execute docker compose commands + executeCommand := func(args ...string) error { + var cmd *exec.Cmd + if useNewStyle { + cmd = exec.Command("docker", append([]string{"compose"}, args...)...) + } else { + cmd = exec.Command("docker-compose", args...) + } + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd.Run() + } + + if err := executeCommand("-f", "docker-compose.yml", "up", "-d"); err != nil { + return fmt.Errorf("failed to start containers: %v", err) + } + + return nil +} + func copyFile(src, dst string) error { source, err := os.Open(src) if err != nil { From 5f95500b6f77d3da6dfb27cc189217adc7d0a884 Mon Sep 17 00:00:00 2001 From: Owen Date: Wed, 19 Feb 2025 21:42:42 -0500 Subject: [PATCH 8/9] Crowdsec installer works? --- install/config.go | 58 +++++++++++++++++++ install/config/docker-compose.yml | 1 + install/crowdsec.go | 95 +++++++++++++------------------ install/main.go | 68 +++++++++++++++++++++- 4 files changed, 165 insertions(+), 57 deletions(-) diff --git a/install/config.go b/install/config.go index f87bb1af..c31149d6 100644 --- a/install/config.go +++ b/install/config.go @@ -333,3 +333,61 @@ func replaceInFile(filepath, oldStr, newStr string) error { return nil } + +func CheckAndAddTraefikLogVolume(composePath string) error { + // Read the docker-compose.yml file + data, err := os.ReadFile(composePath) + if err != nil { + return fmt.Errorf("error reading compose file: %w", err) + } + + // Parse YAML into a generic map + var compose map[string]interface{} + if err := yaml.Unmarshal(data, &compose); err != nil { + return fmt.Errorf("error parsing compose file: %w", err) + } + + // Get services section + services, ok := compose["services"].(map[string]interface{}) + if !ok { + return fmt.Errorf("services section not found or invalid") + } + + // Get traefik service + traefik, ok := services["traefik"].(map[string]interface{}) + if !ok { + return fmt.Errorf("traefik service not found or invalid") + } + + // Check volumes + logVolume := "./config/traefik/logs:/var/log/traefik" + var volumes []interface{} + + if existingVolumes, ok := traefik["volumes"].([]interface{}); ok { + // Check if volume already exists + for _, v := range existingVolumes { + if v.(string) == logVolume { + fmt.Println("Traefik log volume is already configured") + return nil + } + } + volumes = existingVolumes + } + + // Add new volume + volumes = append(volumes, logVolume) + traefik["volumes"] = volumes + + // Write updated config back to file + newData, err := MarshalYAMLWithIndent(compose, 2) + if err != nil { + return fmt.Errorf("error marshaling updated compose file: %w", err) + } + + if err := os.WriteFile(composePath, newData, 0644); err != nil { + return fmt.Errorf("error writing updated compose file: %w", err) + } + + fmt.Println("Added traefik log volume and created logs directory") + return nil +} diff --git a/install/config/docker-compose.yml b/install/config/docker-compose.yml index 42604ab4..f6ce7892 100644 --- a/install/config/docker-compose.yml +++ b/install/config/docker-compose.yml @@ -52,6 +52,7 @@ services: volumes: - ./config/traefik:/etc/traefik:ro # Volume to store the Traefik configuration - ./config/letsencrypt:/letsencrypt # Volume to store the Let's Encrypt certificates + - ./config/traefik/logs:/var/log/traefik # Volume to store Traefik logs networks: default: diff --git a/install/crowdsec.go b/install/crowdsec.go index 6d25a633..d98b2acf 100644 --- a/install/crowdsec.go +++ b/install/crowdsec.go @@ -6,7 +6,6 @@ import ( "os" "os/exec" "strings" - "time" ) func installCrowdsec(config Config) error { @@ -59,70 +58,30 @@ func installCrowdsec(config Config) error { os.Exit(1) } - if err := retrieveBouncerKey(config); err != nil { - return fmt.Errorf("bouncer key retrieval failed: %v", err) + if err := CheckAndAddTraefikLogVolume("docker-compose.yml"); err != nil { + fmt.Printf("Error checking and adding Traefik log volume: %v\n", err) + os.Exit(1) } - // if err := startContainers(); err != nil { - // return fmt.Errorf("failed to start containers: %v", err) - // } - - return nil -} - -func retrieveBouncerKey(config Config) error { - - fmt.Println("Retrieving bouncer key. Please be patient...") - - // Start crowdsec container - cmd := exec.Command("docker", "compose", "up", "-d", "crowdsec") - if err := cmd.Run(); err != nil { - return fmt.Errorf("failed to start crowdsec: %v", err) + if err := startContainers(); err != nil { + return fmt.Errorf("failed to start containers: %v", err) } - // verify that the container is running if not keep waiting for 10 more seconds then return an error - count := 0 - for { - cmd := exec.Command("docker", "inspect", "-f", "{{.State.Running}}", "crowdsec") - output, err := cmd.Output() - if err != nil { - return fmt.Errorf("failed to inspect crowdsec container: %v", err) - } - if strings.TrimSpace(string(output)) == "true" { - break - } - time.Sleep(10 * time.Second) - count++ - - if count > 4 { - return fmt.Errorf("crowdsec container is not running") - } - } - - // Get bouncer key - output, err := exec.Command("docker", "exec", "crowdsec", "cscli", "bouncers", "add", "traefik-bouncer").Output() + // get API key + apiKey, err := GetCrowdSecAPIKey() if err != nil { - return fmt.Errorf("failed to get bouncer key: %v", err) + return fmt.Errorf("failed to get API key: %v", err) + } + config.TraefikBouncerKey = apiKey + + if err := replaceInFile("config/traefik/dynamic_config.yml", "PUT_YOUR_BOUNCER_KEY_HERE_OR_IT_WILL_NOT_WORK", config.TraefikBouncerKey); err != nil { + return fmt.Errorf("failed to replace bouncer key: %v", err) } - // Parse key from output - lines := strings.Split(string(output), "\n") - for _, line := range lines { - if strings.Contains(line, "key:") { - config.TraefikBouncerKey = strings.TrimSpace(strings.Split(line, ":")[1]) - fmt.Println("Bouncer key:", config.TraefikBouncerKey) - break - } + if err := restartContainer("traefik"); err != nil { + return fmt.Errorf("failed to restart containers: %v", err) } - // Stop crowdsec container - cmd = exec.Command("docker", "compose", "down") - if err := cmd.Run(); err != nil { - return fmt.Errorf("failed to stop crowdsec: %v", err) - } - - fmt.Println("Bouncer key retrieved successfully.") - return nil } @@ -136,3 +95,27 @@ func checkIsCrowdsecInstalledInCompose() bool { // Check for crowdsec service return bytes.Contains(content, []byte("crowdsec:")) } + +func GetCrowdSecAPIKey() (string, error) { + // First, ensure the container is running + if err := waitForContainer("crowdsec"); err != nil { + return "", fmt.Errorf("waiting for container: %w", err) + } + + // Execute the command to get the API key + cmd := exec.Command("docker", "exec", "crowdsec", "cscli", "bouncers", "add", "traefik-bouncer", "-o", "raw") + var out bytes.Buffer + cmd.Stdout = &out + + if err := cmd.Run(); err != nil { + return "", fmt.Errorf("executing command: %w", err) + } + + // Trim any whitespace from the output + apiKey := strings.TrimSpace(out.String()) + if apiKey == "" { + return "", fmt.Errorf("empty API key returned") + } + + return apiKey, nil +} diff --git a/install/main.go b/install/main.go index bca19100..9064b4f7 100644 --- a/install/main.go +++ b/install/main.go @@ -4,14 +4,16 @@ import ( "bufio" "embed" "fmt" + "io" "io/fs" "os" - "io" + "time" "os/exec" "path/filepath" "runtime" "strings" "syscall" + "bytes" "text/template" "unicode" @@ -590,6 +592,42 @@ func startContainers() error { return nil } +func restartContainer(container string) error { + fmt.Printf("Restarting %s container...\n", container) + + // Check which docker compose command is available + var useNewStyle bool + checkCmd := exec.Command("docker", "compose", "version") + if err := checkCmd.Run(); err == nil { + useNewStyle = true + } else { + // Check if docker-compose (old style) is available + checkCmd = exec.Command("docker-compose", "version") + if err := checkCmd.Run(); err != nil { + return fmt.Errorf("neither 'docker compose' nor 'docker-compose' command is available: %v", err) + } + } + + // Helper function to execute docker compose commands + executeCommand := func(args ...string) error { + var cmd *exec.Cmd + if useNewStyle { + cmd = exec.Command("docker", append([]string{"compose"}, args...)...) + } else { + cmd = exec.Command("docker-compose", args...) + } + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd.Run() + } + + if err := executeCommand("-f", "docker-compose.yml", "restart", container); err != nil { + return fmt.Errorf("failed to restart %s container: %v", container, err) + } + + return nil +} + func copyFile(src, dst string) error { source, err := os.Open(src) if err != nil { @@ -613,4 +651,32 @@ func moveFile(src, dst string) error { } return os.Remove(src) +} + +func waitForContainer(containerName string) error { + maxAttempts := 30 + retryInterval := time.Second * 2 + + for attempt := 0; attempt < maxAttempts; attempt++ { + // Check if container is running + cmd := exec.Command("docker", "container", "inspect", "-f", "{{.State.Running}}", containerName) + var out bytes.Buffer + cmd.Stdout = &out + + if err := cmd.Run(); err != nil { + // If the container doesn't exist or there's another error, wait and retry + time.Sleep(retryInterval) + continue + } + + isRunning := strings.TrimSpace(out.String()) == "true" + if isRunning { + return nil + } + + // Container exists but isn't running yet, wait and retry + time.Sleep(retryInterval) + } + + return fmt.Errorf("container %s did not start within %v seconds", containerName, maxAttempts*int(retryInterval.Seconds())) } \ No newline at end of file From f59f0ee57dc66e83be73127b626975a1eab8cf29 Mon Sep 17 00:00:00 2001 From: Owen Date: Sun, 23 Feb 2025 21:44:02 -0500 Subject: [PATCH 9/9] Merge yaml files instead? --- install/config.go | 184 +++++++++++++++++--------------------------- install/crowdsec.go | 20 ++--- 2 files changed, 82 insertions(+), 122 deletions(-) diff --git a/install/config.go b/install/config.go index c31149d6..3be62601 100644 --- a/install/config.go +++ b/install/config.go @@ -106,118 +106,6 @@ func findPattern(s, pattern string) int { return bytes.Index([]byte(s), []byte(pattern)) } -func copyEntryPoints(sourceFile, destFile string) error { - // Read source file - sourceData, err := os.ReadFile(sourceFile) - if err != nil { - return fmt.Errorf("error reading source file: %w", err) - } - - // Read destination file - destData, err := os.ReadFile(destFile) - if err != nil { - return fmt.Errorf("error reading destination file: %w", err) - } - - // Parse source YAML - var sourceYAML map[string]interface{} - if err := yaml.Unmarshal(sourceData, &sourceYAML); err != nil { - return fmt.Errorf("error parsing source YAML: %w", err) - } - - // Parse destination YAML - var destYAML map[string]interface{} - if err := yaml.Unmarshal(destData, &destYAML); err != nil { - return fmt.Errorf("error parsing destination YAML: %w", err) - } - - // Get entryPoints section from source - entryPoints, ok := sourceYAML["entryPoints"] - if !ok { - return fmt.Errorf("entryPoints section not found in source file") - } - - // Update entryPoints in destination - destYAML["entryPoints"] = entryPoints - - // Marshal updated destination YAML - // updatedData, err := yaml.Marshal(destYAML) - updatedData, err := MarshalYAMLWithIndent(destYAML, 2) - if err != nil { - return fmt.Errorf("error marshaling updated YAML: %w", err) - } - - // Write updated YAML back to destination file - if err := os.WriteFile(destFile, updatedData, 0644); err != nil { - return fmt.Errorf("error writing to destination file: %w", err) - } - - return nil -} - -func copyWebsecureEntryPoint(sourceFile, destFile string) error { - // Read source file - sourceData, err := os.ReadFile(sourceFile) - if err != nil { - return fmt.Errorf("error reading source file: %w", err) - } - - // Read destination file - destData, err := os.ReadFile(destFile) - if err != nil { - return fmt.Errorf("error reading destination file: %w", err) - } - - // Parse source YAML - var sourceYAML map[string]interface{} - if err := yaml.Unmarshal(sourceData, &sourceYAML); err != nil { - return fmt.Errorf("error parsing source YAML: %w", err) - } - - // Parse destination YAML - var destYAML map[string]interface{} - if err := yaml.Unmarshal(destData, &destYAML); err != nil { - return fmt.Errorf("error parsing destination YAML: %w", err) - } - - // Get entryPoints section from source - entryPoints, ok := sourceYAML["entryPoints"].(map[string]interface{}) - if !ok { - return fmt.Errorf("entryPoints section not found in source file or has invalid format") - } - - // Get websecure configuration - websecure, ok := entryPoints["websecure"] - if !ok { - return fmt.Errorf("websecure entrypoint not found in source file") - } - - // Get or create entryPoints section in destination - destEntryPoints, ok := destYAML["entryPoints"].(map[string]interface{}) - if !ok { - // If entryPoints section doesn't exist, create it - destEntryPoints = make(map[string]interface{}) - destYAML["entryPoints"] = destEntryPoints - } - - // Update websecure in destination - destEntryPoints["websecure"] = websecure - - // Marshal updated destination YAML - // updatedData, err := yaml.Marshal(destYAML) - updatedData, err := MarshalYAMLWithIndent(destYAML, 2) - if err != nil { - return fmt.Errorf("error marshaling updated YAML: %w", err) - } - - // Write updated YAML back to destination file - if err := os.WriteFile(destFile, updatedData, 0644); err != nil { - return fmt.Errorf("error writing to destination file: %w", err) - } - - return nil -} - func copyDockerService(sourceFile, destFile, serviceName string) error { // Read source file sourceData, err := os.ReadFile(sourceFile) @@ -391,3 +279,75 @@ func CheckAndAddTraefikLogVolume(composePath string) error { fmt.Println("Added traefik log volume and created logs directory") return nil } + +// MergeYAML merges two YAML files, where the contents of the second file +// are merged into the first file. In case of conflicts, values from the +// second file take precedence. +func MergeYAML(baseFile, overlayFile string) error { + // Read the base YAML file + baseContent, err := os.ReadFile(baseFile) + if err != nil { + return fmt.Errorf("error reading base file: %v", err) + } + + // Read the overlay YAML file + overlayContent, err := os.ReadFile(overlayFile) + if err != nil { + return fmt.Errorf("error reading overlay file: %v", err) + } + + // Parse base YAML into a map + var baseMap map[string]interface{} + if err := yaml.Unmarshal(baseContent, &baseMap); err != nil { + return fmt.Errorf("error parsing base YAML: %v", err) + } + + // Parse overlay YAML into a map + var overlayMap map[string]interface{} + if err := yaml.Unmarshal(overlayContent, &overlayMap); err != nil { + return fmt.Errorf("error parsing overlay YAML: %v", err) + } + + // Merge the overlay into the base + merged := mergeMap(baseMap, overlayMap) + + // Marshal the merged result back to YAML + mergedContent, err := MarshalYAMLWithIndent(merged, 2) + if err != nil { + return fmt.Errorf("error marshaling merged YAML: %v", err) + } + + // Write the merged content back to the base file + if err := os.WriteFile(baseFile, mergedContent, 0644); err != nil { + return fmt.Errorf("error writing merged YAML: %v", err) + } + + return nil +} + +// mergeMap recursively merges two maps +func mergeMap(base, overlay map[string]interface{}) map[string]interface{} { + result := make(map[string]interface{}) + + // Copy all key-values from base map + for k, v := range base { + result[k] = v + } + + // Merge overlay values + for k, v := range overlay { + // If both maps have the same key and both values are maps, merge recursively + if baseVal, ok := base[k]; ok { + if baseMap, isBaseMap := baseVal.(map[string]interface{}); isBaseMap { + if overlayMap, isOverlayMap := v.(map[string]interface{}); isOverlayMap { + result[k] = mergeMap(baseMap, overlayMap) + continue + } + } + } + // Otherwise, overlay value takes precedence + result[k] = v + } + + return result +} diff --git a/install/crowdsec.go b/install/crowdsec.go index d98b2acf..2d56ecc6 100644 --- a/install/crowdsec.go +++ b/install/crowdsec.go @@ -33,23 +33,23 @@ func installCrowdsec(config Config) error { os.Exit(1) } - if err := copyWebsecureEntryPoint("config/crowdsec/traefik_config.yml", "config/traefik/traefik_config.yml"); err != nil { + if err := MergeYAML("config/traefik/traefik_config.yml", "config/crowdsec/traefik_config.yml"); err != nil { fmt.Printf("Error copying entry points: %v\n", err) os.Exit(1) } + // delete the 2nd file + if err := os.Remove("config/crowdsec/traefik_config.yml"); err != nil { + fmt.Printf("Error removing file: %v\n", err) + os.Exit(1) + } - if err := copyEntryPoints("config/traefik/traefik_config.yml", "config/crowdsec/traefik_config.yml"); err != nil { + if err := MergeYAML("config/traefik/dynamic_config.yml", "config/crowdsec/dynamic_config.yml"); err != nil { fmt.Printf("Error copying entry points: %v\n", err) os.Exit(1) } - - if err := moveFile("config/crowdsec/traefik_config.yml", "config/traefik/traefik_config.yml"); err != nil { - fmt.Printf("Error moving file: %v\n", err) - os.Exit(1) - } - - if err := moveFile("config/crowdsec/dynamic_config.yml", "config/traefik/dynamic_config.yml"); err != nil { - fmt.Printf("Error moving file: %v\n", err) + // delete the 2nd file + if err := os.Remove("config/crowdsec/dynamic_config.yml"); err != nil { + fmt.Printf("Error removing file: %v\n", err) os.Exit(1) }