rspamd: add feature for adjusting options with a file parsed by DMS (#3059)

Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com>

test/config/rspamd-modules.conf

@@ -0,0 +1,42 @@
+# This is suboptimal, but not strictly required either: we do not
+# need the DKIM signing & RBL functionality (which is most likely
+# not set up anyway or it fails (in case of querying DNSBLs with
+# public resolvers)). Hence, we disable it. It is suboptimal
+# since we are testing the functionality of whether disabling
+# a module works at the same time..
+#
+# When testing on ARM64, this is required at the moment due to
+# strange behavior in Rspamd v3.2 (segmentation faults) when DKIM
+# keys are not provided (notes were added to the docs).
+disable-module dkim_signing
+disable-module rbl
+
+# check whether disabling a module works and whether it
+# really overwrites options that came before
+set-option-for-module testmodule1 someoption somevalue
+disable-module testmodule1
+
+# enabling a module and setting some options for it
+enable-module testmodule2
+set-option-for-module testmodule2 someoption somevalue
+set-option-for-module testmodule2 anotheroption whatAvaLue
+
+# overwriting an option (probably unwanted, we emit a warning)
+set-option-for-module testmodule3 someoption somevalue1
+set-option-for-module testmodule3 someoption somevalue2
+
+# check whether adding a line works even with special characters in it
+add-line testmodule4.something some very long line with "weird $charact"ers
+add-line testmodule4.something and! ano. ther &line
+add-line testmodule4.something # some comment
+
+# check whether spaces in front are handles fine, for nested options
+set-option-for-module testmodule_complicated anOption anotherValue
+
+# check whether controller and proxy options are set/overwritten correctly
+set-option-for-controller someOption someValue42
+set-option-for-proxy abcdefg71 RaNDom
+set-option-for-proxy abcdefg71 RAAAANdooM
+
+# check whether setting basic options works (and whether spaces in values work)
+set-common-option OhMy "PraiseBeLinters !"

test/helper/sending.bash

@@ -54,15 +54,15 @@ function _send_mail_and_get_id() {
   _send_email "${TEMPLATE_FILE}"
   _wait_for_empty_mail_queue_in_container
 
+  # The unique ID Postfix (and other services) use may be different in length
+  # on different systems (e.g. amd64 (11) vs aarch64 (10)). Hence, we use a
+  # range to safely capture it.
   MAIL_ID=$(_exec_in_container tac /var/log/mail.log              \
     | grep -E -m 1 'postfix/smtpd.*: [A-Z0-9]+: client=localhost' \
-    | grep -E -o '[A-Z0-9]{11}')
+    | grep -E -o '[A-Z0-9]{9,12}' || true)
 
-  if [[ -z ${MAIL_ID} ]]
-  then
-    echo 'Could not obtain mail ID - aborting!' >&2
-    exit 1
-  fi
+  run bash -c "-z ${MAIL_ID}"
+  assert_success 'Could not obtain mail ID - aborting!'
 
   echo "${MAIL_ID}"
 }

test/tests/parallel/set1/spam_virus/rspamd.bats

@@ -25,6 +25,7 @@ function setup_file() {
 
   _wait_for_service redis
   _wait_for_service rspamd
+  _wait_for_service clamav
   _wait_for_service postfix
   _wait_for_smtp_port_in_container
 
@@ -33,16 +34,19 @@ function setup_file() {
   export MAIL_ID1=$(_send_mail_and_get_id 'existing-user1')
   export MAIL_ID2=$(_send_mail_and_get_id 'rspamd-spam')
   export MAIL_ID3=$(_send_mail_and_get_id 'rspamd-virus')
+
+  # add a nested option to a module
+  _exec_in_container_bash "echo -e 'complicated {\n    anOption = someValue;\n}' >/etc/rspamd/override.d/testmodule_complicated.conf"
 }
 
 function teardown_file() { _default_teardown ; }
 
 @test "Postfix's main.cf was adjusted" {
-  _run_in_container grep -q 'smtpd_milters = inet:localhost:11332' /etc/postfix/main.cf
+  _run_in_container grep -F 'smtpd_milters = inet:localhost:11332' /etc/postfix/main.cf
   assert_success
 }
 
-@test "logs exist and contains proper content" {
+@test 'logs exist and contains proper content' {
   _service_log_should_contain_string 'rspamd' 'rspamd .* is loading configuration'
   _service_log_should_contain_string 'rspamd' 'lua module clickhouse is disabled in the configuration'
   _service_log_should_contain_string 'rspamd' 'lua module elastic is disabled in the configuration'
@@ -53,14 +57,14 @@ function teardown_file() { _default_teardown ; }
   _service_log_should_contain_string 'rspamd' 'lua module metric_exporter is disabled in the configuration'
 }
 
-@test "normal mail passes fine" {
+@test 'normal mail passes fine' {
   _service_log_should_contain_string 'rspamd' 'F \(no action\)'
 
   _print_mail_log_for_id "${MAIL_ID1}"
   assert_output --partial "stored mail into mailbox 'INBOX'"
 }
 
-@test "detects and rejects spam" {
+@test 'detects and rejects spam' {
   _service_log_should_contain_string 'rspamd' 'S \(reject\)'
   _service_log_should_contain_string 'rspamd' 'reject "Gtube pattern"'
 
@@ -69,7 +73,7 @@ function teardown_file() { _default_teardown ; }
   assert_output --partial '5.7.1 Gtube pattern'
 }
 
-@test "detects and rejects virus" {
+@test 'detects and rejects virus' {
   _service_log_should_contain_string 'rspamd' 'T \(reject\)'
   _service_log_should_contain_string 'rspamd' 'reject "ClamAV FOUND VIRUS "Eicar-Signature"'
 
@@ -78,3 +82,74 @@ function teardown_file() { _default_teardown ; }
   assert_output --partial '5.7.1 ClamAV FOUND VIRUS "Eicar-Signature"'
   refute_output --partial "stored mail into mailbox 'INBOX'"
 }
+
+@test 'custom commands work correctly' {
+  # check `testmodule1` which should be disabled
+  local MODULE_PATH='/etc/rspamd/override.d/testmodule1.conf'
+  _run_in_container_bash "[[ -f ${MODULE_PATH} ]]"
+  assert_success
+  _run_in_container grep -F '# documentation: https://rspamd.com/doc/modules/testmodule1.html' "${MODULE_PATH}"
+  assert_success
+  _run_in_container grep -F 'enabled = false;' "${MODULE_PATH}"
+  assert_success
+  _run_in_container grep -F 'someoption = somevalue;' "${MODULE_PATH}"
+  assert_failure
+
+  # check `testmodule2` which should be enabled and it should have extra options set
+  MODULE_PATH='/etc/rspamd/override.d/testmodule2.conf'
+  _run_in_container_bash "[[ -f ${MODULE_PATH} ]]"
+  assert_success
+  _run_in_container grep -F '# documentation: https://rspamd.com/doc/modules/testmodule2.html' "${MODULE_PATH}"
+  assert_success
+  _run_in_container grep -F 'enabled = true;' "${MODULE_PATH}"
+  assert_success
+  _run_in_container grep -F 'someoption = somevalue;' "${MODULE_PATH}"
+  assert_success
+  _run_in_container grep -F 'anotheroption = whatAvaLue;' "${MODULE_PATH}"
+  assert_success
+
+  # check whether writing the same option twice overwrites the first value in `testmodule3`
+  MODULE_PATH='/etc/rspamd/override.d/testmodule3.conf'
+  _run_in_container grep -F 'someoption = somevalue;' "${MODULE_PATH}"
+  assert_failure
+  _run_in_container grep -F 'someoption = somevalue2;' "${MODULE_PATH}"
+  assert_success
+
+  # check whether adding a single line writes the line properly in `testmodule4.something`
+  MODULE_PATH='/etc/rspamd/override.d/testmodule4.something'
+  _run_in_container_bash "[[ -f ${MODULE_PATH} ]]"
+  assert_success
+  _run_in_container grep -F 'some very long line with "weird $charact"ers' "${MODULE_PATH}"
+  assert_success
+  _run_in_container grep -F 'and! ano. ther &line' "${MODULE_PATH}"
+  assert_success
+  _run_in_container grep -F '# some comment' "${MODULE_PATH}"
+  assert_success
+
+  # check whether spaces in front of options are handles properly in `testmodule_complicated`
+  MODULE_PATH='/etc/rspamd/override.d/testmodule_complicated.conf'
+  _run_in_container_bash "[[ -f ${MODULE_PATH} ]]"
+  assert_success
+  _run_in_container grep -F '    anOption = anotherValue;' "${MODULE_PATH}"
+
+  # check whether controller option was written properly
+  MODULE_PATH='/etc/rspamd/override.d/worker-controller.inc'
+  _run_in_container_bash "[[ -f ${MODULE_PATH} ]]"
+  assert_success
+  _run_in_container grep -F 'someOption = someValue42;' "${MODULE_PATH}"
+  assert_success
+
+  # check whether controller option was written properly
+  MODULE_PATH='/etc/rspamd/override.d/worker-proxy.inc'
+  _run_in_container_bash "[[ -f ${MODULE_PATH} ]]"
+  assert_success
+  _run_in_container grep -F 'abcdefg71 = RAAAANdooM;' "${MODULE_PATH}"
+  assert_success
+
+  # check whether basic options are written properly
+  MODULE_PATH='/etc/rspamd/override.d/options.inc'
+  _run_in_container_bash "[[ -f ${MODULE_PATH} ]]"
+  assert_success
+  _run_in_container grep -F 'OhMy = "PraiseBeLinters !";' "${MODULE_PATH}"
+  assert_success
+}