Rspamd: more features (#3159)

test/config/rspamd/postfix-accounts.cf

@@ -0,0 +1,2 @@
+# password is 123
+user1@localhost.localdomain|{SHA512-CRYPT}$6$ARrjj74S83.7c53r$AZXMChav0r03LPEGAJJBmfUt59139rZ1zRIxrPhICSh.Y70Zjq6gClnF/cHDUG95dMoFt4Bkj6N4hvFSZ7L301

test/config/rspamd/user-patches.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+cat >/etc/rspamd/override.d/testmodule_complicated.conf << EOF
+complicated {
+    anOption = someValue;
+}
+EOF
+
+echo "enable_test_patterns = true;" >>/etc/rspamd/local.d/options.inc
+
+echo 'mail_debug = yes' >>/etc/dovecot/dovecot.conf
+sed -i -E '/^}/d' /etc/dovecot/conf.d/90-sieve.conf
+echo -e 'sieve_trace_debug = yes\n}' >>/etc/dovecot/conf.d/90-sieve.conf

test/test-files/email-templates/rspamd-pass.txt

@@ -0,0 +1,12 @@
+HELO mail.external.tld
+MAIL FROM: pass@example.test
+RCPT TO: user1@localhost.localdomain
+DATA
+From: Docker Mail Server <pass@example.test>
+To: Existing Local User <user1@localhost.localdomain>
+Date: Sat, 22 May 2010 07:43:25 -0400
+Subject: Test Message rspamd-pass.txt
+This mail should pass and Rspamd should not mark it.
+
+.
+QUIT

test/test-files/email-templates/rspamd-spam-header.txt

@@ -0,0 +1,12 @@
+HELO mail.example.test
+MAIL FROM: spam-header@example.test
+RCPT TO: user1@localhost.localdomain
+DATA
+From: Docker Mail Server <spam-header@example.test>
+To: Existing Local User <user1@localhost.localdomain>
+Date: Sat, 21 Jan 2023 11:11:11 +0000
+Subject: Test Message rspamd-spam-header.txt
+YJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X
+
+.
+QUIT

test/test-files/email-templates/rspamd-spam.txt

@@ -6,7 +6,6 @@ From: Docker Mail Server <spam@example.test>
 To: Existing Local User <user1@localhost.localdomain>
 Date: Sat, 21 Jan 2023 11:11:11 +0000
 Subject: Test Message rspamd-spam.txt
-This is a test mail.
 XJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X
 
 .

test/test-files/email-templates/rspamd-virus.txt

@@ -2,7 +2,7 @@ HELO mail.example.test
 MAIL FROM: virus@example.test
 RCPT TO: user1@localhost.localdomain
 DATA
-From: Docker Mail Server <dockermailserver@external.tld>
+From: Docker Mail Server <virus@example.test>
 To: Existing Local User <user1@localhost.localdomain>
 Date: Sat, 21 Jan 2023 11:11:11 +0000
 Subject: Test Message rspamd-virus.txt

test/test-files/nc_templates/rspamd_imap_move_to_inbox.txt

@@ -0,0 +1,4 @@
+A LOGIN user1@localhost.localdomain 123
+B SELECT Junk
+A UID MOVE 1:1 INBOX
+A4 LOGOUT

test/test-files/nc_templates/rspamd_imap_move_to_junk.txt

@@ -0,0 +1,4 @@
+A LOGIN user1@localhost.localdomain 123
+B SELECT INBOX
+A UID MOVE 1:1 Junk
+A4 LOGOUT

test/tests/parallel/set1/dovecot/dovecot_sieve.bats

@@ -37,27 +37,24 @@ function teardown_file() { _default_teardown ; }
 
 # dovecot-sieve/dovecot.sieve
 @test "User Sieve - should store mail from 'spam@spam.com' into recipient (user1) mailbox 'INBOX.spam'" {
-  _run_in_container_bash 'ls -A /var/mail/localhost.localdomain/user1/.INBOX.spam/new'
-  assert_success
-  _should_output_number_of_lines 1
+  _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/.INBOX.spam/new 1
 }
 
 # dovecot-sieve/before.dovecot.sieve
 @test "Global Sieve - should have copied mail from 'spam@spam.com' to recipient (user1) inbox" {
-  _run_in_container grep 'Spambot <spam@spam.com>' -R /var/mail/localhost.localdomain/user1/new/
+  _run_in_container grep -R 'Spambot <spam@spam.com>' /var/mail/localhost.localdomain/user1/new/
   assert_success
 }
 
 # dovecot-sieve/sieve-pipe + dovecot-sieve/user2@otherdomain.tld.dovecot.sieve
 @test "Sieve Pipe - should pipe mail received for user2 into '/tmp/pipe-test.out'" {
-  _run_in_container_bash 'ls -A /tmp/pipe-test.out'
+  _run_in_container_bash '[[ -f /tmp/pipe-test.out ]]'
   assert_success
-  _should_output_number_of_lines 1
 }
 
 # Only test coverage for feature is to check that the service is listening on the expected port:
 # https://doc.dovecot.org/admin_manual/pigeonhole_managesieve_server/
 @test "ENV 'ENABLE_MANAGESIEVE' - should have enabled service on port 4190" {
-  _run_in_container_bash 'nc -z 0.0.0.0 4190'
+  _run_in_container nc -z 0.0.0.0 4190
   assert_success
 }

test/tests/parallel/set1/spam_virus/rspamd.bats

@@ -16,8 +16,11 @@ function setup_file() {
     --env ENABLE_OPENDMARC=0
     --env PERMIT_DOCKER=host
     --env LOG_LEVEL=trace
+    --env MOVE_SPAM_TO_JUNK=1
+    --env RSPAMD_LEARN=1
   )
 
+  mv "${TEST_TMP_CONFIG}"/rspamd/* "${TEST_TMP_CONFIG}/"
   _common_container_setup 'CUSTOM_SETUP_ARGUMENTS'
 
   # wait for ClamAV to be fully setup or we will get errors on the log
@@ -31,12 +34,15 @@ function setup_file() {
 
   # We will send 3 emails: the first one should pass just fine; the second one should
   # be rejected due to spam; the third one should be rejected due to a virus.
-  export MAIL_ID1=$(_send_email_and_get_id 'email-templates/existing-user1')
+  export MAIL_ID1=$(_send_email_and_get_id 'email-templates/rspamd-pass')
   export MAIL_ID2=$(_send_email_and_get_id 'email-templates/rspamd-spam')
   export MAIL_ID3=$(_send_email_and_get_id 'email-templates/rspamd-virus')
+  export MAIL_ID4=$(_send_email_and_get_id 'email-templates/rspamd-spam-header')
 
-  # add a nested option to a module
-  _exec_in_container_bash "echo -e 'complicated {\n    anOption = someValue;\n}' >/etc/rspamd/override.d/testmodule_complicated.conf"
+  for ID in MAIL_ID{1,2,3,4}
+  do
+    [[ -n ${!ID} ]] || { echo "${ID} is empty - aborting!" ; return 1 ; }
+  done
 }
 
 function teardown_file() { _default_teardown ; }
@@ -44,6 +50,9 @@ function teardown_file() { _default_teardown ; }
 @test "Postfix's main.cf was adjusted" {
   _run_in_container grep -F 'smtpd_milters = $rspamd_milter' /etc/postfix/main.cf
   assert_success
+  _run_in_container postconf rspamd_milter
+  assert_success
+  assert_output 'rspamd_milter = inet:localhost:11332'
 }
 
 @test 'logs exist and contains proper content' {
@@ -62,6 +71,8 @@ function teardown_file() { _default_teardown ; }
 
   _print_mail_log_for_id "${MAIL_ID1}"
   assert_output --partial "stored mail into mailbox 'INBOX'"
+
+  _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/new/ 1
 }
 
 @test 'detects and rejects spam' {
@@ -71,6 +82,8 @@ function teardown_file() { _default_teardown ; }
   _print_mail_log_for_id "${MAIL_ID2}"
   assert_output --partial 'milter-reject'
   assert_output --partial '5.7.1 Gtube pattern'
+
+  _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/new/ 1
 }
 
 @test 'detects and rejects virus' {
@@ -81,6 +94,8 @@ function teardown_file() { _default_teardown ; }
   assert_output --partial 'milter-reject'
   assert_output --partial '5.7.1 ClamAV FOUND VIRUS "Eicar-Signature"'
   refute_output --partial "stored mail into mailbox 'INBOX'"
+
+  _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/new/ 1
 }
 
 @test 'custom commands work correctly' {
@@ -153,3 +168,78 @@ function teardown_file() { _default_teardown ; }
   _run_in_container grep -F 'OhMy = "PraiseBeLinters !";' "${MODULE_PATH}"
   assert_success
 }
+
+@test 'Check MOVE_SPAM_TO_JUNK works for Rspamd' {
+  _run_in_container_bash '[[ -f /usr/lib/dovecot/sieve-global/after/spam_to_junk.sieve ]]'
+  assert_success
+  _run_in_container_bash '[[ -f /usr/lib/dovecot/sieve-global/after/spam_to_junk.svbin ]]'
+  assert_success
+
+  _service_log_should_contain_string 'rspamd' 'S \(add header\)'
+  _service_log_should_contain_string 'rspamd' 'add header "Gtube pattern"'
+
+  _print_mail_log_for_id "${MAIL_ID4}"
+  assert_output --partial "fileinto action: stored mail into mailbox 'Junk'"
+
+  _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/new/ 1
+  _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/.Junk/new/ 1
+}
+
+@test 'Check RSPAMD_LEARN works' {
+  for FILE in learn-{ham,spam}.{sieve,svbin}
+  do
+    _run_in_container_bash "[[ -f /usr/lib/dovecot/sieve-pipe/${FILE} ]]"
+    assert_success
+  done
+
+  _run_in_container grep 'mail_plugins.*imap_sieve' /etc/dovecot/conf.d/20-imap.conf
+  local SIEVE_CONFIG_FILE='/etc/dovecot/conf.d/90-sieve.conf'
+  _run_in_container grep 'sieve_plugins.*sieve_imapsieve' "${SIEVE_CONFIG_FILE}"
+  _run_in_container grep 'sieve_global_extensions.*\+vnd\.dovecot\.pipe' "${SIEVE_CONFIG_FILE}"
+  _run_in_container grep -F 'sieve_pipe_bin_dir = /usr/lib/dovecot/sieve-pipe' "${SIEVE_CONFIG_FILE}"
+
+  # Move an email to the "Junk" folder from "INBOX"; the first email we
+  # sent should pass fine, hence we can now move it
+  _send_email 'nc_templates/rspamd_imap_move_to_junk' '0.0.0.0 143'
+  sleep 1 # wait for the transaction to finish
+
+  local MOVE_TO_JUNK_LINES=(
+    'imapsieve: mailbox Junk: MOVE event'
+    'imapsieve: Matched static mailbox rule [1]'
+    "sieve: file storage: script: Opened script \`learn-spam'"
+    'sieve: file storage: Using Sieve script path: /usr/lib/dovecot/sieve-pipe/learn-spam.sieve'
+    "sieve: Executing script from \`/usr/lib/dovecot/sieve-pipe/learn-spam.svbin'"
+    "Finished running script \`/usr/lib/dovecot/sieve-pipe/learn-spam.svbin'"
+    'sieve: action pipe: running program: rspamc'
+    "pipe action: piped message to program \`rspamc'"
+    "left message in mailbox 'Junk'"
+  )
+
+  _run_in_container cat /var/log/mail/mail.log
+  assert_success
+  for LINE in "${MOVE_TO_JUNK_LINES[@]}"
+  do
+    assert_output --partial "${LINE}"
+  done
+
+  # Move an email to the "INBOX" folder from "Junk"; there should be two mails
+  # in the "Junk" folder
+  _send_email 'nc_templates/rspamd_imap_move_to_inbox' '0.0.0.0 143'
+  sleep 1 # wait for the transaction to finish
+
+  local MOVE_TO_JUNK_LINES=(
+    'imapsieve: Matched static mailbox rule [2]'
+    "sieve: file storage: script: Opened script \`learn-ham'"
+    'sieve: file storage: Using Sieve script path: /usr/lib/dovecot/sieve-pipe/learn-ham.sieve'
+    "sieve: Executing script from \`/usr/lib/dovecot/sieve-pipe/learn-ham.svbin'"
+    "Finished running script \`/usr/lib/dovecot/sieve-pipe/learn-ham.svbin'"
+    "left message in mailbox 'INBOX'"
+  )
+
+  _run_in_container cat /var/log/mail/mail.log
+  assert_success
+  for LINE in "${MOVE_TO_JUNK_LINES[@]}"
+  do
+    assert_output --partial "${LINE}"
+  done
+}