update BATS & helper + minor updates to BATS variables (#2988)

test/tests/parallel/set1/spam_virus/amavis.bats

@@ -1,7 +1,7 @@
 load "${REPOSITORY_ROOT}/test/helper/common"
 load "${REPOSITORY_ROOT}/test/helper/setup"
 
-TEST_NAME_PREFIX='Amavis:'
+BATS_TEST_NAME_PREFIX='[Amavis] '
 CONTAINER_NAME='dms-test_amavis'
 
 function setup_file() {
@@ -18,8 +18,10 @@ function setup_file() {
 
 function teardown_file() { _default_teardown ; }
 
-@test "${TEST_NAME_PREFIX} Amavis integration should be active" {
-  _run_in_container grep 'ANTI-SPAM-SA' /var/log/mail/mail.log
+@test "SpamAssassin integration should be active" {
+  # give Amavis just a bit of time to print out its full debug log
+  run repeat_in_container_until_success_or_timeout 5 "${CONTAINER_NAME}" grep 'ANTI-SPAM-SA' /var/log/mail/mail.log
+  assert_success
   assert_output --partial 'loaded'
   refute_output --partial 'NOT loaded'
 }

test/tests/parallel/set1/spam_virus/clamav.bats

@@ -1,7 +1,7 @@
 load "${REPOSITORY_ROOT}/test/helper/setup"
 load "${REPOSITORY_ROOT}/test/helper/common"
 
-TEST_NAME_PREFIX='ClamAV:'
+BATS_TEST_NAME_PREFIX='[ClamAV] '
 CONTAINER_NAME='dms-test_clamav'
 
 function setup_file() {
@@ -34,38 +34,38 @@ function setup_file() {
 
 function teardown_file() { _default_teardown ; }
 
-@test "${TEST_NAME_PREFIX} process clamd is running" {
+@test "process clamd is running" {
   run check_if_process_is_running 'clamd'
   assert_success
 }
 
-@test "${TEST_NAME_PREFIX} log files exist at /var/log/mail directory" {
+@test "log files exist at /var/log/mail directory" {
   _run_in_container bash -c "ls -1 /var/log/mail/ | grep -E 'clamav|freshclam|mail.log' | wc -l"
   assert_success
   assert_output 3
 }
 
-@test "${TEST_NAME_PREFIX} should be identified by Amavis" {
+@test "should be identified by Amavis" {
   _run_in_container grep -i 'Found secondary av scanner ClamAV-clamscan' /var/log/mail/mail.log
   assert_success
 }
 
-@test "${TEST_NAME_PREFIX} freshclam cron is enabled" {
+@test "freshclam cron is enabled" {
   _run_in_container bash -c "grep '/usr/bin/freshclam' -r /etc/cron.d"
   assert_success
 }
 
-@test "${TEST_NAME_PREFIX} env CLAMAV_MESSAGE_SIZE_LIMIT is set correctly" {
+@test "env CLAMAV_MESSAGE_SIZE_LIMIT is set correctly" {
   _run_in_container grep -q '^MaxFileSize 30M$' /etc/clamav/clamd.conf
   assert_success
 }
 
-@test "${TEST_NAME_PREFIX} rejects virus" {
+@test "rejects virus" {
   _run_in_container bash -c "grep 'Blocked INFECTED' /var/log/mail/mail.log | grep '<virus@external.tld> -> <user1@localhost.localdomain>'"
   assert_success
 }
 
-@test "${TEST_NAME_PREFIX} process clamd restarts when killed" {
+@test "process clamd restarts when killed" {
   _run_in_container pkill 'clamd'
   assert_success
 

test/tests/parallel/set1/spam_virus/disabled_clamav_spamassassin.bats

@@ -1,7 +1,7 @@
 load "${REPOSITORY_ROOT}/test/helper/setup"
 load "${REPOSITORY_ROOT}/test/helper/common"
 
-TEST_NAME_PREFIX='[ClamAV + SA] (disabled):'
+BATS_TEST_NAME_PREFIX='[ClamAV + SA] (disabled) '
 CONTAINER_NAME='dms-test_clamav-spamassasin_disabled'
 
 function setup_file() {
@@ -24,22 +24,22 @@ function setup_file() {
 
 function teardown_file() { _default_teardown ; }
 
-@test "${TEST_NAME_PREFIX} ClamAV - should be disabled by ENV 'ENABLED_CLAMAV=0'" {
+@test "ClamAV - should be disabled by ENV 'ENABLED_CLAMAV=0'" {
   run check_if_process_is_running 'clamd'
   assert_failure
 }
 
-@test "${TEST_NAME_PREFIX} ClamAV - Amavis integration should not be active" {
+@test "ClamAV - Amavis integration should not be active" {
   _run_in_container grep -i 'Found secondary av scanner ClamAV-clamscan' /var/log/mail/mail.log
   assert_failure
 }
 
-@test "${TEST_NAME_PREFIX} SA - Amavis integration should not be active" {
+@test "SA - Amavis integration should not be active" {
   _run_in_container bash -c "grep -i 'ANTI-SPAM-SA code' /var/log/mail/mail.log | grep 'NOT loaded'"
   assert_success
 }
 
-@test "${TEST_NAME_PREFIX} SA - should not have been called" {
+@test "SA - should not have been called" {
   _run_in_container grep -i 'connect to /var/run/clamav/clamd.ctl failed' /var/log/mail/mail.log
   assert_failure
 }

test/tests/parallel/set1/spam_virus/dnsbl.bats

@@ -1,8 +1,7 @@
 load "${REPOSITORY_ROOT}/test/helper/setup"
 load "${REPOSITORY_ROOT}/test/helper/common"
 
-TEST_NAME_PREFIX='DNSBLs:'
-
+BATS_TEST_NAME_PREFIX='[DNSBLs] '
 CONTAINER1_NAME='dms-test_dnsbl_enabled'
 CONTAINER2_NAME='dms-test_dnsbl_disabled'
 
@@ -29,33 +28,33 @@ function teardown_file() {
 }
 
 # ENABLE_DNSBL=1
-@test "${TEST_NAME_PREFIX} (enabled) Postfix DNS block list zen.spamhaus.org" {
+@test "(enabled) Postfix DNS block list zen.spamhaus.org" {
   run docker exec "${CONTAINER1_NAME}" postconf smtpd_recipient_restrictions
   assert_output --partial 'reject_rbl_client zen.spamhaus.org'
 }
 
-@test "${TEST_NAME_PREFIX} (enabled) Postscreen DNS block lists -> postscreen_dnsbl_action" {
+@test "(enabled) Postscreen DNS block lists -> postscreen_dnsbl_action" {
   run docker exec "${CONTAINER1_NAME}" postconf postscreen_dnsbl_action
   assert_output 'postscreen_dnsbl_action = enforce'
 }
 
-@test "${TEST_NAME_PREFIX} (enabled) Postscreen DNS block lists -> postscreen_dnsbl_sites" {
+@test "(enabled) Postscreen DNS block lists -> postscreen_dnsbl_sites" {
   run docker exec "${CONTAINER1_NAME}" postconf postscreen_dnsbl_sites
   assert_output 'postscreen_dnsbl_sites = zen.spamhaus.org=127.0.0.[2..11]*3 bl.mailspike.net=127.0.0.[2;14;13;12;11;10] b.barracudacentral.org*2 bl.spameatingmonkey.net=127.0.0.2 dnsbl.sorbs.net psbl.surriel.com list.dnswl.org=127.0.[0..255].0*-2 list.dnswl.org=127.0.[0..255].1*-3 list.dnswl.org=127.0.[0..255].[2..3]*-4'
 }
 
 # ENABLE_DNSBL=0
-@test "${TEST_NAME_PREFIX} (disabled) Postfix DNS block list zen.spamhaus.org" {
+@test "(disabled) Postfix DNS block list zen.spamhaus.org" {
   run docker exec "${CONTAINER2_NAME}" postconf smtpd_recipient_restrictions
   refute_output --partial 'reject_rbl_client zen.spamhaus.org'
 }
 
-@test "${TEST_NAME_PREFIX} (disabled) Postscreen DNS block lists -> postscreen_dnsbl_action" {
+@test "(disabled) Postscreen DNS block lists -> postscreen_dnsbl_action" {
   run docker exec "${CONTAINER2_NAME}" postconf postscreen_dnsbl_action
   assert_output 'postscreen_dnsbl_action = ignore'
 }
 
-@test "${TEST_NAME_PREFIX} (disabled) Postscreen DNS block lists -> postscreen_dnsbl_sites" {
+@test "(disabled) Postscreen DNS block lists -> postscreen_dnsbl_sites" {
   run docker exec "${CONTAINER2_NAME}" postconf postscreen_dnsbl_sites
   assert_output 'postscreen_dnsbl_sites ='
 }

test/tests/parallel/set1/spam_virus/fail2ban.bats

@@ -1,7 +1,7 @@
 load "${REPOSITORY_ROOT}/test/helper/setup"
 load "${REPOSITORY_ROOT}/test/helper/common"
 
-TEST_NAME_PREFIX='Fail2Ban:'
+BATS_TEST_NAME_PREFIX='[Fail2Ban] '
 CONTAINER1_NAME='dms-test_fail2ban'
 CONTAINER2_NAME='dms-test_fail2ban_fail-auth-mailer'
 
@@ -33,12 +33,12 @@ function teardown_file() {
   docker rm -f "${CONTAINER1_NAME}" "${CONTAINER2_NAME}"
 }
 
-@test "${TEST_NAME_PREFIX} Fail2Ban is running" {
+@test "Fail2Ban is running" {
   run check_if_process_is_running 'fail2ban-server'
   assert_success
 }
 
-@test "${TEST_NAME_PREFIX} localhost is not banned because ignored" {
+@test "localhost is not banned because ignored" {
   _run_in_container fail2ban-client status postfix-sasl
   assert_success
   refute_output --regexp '.*IP list:.*127\.0\.0\.1.*'
@@ -47,13 +47,13 @@ function teardown_file() {
   assert_success
 }
 
-@test "${TEST_NAME_PREFIX} fail2ban-fail2ban.cf overrides" {
+@test "fail2ban-fail2ban.cf overrides" {
   _run_in_container fail2ban-client get loglevel
   assert_success
   assert_output --partial 'DEBUG'
 }
 
-@test "${TEST_NAME_PREFIX} fail2ban-jail.cf overrides" {
+@test "fail2ban-jail.cf overrides" {
   for FILTER in 'dovecot' 'postfix' 'postfix-sasl'
   do
     _run_in_container fail2ban-client get "${FILTER}" bantime
@@ -76,7 +76,7 @@ function teardown_file() {
 # - After multiple login fails and a slight delay, f2b will ban that IP.
 # - You could hard-code `sleep 5` on both cases to avoid the alternative assertions,
 #   but the polling + piping into grep approach here reliably minimizes the delay.
-@test "${TEST_NAME_PREFIX} ban ip on multiple failed login" {
+@test "ban ip on multiple failed login" {
   CONTAINER1_IP=$(get_container_ip ${CONTAINER1_NAME})
   # Trigger a ban by failing to login twice:
   _run_in_container_explicit "${CONTAINER2_NAME}" bash -c "nc ${CONTAINER1_IP} 25 < /tmp/docker-mailserver-test/auth/smtp-auth-login-wrong.txt"
@@ -95,7 +95,7 @@ function teardown_file() {
 }
 
 # NOTE: Depends on previous test case, if no IP was banned at this point, it passes regardless..
-@test "${TEST_NAME_PREFIX} unban ip works" {
+@test "unban ip works" {
   CONTAINER2_IP=$(get_container_ip ${CONTAINER2_NAME})
   _run_in_container fail2ban-client set postfix-sasl unbanip "${CONTAINER2_IP}"
   assert_success
@@ -110,7 +110,7 @@ function teardown_file() {
   refute_output --partial "${CONTAINER2_IP}"
 }
 
-@test "${TEST_NAME_PREFIX} bans work properly (single IP)" {
+@test "bans work properly (single IP)" {
   _run_in_container fail2ban ban 192.0.66.7
   assert_success
   assert_output 'Banned custom IP: 1'
@@ -131,7 +131,7 @@ function teardown_file() {
   refute_output --partial '192.0.66.7'
 }
 
-@test "${TEST_NAME_PREFIX} bans work properly (subnet)" {
+@test "bans work properly (subnet)" {
   _run_in_container fail2ban ban 192.0.66.0/24
   assert_success
   assert_output 'Banned custom IP: 1'
@@ -152,7 +152,7 @@ function teardown_file() {
   refute_output --partial '192.0.66.0/24'
 }
 
-@test "${TEST_NAME_PREFIX} FAIL2BAN_BLOCKTYPE is really set to drop" {
+@test "FAIL2BAN_BLOCKTYPE is really set to drop" {
   # ban IPs here manually so we can be sure something is inside the jails
   for JAIL in dovecot postfix-sasl custom; do
     _run_in_container fail2ban-client set "${JAIL}" banip 192.33.44.55
@@ -172,7 +172,7 @@ function teardown_file() {
   done
 }
 
-@test "${TEST_NAME_PREFIX} setup.sh fail2ban" {
+@test "setup.sh fail2ban" {
   _run_in_container fail2ban-client set dovecot banip 192.0.66.4
   _run_in_container fail2ban-client set dovecot banip 192.0.66.5
 
@@ -194,7 +194,7 @@ function teardown_file() {
   assert_output --partial 'You need to specify an IP address: Run'
 }
 
-@test "${TEST_NAME_PREFIX} restart of Fail2Ban" {
+@test "restart of Fail2Ban" {
   _run_in_container pkill fail2ban
   assert_success
 

test/tests/parallel/set1/spam_virus/postgrey_enabled.bats

@@ -1,7 +1,7 @@
 load "${REPOSITORY_ROOT}/test/helper/setup"
 load "${REPOSITORY_ROOT}/test/helper/common"
 
-TEST_NAME_PREFIX='Postgrey (enabled):'
+BATS_TEST_NAME_PREFIX='[Postgrey] (enabled) '
 CONTAINER_NAME='dms-test_postgrey_enabled'
 
 function setup_file() {
@@ -24,13 +24,13 @@ function setup_file() {
 
 function teardown_file() { _default_teardown ; }
 
-@test "${TEST_NAME_PREFIX} should have added Postgrey to 'main.cf:check_policy_service'" {
+@test "should have added Postgrey to 'main.cf:check_policy_service'" {
   _run_in_container grep -F 'check_policy_service inet:127.0.0.1:10023' /etc/postfix/main.cf
   assert_success
   _should_output_number_of_lines 1
 }
 
-@test "${TEST_NAME_PREFIX} should have configured /etc/default/postgrey with default values and ENV overrides" {
+@test "should have configured /etc/default/postgrey with default values and ENV overrides" {
   _run_in_container grep -F 'POSTGREY_OPTS="--inet=127.0.0.1:10023 --delay=3 --max-age=35 --auto-whitelist-clients=5"' /etc/default/postgrey
   assert_success
   _should_output_number_of_lines 1
@@ -40,12 +40,12 @@ function teardown_file() { _default_teardown ; }
   _should_output_number_of_lines 1
 }
 
-@test "${TEST_NAME_PREFIX} Postgrey is running" {
+@test "Postgrey is running" {
   run check_if_process_is_running 'postgrey'
   assert_success
 }
 
-@test "${TEST_NAME_PREFIX} should initially reject (greylist) mail from 'user@external.tld'" {
+@test "should initially reject (greylist) mail from 'user@external.tld'" {
   # Modify the postfix config in order to ensure that postgrey handles the test e-mail.
   # The other spam checks in `main.cf:smtpd_recipient_restrictions` would interfere with testing postgrey.
   _run_in_container bash -c "sed -ie 's/permit_sasl_authenticated.*policyd-spf,$//g' /etc/postfix/main.cf"
@@ -69,8 +69,8 @@ function teardown_file() { _default_teardown ; }
 }
 
 # NOTE: This test case depends on the previous one
-@test "${TEST_NAME_PREFIX} should accept mail from 'user@external.tld' after POSTGREY_DELAY duration" {
-  # Wait until `$POSTGREY_DELAY` seconds pass before trying again: 
+@test "should accept mail from 'user@external.tld' after POSTGREY_DELAY duration" {
+  # Wait until `$POSTGREY_DELAY` seconds pass before trying again:
   sleep 3
   # Retry delivering test mail (it should be trusted this time):
   _send_test_mail '/tmp/docker-mailserver-test/email-templates/postgrey.txt' '25'
@@ -90,7 +90,7 @@ function teardown_file() { _default_teardown ; }
 #   - However this does not help verify that the actual client HELO address is properly whitelisted?
 #   - It'd also cause the earlier greylist test to fail.
 # - TODO: Actually confirm whitelist feature works correctly as these test cases are using a workaround:
-@test "${TEST_NAME_PREFIX} should whitelist sender 'user@whitelist.tld'" {
+@test "should whitelist sender 'user@whitelist.tld'" {
   _send_test_mail '/tmp/docker-mailserver-test/nc_templates/postgrey_whitelist.txt' '10023'
 
   _should_have_log_entry \
@@ -99,7 +99,7 @@ function teardown_file() { _default_teardown ; }
     'client_address=127.0.0.1/32, sender=test@whitelist.tld, recipient=user1@localhost.localdomain'
 }
 
-@test "${TEST_NAME_PREFIX} should whitelist recipient 'user2@otherdomain.tld'" {
+@test "should whitelist recipient 'user2@otherdomain.tld'" {
   _send_test_mail '/tmp/docker-mailserver-test/nc_templates/postgrey_whitelist_recipients.txt' '10023'
 
   _should_have_log_entry \

test/tests/parallel/set1/spam_virus/postscreen.bats

@@ -1,7 +1,7 @@
 load "${REPOSITORY_ROOT}/test/helper/setup"
 load "${REPOSITORY_ROOT}/test/helper/common"
 
-TEST_NAME_PREFIX='Postscreen:'
+BATS_TEST_NAME_PREFIX='[Postscreen] '
 CONTAINER1_NAME='dms-test_postscreen_enforce'
 CONTAINER2_NAME='dms-test_postscreen_sender'
 
@@ -37,7 +37,7 @@ function teardown_file() {
   docker rm -f "${CONTAINER1_NAME}" "${CONTAINER2_NAME}"
 }
 
-@test "${TEST_NAME_PREFIX} should fail login when talking out of turn" {
+@test "should fail login when talking out of turn" {
   _run_in_container_explicit "${CONTAINER2_NAME}" bash -c "nc ${CONTAINER1_IP} 25 < /tmp/docker-mailserver-test/auth/smtp-auth-login.txt"
   assert_success
   assert_output --partial '502 5.5.2 Error: command not recognized'
@@ -47,7 +47,7 @@ function teardown_file() {
   assert_output --partial 'COMMAND PIPELINING'
 }
 
-@test "${TEST_NAME_PREFIX} should successfully login (respecting postscreen_greet_wait time)" {
+@test "should successfully login (respecting postscreen_greet_wait time)" {
   # NOTE: Sometimes fails on first attempt (trying too soon?),
   # Instead of a `run` + asserting partial, Using repeat + internal grep match:
   repeat_until_success_or_timeout 10 _should_wait_turn_speaking_smtp \

test/tests/parallel/set1/spam_virus/spam_bounced.bats

@@ -1,7 +1,7 @@
 load "${REPOSITORY_ROOT}/test/helper/setup"
 load "${REPOSITORY_ROOT}/test/helper/common"
 
-TEST_NAME_PREFIX='Spam bounced:'
+BATS_TEST_NAME_PREFIX='[Spam] (bounced) '
 CONTAINER_NAME='dms-test_spam-bounced'
 
 function setup_file() {
@@ -25,7 +25,7 @@ function teardown_file() { _default_teardown ; }
 # When SPAMASSASSIN_SPAM_TO_INBOX=0, spam messages must be bounced (rejected).
 # SPAMASSASSIN_SPAM_TO_INBOX=1 is covered in `mail_spam_junk_folder.bats`.
 # Original test PR: https://github.com/docker-mailserver/docker-mailserver/pull/1485
-@test "${TEST_NAME_PREFIX} spam message is bounced (rejected)" {
+@test "spam message is bounced (rejected)" {
   # send a spam message
   _run_in_container /bin/sh -c "nc 0.0.0.0 25 < /tmp/docker-mailserver-test/email-templates/amavis-spam.txt"
   assert_success

test/tests/parallel/set1/spam_virus/spam_junk_folder.bats

@@ -1,7 +1,7 @@
 load "${REPOSITORY_ROOT}/test/helper/setup"
 load "${REPOSITORY_ROOT}/test/helper/common"
 
-TEST_NAME_PREFIX='Spam junk folder:'
+BATS_TEST_NAME_PREFIX='[Spam] (junk folder) '
 CONTAINER1_NAME='dms-test_spam-junk-folder_1'
 CONTAINER2_NAME='dms-test_spam-junk-folder_2'
 
@@ -12,7 +12,7 @@ function teardown() { _default_teardown ; }
 # When SPAMASSASSIN_SPAM_TO_INBOX=1, spam messages must be delivered
 # and eventually (MOVE_SPAM_TO_JUNK=1) moved to the Junk folder.
 
-@test "${TEST_NAME_PREFIX} (Amavis) spam message delivered & moved to Junk folder" {
+@test "(Amavis) spam message delivered & moved to Junk folder" {
   export CONTAINER_NAME=${CONTAINER1_NAME}
   local CUSTOM_SETUP_ARGUMENTS=(
     --env ENABLE_AMAVIS=1
@@ -39,7 +39,7 @@ function teardown() { _default_teardown ; }
   assert_success
 }
 
-@test "${TEST_NAME_PREFIX} (Amavis) spam message delivered to INBOX" {
+@test "(Amavis) spam message delivered to INBOX" {
   export CONTAINER_NAME=${CONTAINER2_NAME}
   local CUSTOM_SETUP_ARGUMENTS=(
     --env ENABLE_AMAVIS=1

test/tests/parallel/set1/spam_virus/undef_spam_subject.bats

@@ -1,7 +1,7 @@
 load "${REPOSITORY_ROOT}/test/helper/setup"
 load "${REPOSITORY_ROOT}/test/helper/common"
 
-TEST_NAME_PREFIX='Undefined spam subject:'
+BATS_TEST_NAME_PREFIX='[Spam] (undefined subject) '
 
 CONTAINER1_NAME='dms-test_spam-undef-subject_1'
 CONTAINER2_NAME='dms-test_spam-undef-subject_2'
@@ -9,7 +9,7 @@ CONTAINER_NAME=${CONTAINER2_NAME}
 
 function teardown() { _default_teardown ; }
 
-@test "${TEST_NAME_PREFIX} 'SA_SPAM_SUBJECT=undef' should update Amavis config" {
+@test "'SA_SPAM_SUBJECT=undef' should update Amavis config" {
   export CONTAINER_NAME=${CONTAINER1_NAME}
   local CUSTOM_SETUP_ARGUMENTS=(
     --env ENABLE_AMAVIS=1
@@ -24,7 +24,7 @@ function teardown() { _default_teardown ; }
 }
 
 # TODO: Unclear why some of these ENV are relevant for the test?
-@test "${TEST_NAME_PREFIX} Docker env variables are set correctly (custom)" {
+@test "Docker env variables are set correctly (custom)" {
   export CONTAINER_NAME=${CONTAINER2_NAME}
 
   local CUSTOM_SETUP_ARGUMENTS=(