tests: new sending and filtering functions (#3786)

* move log/filter functions into own file

* add ShellCheck global directives

* use new function for tracking logs

The new function, called `_send_email_with_mid`, aligns with suggestions
from @polarethene and is heavily simplified compared to its predecessor
`_send_email_and_get_id`. New helpers will be introduced to filter logs
according to the MID constructed in this function.

* new filters for searching logs with MID

* use new filters (and sending) functions

* add new helper for asserting non-existence of log message

* use new filters in tests

* Apply suggestions from code review

- `_mid` / `MID` => `_msgid` / `MSG_ID`
- Revised documentation / tooltip comments

* Apply suggestions from code review

* fix tests

* use more distinct names for MSG_ID headers

* update `_filter_service_log` to not use `-i -E`

Moreover, I added a function to print the whole mail log. Appropriate
comments were added to this function to indicate that one should only
use this function when necessary.

* adjust helpers to new helper filter

* follow-up of previous commit

* add CHANGELOG entry

* Apply suggestions from code review

* chore: Update OAuth2 to use new log helper

* Apply suggestions from code review

Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com>

* added explicit `_regexp` filters for logs

* Apply suggestions from code review

---------

Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com>

test/tests/parallel/set3/mta/dsn.bats

@@ -54,7 +54,7 @@ function teardown_file() {
   _nc_wrapper 'emails/nc_raw/dsn/authenticated.txt' '0.0.0.0 587'
   _wait_for_empty_mail_queue_in_container
 
-  _run_in_container grep "${LOG_DSN}" /var/log/mail/mail.log
+  _filter_service_log 'mail' "${LOG_DSN}"
   _should_output_number_of_lines 3
 }
 
@@ -70,15 +70,14 @@ function teardown_file() {
   #
   # Although external requests are discarded, anyone who has requested a DSN
   # will still receive it, but it will come from the sending mail server, not this one.
-  _run_in_container grep "${LOG_DSN}" /var/log/mail/mail.log
-  assert_failure
+  _service_log_should_not_contain_string 'mail' "${LOG_DSN}"
 
   # These ports are excluded via master.cf.
   _nc_wrapper 'emails/nc_raw/dsn/authenticated.txt' '0.0.0.0 465'
   _nc_wrapper 'emails/nc_raw/dsn/authenticated.txt' '0.0.0.0 587'
   _wait_for_empty_mail_queue_in_container
 
-  _run_in_container grep "${LOG_DSN}" /var/log/mail/mail.log
+  _service_log_should_contain_string 'mail' "${LOG_DSN}"
   _should_output_number_of_lines 2
 }
 
@@ -92,6 +91,5 @@ function teardown_file() {
 
   # DSN requests are rejected regardless of origin.
   # This is usually a bad idea, as you won't get them either.
-  _run_in_container grep "${LOG_DSN}" /var/log/mail/mail.log
-  assert_failure
+  _service_log_should_not_contain_string 'mail' "${LOG_DSN}"
 }

test/tests/parallel/set3/mta/smtp_delivery.bats

@@ -174,33 +174,28 @@ function _successful() {
 }
 
 @test "delivers mail to existing alias" {
-  _run_in_container grep 'to=<user1@localhost.localdomain>, orig_to=<alias1@localhost.localdomain>' /var/log/mail/mail.log
-  assert_success
+  _service_log_should_contain_string 'mail' 'to=<user1@localhost.localdomain>, orig_to=<alias1@localhost.localdomain>'
   assert_output --partial 'status=sent'
   _should_output_number_of_lines 1
 }
 
 @test "delivers mail to existing alias with recipient delimiter" {
-  _run_in_container grep 'to=<user1~test@localhost.localdomain>, orig_to=<alias1~test@localhost.localdomain>' /var/log/mail/mail.log
-  assert_success
+  _service_log_should_contain_string 'mail' 'to=<user1~test@localhost.localdomain>, orig_to=<alias1~test@localhost.localdomain>'
   assert_output --partial 'status=sent'
   _should_output_number_of_lines 1
 
-  _run_in_container grep 'to=<user1~test@localhost.localdomain>' /var/log/mail/mail.log
-  assert_success
+  _service_log_should_contain_string 'mail' 'to=<user1~test@localhost.localdomain>'
   refute_output --partial 'status=bounced'
 }
 
 @test "delivers mail to existing catchall" {
-  _run_in_container grep 'to=<user1@localhost.localdomain>, orig_to=<wildcard@localdomain2.com>' /var/log/mail/mail.log
-  assert_success
+  _service_log_should_contain_string 'mail' 'to=<user1@localhost.localdomain>, orig_to=<wildcard@localdomain2.com>'
   assert_output --partial 'status=sent'
   _should_output_number_of_lines 1
 }
 
 @test "delivers mail to regexp alias" {
-  _run_in_container grep 'to=<user1@localhost.localdomain>, orig_to=<test123@localhost.localdomain>' /var/log/mail/mail.log
-  assert_success
+  _service_log_should_contain_string 'mail' 'to=<user1@localhost.localdomain>, orig_to=<test123@localhost.localdomain>'
   assert_output --partial 'status=sent'
   _should_output_number_of_lines 1
 }
@@ -227,23 +222,20 @@ function _successful() {
 }
 
 @test "rejects mail to unknown user" {
-  _run_in_container grep '<nouser@localhost.localdomain>: Recipient address rejected: User unknown in virtual mailbox table' /var/log/mail/mail.log
-  assert_success
+  _service_log_should_contain_string 'mail' '<nouser@localhost.localdomain>: Recipient address rejected: User unknown in virtual mailbox table'
   _should_output_number_of_lines 1
 }
 
 @test "redirects mail to external aliases" {
-  _run_in_container_bash "grep 'Passed CLEAN {RelayedInbound}' /var/log/mail/mail.log | grep -- '-> <external1@otherdomain.tld>'"
-  assert_success
-  assert_output --partial '<user@external.tld> -> <external1@otherdomain.tld>'
+  _service_log_should_contain_string 'mail' 'Passed CLEAN {RelayedInbound}'
+  run bash -c "grep '<user@external.tld> -> <external1@otherdomain.tld>' <<< '${output}'"
   _should_output_number_of_lines 2
   # assert_output --partial 'external.tld=user@example.test> -> <external1@otherdomain.tld>'
 }
 
 # TODO: Add a test covering case SPAMASSASSIN_SPAM_TO_INBOX=1 (default)
 @test "rejects spam" {
-  _run_in_container grep 'Blocked SPAM {NoBounceInbound,Quarantined}' /var/log/mail/mail.log
-  assert_success
+  _service_log_should_contain_string 'mail' 'Blocked SPAM {NoBounceInbound,Quarantined}'
   assert_output --partial '<spam@external.tld> -> <user1@localhost.localdomain>'
   _should_output_number_of_lines 1