Merge branch 'master' into patch-3

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -4,6 +4,8 @@ title: 'feature request: '
 labels:
   - kind/new feature
   - meta/needs triage
+projects:
+  - DMS Core Backlog
 
 body:
   - type: markdown

.github/workflows/generic_build.yml

@@ -79,11 +79,11 @@ jobs:
           platforms: arm64
 
       - name: 'Set up Docker Buildx'
-        uses: docker/setup-buildx-action@v3.6.1
+        uses: docker/setup-buildx-action@v3.7.1
 
       # NOTE: AMD64 can build within 2 minutes
       - name: 'Build images'
-        uses: docker/build-push-action@v6.7.0
+        uses: docker/build-push-action@v6.9.0
         with:
           context: .
           # Build at least the AMD64 image (which runs against the test suite).

.github/workflows/generic_publish.yml

@@ -40,7 +40,7 @@ jobs:
           platforms: arm64
 
       - name: 'Set up Docker Buildx'
-        uses: docker/setup-buildx-action@v3.6.1
+        uses: docker/setup-buildx-action@v3.7.1
 
       # Try get the cached build layers from a prior `generic_build.yml` job.
       # NOTE: Until adopting `type=gha` scoped cache exporter (in `docker/build-push-action`),
@@ -67,7 +67,7 @@ jobs:
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: 'Build and publish images'
-        uses: docker/build-push-action@v6.7.0
+        uses: docker/build-push-action@v6.9.0
         with:
           context: .
           build-args: |

.github/workflows/generic_test.yml

@@ -38,12 +38,12 @@ jobs:
       # Ensures consistent BuildKit version (not coupled to Docker Engine),
       # and increased compatibility of the build cache vs mixing buildx drivers.
       - name: 'Set up Docker Buildx'
-        uses: docker/setup-buildx-action@v3.6.1
+        uses: docker/setup-buildx-action@v3.7.1
 
       # Importing from the cache should create the image within approx 30 seconds:
       # NOTE: `qemu` step is not needed as we only test for AMD64.
       - name: 'Build AMD64 image from cache'
-        uses: docker/build-push-action@v6.7.0
+        uses: docker/build-push-action@v6.9.0
         with:
           context: .
           tags: mailserver-testing:ci

.github/workflows/generic_vulnerability-scan.yml

@@ -37,12 +37,12 @@ jobs:
       # Ensures consistent BuildKit version (not coupled to Docker Engine),
       # and increased compatibility of the build cache vs mixing buildx drivers.
       - name: 'Set up Docker Buildx'
-        uses: docker/setup-buildx-action@v3.6.1
+        uses: docker/setup-buildx-action@v3.7.1
 
       # Importing from the cache should create the image within approx 30 seconds:
       # NOTE: `qemu` step is not needed as we only test for AMD64.
       - name: 'Build AMD64 image from cache'
-        uses: docker/build-push-action@v6.7.0
+        uses: docker/build-push-action@v6.9.0
         with:
           context: .
           tags: mailserver-testing:ci

CHANGELOG.md

@@ -42,18 +42,23 @@ All notable changes to this project will be documented in this file. The format
   - Add caveat for `DMS_VMAIL_UID` not being compatible with `0` / root ([#4143](https://github.com/docker-mailserver/docker-mailserver/pull/4143))
 - **Postfix:**
   - Disable Microsoft reactions to outgoing mail ([#4120](https://github.com/docker-mailserver/docker-mailserver/pull/4120))
+- bumped `jaq` version from 1.3.0 to 1.6.0 ([#4190](https://github.com/docker-mailserver/docker-mailserver/pull/4190))
+- updated Rspamd GTube settings and tests ([#4191](https://github.com/docker-mailserver/docker-mailserver/pull/4191))
 
 ### Fixes
 
 - **Dovecot:**
   - Update logwatch `ignore.conf` to exclude Xapian messages about pending documents ([#4060](https://github.com/docker-mailserver/docker-mailserver/pull/4060))
   - `dovecot-fts-xapian` plugin was updated to `1.7.13`, fixing a regression with indexing ([#4095](https://github.com/docker-mailserver/docker-mailserver/pull/4095))
+  - The Dovecot Quota support "dummy account" workaround no longer treats the alias as a regex when checking the Dovecot UserDB ([#4222](https://github.com/docker-mailserver/docker-mailserver/pull/4222))
 - **LDAP:**
   - A previous compatibility fix for OAuth2 in v13.3.1 had not applied the actual LDAP config changes. This has been corrected ([#4175](https://github.com/docker-mailserver/docker-mailserver/pull/4175))
 - **Internal:**
   - The main `mail.log` which is piped to stdout via `tail` now correctly begins from the first log line of the active container run. Previously some daemon logs and potential warnings/errors were omitted. ([#4146](https://github.com/docker-mailserver/docker-mailserver/pull/4146))
   - Fixed a regression introduced in v14 where `postfix-main.cf` appended `stderr` output into `/etc/postfix/main.cf`, causing Postfix startup to fail ([#4147](https://github.com/docker-mailserver/docker-mailserver/pull/4147))
   - Unused `shopt -s inherit_errexit` removed from `start-mailserver.sh` ([#4161](https://github.com/docker-mailserver/docker-mailserver/pull/4161))
+- **Rspamd:**
+  - DKIM private key path checking is now performed only on paths that do not contain "$" ([#4201](https://github.com/docker-mailserver/docker-mailserver/pull/4201))
 
 ### CI
 

docs/content/config/security/rspamd.md

@@ -251,7 +251,7 @@ There is a dedicated [section for setting up DKIM with Rspamd in our documentati
 
 This subsection provides information about the integration of [Abusix][abusix-web], "a set of blocklists that work as an additional email security layer for your existing mail environment". The setup is straight-forward and well documented:
 
-1. [Create an account](https://app.abusix.com/signup)
+1. [Create an account](https://app.abusix.com/)
 2. Retrieve your API key
 3. Navigate to the ["Getting Started" documentation for Rspamd][abusix-docs::rspamd-integration] and follow the steps described there
 4. Make sure to change `<APIKEY>` to your private API key

target/scripts/build/packages.sh

@@ -38,7 +38,7 @@ function _pre_installation_steps() {
 function _install_utils() {
   _log 'debug' 'Installing utils sourced from Github'
   _log 'trace' 'Installing jaq'
-  local JAQ_TAG='v1.3.0'
+  local JAQ_TAG='v1.6.0'
   curl -sSfL "https://github.com/01mf02/jaq/releases/download/${JAQ_TAG}/jaq-${JAQ_TAG}-$(uname -m)-unknown-linux-gnu" -o /usr/bin/jaq
   chmod +x /usr/bin/jaq
 

target/scripts/helpers/accounts.sh

@@ -135,7 +135,8 @@ function _create_dovecot_alias_dummy_accounts() {
       fi
 
       DOVECOT_USERDB_LINE="${ALIAS}:${REAL_ACC[1]}:${DMS_VMAIL_UID}:${DMS_VMAIL_GID}::/var/mail/${REAL_DOMAINNAME}/${REAL_USERNAME}/home::${REAL_ACC[2]:-}"
-      if grep -qi "^${ALIAS}:" "${DOVECOT_USERDB_FILE}"; then
+      # Match a full line with `-xF` to avoid regex patterns introducing false positives matching `ALIAS`:
+      if grep -qixF "${DOVECOT_USERDB_LINE}" "${DOVECOT_USERDB_FILE}"; then
         _log 'warn' "Alias '${ALIAS}' will not be added to '${DOVECOT_USERDB_FILE}' twice"
       else
         echo "${DOVECOT_USERDB_LINE}" >>"${DOVECOT_USERDB_FILE}"

target/scripts/startup/setup.d/postfix.sh

@@ -79,6 +79,8 @@ EOF
     if [[ ${ACCOUNT_PROVISIONER} == 'FILE' ]]; then
       postconf 'virtual_mailbox_maps = texthash:/etc/postfix/vmailbox'
     fi
+    # Historical context regarding decision to use LMTP instead of LDA (do not change this):
+    # https://github.com/docker-mailserver/docker-mailserver/issues/4178#issuecomment-2375489302
     postconf 'virtual_transport = lmtp:unix:/var/run/dovecot/lmtp'
   fi
 

target/scripts/startup/setup.d/security/rspamd.sh

@@ -76,8 +76,9 @@ function __rspamd__run_early_setup_and_checks() {
   mkdir -p /var/lib/rspamd/
   : >/var/lib/rspamd/stats.ucl
 
-  if [[ -d ${RSPAMD_DMS_OVERRIDE_D} ]]; then
-    cp "${RSPAMD_DMS_OVERRIDE_D}"/* "${RSPAMD_OVERRIDE_D}"
+  # Copy if directory exists and is not empty
+  if [[ -d ${RSPAMD_DMS_OVERRIDE_D} ]] && [[ -z $(find "${RSPAMD_DMS_OVERRIDE_D}" -maxdepth 0 -empty) ]]; then
+    cp "${RSPAMD_DMS_OVERRIDE_D}/"* "${RSPAMD_OVERRIDE_D}"
   fi
 
   if [[ ${ENABLE_AMAVIS} -eq 1 ]] || [[ ${ENABLE_SPAMASSASSIN} -eq 1 ]]; then
@@ -319,8 +320,7 @@ function __rspamd__setup_check_authenticated() {
   local MODULE_FILE="${RSPAMD_LOCAL_D}/settings.conf"
   readonly MODULE_FILE
   if _env_var_expect_zero_or_one 'RSPAMD_CHECK_AUTHENTICATED' \
-  && [[ ${RSPAMD_CHECK_AUTHENTICATED} -eq 0 ]]
-  then
+  && [[ ${RSPAMD_CHECK_AUTHENTICATED} -eq 0 ]]; then
     __rspamd__log 'debug' 'Content checks for authenticated users are disabled'
   else
     __rspamd__log 'debug' 'Enabling content checks for authenticated users'
@@ -332,32 +332,22 @@ function __rspamd__setup_check_authenticated() {
 
 # This function performs a simple check: go through DKIM configuration files, acquire
 # all private key file locations and check whether they exist and whether they can be
-# accessed by Rspamd.
+# accessed by Rspamd. We are not checking paths that conatain the '$' symbol.
 function __rspamd__check_dkim_permissions() {
-  local DKIM_CONF_FILES DKIM_KEY_FILES
-  [[ -f ${RSPAMD_LOCAL_D}/dkim_signing.conf ]] && DKIM_CONF_FILES+=("${RSPAMD_LOCAL_D}/dkim_signing.conf")
-  [[ -f ${RSPAMD_OVERRIDE_D}/dkim_signing.conf ]] && DKIM_CONF_FILES+=("${RSPAMD_OVERRIDE_D}/dkim_signing.conf")
-
-  # Here, we populate DKIM_KEY_FILES which we later iterate over. DKIM_KEY_FILES
-  # contains all keys files configured by the user.
-  local FILE
-  for FILE in "${DKIM_CONF_FILES[@]}"; do
-    readarray -t DKIM_KEY_FILES_TMP < <(grep -o -E 'path = .*' "${FILE}" | cut -d '=' -f 2 | tr -d ' ";')
-    DKIM_KEY_FILES+=("${DKIM_KEY_FILES_TMP[@]}")
-  done
-
-  for FILE in "${DKIM_KEY_FILES[@]}"; do
-    if [[ -f ${FILE} ]]; then
-      __rspamd__log 'trace' "Checking DKIM file '${FILE}'"
+  local KEY_FILE
+  while read -r KEY_FILE; do
+    if [[ -f ${KEY_FILE} ]]; then
+      __rspamd__log 'trace' "Checking DKIM file '${KEY_FILE}'"
       # See https://serverfault.com/a/829314 for an explanation on `-exec false {} +`
       # We additionally resolve symbolic links to check the permissions of the actual files
-      if find "$(realpath -eL "${FILE}")" \( -user _rspamd -or -group _rspamd -or -perm -o=r \) -exec false {} +; then
-        __rspamd__log 'warn' "Rspamd DKIM private key file '${FILE}' does not appear to have correct permissions/ownership for Rspamd to use it"
+      if find "$(realpath -L "${KEY_FILE}")" \( -user _rspamd -or -group _rspamd -or -perm -o=r \) \
+        -exec false {} +; then
+        __rspamd__log 'warn' "Rspamd DKIM private key file '${KEY_FILE}' does not appear to have correct permissions/ownership for Rspamd to use it"
       else
-        __rspamd__log 'trace' "DKIM file '${FILE}' permissions and ownership appear correct"
+        __rspamd__log 'trace' "DKIM file '${KEY_FILE}' permissions and ownership appear correct"
       fi
     else
-      __rspamd__log 'warn' "Rspamd DKIM private key file '${FILE}' is configured for usage, but does not appear to exist"
+      __rspamd__log 'warn' "Rspamd DKIM private key file '${KEY_FILE}' is configured for usage, but does not appear to exist"
     fi
-  done
+  done < <(rspamadm configdump dkim_signing | grep 'path =' | grep -v -F '$' | awk '{print $3}' | tr -d ';"')
 }

test/config/postfix-virtual.cf

@@ -3,3 +3,13 @@ alias1@localhost.localdomain user1@localhost.localdomain
     # this is also a test comment, :O
 alias2@localhost.localdomain external1@otherdomain.tld
 @localdomain2.com user1@localhost.localdomain
+
+## Dovecot "dummy accounts" for quota support (handled in `helpers/accounts.sh`)
+# Do not filter alias by substring condition (longer prefix must be before substring alias):
+# https://github.com/docker-mailserver/docker-mailserver/issues/2639
+prefixtest@localhost.localdomain user2@otherdomain.tld
+test@localhost.localdomain user2@otherdomain.tld
+# Do not filter alias when input be treated as regex tokens (eg `.`):
+# https://github.com/docker-mailserver/docker-mailserver/issues/4170
+first-name@localhost.localdomain user2@otherdomain.tld
+first.name@localhost.localdomain user2@otherdomain.tld

test/config/rspamd_full/user-patches.sh

@@ -5,7 +5,7 @@
 #
 # We do not use `custom-commands.conf` because this a feature
 # we are testing too.
-echo "enable_test_patterns = true;" >>/etc/rspamd/local.d/options.inc
+echo 'gtube_patterns = "all"' >>/etc/rspamd/local.d/options.inc
 
 # We want Dovecot to be very detailed about what it is doing,
 # specifically for Sieve because we need to check whether the

test/tests/parallel/set1/spam_virus/rspamd_full.bats

@@ -45,8 +45,10 @@ function setup_file() {
   _wait_for_smtp_port_in_container
 
   # We will send 5 emails:
-  #   1. The first one should pass just fine
+  #   1. The first ones should pass just fine
   _send_email_with_msgid 'rspamd-test-email-pass'
+  _send_email_with_msgid 'rspamd-test-email-pass-gtube' \
+    --body 'AJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X'
   #   2. The second one should be rejected (Rspamd-specific GTUBE pattern for rejection)
   _send_spam --expect-rejection
   #   3. The third one should be rejected due to a virus (ClamAV EICAR pattern)
@@ -54,7 +56,7 @@ function setup_file() {
   _send_email_with_msgid 'rspamd-test-email-virus' --expect-rejection \
     --body 'X5O!P%@AP[4\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-FILE!$H+H*'
   #   4. The fourth one will receive an added header (Rspamd-specific GTUBE pattern for adding a spam header)
-  #      ref: https://rspamd.com/doc/gtube_patterns.html
+  #      ref: https://rspamd.com/doc/other/gtube_patterns.html
   _send_email_with_msgid 'rspamd-test-email-header' \
     --body "YJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X"
   #   5. The fifth one will have its subject rewritten, but now spam header is applied.
@@ -134,11 +136,12 @@ function teardown_file() { _default_teardown ; }
 
 @test 'normal mail passes fine' {
   _service_log_should_contain_string 'rspamd' 'F (no action)'
+  _service_log_should_contain_string 'rspamd' 'S (no action)'
 
   _print_mail_log_for_msgid 'rspamd-test-email-pass'
   assert_output --partial "stored mail into mailbox 'INBOX'"
 
-  _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/new/ 2
+  _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/new/ 3
 }
 
 @test 'detects and rejects spam' {
@@ -153,7 +156,7 @@ function teardown_file() { _default_teardown ; }
   refute_output --partial "stored mail into mailbox 'INBOX'"
   assert_failure
 
-  _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/new/ 2
+  _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/new/ 3
 }
 
 @test 'detects and rejects virus' {
@@ -168,7 +171,7 @@ function teardown_file() { _default_teardown ; }
   refute_output --partial "stored mail into mailbox 'INBOX'"
   assert_failure
 
-  _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/new/ 2
+  _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/new/ 3
 }
 
 @test 'custom commands work correctly' {
@@ -246,7 +249,7 @@ function teardown_file() { _default_teardown ; }
   _print_mail_log_for_msgid 'rspamd-test-email-header'
   assert_output --partial "fileinto action: stored mail into mailbox [SPECIAL-USE \\Junk]"
 
-  _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/new/ 2
+  _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/new/ 3
   _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/.Junk/new/ 1
 }
 

test/tests/parallel/set3/mta/account_management.bats

@@ -29,7 +29,12 @@ function teardown_file() { _default_teardown ; }
   assert_line --index 5 'alias1@localhost.localdomain'
   # TODO: Probably not intentional?:
   assert_line --index 6 '@localdomain2.com'
-  _should_output_number_of_lines 7
+  # Dovecot "dummy accounts" for quota support, see `test/config/postfix-virtual.cf` for more context
+  assert_line --index 7 'prefixtest@localhost.localdomain'
+  assert_line --index 8 'test@localhost.localdomain'
+  assert_line --index 9 'first-name@localhost.localdomain'
+  assert_line --index 10 'first.name@localhost.localdomain'
+  _should_output_number_of_lines 11
 
   # Relevant log output from scripts/helpers/accounts.sh:_create_dovecot_alias_dummy_accounts():
   # [  DEBUG  ]  Adding alias 'alias1@localhost.localdomain' for user 'user1@localhost.localdomain' to Dovecot's userdb
@@ -37,6 +42,19 @@ function teardown_file() { _default_teardown ; }
   # [  DEBUG  ]  Adding alias '@localdomain2.com' for user 'user1@localhost.localdomain' to Dovecot's userdb
 }
 
+# Dovecot "dummy accounts" for quota support, see `test/config/postfix-virtual.cf` for more context
+@test "should create all dovecot dummy accounts" {
+  run docker logs "${CONTAINER_NAME}"
+  assert_success
+  assert_line --partial "Adding alias 'prefixtest@localhost.localdomain' for user 'user2@otherdomain.tld' to Dovecot's userdb"
+  assert_line --partial "Adding alias 'test@localhost.localdomain' for user 'user2@otherdomain.tld' to Dovecot's userdb"
+  refute_line --partial "Alias 'test@localhost.localdomain' will not be added to '/etc/dovecot/userdb' twice"
+
+  assert_line --partial "Adding alias 'first-name@localhost.localdomain' for user 'user2@otherdomain.tld' to Dovecot's userdb"
+  assert_line --partial "Adding alias 'first.name@localhost.localdomain' for user 'user2@otherdomain.tld' to Dovecot's userdb"
+  refute_line --partial "Alias 'first.name@localhost.localdomain' will not be added to '/etc/dovecot/userdb' twice"
+}
+
 @test "should have created maildir for 'user1@localhost.localdomain'" {
   _run_in_container_bash '[[ -d /var/mail/localhost.localdomain/user1 ]]'
   assert_success