Merge branch 'master' into feat/env-vars-from-files

target/scripts/build/packages.sh

@@ -36,20 +36,46 @@ function _pre_installation_steps() {
   apt-get "${QUIET}" install --no-install-recommends "${EARLY_PACKAGES[@]}" 2>/dev/null
 }
 
+# Install third-party commands to /usr/local/bin
 function _install_utils() {
+  local ARCH_A
+  ARCH_A=$(uname --machine)
+  # Alternate naming convention support: x86_64 (amd64) / aarch64 (arm64)
+  # https://en.wikipedia.org/wiki/X86-64#Industry_naming_conventions
+  local ARCH_B
+  case "${ARCH_A}" in
+    ( 'x86_64'  ) ARCH_B='amd64' ;;
+    ( 'aarch64' ) ARCH_B='arm64' ;;
+    ( * )
+      _log 'error' "Unsupported arch: '${ARCH_A}'"
+      return 1
+      ;;
+  esac
+
+  # TIP: `*.tar.gz` releases tend to forget to reset UID/GID ownership when archiving.
+  # When extracting with `tar` as `root` the archived UID/GID is kept, unless using `--no-same-owner`.
+  # Likewise when the binary is in a nested location the full archived path
+  # must be provided + `--strip-components` to extract the file to the target directory.
+  # Doing this avoids the need for (`mv` + `rm`) or (`--to-stdout` + `chmod +x`)
   _log 'debug' 'Installing utils sourced from Github'
+
   _log 'trace' 'Installing jaq'
   local JAQ_TAG='v2.1.0'
-  curl -sSfL "https://github.com/01mf02/jaq/releases/download/${JAQ_TAG}/jaq-$(uname -m)-unknown-linux-gnu" -o /usr/bin/jaq
-  chmod +x /usr/bin/jaq
+  curl -sSfL "https://github.com/01mf02/jaq/releases/download/${JAQ_TAG}/jaq-$(uname -m)-unknown-linux-gnu" -o /usr/local/bin/jaq
+  chmod +x /usr/local/bin/jaq
+
+  _log 'trace' 'Installing step'
+  local STEP_RELEASE='0.28.2'
+  curl -sSfL "https://github.com/smallstep/cli/releases/download/v${STEP_RELEASE}/step_linux_${STEP_RELEASE}_${ARCH_B}.tar.gz" \
+    | tar -xz --directory /usr/local/bin --no-same-owner --strip-components=2 "step_${STEP_RELEASE}/bin/step"
 
   _log 'trace' 'Installing swaks'
+  # `perl-doc` is required for `swaks --help` to work:
   apt-get "${QUIET}" install --no-install-recommends perl-doc
   local SWAKS_VERSION='20240103.0'
   local SWAKS_RELEASE="swaks-${SWAKS_VERSION}"
-  curl -sSfL "https://github.com/jetmore/swaks/releases/download/v${SWAKS_VERSION}/${SWAKS_RELEASE}.tar.gz" | tar -xz
-  mv "${SWAKS_RELEASE}/swaks" /usr/local/bin
-  rm -r "${SWAKS_RELEASE}"
+  curl -sSfL "https://github.com/jetmore/swaks/releases/download/v${SWAKS_VERSION}/${SWAKS_RELEASE}.tar.gz" \
+    | tar -xz --directory /usr/local/bin --no-same-owner --strip-components=1 "${SWAKS_RELEASE}/swaks"
 }
 
 function _install_postfix() {

target/scripts/check-for-changes.sh

@@ -211,7 +211,7 @@ function _rspamd_changes() {
 
 while true; do
   _check_for_changes
-  sleep 2
+  sleep "${DMS_CONFIG_POLL:-2}"
 done
 
 exit 0

target/scripts/helpers/change-detection.sh

@@ -43,7 +43,7 @@ function _monitored_files_checksums() {
 
     # Check whether Rspamd is used and if so, monitor it's changes as well
     if [[ ${ENABLE_RSPAMD} -eq 1 ]] && [[ -d ${RSPAMD_DMS_D} ]]; then
-      readarray -d '' STAGING_FILES_RSPAMD < <(find "${RSPAMD_DMS_D}" -type f -name "*.sh" -print0)
+      readarray -d '' STAGING_FILES_RSPAMD < <(find "${RSPAMD_DMS_D}" -type f -print0)
       STAGING_FILES+=("${STAGING_FILES_RSPAMD[@]}")
     fi
   fi

target/scripts/helpers/error.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 function _exit_with_error() {
-  if [[ -n ${1+set} ]]; then
+  if [[ -n ${1:-} ]]; then
     _log 'error' "${1}"
   else
     _log 'error' "Call to '_exit_with_error' is missing a message to log"

target/scripts/helpers/log.sh

@@ -43,12 +43,12 @@ RESET=$(echo -ne   '\e[0m')
 # message is logged. Likewise when the second argument
 # is missing. Both failures will return with exit code '1'.
 function _log() {
-  if [[ -z ${1+set} ]]; then
+  if [[ -z ${1:-} ]]; then
     _log 'error' "Call to '_log' is missing a valid log level"
     return 1
   fi
 
-  if [[ -z ${2+set} ]]; then
+  if [[ -z ${2:-} ]]; then
     _log 'error' "Call to '_log' is missing a message to log"
     return 1
   fi
@@ -116,7 +116,7 @@ function _log() {
 # variables file. If this does not yield a value either,
 # use the default log level.
 function _get_log_level_or_default() {
-  if [[ -n ${LOG_LEVEL+set} ]]; then
+  if [[ -n ${LOG_LEVEL:-} ]]; then
     echo "${LOG_LEVEL}"
   elif [[ -e /etc/dms-settings ]] && grep -q -E "^LOG_LEVEL='[a-z]+'" /etc/dms-settings; then
     grep '^LOG_LEVEL=' /etc/dms-settings | cut -d "'" -f 2

target/scripts/helpers/rspamd.sh

@@ -111,14 +111,6 @@ function _rspamd_handle_user_modules_adjustments() {
     fi
   }
 
-  # We check for usage of the previous location of the commands file.
-  # TODO This can be removed after the release of v14.0.0.
-  local RSPAMD_DMS_CUSTOM_COMMANDS_F_OLD="${RSPAMD_DMS_D}-modules.conf"
-  readonly RSPAMD_DMS_CUSTOM_COMMANDS_F_OLD
-  if [[ -f ${RSPAMD_DMS_CUSTOM_COMMANDS_F_OLD} ]]; then
-    _dms_panic__general "Old custom command file location '${RSPAMD_DMS_CUSTOM_COMMANDS_F_OLD}' is deprecated (use '${RSPAMD_DMS_CUSTOM_COMMANDS_F}' now)" 'Rspamd setup'
-  fi
-
   if [[ -f "${RSPAMD_DMS_CUSTOM_COMMANDS_F}" ]]; then
     __rspamd__log 'debug' "Found file '${RSPAMD_DMS_CUSTOM_COMMANDS_F}' - parsing and applying it"
 

target/scripts/helpers/utils.sh

@@ -131,9 +131,9 @@ function _reload_postfix() {
 # 1. No first and second argument is supplied
 # 2. The second argument is a path to a file that does not exist
 function _replace_by_env_in_file() {
-  if [[ -z ${1+set} ]]; then
+  if [[ -z ${1:-} ]]; then
     _dms_panic__invalid_value 'first argument unset' 'utils.sh:_replace_by_env_in_file'
-  elif [[ -z ${2+set} ]]; then
+  elif [[ -z ${2:-} ]]; then
     _dms_panic__invalid_value 'second argument unset' 'utils.sh:_replace_by_env_in_file'
   elif [[ ! -f ${2} ]]; then
     _dms_panic__invalid_value "file '${2}' does not exist" 'utils.sh:_replace_by_env_in_file'

target/scripts/start-mailserver.sh

@@ -43,7 +43,6 @@ function _register_functions() {
   # ? >> Setup
 
   _register_setup_function '_setup_vmail_id'
-  _register_setup_function '_setup_logs_general'
   _register_setup_function '_setup_timezone'
 
   if [[ ${SMTP_ONLY} -ne 1 ]]; then
@@ -182,6 +181,9 @@ if [[ -f /CONTAINER_START ]]; then
   # We cannot skip all setup routines because some need to run _after_
   # the initial setup (and hence, they cannot be moved to the check stack).
   _setup_directory_and_file_permissions
+
+  # shellcheck source=./startup/setup.d/mail_state.sh
+  source /usr/local/bin/setup.d/mail_state.sh
   _setup_adjust_state_permissions
 else
   _setup

target/scripts/startup/setup-stack.sh

@@ -82,6 +82,8 @@ function _setup_timezone() {
   fi
 }
 
+# Misc checks and fixes migrated here until next refactor:
+# NOTE: `start-mailserver.sh` runs this along with `mail-state.sh` during container restarts
 function _setup_directory_and_file_permissions() {
   _log 'trace' 'Removing leftover PID files from a stop/start'
   find /var/run/ -not -name 'supervisord.pid' -name '*.pid' -delete
@@ -101,6 +103,8 @@ function _setup_directory_and_file_permissions() {
     _log 'debug' "Ensuring '${RSPAMD_DMS_DKIM_D}' is owned by '_rspamd:_rspamd'"
     chown -R _rspamd:_rspamd "${RSPAMD_DMS_DKIM_D}"
   fi
+
+  __log_fixes
 }
 
 function _setup_run_user_patches() {
@@ -113,3 +117,32 @@ function _setup_run_user_patches() {
     _log 'trace' "No optional '${USER_PATCHES}' provided"
   fi
 }
+
+function __log_fixes() {
+  _log 'debug' 'Ensuring /var/log/mail owneership + permissions are correct'
+
+  # File/folder permissions are fine when using docker volumes, but may be wrong
+  # when file system folders are mounted into the container.
+  # Set the expected values and create missing folders/files just in case.
+  mkdir -p /var/log/{mail,supervisor}
+
+  # TODO: Remove these lines in a future release once concerns are resolved:
+  # https://github.com/docker-mailserver/docker-mailserver/pull/4370#issuecomment-2661762043
+  chown syslog:root /var/log/mail
+
+  if [[ ${ENABLE_CLAMAV} -eq 1 ]]; then
+    # TODO: Consider assigning /var/log/mail a writable non-root group for other processes like ClamAV?
+    # - Check if ClamAV is capable of creating files itself when they're missing?
+    # - Alternatively a symlink to /var/log/mail from the original intended location would allow write access
+    #   as a user to the symlink location, while keeping ownership as root at /var/log/mail
+    # - `LogSyslog false` for clamd.conf + freshclam.conf could possibly be enabled instead of log files?
+    #   However without better filtering in place (once Vector is adopted), this should be avoided.
+    touch /var/log/mail/{clamav,freshclam}.log
+    chown clamav:adm /var/log/mail/{clamav,freshclam}.log
+  fi
+
+  # Volume permissions should be corrected:
+  # https://github.com/docker-mailserver/docker-mailserver-helm/issues/137
+  chmod 755 /var/log/mail/
+  find /var/log/mail/ -type f -exec chmod 640 {} +
+}

target/scripts/startup/setup.d/dmarc_dkim_spf.sh

@@ -23,7 +23,11 @@ function _setup_opendkim() {
     # check if any keys are available
     if [[ -e /tmp/docker-mailserver/opendkim/KeyTable ]]; then
       cp -a /tmp/docker-mailserver/opendkim/* /etc/opendkim/
-      _log 'trace' "DKIM keys added for: $(find /etc/opendkim/keys/ -maxdepth 1 -type f -printf '%f ')"
+
+      local DKIM_DOMAINS
+      DKIM_DOMAINS=$(find /etc/opendkim/keys/ -maxdepth 1 -type f -printf '%f ')
+      _log 'trace' "DKIM keys added for: ${DKIM_DOMAINS}"
+
       chown -R opendkim:opendkim /etc/opendkim/
       chmod -R 0700 /etc/opendkim/keys/
     else

target/scripts/startup/setup.d/log.sh

@@ -1,15 +1,5 @@
 #!/bin/bash
 
-function _setup_logs_general() {
-  _log 'debug' 'Setting up general log files'
-
-  # File/folder permissions are fine when using docker volumes, but may be wrong
-  # when file system folders are mounted into the container.
-  # Set the expected values and create missing folders/files just in case.
-  mkdir -p /var/log/{mail,supervisor}
-  chown syslog:root /var/log/mail
-}
-
 function _setup_logrotate() {
   _log 'debug' 'Setting up logrotate'
 

target/scripts/startup/setup.d/mail_state.sh

@@ -1,9 +1,11 @@
 #!/bin/bash
 
+DMS_STATE_DIR='/var/mail-state'
+
 # Consolidate all states into a single directory
 # (/var/mail-state) to allow persistence using docker volumes
 function _setup_save_states() {
-  if [[ ! -d ${DMS_STATE_DIR:?DMS_STATE_DIR is not set} ]]; then
+  if [[ ! -d ${DMS_STATE_DIR} ]]; then
     _log 'debug' "'${DMS_STATE_DIR}' is not present - not consolidating state"
     return 0
   fi
@@ -91,7 +93,12 @@ function _setup_save_states() {
 # These corrections are to fix changes to UID/GID values between upgrades,
 # or when ownership/permissions were altered externally on the host (eg: migration or system scripts)
 function _setup_adjust_state_permissions() {
-  [[ ! -d ${DMS_STATE_DIR:?DMS_STATE_DIR is not set} ]] && return 0
+  [[ ! -d ${DMS_STATE_DIR} ]] && return 0
+
+  # Parent directories must have executable bit set to descend the file tree for access,
+  # as each service running as a non-root user requires this to access their state directory,
+  # `/var/mail-state` must allow all users `+x`:
+  chmod +x "${DMS_STATE_DIR}"
 
   # This ensures the user and group of the files from the external mount have their
   # numeric ID values in sync. New releases where the installed packages order changes

target/scripts/startup/setup.d/postfix.sh

@@ -93,13 +93,19 @@ EOF
 function _setup_postfix_late() {
   _log 'debug' 'Configuring Postfix (late setup)'
 
+  # These two config files are `access` database tables managed via `setup email restrict`:
+  # NOTE: Prepends to existing restrictions, thus has priority over other permit/reject policies that follow.
+  # https://www.postfix.org/postconf.5.html#smtpd_sender_restrictions
+  # https://www.postfix.org/access.5.html
   __postfix__log 'trace' 'Configuring user access'
   if [[ -f /tmp/docker-mailserver/postfix-send-access.cf ]]; then
-    sed -i -E 's|(smtpd_sender_restrictions =)|\1 check_sender_access texthash:/tmp/docker-mailserver/postfix-send-access.cf,|' /etc/postfix/main.cf
+    # Prefer to prepend to our specialized variant instead:
+    # https://github.com/docker-mailserver/docker-mailserver/pull/4379
+    sed -i -E 's|^(dms_smtpd_sender_restrictions =)|\1 check_sender_access texthash:/tmp/docker-mailserver/postfix-send-access.cf,|' /etc/postfix/main.cf
   fi
 
   if [[ -f /tmp/docker-mailserver/postfix-receive-access.cf ]]; then
-    sed -i -E 's|(smtpd_recipient_restrictions =)|\1 check_recipient_access texthash:/tmp/docker-mailserver/postfix-receive-access.cf,|' /etc/postfix/main.cf
+    sed -i -E 's|^(smtpd_recipient_restrictions =)|\1 check_recipient_access texthash:/tmp/docker-mailserver/postfix-receive-access.cf,|' /etc/postfix/main.cf
   fi
 
   __postfix__log 'trace' 'Configuring relay host'

target/scripts/startup/setup.d/security/misc.sh

@@ -155,13 +155,6 @@ function __setup__security__clamav() {
   if [[ ${ENABLE_CLAMAV} -eq 1 ]]; then
     _log 'debug' 'Enabling and configuring ClamAV'
 
-    local FILE
-    for FILE in /var/log/mail/{clamav,freshclam}.log; do
-      touch "${FILE}"
-      chown clamav:adm "${FILE}"
-      chmod 640 "${FILE}"
-    done
-
     if [[ ${CLAMAV_MESSAGE_SIZE_LIMIT} != '25M' ]]; then
       _log 'trace' "Setting ClamAV message scan size limit to '${CLAMAV_MESSAGE_SIZE_LIMIT}'"
 

target/scripts/startup/variables-stack.sh

@@ -17,17 +17,6 @@ function _early_variables_setup() {
   __environment_variables_export
 }
 
-# Declare a variable as readonly if it is not already set.
-function __declare_readonly() {
-  local VARIABLE_NAME=${1:?Variable name required when declaring a variable as readonly}
-  local VARIABLE_VALUE=${2:?Variable value required when declaring a variable as readonly}
-
-  if [[ ! -v ${VARIABLE_NAME} ]]; then
-    readonly "${VARIABLE_NAME}=${VARIABLE_VALUE}"
-    VARS[${VARIABLE_NAME}]="${VARIABLE_VALUE}"
-  fi
-}
-
 # This function handles variables that are deprecated. This allows a
 # smooth transition period, without the need of removing a variable
 # completely with a single version.
@@ -74,11 +63,7 @@ function __environment_variables_general_setup() {
   VARS[DMS_VMAIL_UID]="${DMS_VMAIL_UID:=5000}"
   VARS[DMS_VMAIL_GID]="${DMS_VMAIL_GID:=5000}"
 
-  # internal variables are next
-
-  __declare_readonly 'DMS_STATE_DIR' '/var/mail-state'
-
-  # user-customizable are last
+  # user-customizable are next
 
   _log 'trace' 'Setting anti-spam & anti-virus environment variables'
 
@@ -122,7 +107,6 @@ function __environment_variables_general_setup() {
   VARS[ENABLE_POP3]="${ENABLE_POP3:=0}"
   VARS[ENABLE_IMAP]="${ENABLE_IMAP:=1}"
   VARS[ENABLE_POSTGREY]="${ENABLE_POSTGREY:=0}"
-  VARS[ENABLE_QUOTAS]="${ENABLE_QUOTAS:=1}"
   VARS[ENABLE_RSPAMD]="${ENABLE_RSPAMD:=0}"
   VARS[ENABLE_RSPAMD_REDIS]="${ENABLE_RSPAMD_REDIS:=${ENABLE_RSPAMD}}"
   VARS[ENABLE_SASLAUTHD]="${ENABLE_SASLAUTHD:=0}"
@@ -165,6 +149,7 @@ function __environment_variables_general_setup() {
   _log 'trace' 'Setting miscellaneous environment variables'
 
   VARS[ACCOUNT_PROVISIONER]="${ACCOUNT_PROVISIONER:=FILE}"
+  VARS[DMS_CONFIG_POLL]="${DMS_CONFIG_POLL:=2}"
   VARS[FETCHMAIL_PARALLEL]="${FETCHMAIL_PARALLEL:=0}"
   VARS[FETCHMAIL_POLL]="${FETCHMAIL_POLL:=300}"
   VARS[GETMAIL_POLL]="${GETMAIL_POLL:=5}"
@@ -182,6 +167,18 @@ function __environment_variables_general_setup() {
   VARS[SUPERVISOR_LOGLEVEL]="${SUPERVISOR_LOGLEVEL:=warn}"
   VARS[TZ]="${TZ:=}"
   VARS[UPDATE_CHECK_INTERVAL]="${UPDATE_CHECK_INTERVAL:=1d}"
+
+  _log 'trace' 'Setting environment variables that require other variables to be set first'
+
+  # The Dovecot Quotas feature is presently only supported with the default FILE account provisioner,
+  # Enforce disabling the feature, unless it's been explicitly set via ENV (to avoid mismatch between
+  # explicit ENV and sourcing from /etc/dms-settings)
+  if [[ ${ACCOUNT_PROVISIONER} != 'FILE' || ${SMTP_ONLY} -eq 1 ]] && [[ ${ENABLE_QUOTAS:-1} -eq 1 ]]; then
+    _log 'debug' "The 'ENABLE_QUOTAS' feature is enabled (by default) but is not compatible with your config. Disabling"
+    VARS[ENABLE_QUOTAS]="${ENABLE_QUOTAS:=0}"
+  else
+    VARS[ENABLE_QUOTAS]="${ENABLE_QUOTAS:=1}"
+  fi
 }
 
 function __environment_variables_log_level() {