feat: Enable reading env vars from files (#4359)

Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com>
Co-authored-by: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com>

CHANGELOG.md

@@ -8,6 +8,8 @@ All notable changes to this project will be documented in this file. The format
 
 ### Added
 
+- **Environment Variables:**
+  - [ENV can be declared with a `__FILE` suffix](https://docker-mailserver.github.io/docker-mailserver/v15.1/config/environment/) to read a value from a file during initial DMS setup scripts ([#4359](https://github.com/docker-mailserver/docker-mailserver/pull/4359))
 - **Internal:**
   - [`DMS_CONFIG_POLL`](https://docker-mailserver.github.io/docker-mailserver/v15.0/config/environment/#dms_config_poll) supports adjusting the polling rate (seconds) for the change detection service `check-for-changes.sh` ([#4450](https://github.com/docker-mailserver/docker-mailserver/pull/4450))
 

docs/content/config/environment.md

@@ -6,6 +6,10 @@ title: Environment Variables
 
     Values in **bold** are the default values. If an option doesn't work as documented here, check if you are running the latest image. The current `master` branch corresponds to the image `ghcr.io/docker-mailserver/docker-mailserver:edge`.
 
+!!! tip
+
+    If an environment variable `<VAR>__FILE` is set with a valid file path as the value, the content of that file will become the value for `<VAR>` (_provided `<VAR>` has not already been set_).
+
 #### General
 
 ##### OVERRIDE_HOSTNAME

target/scripts/startup/variables-stack.sh

@@ -5,6 +5,7 @@ declare -A VARS
 
 function _early_variables_setup() {
   __environment_variables_log_level
+  __environment_variables_from_files
   _obtain_hostname_and_domainname
   __environment_variables_backwards_compatibility
   __environment_variables_general_setup
@@ -244,3 +245,34 @@ function __environment_variables_export() {
   sort -o /root/.bashrc     /root/.bashrc
   sort -o /etc/dms-settings /etc/dms-settings
 }
+
+# This function sets any environment variable with a value from a referenced file
+# when an equivalent ENV with a `__FILE` suffix exists with a valid file path as the value.
+function __environment_variables_from_files() {
+  # Iterate through all ENV found with a `__FILE` suffix:
+  while read -r ENV_WITH_FILE_REF; do
+    # Store the value of the `__FILE` ENV:
+    local FILE_PATH="${!ENV_WITH_FILE_REF}"
+    # Store the ENV name without the `__FILE` suffix:
+    local TARGET_ENV_NAME="${ENV_WITH_FILE_REF/__FILE/}"
+    # Assign a value representing a variable name,
+    # `-n` will alias `TARGET_ENV` so that it is treated as if it were the referenced variable:
+    local -n TARGET_ENV="${TARGET_ENV_NAME}"
+
+    # Skip if the target ENV is already set:
+    if [[ -v TARGET_ENV ]]; then
+      _log 'warn' "ENV value will not be sourced from '${ENV_WITH_FILE_REF}' since '${TARGET_ENV_NAME}' is already set"
+      continue
+    fi
+
+    # Skip if the file path provided is invalid:
+    if [[ ! -f ${FILE_PATH} ]]; then
+      _log 'warn' "File defined for secret '${TARGET_ENV_NAME}' with path '${FILE_PATH}' does not exist"
+      continue
+    fi
+
+    # Read the value from a file and assign it to the intended ENV:
+    _log 'info' "Getting secret '${TARGET_ENV_NAME}' from '${FILE_PATH}'"
+    TARGET_ENV="$(< "${FILE_PATH}")"
+  done < <(env | grep -Po '^.+?__FILE')
+}

test/tests/parallel/set3/container_configuration/env_vars_from_files.bats

@@ -0,0 +1,89 @@
+load "${REPOSITORY_ROOT}/test/helper/setup"
+load "${REPOSITORY_ROOT}/test/helper/common"
+
+# Feature (ENV value sourced from file):
+# - An ENV with a `__FILE` suffix will read a value from a referenced file path to set the actual ENV (assuming it is empty)
+# - Feature implemented at: `variables-stack.sh:__environment_variables_from_files()`
+# - Feature PR: https://github.com/docker-mailserver/docker-mailserver/pull/4359
+
+BATS_TEST_NAME_PREFIX='[Configuration] (ENV __FILE support) '
+CONTAINER1_NAME='dms-test_env-files_success'
+CONTAINER2_NAME='dms-test_env-files_warning'
+CONTAINER3_NAME='dms-test_env-files_error'
+
+function setup_file() {
+  export CONTAINER_NAME
+  export FILEPATH_VALID='/tmp/file-with-value'
+  export FILEPATH_INVALID='/path/to/non-existent-file'
+  # Each `_init_with_defaults` call updates the `TEST_TMP_CONFIG` location to create a container specific file:
+  local FILE_WITH_VALUE
+
+  # ENV is set via file content (valid file path):
+  CONTAINER_NAME=${CONTAINER1_NAME}
+  _init_with_defaults
+  FILE_WITH_VALUE=${TEST_TMP_CONFIG}/test_secret
+  echo 1 > "${FILE_WITH_VALUE}"
+  local CUSTOM_SETUP_ARGUMENTS=(
+    --env ENABLE_POP3__FILE="${FILEPATH_VALID}"
+    -v "${FILE_WITH_VALUE}:${FILEPATH_VALID}"
+  )
+  _common_container_setup 'CUSTOM_SETUP_ARGUMENTS'
+
+  # ENV is already set explicitly, a warning should be logged:
+  CONTAINER_NAME=${CONTAINER2_NAME}
+  _init_with_defaults
+  FILE_WITH_VALUE=${TEST_TMP_CONFIG}/test_secret
+  echo 1 > "${FILE_WITH_VALUE}"
+  local CUSTOM_SETUP_ARGUMENTS=(
+    --env ENABLE_POP3="0"
+    --env ENABLE_POP3__FILE="${FILEPATH_VALID}"
+    -v "${FILE_WITH_VALUE}:${FILEPATH_VALID}"
+  )
+  _common_container_setup 'CUSTOM_SETUP_ARGUMENTS'
+
+  # ENV is not set by file content (invalid file path):
+  CONTAINER_NAME=${CONTAINER3_NAME}
+  _init_with_defaults
+  local CUSTOM_SETUP_ARGUMENTS=(
+    --env ENABLE_POP3__FILE="${FILEPATH_INVALID}"
+  )
+  _common_container_setup 'CUSTOM_SETUP_ARGUMENTS'
+}
+
+function teardown_file() {
+  docker rm -f "${CONTAINER1_NAME}" "${CONTAINER2_NAME}" "${CONTAINER3_NAME}"
+}
+
+@test "ENV can be set from a file" {
+  export CONTAINER_NAME=${CONTAINER1_NAME}
+
+  # /var/log/mail/mail.log is not equivalent to stdout content,
+  # Relevant log content only available via docker logs:
+  run docker logs "${CONTAINER_NAME}"
+  assert_success
+  assert_line --partial "Getting secret 'ENABLE_POP3' from '${FILEPATH_VALID}'"
+
+  # Verify ENABLE_POP3 was enabled (disabled by default), by checking this file path is valid:
+  _run_in_container [ -f /etc/dovecot/protocols.d/pop3d.protocol ]
+  assert_success
+}
+
+@test "Non-empty ENV have precedence over their __FILE variant" {
+  export CONTAINER_NAME=${CONTAINER2_NAME}
+
+  # /var/log/mail/mail.log is not equivalent to stdout content,
+  # Relevant log content only available via docker logs:
+  run docker logs "${CONTAINER_NAME}"
+  assert_success
+  assert_line --partial "ENV value will not be sourced from 'ENABLE_POP3__FILE' since 'ENABLE_POP3' is already set"
+}
+
+@test "Referencing a non-existent file logs an error" {
+  export CONTAINER_NAME=${CONTAINER3_NAME}
+
+  # /var/log/mail/mail.log is not equivalent to stdout content,
+  # Relevant log content only available via docker logs:
+  run docker logs "${CONTAINER_NAME}"
+  assert_success
+  assert_line --partial "File defined for secret 'ENABLE_POP3' with path '${FILEPATH_INVALID}' does not exist"
+}