Final Migration Step (#6)

* first migration steps
  * altered issue templates
  * altered README
  * removed .travis.yml
* adjusting registry & repository, Dockerfile and compose.env
* Close stale issues automatically
* Integrated CI with Github Actions (#3)
* feat: integrated ci with github actions
* fix: use secrets for docker org and update image
* docs: clarify why we use -t if no tty exists
* fix: correct remaining references to old repo
chore: prettier automatically updated markdown as well
* fix: hardcode docker org
* change testing image to just testing
* ci: add armv7 as a supported platform
* finished migration steps
* corrected linting in build-push action
* corrected linting in build-push action (2)
* minor preps for PR
* correcting push on pull request and minor details
* adjusted workflows to adhere closer to @wernerfred's diagram
* minor patches
* adjusting Dockerfile's installation of base packages
* adjusting schedule for stale issue action
* reverting license text
* improving CONTRIBUTING.md PR text
* Update CONTRIBUTING.md
* a bigger patch at the end
  * moved all scripts into one directory under target/scripts/
  * moved the quota-warning.sh script into target/scripts/ and removed empty directory /target/dovecot/scripts
  * minor fixes here and there
  * adjusted workflows for use a fully qualified name (i.e. docker.io/...)
  * improved on the Dockerfile layer count
  * corrected local tests - now they (actually) work (fine)!
  * corrected start-mailserver.sh to make use of defaults consistently
  * removed very old, deprecated variables (actually only one)
* various smaller improvements in the end
* last commit before merging #6
* rearranging variables to use alphabetic order

Co-authored-by: casperklein <casperklein@users.noreply.github.com>
Co-authored-by: Nick Pappas <radicand@users.noreply.github.com>
Co-authored-by: William Desportes <williamdes@wdes.fr>
This commit is contained in:
Georg Lauterbach 2021-01-16 10:16:05 +01:00 committed by GitHub
parent 66422cbcb0
commit 189e5376cc
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
57 changed files with 1644 additions and 1671 deletions

View file

@ -0,0 +1,236 @@
#! /bin/bash
# shellcheck source=./helper-functions.sh
. /usr/local/bin/helper-functions.sh
LOG_DATE=$(date +"%Y-%m-%d %H:%M:%S ")
echo "${LOG_DATE} Start check-for-changes script."
# ? Checks
cd /tmp/docker-mailserver || exit 1
# check postfix-accounts.cf exist else break
if [[ ! -f postfix-accounts.cf ]]
then
echo "${LOG_DATE} postfix-accounts.cf is missing! This should not run! Exit!"
exit 0
fi
# verify checksum file exists; must be prepared by start-mailserver.sh
if [[ ! -f ${CHKSUM_FILE} ]]
then
echo "${LOG_DATE} ${CHKSUM_FILE} is missing! Start script failed? Exit!"
exit 0
fi
# ? Actual script begins
# determine postmaster address, duplicated from start-mailserver.sh
# this script previously didn't work when POSTMASTER_ADDRESS was empty
if [[ -n ${OVERRIDE_HOSTNAME} ]]
then
DOMAINNAME="${OVERRIDE_HOSTNAME#*.}"
else
DOMAINNAME="$(hostname -d)"
fi
PM_ADDRESS="${POSTMASTER_ADDRESS:=postmaster@${DOMAINNAME}}"
echo "${LOG_DATE} Using postmaster address ${PM_ADDRESS}"
sleep 10
while true
do
LOG_DATE=$(date +"%Y-%m-%d %H:%M:%S ")
# get chksum and check it, no need to lock config yet
_monitored_files_checksums >"${CHKSUM_FILE}.new"
if ! cmp --silent -- "${CHKSUM_FILE}" "${CHKSUM_FILE}.new"
then
echo "${LOG_DATE} Change detected"
CHANGED=$(grep -Fxvf "${CHKSUM_FILE}" "${CHKSUM_FILE}.new" | sed 's/^[^ ]\+ //')
# Bug alert! This overwrites the alias set by start-mailserver.sh
# Take care that changes in one script are propagated to the other
# ! NEEDS FIX -----------------------------------------
# TODO FIX --------------------------------------------
# ! NEEDS EXTENSIONS ----------------------------------
# TODO Perform updates below conditionally too --------
# Also note that changes are performed in place and are not atomic
# We should fix that and write to temporary files, stop, swap and start
# Lock configuration while working
(
flock -e 200
for FILE in ${CHANGED}
do
case ${FILE} in
/etc/letsencrypt/acme.json)
for certdomain in ${SSL_DOMAIN} ${HOSTNAME} ${DOMAINNAME}
do
if _extract_certs_from_acme "${certdomain}"
then
break
fi
done
;;
* ) _notify 'warn' 'file not found for certificate in check_for_changes.sh' ;;
esac
done
# regenerate postix aliases
echo "root: ${PM_ADDRESS}" >/etc/aliases
if [[ -f /tmp/docker-mailserver/postfix-aliases.cf ]]
then
cat /tmp/docker-mailserver/postfix-aliases.cf >>/etc/aliases
fi
postalias /etc/aliases
# regenerate postfix accounts
: >/etc/postfix/vmailbox
: >/etc/dovecot/userdb
if [[ -f /tmp/docker-mailserver/postfix-accounts.cf ]] && [[ ${ENABLE_LDAP} -ne 1 ]]
then
sed -i 's/\r//g' /tmp/docker-mailserver/postfix-accounts.cf
echo "# WARNING: this file is auto-generated. Modify config/postfix-accounts.cf to edit user list." >/etc/postfix/vmailbox
# Checking that /tmp/docker-mailserver/postfix-accounts.cf ends with a newline
# shellcheck disable=SC1003
sed -i -e '$a\' /tmp/docker-mailserver/postfix-accounts.cf
chown dovecot:dovecot /etc/dovecot/userdb
chmod 640 /etc/dovecot/userdb
sed -i -e '/\!include auth-ldap\.conf\.ext/s/^/#/' /etc/dovecot/conf.d/10-auth.conf
sed -i -e '/\!include auth-passwdfile\.inc/s/^#//' /etc/dovecot/conf.d/10-auth.conf
# rebuild relay host
if [[ -n ${RELAY_HOST} ]]
then
# keep old config
: >/etc/postfix/sasl_passwd
if [[ -n ${SASL_PASSWD} ]]
then
echo "${SASL_PASSWD}" >>/etc/postfix/sasl_passwd
fi
# add domain-specific auth from config file
if [[ -f /tmp/docker-mailserver/postfix-sasl-password.cf ]]
then
while read -r LINE
do
if ! echo "${LINE}" | grep -q -e "\s*#"
then
echo "${LINE}" >>/etc/postfix/sasl_passwd
fi
done < <(grep -v "^\s*$\|^\s*\#" /tmp/docker-mailserver/postfix-sasl-password.cf || true)
fi
# add default relay
if [[ -n "${RELAY_USER}" ]] && [[ -n "${RELAY_PASSWORD}" ]]
then
echo "[${RELAY_HOST}]:${RELAY_PORT} ${RELAY_USER}:${RELAY_PASSWORD}" >>/etc/postfix/sasl_passwd
fi
fi
# creating users ; 'pass' is encrypted
# comments and empty lines are ignored
while IFS=$'|' read -r LOGIN PASS
do
USER=$(echo "${LOGIN}" | cut -d @ -f1)
DOMAIN=$(echo "${LOGIN}" | cut -d @ -f2)
user_attributes=""
# test if user has a defined quota
if [[ -f /tmp/docker-mailserver/dovecot-quotas.cf ]]
then
declare -a USER_QUOTA
IFS=':' ; read -r -a USER_QUOTA < <(grep "${USER}@${DOMAIN}:" -i /tmp/docker-mailserver/dovecot-quotas.cf)
unset IFS
[[ ${#USER_QUOTA[@]} -eq 2 ]] && user_attributes="${user_attributes}userdb_quota_rule=*:bytes=${USER_QUOTA[1]}"
fi
echo "${LOGIN} ${DOMAIN}/${USER}/" >>/etc/postfix/vmailbox
# user database for dovecot has the following format:
# user:password:uid:gid:(gecos):home:(shell):extra_fields
# example :
# ${LOGIN}:${PASS}:5000:5000::/var/mail/${DOMAIN}/${USER}::userdb_mail=maildir:/var/mail/${DOMAIN}/${USER}
echo "${LOGIN}:${PASS}:5000:5000::/var/mail/${DOMAIN}/${USER}::${user_attributes}" >>/etc/dovecot/userdb
mkdir -p "/var/mail/${DOMAIN}/${USER}"
if [[ -e /tmp/docker-mailserver/${LOGIN}.dovecot.sieve ]]
then
cp "/tmp/docker-mailserver/${LOGIN}.dovecot.sieve" "/var/mail/${DOMAIN}/${USER}/.dovecot.sieve"
fi
echo "${DOMAIN}" >>/tmp/vhost.tmp
done < <(grep -v "^\s*$\|^\s*\#" /tmp/docker-mailserver/postfix-accounts.cf)
fi
[[ -n ${RELAY_HOST} ]] && _populate_relayhost_map
if [[ -f /etc/postfix/sasl_passwd ]]
then
chown root:root /etc/postfix/sasl_passwd
chmod 0600 /etc/postfix/sasl_passwd
fi
if [[ -f postfix-virtual.cf ]]
then
# regenerate postfix aliases
: >/etc/postfix/virtual
: >/etc/postfix/regexp
if [[ -f /tmp/docker-mailserver/postfix-virtual.cf ]]
then
cp -f /tmp/docker-mailserver/postfix-virtual.cf /etc/postfix/virtual
# the `to` seems to be important; don't delete it
# shellcheck disable=SC2034
while read -r FROM TO
do
UNAME=$(echo "${FROM}" | cut -d @ -f1)
DOMAIN=$(echo "${FROM}" | cut -d @ -f2)
# if they are equal it means the line looks like: "user1 other@domain.tld"
[ "${UNAME}" != "${DOMAIN}" ] && echo "${DOMAIN}" >>/tmp/vhost.tmp
done < <(grep -v "^\s*$\|^\s*\#" /tmp/docker-mailserver/postfix-virtual.cf || true)
fi
if [[ -f /tmp/docker-mailserver/postfix-regexp.cf ]]
then
cp -f /tmp/docker-mailserver/postfix-regexp.cf /etc/postfix/regexp
sed -i -e '/^virtual_alias_maps/{
s/ regexp:.*//
s/$/ regexp:\/etc\/postfix\/regexp/
}' /etc/postfix/main.cf
fi
fi
if [[ -f /tmp/vhost.tmp ]]
then
# shellcheck disable=SC2002
cat /tmp/vhost.tmp | sort | uniq >/etc/postfix/vhost && rm /tmp/vhost.tmp
fi
if [[ $(find /var/mail -maxdepth 3 -a \( \! -user 5000 -o \! -group 5000 \) | grep -c .) -ne 0 ]]
then
chown -R 5000:5000 /var/mail
fi
supervisorctl restart postfix
# prevent restart of dovecot when smtp_only=1
[[ ${SMTP_ONLY} -ne 1 ]] && supervisorctl restart dovecot
) 200<postfix-accounts.cf # end lock
# mark changes as applied
mv "${CHKSUM_FILE}.new" "${CHKSUM_FILE}"
fi
sleep 1
done

View file

@ -0,0 +1,31 @@
#! /bin/bash
# You cannot start fail2ban in some foreground mode and
# it's more or less important that docker doesn't kill
# fail2ban and its chilren if you stop the container.
#
# Use this script with supervisord and it will take
# care about starting and stopping fail2ban correctly.
#
# supervisord config snippet for fail2ban-wrapper:
#
# [program:fail2ban]
# process_name = fail2ban
# command = /path/to/fail2ban-wrapper.sh
# startsecs = 0
# autorestart = false
#
trap "/usr/bin/fail2ban-client stop" SIGINT
trap "/usr/bin/fail2ban-client stop" SIGTERM
trap "/usr/bin/fail2ban-client reload" SIGHUP
/usr/bin/fail2ban-client start
sleep 5
# wait until fail2ban is dead (triggered by trap)
while kill -0 "$(< /var/run/fail2ban/fail2ban.pid)"
do
sleep 5
done

View file

@ -0,0 +1,191 @@
#! /bin/bash
DMS_DEBUG="${DMS_DEBUG:=0}"
# ? BIN HELPER
function errex
{
echo "${@}" 1>&2
exit 1
}
function escape
{
echo "${1//./\\.}"
}
# ? IP & CIDR
function _mask_ip_digit
{
if [[ ${1} -ge 8 ]]
then
MASK=255
elif [[ ${1} -le 0 ]]
then
MASK=0
else
VALUES=(0 128 192 224 240 248 252 254 255)
MASK=${VALUES[${1}]}
fi
local DVAL=${2}
((DVAL&=MASK))
echo "${DVAL}"
}
# Transforms a specific IP with CIDR suffix
# like 1.2.3.4/16 to subnet with cidr suffix
# like 1.2.0.0/16.
# Assumes correct IP and subnet are provided.
function _sanitize_ipv4_to_subnet_cidr
{
local DIGIT_PREFIX_LENGTH="${1#*/}"
declare -a MASKED_DIGITS DIGITS
IFS='.' ; read -r -a DIGITS < <(echo "${1%%/*}") ; unset IFS
for ((i = 0 ; i < 4 ; i++))
do
MASKED_DIGITS[i]=$(_mask_ip_digit "${DIGIT_PREFIX_LENGTH}" "${DIGITS[i]}")
DIGIT_PREFIX_LENGTH=$((DIGIT_PREFIX_LENGTH - 8))
done
echo "${MASKED_DIGITS[0]}.${MASKED_DIGITS[1]}.${MASKED_DIGITS[2]}.${MASKED_DIGITS[3]}/${1#*/}"
}
export -f _sanitize_ipv4_to_subnet_cidr
# ? ACME
function _extract_certs_from_acme
{
local KEY
# shellcheck disable=SC2002
KEY=$(cat /etc/letsencrypt/acme.json | python -c "
import sys,json
acme = json.load(sys.stdin)
for key, value in acme.items():
certs = value['Certificates']
if certs is not None:
for cert in certs:
if 'domain' in cert and 'key' in cert:
if 'main' in cert['domain'] and cert['domain']['main'] == '${1}' or 'sans' in cert['domain'] and '${1}' in cert['domain']['sans']:
print cert['key']
break
")
local CERT
# shellcheck disable=SC2002
CERT=$(cat /etc/letsencrypt/acme.json | python -c "
import sys,json
acme = json.load(sys.stdin)
for key, value in acme.items():
certs = value['Certificates']
if certs is not None:
for cert in certs:
if 'domain' in cert and 'certificate' in cert:
if 'main' in cert['domain'] and cert['domain']['main'] == '${1}' or 'sans' in cert['domain'] and '${1}' in cert['domain']['sans']:
print cert['certificate']
break
")
if [[ -n "${KEY}${CERT}" ]]
then
mkdir -p "/etc/letsencrypt/live/${HOSTNAME}/"
echo "${KEY}" | base64 -d >/etc/letsencrypt/live/"${HOSTNAME}"/key.pem || exit 1
echo "${CERT}" | base64 -d >/etc/letsencrypt/live/"${HOSTNAME}"/fullchain.pem || exit 1
echo "Cert found in /etc/letsencrypt/acme.json for ${1}"
return 0
else
return 1
fi
}
export -f _extract_certs_from_acme
# ? Notifications
function _notify
{
local FINAL_MSG=''
local MSG="${2:-}"
local TYPE="${1:-}"
case "${TYPE}" in
'none' ) FINAL_MSG=' ' ;;
'tasklog' ) FINAL_MSG="[ \e[0;92mTASKLOG\e[0m ] ${MSG}" ;; # light green
'warn' ) FINAL_MSG="[ \e[0;93mWARNING\e[0m ] ${MSG}" ;; # light yellow
'err' ) FINAL_MSG="[ \e[0;31mERROR\e[0m ] ${MSG}" ;; # light red
'fatal' ) FINAL_MSG="[ \e[0;91mFATAL\e[0m ] ${MSG}" ;; # red
'inf' ) [[ ${DMS_DEBUG} -eq 1 ]] && FINAL_MSG="[[ \e[0;34mINFO\e[0m ]] ${MSG}" ;; # light blue
'task' ) [[ ${DMS_DEBUG} -eq 1 ]] && FINAL_MSG="[[ \e[0;37mTASK\e[0m ]] ${MSG}" ;; # light grey
* ) ;;
esac
[[ -n ${FINAL_MSG} ]] && echo "-e${3:-}" "${FINAL_MSG}"
}
export -f _notify
# ? Relay Host Map
# setup /etc/postfix/relayhost_map
# --
# @domain1.com [smtp.mailgun.org]:587
# @domain2.com [smtp.mailgun.org]:587
# @domain3.com [smtp.mailgun.org]:587
function _populate_relayhost_map
{
: >/etc/postfix/relayhost_map
chown root:root /etc/postfix/relayhost_map
chmod 0600 /etc/postfix/relayhost_map
if [[ -f /tmp/docker-mailserver/postfix-relaymap.cf ]]
then
_notify 'inf' "Adding relay mappings from postfix-relaymap.cf"
# keep lines which are not a comment *and* have a destination.
sed -n '/^\s*[^#[:space:]]\S*\s\+\S/p' /tmp/docker-mailserver/postfix-relaymap.cf >> /etc/postfix/relayhost_map
fi
{
# note: won't detect domains when lhs has spaces (but who does that?!)
sed -n '/^\s*[^#[:space:]]/ s/^[^@|]*@\([^|]\+\)|.*$/\1/p' /tmp/docker-mailserver/postfix-accounts.cf
[ -f /tmp/docker-mailserver/postfix-virtual.cf ] && sed -n '/^\s*[^#[:space:]]/ s/^\s*[^@[:space:]]*@\(\S\+\)\s.*/\1/p' /tmp/docker-mailserver/postfix-virtual.cf
} | while read -r domain
do
# domain not already present *and* not ignored
if ! grep -q -e "^@${domain}\b" /etc/postfix/relayhost_map && ! grep -qs -e "^\s*@${domain}\s*$" /tmp/docker-mailserver/postfix-relaymap.cf
then
_notify 'inf' "Adding relay mapping for ${domain}"
echo "@${domain} [${RELAY_HOST}]:${RELAY_PORT}" >> /etc/postfix/relayhost_map
fi
done
}
export -f _populate_relayhost_map
# ? File Checksums
# file storing the checksums of the monitored files.
# shellcheck disable=SC2034
CHKSUM_FILE=/tmp/docker-mailserver-config-chksum
# Compute checksums of monitored files.
function _monitored_files_checksums
{
(
cd /tmp/docker-mailserver || exit 1
exec sha512sum 2>/dev/null -- \
postfix-accounts.cf \
postfix-virtual.cf \
postfix-aliases.cf \
dovecot-quotas.cf \
/etc/letsencrypt/acme.json \
"/etc/letsencrypt/live/${HOSTNAME}/key.pem" \
"/etc/letsencrypt/live/${HOSTNAME}/privkey.pem" \
"/etc/letsencrypt/live/${HOSTNAME}/fullchain.pem"
)
}
export -f _monitored_files_checksums

View file

@ -0,0 +1,30 @@
#! /bin/bash
# You cannot start postfix in some foreground mode and
# it's more or less important that docker doesn't kill
# postfix and its chilren if you stop the container.
#
# Use this script with supervisord and it will take
# care about starting and stopping postfix correctly.
#
# supervisord config snippet for postfix-wrapper:
#
# [program:postfix]
# process_name = postfix
# command = /path/to/postfix-wrapper.sh
# startsecs = 0
# autorestart = false
#
trap "service postfix stop" SIGINT
trap "service postfix stop" SIGTERM
trap "service postfix reload" SIGHUP
service postfix start
sleep 5
# wait until postfix is dead (triggered by trap)
while kill -0 "$(< /var/spool/postfix/pid/master.pid)"
do
sleep 5
done

View file

@ -0,0 +1,48 @@
#! /bin/bash
function _generate_secret { ( umask 0077 ; dd if=/dev/urandom bs=24 count=1 2>/dev/null | base64 -w0 > "${1}" ; ) ; }
if [[ -n ${SRS_DOMAINNAME} ]]
then
NEW_DOMAIN_NAME="${SRS_DOMAINNAME}"
elif [[ -n ${OVERRIDE_HOSTNAME} ]]
then
NEW_DOMAIN_NAME="${OVERRIDE_HOSTNAME#*.}"
elif [[ -n ${DOMAINNAME} ]]
then
NEW_DOMAIN_NAME="${DOMAINNAME}"
else
NEW_DOMAIN_NAME=$(hostname -d)
fi
sed -i -e "s/localdomain/${NEW_DOMAIN_NAME}/g" /etc/default/postsrsd
POSTSRSD_SECRET_FILE='/etc/postsrsd.secret'
POSTSRSD_STATE_DIR='/var/mail-state/etc-postsrsd'
POSTSRSD_STATE_SECRET_FILE="${POSTSRSD_STATE_DIR}/postsrsd.secret"
if [[ -n ${SRS_SECRET} ]]
then
( umask 0077 ; echo "${SRS_SECRET}" | tr ',' '\n' > "${POSTSRSD_SECRET_FILE}" ; )
else
if [[ ${ONE_DIR} -eq 1 ]]
then
if [[ ! -f ${POSTSRSD_STATE_SECRET_FILE} ]]
then
install -d -m 0775 "${POSTSRSD_STATE_DIR}"
_generate_secret "${POSTSRSD_STATE_SECRET_FILE}"
fi
install -m 0400 "${POSTSRSD_STATE_SECRET_FILE}" "${POSTSRSD_SECRET_FILE}"
elif [[ ! -f ${POSTSRSD_SECRET_FILE} ]]
then
_generate_secret "${POSTSRSD_SECRET_FILE}"
fi
fi
if [[ -n ${SRS_EXCLUDE_DOMAINS} ]]
then
sed -i -e "s/^#\?SRS_EXCLUDE_DOMAINS=.*$/SRS_EXCLUDE_DOMAINS=${SRS_EXCLUDE_DOMAINS}/g" /etc/default/postsrsd
fi
/etc/init.d/postsrsd start

14
target/scripts/quota-warning.sh Executable file
View file

@ -0,0 +1,14 @@
#! /bin/bash
# Report a quota usage warning to an user
PERCENT="${1}"
USER="${2}"
DOMAIN="${3}"
cat << EOF | /usr/lib/dovecot/dovecot-lda -d "${USER}" -o "plugin/quota=maildir:User quota:noenforcing"
From: postmaster@${DOMAIN}
Subject: quota warning
Your mailbox is now ${PERCENT}% full.
EOF

2149
target/scripts/start-mailserver.sh Executable file

File diff suppressed because it is too large Load diff