144 lines
6.9 KiB
Bash
Executable file
144 lines
6.9 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# =====================================================================================
|
|
# Disaster-recovery drill - self-contained and paranoid.
|
|
#
|
|
# Proves that a restore actually recovers THE data, not just a working-looking directory:
|
|
# lab DC -> marker OU -> online backup -> restore as a NEW DC -> marker + SID must match
|
|
#
|
|
# This exists because `samba-tool domain backup restore` can silently leave you with a
|
|
# *fresh forest* if the restore step is skipped and the container then self-provisions.
|
|
# A restore that passes health checks but lost your data is the worst failure mode there is.
|
|
#
|
|
# ./tests/dr-drill.sh # tear down afterwards
|
|
# KEEP=1 ./tests/dr-drill.sh # leave lab + restored DC running
|
|
# =====================================================================================
|
|
set -uo pipefail
|
|
cd "$(dirname "$0")/.."
|
|
|
|
IMG=local/samba-ad-dc:4.22-debian13
|
|
PW='Lab!Only9Tester#2026'
|
|
REALM=TESTLAB.LOCAL
|
|
ZONE=testlab.local
|
|
BASEDN="DC=testlab,DC=local"
|
|
LAB=ad-lab
|
|
DRILL=ad-drill
|
|
DRILL_IP=172.28.90.30
|
|
STAMP="$(date -u +%H%M%S)"
|
|
MARKER="DR-MARKER-${STAMP}"
|
|
FAIL=0
|
|
|
|
note() { printf '\n\033[1m==> %s\033[0m\n' "$*"; }
|
|
die() { printf '\033[31mDRILL FAILED: %s\033[0m\n' "$*" >&2; FAIL=1; }
|
|
sid_of() { # $1 container -> base64 domain objectSid (authenticates inside the container)
|
|
docker exec "$1" bash -c "printf '%s\n' '$PW' | kinit administrator@${REALM} >/dev/null 2>&1; \
|
|
ldapsearch -H ldap://127.0.0.1 -Y GSSAPI -s base -b '${BASEDN}' '(objectClass=domainDNS)' objectSid 2>/dev/null \
|
|
| sed -n 's/^[Oo]bjectSid:: //p' | tr -d '\r'"
|
|
}
|
|
|
|
cleanup() {
|
|
if [ "${KEEP:-0}" != 1 ]; then
|
|
docker rm -f "$DRILL" >/dev/null 2>&1 || true
|
|
docker volume rm drill-lib drill-conf >/dev/null 2>&1 || true
|
|
docker compose -f docker-compose.lab.yml down -v >/dev/null 2>&1 || true
|
|
fi
|
|
}
|
|
trap cleanup EXIT INT TERM
|
|
|
|
caps=(--cap-drop ALL)
|
|
for c in CHOWN FOWNER SETUID SETGID DAC_OVERRIDE NET_BIND_SERVICE SYS_NICE SYS_RESOURCE AUDIT_WRITE SYS_ADMIN; do
|
|
caps+=(--cap-add "$c")
|
|
done
|
|
|
|
note "1/7 fresh lab DC (this is the 'production' we are protecting)"
|
|
docker compose -f docker-compose.lab.yml up -d >/dev/null 2>&1 || die "could not start the lab"
|
|
h=none
|
|
for _ in $(seq 1 60); do
|
|
h="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$LAB" 2>/dev/null)"
|
|
[ "$h" = healthy ] && break
|
|
sleep 4
|
|
done
|
|
[ "$h" = healthy ] || die "lab not healthy ($h)"
|
|
src_sid="$(sid_of "$LAB")"
|
|
[ -n "$src_sid" ] || die "could not read the source domain SID"
|
|
printf ' %s healthy, domain SID %s\n' "$LAB" "$src_sid"
|
|
|
|
note "2/7 plant a marker OU so we can prove data survived"
|
|
docker exec "$LAB" bash -c "printf '%s\n' '$PW' | kinit administrator@${REALM} 2>/dev/null && samba-tool ou create 'OU=${MARKER}'" >/dev/null 2>&1 \
|
|
|| die "could not create the marker OU"
|
|
printf ' planted OU=%s\n' "$MARKER"
|
|
|
|
note "3/7 online backup"
|
|
AD_CONTAINER="$LAB" AD_REALM="$REALM" AD_DNS_DOMAIN="$ZONE" AD_HOSTNAME=dc1 AD_ADMIN_PASSWORD="$PW" \
|
|
./scripts/backup.sh >/tmp/dr-backup.log 2>&1 || { tail -20 /tmp/dr-backup.log; die "backup.sh failed"; }
|
|
BK="$(ls -1t backups/*.tar.bz2 backups/*.tgz 2>/dev/null | head -1)"
|
|
[ -n "$BK" ] || die "no archive produced"
|
|
printf ' %s (%s)\n' "$(basename "$BK")" "$(du -h "$BK" | cut -f1)"
|
|
|
|
note "4/7 restore into a fresh volume AS A NEW DC (dc3) - samba's restore does a join"
|
|
# Remove the drill container FIRST: a still-mounted volume cannot be removed, and reusing a
|
|
# stale volume would silently test the wrong tree.
|
|
docker rm -f "$DRILL" >/dev/null 2>&1 || true
|
|
docker volume rm drill-lib drill-conf >/dev/null 2>&1 || true
|
|
if docker volume inspect drill-lib >/dev/null 2>&1; then
|
|
die "could not remove the drill volume (something still holds it) - refusing to test a stale tree"
|
|
fi
|
|
docker run --rm --network adlab --ip "$DRILL_IP" \
|
|
--cap-drop ALL --cap-add SYS_ADMIN --cap-add CHOWN --cap-add FOWNER \
|
|
--cap-add DAC_OVERRIDE --cap-add SETUID --cap-add SETGID \
|
|
-v "$PWD/backups:/restore:ro" -v drill-lib:/var/lib/samba -v drill-conf:/etc/samba \
|
|
-e AD_REALM="$REALM" -e AD_DNS_DOMAIN="$ZONE" -e AD_HOSTNAME=dc3 -e AD_IP="$DRILL_IP" \
|
|
-e AD_RESTORE_FROM="/restore/$(basename "$BK")" -e AD_RESTORE_CONFIRM=yes \
|
|
"$IMG" >/tmp/dr-restore.log 2>&1
|
|
rc=$?
|
|
tail -4 /tmp/dr-restore.log | sed 's/^/ /'
|
|
[ "$rc" -eq 0 ] || die "restore exited $rc"
|
|
grep -q 'restore complete' /tmp/dr-restore.log || die "restore did not report completion"
|
|
|
|
note "5/7 the fresh volume must already hold sam.ldb BEFORE anything starts serving"
|
|
has=$(docker run --rm -v drill-lib:/var/lib/samba --entrypoint bash "$IMG" \
|
|
-c '[ -f /var/lib/samba/private/sam.ldb ] && echo yes || echo no' | tail -1)
|
|
[ "$has" = yes ] || die "restored volume has no sam.ldb"
|
|
printf ' sam.ldb present; marker: %s\n' "$(docker run --rm -v drill-lib:/var/lib/samba --entrypoint bash "$IMG" -c 'head -1 /var/lib/samba/.ad-provisioned')"
|
|
|
|
note "6/7 serve the restored DC"
|
|
docker rm -f "$DRILL" >/dev/null 2>&1 || true
|
|
docker run -d --name "$DRILL" --hostname "dc3.${ZONE}" "${caps[@]}" --network adlab --ip "$DRILL_IP" \
|
|
-v drill-lib:/var/lib/samba -v drill-conf:/etc/samba \
|
|
-e AD_REALM="$REALM" -e AD_DNS_DOMAIN="$ZONE" -e AD_HOSTNAME=dc3 -e AD_IP="$DRILL_IP" \
|
|
-e AD_ADMIN_PASSWORD="$PW" -e AD_DNS_FORWARDER='1.1.1.1' "$IMG" >/dev/null || die "could not start restored DC"
|
|
h=none
|
|
for _ in $(seq 1 60); do
|
|
h="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$DRILL" 2>/dev/null)"
|
|
[ "$h" = healthy ] && break
|
|
sleep 4
|
|
done
|
|
[ "$h" = healthy ] || { docker logs --tail 40 "$DRILL"; die "restored DC not healthy"; }
|
|
printf ' %s healthy on %s\n' "$DRILL" "$DRILL_IP"
|
|
|
|
note "7/7 integrity assertions"
|
|
if docker logs "$DRILL" 2>&1 | grep -q 'provisioning NEW forest'; then
|
|
die "restored DC provisioned a NEW forest instead of serving the restore"
|
|
fi
|
|
new_sid="$(sid_of "$DRILL")"
|
|
if [ -z "$new_sid" ]; then
|
|
die "could not read the restored domain SID (empty) - refusing to call this a match"
|
|
elif [ "$new_sid" = "$src_sid" ]; then
|
|
printf ' SID matches the source domain ✔ (%s)\n' "$new_sid"
|
|
else
|
|
die "domain SID mismatch (src=$src_sid restored=$new_sid)"
|
|
fi
|
|
found=$(docker exec "$DRILL" bash -c "printf '%s\n' '$PW' | kinit administrator@${REALM} >/dev/null 2>&1; ldapsearch -H ldap://127.0.0.1 -Y GSSAPI -s base -b 'OU=${MARKER},${BASEDN}' dn 2>/dev/null | grep -c '^dn:'" || true)
|
|
if [ "${found:-0}" -ge 1 ]; then
|
|
printf ' marker OU survived the restore ✔\n'
|
|
else
|
|
die "marker OU '${MARKER}' is missing after restore - data loss"
|
|
fi
|
|
AD_SKIP_DOTENV=1 AD_CONTAINER="$DRILL" AD_REALM="$REALM" AD_DNS_DOMAIN="$ZONE" \
|
|
AD_HOSTNAME=dc3 AD_ADMIN_PASSWORD="$PW" ./scripts/verify.sh || die "verify failed against restored DC"
|
|
|
|
if [ "$FAIL" -eq 0 ]; then
|
|
printf '\n\033[32mDR DRILL PASSED\033[0m archive=%s -> %s (dc3)\n' "$(basename "$BK")" "$DRILL"
|
|
else
|
|
printf '\n\033[31mDR DRILL FAILED\033[0m see messages above\n'
|
|
fi
|
|
exit "$FAIL"
|