First Commit

2026-09-24 11:29:54 -05:00 · 2026-09-24 11:29:54 -05:00 · 2e5a4751a7
commit 2e5a4751a7
30 changed files with 2517 additions and 0 deletions
--- a/tests/dr-drill.sh
+++ b/tests/dr-drill.sh
@ -0,0 +1,144 @@
+#!/usr/bin/env bash
+# =====================================================================================
+# Disaster-recovery drill - self-contained and paranoid.
+#
+# Proves that a restore actually recovers THE data, not just a working-looking directory:
+#   lab DC -> marker OU -> online backup -> restore as a NEW DC -> marker + SID must match
+#
+# This exists because `samba-tool domain backup restore` can silently leave you with a
+# *fresh forest* if the restore step is skipped and the container then self-provisions.
+# A restore that passes health checks but lost your data is the worst failure mode there is.
+#
+#   ./tests/dr-drill.sh          # tear down afterwards
+#   KEEP=1 ./tests/dr-drill.sh   # leave lab + restored DC running
+# =====================================================================================
+set -uo pipefail
+cd "$(dirname "$0")/.."
+
+IMG=local/samba-ad-dc:4.22-debian13
+PW='Lab!Only9Tester#2026'
+REALM=TESTLAB.LOCAL
+ZONE=testlab.local
+BASEDN="DC=testlab,DC=local"
+LAB=ad-lab
+DRILL=ad-drill
+DRILL_IP=172.28.90.30
+STAMP="$(date -u +%H%M%S)"
+MARKER="DR-MARKER-${STAMP}"
+FAIL=0
+
+note() { printf '\n\033[1m==> %s\033[0m\n' "$*"; }
+die()  { printf '\033[31mDRILL FAILED: %s\033[0m\n' "$*" >&2; FAIL=1; }
+sid_of() {  # $1 container -> base64 domain objectSid (authenticates inside the container)
+    docker exec "$1" bash -c "printf '%s\n' '$PW' | kinit administrator@${REALM} >/dev/null 2>&1; \
+        ldapsearch -H ldap://127.0.0.1 -Y GSSAPI -s base -b '${BASEDN}' '(objectClass=domainDNS)' objectSid 2>/dev/null \
+        | sed -n 's/^[Oo]bjectSid:: //p' | tr -d '\r'"
+}
+
+cleanup() {
+    if [ "${KEEP:-0}" != 1 ]; then
+        docker rm -f "$DRILL" >/dev/null 2>&1 || true
+        docker volume rm drill-lib drill-conf >/dev/null 2>&1 || true
+        docker compose -f docker-compose.lab.yml down -v >/dev/null 2>&1 || true
+    fi
+}
+trap cleanup EXIT INT TERM
+
+caps=(--cap-drop ALL)
+for c in CHOWN FOWNER SETUID SETGID DAC_OVERRIDE NET_BIND_SERVICE SYS_NICE SYS_RESOURCE AUDIT_WRITE SYS_ADMIN; do
+    caps+=(--cap-add "$c")
+done
+
+note "1/7 fresh lab DC (this is the 'production' we are protecting)"
+docker compose -f docker-compose.lab.yml up -d >/dev/null 2>&1 || die "could not start the lab"
+h=none
+for _ in $(seq 1 60); do
+    h="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$LAB" 2>/dev/null)"
+    [ "$h" = healthy ] && break
+    sleep 4
+done
+[ "$h" = healthy ] || die "lab not healthy ($h)"
+src_sid="$(sid_of "$LAB")"
+[ -n "$src_sid" ] || die "could not read the source domain SID"
+printf '   %s healthy, domain SID %s\n' "$LAB" "$src_sid"
+
+note "2/7 plant a marker OU so we can prove data survived"
+docker exec "$LAB" bash -c "printf '%s\n' '$PW' | kinit administrator@${REALM} 2>/dev/null && samba-tool ou create 'OU=${MARKER}'" >/dev/null 2>&1 \
+    || die "could not create the marker OU"
+printf '   planted OU=%s\n' "$MARKER"
+
+note "3/7 online backup"
+AD_CONTAINER="$LAB" AD_REALM="$REALM" AD_DNS_DOMAIN="$ZONE" AD_HOSTNAME=dc1 AD_ADMIN_PASSWORD="$PW" \
+    ./scripts/backup.sh >/tmp/dr-backup.log 2>&1 || { tail -20 /tmp/dr-backup.log; die "backup.sh failed"; }
+BK="$(ls -1t backups/*.tar.bz2 backups/*.tgz 2>/dev/null | head -1)"
+[ -n "$BK" ] || die "no archive produced"
+printf '   %s (%s)\n' "$(basename "$BK")" "$(du -h "$BK" | cut -f1)"
+
+note "4/7 restore into a fresh volume AS A NEW DC (dc3) - samba's restore does a join"
+# Remove the drill container FIRST: a still-mounted volume cannot be removed, and reusing a
+# stale volume would silently test the wrong tree.
+docker rm -f "$DRILL" >/dev/null 2>&1 || true
+docker volume rm drill-lib drill-conf >/dev/null 2>&1 || true
+if docker volume inspect drill-lib >/dev/null 2>&1; then
+    die "could not remove the drill volume (something still holds it) - refusing to test a stale tree"
+fi
+docker run --rm --network adlab --ip "$DRILL_IP" \
+    --cap-drop ALL --cap-add SYS_ADMIN --cap-add CHOWN --cap-add FOWNER \
+    --cap-add DAC_OVERRIDE --cap-add SETUID --cap-add SETGID \
+    -v "$PWD/backups:/restore:ro" -v drill-lib:/var/lib/samba -v drill-conf:/etc/samba \
+    -e AD_REALM="$REALM" -e AD_DNS_DOMAIN="$ZONE" -e AD_HOSTNAME=dc3 -e AD_IP="$DRILL_IP" \
+    -e AD_RESTORE_FROM="/restore/$(basename "$BK")" -e AD_RESTORE_CONFIRM=yes \
+    "$IMG" >/tmp/dr-restore.log 2>&1
+rc=$?
+tail -4 /tmp/dr-restore.log | sed 's/^/   /'
+[ "$rc" -eq 0 ] || die "restore exited $rc"
+grep -q 'restore complete' /tmp/dr-restore.log || die "restore did not report completion"
+
+note "5/7 the fresh volume must already hold sam.ldb BEFORE anything starts serving"
+has=$(docker run --rm -v drill-lib:/var/lib/samba --entrypoint bash "$IMG" \
+        -c '[ -f /var/lib/samba/private/sam.ldb ] && echo yes || echo no' | tail -1)
+[ "$has" = yes ] || die "restored volume has no sam.ldb"
+printf '   sam.ldb present; marker: %s\n' "$(docker run --rm -v drill-lib:/var/lib/samba --entrypoint bash "$IMG" -c 'head -1 /var/lib/samba/.ad-provisioned')"
+
+note "6/7 serve the restored DC"
+docker rm -f "$DRILL" >/dev/null 2>&1 || true
+docker run -d --name "$DRILL" --hostname "dc3.${ZONE}" "${caps[@]}" --network adlab --ip "$DRILL_IP" \
+    -v drill-lib:/var/lib/samba -v drill-conf:/etc/samba \
+    -e AD_REALM="$REALM" -e AD_DNS_DOMAIN="$ZONE" -e AD_HOSTNAME=dc3 -e AD_IP="$DRILL_IP" \
+    -e AD_ADMIN_PASSWORD="$PW" -e AD_DNS_FORWARDER='1.1.1.1' "$IMG" >/dev/null || die "could not start restored DC"
+h=none
+for _ in $(seq 1 60); do
+    h="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$DRILL" 2>/dev/null)"
+    [ "$h" = healthy ] && break
+    sleep 4
+done
+[ "$h" = healthy ] || { docker logs --tail 40 "$DRILL"; die "restored DC not healthy"; }
+printf '   %s healthy on %s\n' "$DRILL" "$DRILL_IP"
+
+note "7/7 integrity assertions"
+if docker logs "$DRILL" 2>&1 | grep -q 'provisioning NEW forest'; then
+    die "restored DC provisioned a NEW forest instead of serving the restore"
+fi
+new_sid="$(sid_of "$DRILL")"
+if [ -z "$new_sid" ]; then
+    die "could not read the restored domain SID (empty) - refusing to call this a match"
+elif [ "$new_sid" = "$src_sid" ]; then
+    printf '   SID matches the source domain  ✔  (%s)\n' "$new_sid"
+else
+    die "domain SID mismatch (src=$src_sid restored=$new_sid)"
+fi
+found=$(docker exec "$DRILL" bash -c "printf '%s\n' '$PW' | kinit administrator@${REALM} >/dev/null 2>&1; ldapsearch -H ldap://127.0.0.1 -Y GSSAPI -s base -b 'OU=${MARKER},${BASEDN}' dn 2>/dev/null | grep -c '^dn:'" || true)
+if [ "${found:-0}" -ge 1 ]; then
+    printf '   marker OU survived the restore  ✔\n'
+else
+    die "marker OU '${MARKER}' is missing after restore - data loss"
+fi
+AD_SKIP_DOTENV=1 AD_CONTAINER="$DRILL" AD_REALM="$REALM" AD_DNS_DOMAIN="$ZONE" \
+AD_HOSTNAME=dc3 AD_ADMIN_PASSWORD="$PW" ./scripts/verify.sh || die "verify failed against restored DC"
+
+if [ "$FAIL" -eq 0 ]; then
+    printf '\n\033[32mDR DRILL PASSED\033[0m  archive=%s -> %s (dc3)\n' "$(basename "$BK")" "$DRILL"
+else
+    printf '\n\033[31mDR DRILL FAILED\033[0m  see messages above\n'
+fi
+exit "$FAIL"
--- a/tests/ldaps-test.sh
+++ b/tests/ldaps-test.sh
@ -0,0 +1,77 @@
+#!/usr/bin/env bash
+# =====================================================================================
+# LDAPS test. Proves three separate things:
+#   * the DC serves 636 and the chain verifies against our CA
+#   * the SAN actually matches the DC FQDN (Windows requires SAN, not CN)
+#   * a client WITHOUT our CA is rejected (i.e. we are not accidentally self-trusted)
+# Run ./scripts/tls.sh first to create secrets/tls/.
+#   ./tests/ldaps-test.sh          # tears down afterwards
+# =====================================================================================
+# LDAPS end-to-end check against a freshly provisioned DC with our generated certs.
+set -uo pipefail
+cd /dockers/WindowsAD
+PW='Lab!Only9Tester#2026'
+LABTLS="$PWD/secrets/tls-lab"
+trap '[ "${KEEP:-0}" = 1 ] || { docker rm -f ad-tlslab >/dev/null 2>&1; docker volume rm tlslab-lib >/dev/null 2>&1; rm -rf "$LABTLS"; }' EXIT
+
+# certs for THIS lab only. Never reuse secrets/tls: that is the production chain,
+# regenerating it here would silently destroy the trust anchor every client has installed.
+echo "=== generating throwaway testlab CA + leaf (isolated dir, no SYSVOL publish) ==="
+mkdir -p "$LABTLS"
+AD_TLS_DIR="$LABTLS" AD_TLS_PUBLISH=no AD_REALM=TESTLAB.LOCAL AD_DNS_DOMAIN=testlab.local \
+AD_HOSTNAME=dc1 AD_IP=172.28.90.10 AD_CONTAINER=ad-tlslab-absent ./scripts/tls.sh >/tmp/ldaps-tls.log 2>&1 \
+    || { echo "cert generation failed:"; tail -15 /tmp/ldaps-tls.log; exit 1; }
+# Guard: refuse to run against a cert that is not ours, otherwise this test silently tests the
+# production chain and can "pass" for the wrong reason (or fail confusingly).
+openssl x509 -in "$LABTLS/dc.crt" -noout -subject | grep -q 'testlab.local' \
+    || { echo "!! $LABTLS/dc.crt is not a testlab cert - refusing to run this test"; exit 1; }
+openssl x509 -in "$LABTLS/dc.crt" -noout -ext subjectAltName | sed 's/^/   /'
+
+docker rm -f ad-tlslab >/dev/null 2>&1 || true
+docker volume rm tlslab-lib >/dev/null 2>&1 || true
+docker network rm adlab >/dev/null 2>&1 || true
+docker network create --subnet 172.28.90.0/24 --internal adlab >/dev/null
+
+docker run -d --name ad-tlslab --hostname dc1.testlab.local --network adlab --ip 172.28.90.10 \
+    --cap-drop ALL --cap-add SYS_ADMIN --cap-add CHOWN --cap-add FOWNER --cap-add DAC_OVERRIDE \
+    --cap-add SETUID --cap-add SETGID --cap-add NET_BIND_SERVICE \
+    -v "$PWD/secrets:/run/ad-secrets:ro" -v "$LABTLS:/run/ad-secrets/tls:ro" -v tlslab-lib:/var/lib/samba \
+    -e AD_REALM=TESTLAB.LOCAL -e AD_DNS_DOMAIN=testlab.local -e AD_HOSTNAME=dc1 \
+    -e AD_IP=172.28.90.10 -e AD_ADMIN_PASSWORD="$PW" \
+    -e AD_TLS_CERT=/run/ad-secrets/tls/dc.crt -e AD_TLS_KEY=/run/ad-secrets/tls/dc.key \
+    -e AD_TLS_CA=/run/ad-secrets/tls/ca.crt \
+    local/samba-ad-dc:4.22-debian13 >/dev/null || { echo "start failed"; exit 1; }
+
+h=none
+for _ in $(seq 1 60); do
+    h="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' ad-tlslab 2>/dev/null || echo gone)"
+    [ "$h" = healthy ] && break
+    sleep 4
+done
+echo "health: $h"
+st="$(docker inspect -f '{{.State.Status}} exit={{.State.ExitCode}}' ad-tlslab 2>/dev/null)"
+echo "state: $st"
+[ "$h" = healthy ] || { echo "--- log ---"; docker logs ad-tlslab 2>&1 | grep -vE 'Applied |Skip ' | tail -20; exit 1; }
+
+echo "=== 636 listening? ==="
+docker exec ad-tlslab ss -ltn | grep ':636' || { echo "636 not listening"; exit 1; }
+
+echo "=== handshake, trusting our CA (must verify) ==="
+docker exec ad-tlslab openssl s_client -connect 127.0.0.1:636 \
+    -CAfile /run/ad-secrets/tls/ca.crt -verify_return_error </dev/null 2>&1 \
+    | grep -E 'Verify return code|subject=|CN' | head -4
+
+echo "=== handshake with default trust store (must FAIL, proves certs are not self-trusted) ==="
+if docker exec ad-tlslab openssl s_client -connect 127.0.0.1:636 -verify_return_error </dev/null >/dev/null 2>&1; then
+    echo "UNEXPECTED: untrusted handshake succeeded"; RC=1
+else
+    echo "rejected as expected  ✔"
+fi
+
+echo "=== full battery with LDAPS enabled ==="
+AD_SKIP_DOTENV=1 AD_CONTAINER=ad-tlslab AD_REALM=TESTLAB.LOCAL AD_DNS_DOMAIN=testlab.local \
+AD_HOSTNAME=dc1 AD_ADMIN_PASSWORD="$PW" \
+AD_TLS_CERT=/run/ad-secrets/tls/dc.crt AD_TLS_CA=/run/ad-secrets/tls/ca.crt \
+    ./scripts/verify.sh | grep -E 'LDAPS|passed,'
+RC=${RC:-0}
+exit "$RC"
--- a/tests/smoke.sh
+++ b/tests/smoke.sh
@ -0,0 +1,109 @@
+#!/usr/bin/env bash
+# =====================================================================================
+# End-to-end smoke test. Fully isolated: realm TESTLAB.LOCAL on an internal-only docker
+# network, named volumes, no host ports, throwaway credentials.
+#
+# Proves, in order:
+#   1. the image builds
+#   2. a fresh volume provisions a working AD DC (DNS/Kerberos/LDAP/GC/SYSVOL)
+#   3. hardening params actually land in smb.conf
+#   4. a restart REUSES the database instead of re-provisioning (idempotency)
+#   5. the full verify battery passes both before and after the restart
+#
+#   ./tests/smoke.sh            # build + test + destroy
+#   KEEP=1 ./tests/smoke.sh     # leave the lab container running for poking at
+# =====================================================================================
+set -uo pipefail
+cd "$(dirname "$0")/.."
+
+LAB_FILE=docker-compose.lab.yml
+CID=ad-lab
+LAB_PW='Lab!Only9Tester#2026'
+IMAGE=local/samba-ad-dc:4.22-debian13
+fail_hard() {
+    echo
+    echo "!! ${1}" >&2
+    echo "---- recent log (schema-apply noise filtered) ----" >&2
+    docker compose -f "$LAB_FILE" logs --tail 400 2>&1 \
+        | grep -vE '(Applied|Skip) (Domain|Forest) Update' | tail -40 | sed 's/^/   /' >&2
+    [ "${KEEP:-0}" = 1 ] || docker compose -f "$LAB_FILE" down -v >/dev/null 2>&1 || true
+    exit 1
+}
+
+echo "==> 1/6 build image"
+if ! docker build -t "$IMAGE" -f image/Dockerfile . ; then
+    echo "!! image build failed" >&2
+    exit 1
+fi
+
+echo "==> 2/6 start lab DC"
+docker compose -f "$LAB_FILE" down -v >/dev/null 2>&1 || true   # always start from a clean slate
+docker compose -f "$LAB_FILE" up -d >/dev/null 2>&1 || fail_hard "compose up failed"
+
+wait_healthy() {  # $1 = seconds
+    local deadline=$(( SECONDS + ${1:-240} )) st=""
+    while [ "$SECONDS" -lt "$deadline" ]; do
+        st="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$CID" 2>/dev/null || echo missing)"
+        [ "$st" = healthy ] && return 0
+        [ "$st" = unhealthy ] && return 2
+        printf '.'
+        sleep 4
+    done
+    return 1
+}
+
+printf '    waiting for healthy '
+wait_healthy 300; rc=$?
+echo " -> $rc"
+[ "$rc" = 0 ] || fail_hard "lab DC never became healthy (rc=$rc)"
+
+if ! docker compose -f "$LAB_FILE" logs 2>&1 | grep -q 'provisioning complete'; then
+    fail_hard "expected the entrypoint to report a fresh provisioning run"
+fi
+echo "    fresh provisioning reported"
+
+verify() {
+    AD_SKIP_DOTENV=1 AD_CONTAINER="$CID" \
+    AD_REALM=TESTLAB.LOCAL AD_DNS_DOMAIN=testlab.local AD_HOSTNAME=dc1 \
+    AD_ADMIN_PASSWORD="$LAB_PW" ./scripts/verify.sh
+}
+
+echo "==> 3/6 verification battery (fresh forest)"
+verify || fail_hard "verification failed on a freshly provisioned forest"
+
+echo "==> 4/6 hardening landed in smb.conf"
+for kv in 'server signing = required' 'smb encrypt = desired' 'ntlm auth = ntlmv2-only' 'restrict anonymous = 2'; do
+    if docker exec "$CID" grep -q "^[[:space:]]*${kv}[[:space:]]*$" /etc/samba/smb.conf; then
+        echo "    ok: $kv"
+    else
+        fail_hard "missing '$kv' in rendered smb.conf"
+    fi
+done
+
+echo "==> 5/6 restart must NOT re-provision"
+# Count log lines first and diff afterwards: `docker logs --since` is interpreted in the
+# daemon's local timezone, which silently mis-filters when the host is not UTC.
+log_before="$(docker compose -f "$LAB_FILE" logs 2>/dev/null | wc -l)"
+docker restart "$CID" >/dev/null 2>&1 || fail_hard "restart failed"
+printf '    waiting for healthy after restart '
+wait_healthy 180; rc=$?
+echo " -> $rc"
+[ "$rc" = 0 ] || fail_hard "not healthy after restart"
+log="$(docker compose -f "$LAB_FILE" logs 2>/dev/null | tail -n +$((log_before + 1)))"
+if printf '%s' "$log" | grep -q 'provisioning NEW forest'; then
+    fail_hard "restart tried to re-provision the domain (idempotency broken)"
+fi
+printf '%s' "$log" | grep -q 'existing AD database found' || fail_hard "restart did not detect the existing database"
+echo "    entrypoint reused the existing database"
+
+echo "==> 6/6 verification battery (after restart)"
+verify || fail_hard "verification failed after restart"
+
+echo
+echo -e "\033[32mSMOKE TEST PASSED\033[0m  (image: $IMAGE)"
+if [ "${KEEP:-0}" = 1 ]; then
+    echo "lab left running: docker compose -f $LAB_FILE logs -f | docker exec -it $CID bash"
+else
+    docker compose -f "$LAB_FILE" down -v >/dev/null 2>&1 || true
+    echo "lab destroyed"
+fi