#!/usr/bin/env bash # ===================================================================================== # Post-deployment verification battery for the AD DC container. # Runs everything through `docker exec` inside the running container so the checks hit # the same code paths a client would, without needing host-level AD tooling. # # ./scripts/verify.sh # standard checks # ./scripts/verify.sh --full # + ldb consistency check (slow, DB heavy) # ===================================================================================== set -uo pipefail cd "$(dirname "$0")/.." . "$(dirname "$0")/envfile.sh"; ad_env_load .env # parsed, never executed (see envfile.sh) FULL=0; [ "${1:-}" = "--full" ] && FULL=1 CID="${AD_CONTAINER:-ad-dc}" cenv(){ docker exec "$CID" printenv "$1" 2>/dev/null | head -n1; } # fall back to container env REALM="${AD_REALM:-$(cenv AD_REALM)}" ZONE="${AD_DNS_DOMAIN:-${REALM,,}}" HOST="${AD_HOSTNAME:-$(cenv AD_HOSTNAME)}" [ -n "$REALM" ] || { echo "AD_REALM unknown (set it, or run against a container that has it)" >&2; exit 2; } [ -n "$ZONE" ] || { echo "AD_DNS_DOMAIN unknown" >&2; exit 2; } FQDN="${HOST}.${ZONE}" BASEDN="DC=$(printf '%s' "$ZONE" | sed 's/\./,DC=/g')" CC=/tmp/verify_ccache PW="${AD_ADMIN_PASSWORD:-}" if [ -z "$PW" ]; then for f in secrets/ad_admin.password /run/ad-secrets/ad_admin.password; do [ -r "$f" ] && { PW="$(tr -d '\r\n' < "$f")"; break; } done fi [ -n "$PW" ] || PW="$(cenv AD_ADMIN_PASSWORD)" [ -n "$PW" ] || { echo "cannot find administrator password (AD_ADMIN_PASSWORD or secrets/ad_admin.password)" >&2; exit 2; } pass=0; fail=0; skip=0 ok() { printf ' \033[32m✔\033[0m %s\n' "$1"; pass=$((pass+1)); } bad() { printf ' \033[31m✘\033[0m %s\n' "$1"; [ -n "${2:-}" ] && printf ' %s\n' "$(printf '%s' "$2" | tail -4 | tr '\n' '|')"; fail=$((fail+1)); } skp() { printf ' \033[33m○\033[0m %s\n' "$1"; skip=$((skip+1)); } dexec(){ docker exec -i -e "KRB5CCNAME=FILE:${CC}" "$CID" "$@" 2>&1; } # expect a non-zero exit (used for "this must be denied" checks) dexec_expect_denied(){ ! docker exec -i "$CID" "$@" >/dev/null 2>&1; } printf '\n\033[1mVerifying %s (realm %s, container %s)\033[0m\n' "$FQDN" "$REALM" "$CID" echo "----------------------------------------------------------------" # --- 0. container / state ---------------------------------------------------------- st="$(docker inspect -f '{{.State.Status}}' "$CID" 2>/dev/null || echo missing)" [ "$st" = running ] && ok "container running" || { bad "container not running (state: $st)"; echo; exit 1; } health="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$CID")" case "$health" in healthy) ok "docker health: healthy" ;; starting) skp "docker health: starting (re-run in ~60s)" ;; *) bad "docker health: $health" "$(docker inspect -f '{{range .State.Health.Log}}{{.Output}}{{end}}' "$CID" | tail -5)" ;; esac dexec test -f /var/lib/samba/private/sam.ldb && ok "AD database present (sam.ldb)" \ || { bad "no AD database - provision did not complete"; echo; exit 1; } # --- 1. DNS: SRV bootstrap records Windows depends on ------------------------------ # SRV records a Windows 11 client actually needs for domain/DC location. Verified against # this Samba build. Samba does NOT register the NT4-era `_pdc._tcp` under the domain, the # `_gc._tcp.dc._msdcs` variant, or MIT-style `_kerberos-adm._tcp` - clients don't ask. for rec in _ldap._tcp.dc._msdcs _kerberos._tcp.dc._msdcs _ldap._tcp.gc._msdcs \ _ldap._tcp.pdc._msdcs _gc._tcp _ldap._tcp _kerberos._udp _kerberos._tcp \ _kpasswd._udp _kpasswd._tcp; do out="$(dexec dig +time=3 +tries=1 +short @127.0.0.1 "${rec}.${ZONE}" SRV)" if [ -n "$out" ]; then printf ' \033[32m✔\033[0m SRV %-34s -> %s\n' "$rec" "$(printf '%s' "$out" | head -1 | tr -s ' ')" pass=$((pass+1)) else printf ' \033[31m✘\033[0m SRV %-34s -> (empty)\n' "$rec" fail=$((fail+1)) fi done a_rec="$(dexec dig +time=3 +tries=1 +short @127.0.0.1 "$FQDN" A)" [ -n "$a_rec" ] && ok "A record ${FQDN} -> $(echo "$a_rec" | tr '\n' ' ')" || bad "A record for ${FQDN} missing" # Regression guard: a multi-homed Docker host (docker0/bridges/stale DHCP) can leak # unreachable addresses into the DC's A record, and clients that pick one of those fail in # ways that look like Kerberos bugs. The recordset must be exactly what the DC advertises. # The container's own AD_IP is the source of truth - the host .env may describe a different # DC (lab container, replica), and AD_IP_EXPECT overrides explicitly. EXPECT_IP="${AD_IP_EXPECT:-$(cenv AD_IP)}" [ -n "$EXPECT_IP" ] || EXPECT_IP="${AD_IP:-}" if [ -z "$EXPECT_IP" ]; then skp "A recordset pin check skipped (no AD_IP known)" else extra_ips="$(printf '%s\n' $a_rec | grep -v "^${EXPECT_IP}$" | tr '\n' ' ')" if [ -n "${extra_ips// /}" ]; then bad "A recordset for ${FQDN} has non-advertised address(es): ${extra_ips}(only ${EXPECT_IP} may be published)" else ok "A recordset is exactly the advertised ${EXPECT_IP} (no Docker bridge / stale addresses)" fi fi # --- 2. Kerberos ----------------------------------------------------------------- out="$(printf '%s\n' "$PW" | dexec kinit "${AD_ADMIN_USER:-administrator}@${REALM}")" \ && ok "kinit ${AD_ADMIN_USER:-administrator}@${REALM} issued a TGT" \ || bad "kinit failed" "$out" out="$(dexec kvno "cifs/${FQDN}")" if printf '%s' "$out" | grep -q 'kvno = '; then enc="$(dexec klist | grep -oiE 'aes[0-9]+[a-z0-9-]*' | sort -u | tr '\n' ' ')" ok "service ticket issued for cifs/${FQDN}${enc:+ (${enc})}" else bad "kvno cifs/${FQDN} failed" "$out" fi # --- 3. LDAP / GC over GSSAPI ---------------------------------------------------- out="$(dexec ldapsearch -H "ldap://${FQDN}" -Y GSSAPI -b "$BASEDN" '(objectClass=domain)' dn)" printf '%s' "$out" | grep -q "^dn: ${BASEDN}" && ok "LDAP + GSSAPI bind, domain object found" || bad "LDAP GSSAPI search failed" "$out" out="$(dexec ldapsearch -H "ldap://${FQDN}:3268" -Y GSSAPI -b '' -s base defaultNamingContext)" printf '%s' "$out" | grep -q "^defaultNamingContext: ${BASEDN}" && ok "Global Catalog (3268) serving ${BASEDN}" || bad "GC on 3268 failed" "$out" out="$(dexec ldapsearch -H "ldap://${FQDN}" -Y GSSAPI -b "$BASEDN" '(sAMAccountName=Domain Computers)' dn)" printf '%s' "$out" | grep -q 'dn: ' && ok "machine-account group readable (join will work)" || bad "cannot read Domain Computers group" "$out" # --- 4. LDAPS ------------------------------------------------------------------- # NB: the OpenLDAP client reads LDAPTLS_* environment variables (bare TLS_* is the # ldap.conf keyword, not the env var). Using the wrong name silently leaves TLS_REQCERT at # its default, and a verification failure then surfaces as the very misleading # "Can't contact LDAP server (-1)". # LDAPS status and the CA path are read from the CONTAINER we are verifying, not the host # .env - a host .env describing the production DC must not make us run LDAPS assertions # against a lab/replica container that has no TLS configured. LDAPS_ON="$(cenv AD_TLS_CERT)" LDAPS_CA="$(cenv AD_TLS_CA)" [ -n "$LDAPS_CA" ] || LDAPS_CA="/run/ad-secrets/tls/ca.crt" if [ -n "$LDAPS_ON" ]; then out="$(dexec openssl s_client -connect "${FQDN}:636" -CAfile "$LDAPS_CA" -verify_return_error &1)" printf '%s' "$out" | grep -q 'Verify return code: 0 (ok)' \ && ok "LDAPS 636 chain verifies against the CA" \ || bad "LDAPS verification failed" "$out" # Real TLS-strict authenticated search: proves chain AND hostname match AND a bind. out="$(docker exec -e LDAPTLS_CACERT="$LDAPS_CA" -e LDAPTLS_REQCERT=demand "$CID" \ ldapsearch -H "ldaps://${FQDN}" -D "CN=${AD_ADMIN_USER:-administrator},CN=Users,${BASEDN}" \ -w "$PW" -b "$BASEDN" '(objectClass=domainDNS)' dn 2>&1)" printf '%s' "$out" | grep -q '^dn: ' \ && ok "LDAPS authenticated search with TLS_REQCERT=demand" \ || bad "LDAPS authenticated search failed" "$out" # Negative controls: strict trust must actually be strict, or the whole exercise is theatre. out="$(docker exec -e LDAPTLS_REQCERT=demand -e LDAPTLS_CACERT=/nonexistent "$CID" \ ldapsearch -H "ldaps://${FQDN}" -x -b "$BASEDN" '(objectClass=domainDNS)' dn 2>&1)" printf '%s' "$out" | grep -qi "Can't contact LDAP server" \ && ok "strict client WITHOUT the CA is rejected" \ || bad "untrusted LDAPS client was NOT rejected (trust is not being enforced)" "$out" out="$(docker exec -e LDAPTLS_CACERT="$LDAPS_CA" -e LDAPTLS_REQCERT=demand "$CID" \ ldapsearch -H "ldaps://127.0.0.1" -x -b "$BASEDN" '(objectClass=domainDNS)' dn 2>&1)" printf '%s' "$out" | grep -qi "Can't contact LDAP server" \ && ok "hostname/SAN mismatch is rejected (dialled 127.0.0.1)" \ || bad "SAN mismatch NOT rejected - cert accepted for a name it does not own" "$out" else skp "LDAPS not configured on ${CID} (set AD_TLS_* and run ./scripts/tls.sh)" fi # --- 5. SYSVOL / NETLOGON -------------------------------------------------------- out="$(dexec samba-tool ntacl sysvolcheck)" printf '%s' "$out" | grep -qi 'ERROR' \ && bad "SYSVOL ACL drift (fix: samba-tool ntacl sysvolreset)" "$out" \ || ok "SYSVOL ACLs consistent" out="$(printf 'username=%s\npassword=%s\n' "${AD_ADMIN_USER:-administrator}" "$PW" \ | dexec smbclient -L //127.0.0.1 -A /dev/stdin -m SMB3)" printf '%s' "$out" | grep -qi 'NETLOGON' && printf '%s' "$out" | grep -qi 'SYSVOL' \ && ok "SMB3 lists NETLOGON + SYSVOL (GPO/slogans path)" \ || bad "NETLOGON/SYSVOL not listed over SMB3" "$out" # --- 6. Replication / DB --------------------------------------------------------- out="$(dexec samba-tool drs showrepl)" if printf '%s' "$out" | grep -q 'DSA object GUID:'; then failing="$(printf '%s' "$out" | grep -cE 'failing (inbound|outbound)' || true)" if [ "${failing:-0}" -eq 0 ]; then ok "DRS healthy (no failing replication requests)" else bad "DRS shows ${failing} failing replication request(s)" "$out" fi else bad "DRS did not report the DSA object" "$out" fi out="$(dexec samba-tool domain level show)" printf ' %s\n' "$(printf '%s' "$out" | tr -d '\r' | grep -iE 'function|level' | head -3 | tr '\n' ' ')" ok "functional levels readable" if [ "$FULL" = 1 ]; then echo " .. running ldb consistency check (this can take minutes)" out="$(dexec samba-tool dbcheck --cross-ncs --yes)" printf '%s' "$out" | grep -qiE 'ERROR|bad referral' \ && bad "dbcheck reported errors" "$out" || ok "dbcheck clean" else skp "dbcheck skipped (use --full)" fi # --- 7. Negative checks: what must NOT be reachable ------------------------------ if dexec_expect_denied ldapsearch -x -H "ldap://127.0.0.1:389" -b "$BASEDN" '(objectClass=user)' dn; then ok "anonymous LDAP search of user objects denied" else bad "anonymous LDAP search of user objects SUCCEEDED (restrict anonymous?)" fi if dexec_expect_denied smbclient -L //127.0.0.1 -g -m SMB1; then ok "SMB1 refused" else bad "SMB1 not refused" fi echo "----------------------------------------------------------------" printf '\033[1m%d passed, %d failed, %d skipped\033[0m\n' "$pass" "$fail" "$skip" [ "$fail" -eq 0 ] || echo "AD NOT READY - fix failures above before joining Windows hosts" exit "$fail"