#!/bin/sh # tools-sync: gitea server-side triage + repair. Runs ON the gitea host (POSIX sh). # # Driven by gitea_doctor.ps1, which pipes this in over ssh. Safe to run by hand too. # # Every check prints "PASS " or "FAIL " so the caller can # parse the outcome without re-deriving it. Repairs only happen with --fix, and only # the reversible ones; anything that could lose data needs --allow-destructive. # # Exit: 0 = healthy (after any repairs), 1 = still broken, 2 = usage/precondition error. CONTAINER=gitea DOCKER=/usr/local/bin/docker FIX=0 DESTRUCTIVE=0 REPO="" while [ $# -gt 0 ]; do case "$1" in --container) CONTAINER="$2"; shift 2 ;; --docker) DOCKER="$2"; shift 2 ;; --repo) REPO="$2"; shift 2 ;; --fix) FIX=1; shift ;; --allow-destructive) DESTRUCTIVE=1; shift ;; *) echo "FAIL usage unknown argument: $1"; exit 2 ;; esac done [ -x "$DOCKER" ] || { echo "FAIL docker $DOCKER not executable (ssh PATH is not the login PATH)"; exit 2; } D="$DOCKER" TS=$(date +%Y%m%d-%H%M%S) BROKEN=0 REPAIRED="" note() { echo "PASS $1 $2"; } bad() { echo "FAIL $1 $2"; BROKEN=1; } fixed() { echo "FIXED $1 $2"; REPAIRED="$REPAIRED $1"; } dex() { $D exec "$CONTAINER" sh -c "$1" 2>&1; } echo "=== gitea triage $TS (container=$CONTAINER fix=$FIX destructive=$DESTRUCTIVE) ===" # ---------- A. container is up ---------- STATUS=$($D inspect "$CONTAINER" --format '{{.State.Status}}' 2>/dev/null) if [ -z "$STATUS" ]; then bad container "no such container: $CONTAINER" echo "--- containers present ---"; $D ps -a --format '{{.Names}} {{.Image}} {{.Status}}' exit 1 fi if [ "$STATUS" != "running" ]; then bad container "status=$STATUS" if [ "$FIX" = 1 ]; then $D start "$CONTAINER" >/dev/null 2>&1 && sleep 8 STATUS=$($D inspect "$CONTAINER" --format '{{.State.Status}}' 2>/dev/null) [ "$STATUS" = "running" ] && fixed container "started" || { echo "FAIL container could not start"; exit 1; } else exit 1 fi else note container "running ($($D inspect "$CONTAINER" --format '{{.Config.Image}}'))" fi # Where gitea's own HOME (and therefore its global gitconfig) lives. It is NOT /data/git. GHOME=$(dex 'getent passwd git 2>/dev/null | cut -d: -f6') [ -n "$GHOME" ] || GHOME=/data/git GCFG=$(dex 'ls /data/gitea/home/.gitconfig >/dev/null 2>&1 && echo /data/gitea/home/.gitconfig || echo '"$GHOME"'/.gitconfig') GCFG=$(echo "$GCFG" | tr -d '\r') echo "INFO gitconfig $GCFG" # ---------- B. disk ---------- FREE=$(dex 'df -P /data | tail -1 | awk "{print \$4}"') FREE=$(echo "$FREE" | tr -d '\r ') if [ -n "$FREE" ] && [ "$FREE" -lt 1048576 ] 2>/dev/null; then bad disk "only ${FREE}KB free on /data" else note disk "${FREE}KB free on /data" fi # ---------- C. global gitconfig sanity ---------- # The 2026-09-02 outage: [uploadpack] packObjectsHook replaced git pack-objects with a # script that produced no pack, so every clone got an empty one. Any hook-ish config # here is remote code execution by design, so treat its mere presence as a failure. CFG_SIZE=$(dex "wc -c < $GCFG 2>/dev/null" | tr -d '\r ') [ -n "$CFG_SIZE" ] || CFG_SIZE=0 HOOKCFG=$(dex "grep -icE 'packObjectsHook|core\\.hooksPath|sshCommand|credential\\.helper *= *!' $GCFG 2>/dev/null" | tr -d '\r ') [ -n "$HOOKCFG" ] || HOOKCFG=0 if [ "$HOOKCFG" != 0 ] || [ "$CFG_SIZE" -gt 65536 ] 2>/dev/null; then bad gitconfig "size=${CFG_SIZE}B suspicious-directives=${HOOKCFG} (expected a few hundred B, 0)" dex "grep -inE 'packObjectsHook|core\\.hooksPath|sshCommand' $GCFG | head -5" if [ "$FIX" = 1 ]; then # Preserve both the injected config and its referenced executable as # reversible evidence before startup rebuilds the global config. HOOK_SCRIPT=$(dex "grep -oE '/data/gitea/home/[A-Za-z0-9_.-]+\\.sh' $GCFG | head -1" | tr -d '\r') if [ -n "$HOOK_SCRIPT" ]; then case "$HOOK_SCRIPT" in /data/gitea/home/*.sh) if dex "[ -f $HOOK_SCRIPT ] && [ ! -e $HOOK_SCRIPT.quarantine.$TS ]" >/dev/null 2>&1; then dex "mv $HOOK_SCRIPT $HOOK_SCRIPT.quarantine.$TS" echo "INFO injected hook script quarantined: $HOOK_SCRIPT.quarantine.$TS" fi ;; esac fi dex "[ ! -e $GCFG.quarantine.$TS ] && mv $GCFG $GCFG.quarantine.$TS" $D restart "$CONTAINER" >/dev/null 2>&1 sleep 12 NEW=$(dex "wc -c < $GCFG 2>/dev/null" | tr -d '\r ') NEWHOOK=$(dex "grep -icE 'packObjectsHook|core\\.hooksPath' $GCFG 2>/dev/null" | tr -d '\r ') [ -n "$NEWHOOK" ] || NEWHOOK=0 if [ "$NEWHOOK" = 0 ] && [ -n "$NEW" ]; then fixed gitconfig "quarantined to .gitconfig.quarantine.$TS, gitea rebuilt a clean ${NEW}B one" else echo "FAIL gitconfig rebuild did not produce a clean file (size=$NEW hooks=$NEWHOOK)" fi fi else note gitconfig "${CFG_SIZE}B, no hook directives" fi # ---------- D. tampered repo hooks ---------- TAMPER=$(dex "grep -rlE '/tmp/[A-Za-z0-9_]+' /data/git/repositories 2>/dev/null | head -20") TAMPER=$(echo "$TAMPER" | tr -d '\r' | sed '/^$/d') if [ -n "$TAMPER" ]; then bad hooks "hook files referencing /tmp:" echo "$TAMPER" | sed 's/^/ /' if [ "$FIX" = 1 ]; then for h in $TAMPER; do base=$(basename "$h") tpl="/usr/share/git-core/templates/hooks/$base" if dex "[ -f $tpl ]" >/dev/null 2>&1; then dex "cp $tpl $h && chmod 755 $h" else dex "grep -v '/tmp/' $h > $h.clean && mv $h.clean $h && chmod 755 $h" fi done LEFT=$(dex "grep -rlE '/tmp/[A-Za-z0-9_]+' /data/git/repositories 2>/dev/null | head -5" | tr -d '\r' | sed '/^$/d') [ -z "$LEFT" ] && fixed hooks "restored from git templates" || echo "FAIL hooks still tampered: $LEFT" fi else note hooks "no hook references /tmp" fi # ---------- E. dropped payloads ---------- PAYLOAD=$(dex "ls /tmp/giteahook* /tmp/col_* 2>/dev/null | head -20" | tr -d '\r' | sed '/^$/d') if [ -n "$PAYLOAD" ]; then bad payload "executable drops in /tmp:" echo "$PAYLOAD" | sed 's/^/ /' [ "$FIX" = 1 ] && { dex "rm -f /tmp/giteahook* /tmp/col_*" >/dev/null 2>&1; fixed payload "removed"; } else note payload "none (known names)" fi # The names change every round (2026-10-01: /data/gitea/home/.pwn + /tmp/.k binaries, # RootHawk-* in /tmp) and the name list above said "none" through all of it. So also look # by shape, name-independent. Report only: these are evidence of a compromise, and what to # do with a compromised container (rebuild, rotate credentials) is the user's decision. DROPS=$(dex "find /tmp -maxdepth 1 -type f -perm -u+x 2>/dev/null | head -20; \ ls -A /data/gitea/home 2>/dev/null | grep -Ev '^(\.cache|\.local|\.ssh|\.gitconfig(\..*)?)$' | sed 's|^|/data/gitea/home/|'" \ | tr -d '\r' | sed '/^$/d' | grep -v '\.quarantine\.') if [ -n "$DROPS" ]; then bad payload "unexpected files (executables in /tmp, strays in gitea HOME) - treat the container as compromised:" echo "$DROPS" | sed 's/^/ /' fi # ---------- F. git binary not shimmed ---------- # A leftover diagnostic wrapper (git.real present) silently changes every git call. if dex "[ -e /usr/bin/git.real ]" >/dev/null 2>&1; then bad gitbin "/usr/bin/git.real exists - /usr/bin/git is a wrapper, not the real binary" if [ "$FIX" = 1 ]; then dex "cp /usr/bin/git.real /usr/bin/git && rm -f /usr/bin/git.real" >/dev/null 2>&1 fixed gitbin "real git restored" fi else note gitbin "$(dex 'git --version' | tr -d '\r')" fi # ---------- G. pack generation (the ladder's decisive rung) ---------- # Must use --stateless-rpc: that is the code path HTTP takes. A `clone file://` proves # nothing here, which is exactly how this fault stayed misdiagnosed for a week. if [ -z "$REPO" ]; then REPO=$(dex 'ls -d /data/git/repositories/*/*.git 2>/dev/null | head -1' | tr -d '\r') fi if [ -n "$REPO" ]; then SHA=$(dex "git --git-dir=$REPO rev-parse HEAD 2>/dev/null" | tr -d '\r') if [ -n "$SHA" ]; then BYTES=$(dex "printf '0032want $SHA\n00000009done\n' | git upload-pack --stateless-rpc $REPO 2>/dev/null | wc -c" | tr -d '\r ') [ -n "$BYTES" ] || BYTES=0 if [ "$BYTES" -lt 512 ] 2>/dev/null; then bad packgen "upload-pack --stateless-rpc produced only ${BYTES}B for $REPO (a real pack is KBs+)" echo " -> git itself is being told not to work; check the gitconfig directives above" else note packgen "${BYTES}B pack from $REPO" fi else bad packgen "cannot resolve HEAD of $REPO" fi else bad packgen "no repository found under /data/git/repositories" fi # ---------- H. end-to-end clone inside the container ---------- # Bypasses frp, any reverse proxy and docker port forwarding: a failure here is gitea's. PUB=$(dex 'ls -d /data/git/repositories/*/*.git 2>/dev/null | head -5' | tr -d '\r') CLONE_OK=0 CLONE_MSG="no public repo reachable anonymously" CLONE_DIR="/tmp/tools-sync-httpclone.$TS.$$" for r in $PUB; do owner=$(basename "$(dirname "$r")") name=$(basename "$r" .git) out=$(dex "test ! -e $CLONE_DIR || exit 3; GIT_TERMINAL_PROMPT=0 git clone -q --filter=tree:0 --depth 1 http://localhost:3000/$owner/$name.git $CLONE_DIR 2>&1; ls $CLONE_DIR/.git >/dev/null 2>&1 && echo TSDOC_OK; rm -rf $CLONE_DIR") case "$out" in *TSDOC_OK*) CLONE_OK=1; CLONE_MSG="$owner/$name"; break ;; *"could not read Username"*|*"Authentication"*) continue ;; *) CLONE_MSG="$owner/$name: $(echo "$out" | tr -d '\r' | tr '\n' ' ' | cut -c1-160)" ;; esac done [ "$CLONE_OK" = 1 ] && note httpclone "$CLONE_MSG" || bad httpclone "$CLONE_MSG" # ---------- verdict ---------- echo "=== verdict ===" if [ "$BROKEN" = 0 ]; then echo "RESULT healthy"; exit 0 fi if [ -n "$REPAIRED" ]; then echo "RESULT repaired:$REPAIRED (caller must re-verify from the client side)"; exit 0 fi echo "RESULT still-broken (run with --fix to apply the reversible repairs)"; exit 1