204 lines
8.7 KiB
Bash
204 lines
8.7 KiB
Bash
#!/bin/sh
|
|
# tools-sync: gitea server-side triage + repair. Runs ON the gitea host (POSIX sh).
|
|
#
|
|
# Driven by gitea_doctor.ps1, which pipes this in over ssh. Safe to run by hand too.
|
|
#
|
|
# Every check prints "PASS <id> <detail>" or "FAIL <id> <detail>" so the caller can
|
|
# parse the outcome without re-deriving it. Repairs only happen with --fix, and only
|
|
# the reversible ones; anything that could lose data needs --allow-destructive.
|
|
#
|
|
# Exit: 0 = healthy (after any repairs), 1 = still broken, 2 = usage/precondition error.
|
|
|
|
CONTAINER=gitea
|
|
DOCKER=/usr/local/bin/docker
|
|
FIX=0
|
|
DESTRUCTIVE=0
|
|
REPO=""
|
|
|
|
while [ $# -gt 0 ]; do
|
|
case "$1" in
|
|
--container) CONTAINER="$2"; shift 2 ;;
|
|
--docker) DOCKER="$2"; shift 2 ;;
|
|
--repo) REPO="$2"; shift 2 ;;
|
|
--fix) FIX=1; shift ;;
|
|
--allow-destructive) DESTRUCTIVE=1; shift ;;
|
|
*) echo "FAIL usage unknown argument: $1"; exit 2 ;;
|
|
esac
|
|
done
|
|
|
|
[ -x "$DOCKER" ] || { echo "FAIL docker $DOCKER not executable (ssh PATH is not the login PATH)"; exit 2; }
|
|
D="$DOCKER"
|
|
TS=$(date +%Y%m%d-%H%M%S)
|
|
BROKEN=0
|
|
REPAIRED=""
|
|
|
|
note() { echo "PASS $1 $2"; }
|
|
bad() { echo "FAIL $1 $2"; BROKEN=1; }
|
|
fixed() { echo "FIXED $1 $2"; REPAIRED="$REPAIRED $1"; }
|
|
dex() { $D exec "$CONTAINER" sh -c "$1" 2>&1; }
|
|
|
|
echo "=== gitea triage $TS (container=$CONTAINER fix=$FIX destructive=$DESTRUCTIVE) ==="
|
|
|
|
# ---------- A. container is up ----------
|
|
STATUS=$($D inspect "$CONTAINER" --format '{{.State.Status}}' 2>/dev/null)
|
|
if [ -z "$STATUS" ]; then
|
|
bad container "no such container: $CONTAINER"
|
|
echo "--- containers present ---"; $D ps -a --format '{{.Names}} {{.Image}} {{.Status}}'
|
|
exit 1
|
|
fi
|
|
if [ "$STATUS" != "running" ]; then
|
|
bad container "status=$STATUS"
|
|
if [ "$FIX" = 1 ]; then
|
|
$D start "$CONTAINER" >/dev/null 2>&1 && sleep 8
|
|
STATUS=$($D inspect "$CONTAINER" --format '{{.State.Status}}' 2>/dev/null)
|
|
[ "$STATUS" = "running" ] && fixed container "started" || { echo "FAIL container could not start"; exit 1; }
|
|
else
|
|
exit 1
|
|
fi
|
|
else
|
|
note container "running ($($D inspect "$CONTAINER" --format '{{.Config.Image}}'))"
|
|
fi
|
|
|
|
# Where gitea's own HOME (and therefore its global gitconfig) lives. It is NOT /data/git.
|
|
GHOME=$(dex 'getent passwd git 2>/dev/null | cut -d: -f6')
|
|
[ -n "$GHOME" ] || GHOME=/data/git
|
|
GCFG=$(dex 'ls /data/gitea/home/.gitconfig >/dev/null 2>&1 && echo /data/gitea/home/.gitconfig || echo '"$GHOME"'/.gitconfig')
|
|
GCFG=$(echo "$GCFG" | tr -d '\r')
|
|
echo "INFO gitconfig $GCFG"
|
|
|
|
# ---------- B. disk ----------
|
|
FREE=$(dex 'df -P /data | tail -1 | awk "{print \$4}"')
|
|
FREE=$(echo "$FREE" | tr -d '\r ')
|
|
if [ -n "$FREE" ] && [ "$FREE" -lt 1048576 ] 2>/dev/null; then
|
|
bad disk "only ${FREE}KB free on /data"
|
|
else
|
|
note disk "${FREE}KB free on /data"
|
|
fi
|
|
|
|
# ---------- C. global gitconfig sanity ----------
|
|
# The 2026-09-02 outage: [uploadpack] packObjectsHook replaced git pack-objects with a
|
|
# script that produced no pack, so every clone got an empty one. Any hook-ish config
|
|
# here is remote code execution by design, so treat its mere presence as a failure.
|
|
CFG_SIZE=$(dex "wc -c < $GCFG 2>/dev/null" | tr -d '\r ')
|
|
[ -n "$CFG_SIZE" ] || CFG_SIZE=0
|
|
HOOKCFG=$(dex "grep -icE 'packObjectsHook|core\\.hooksPath|sshCommand|credential\\.helper *= *!' $GCFG 2>/dev/null" | tr -d '\r ')
|
|
[ -n "$HOOKCFG" ] || HOOKCFG=0
|
|
if [ "$HOOKCFG" != 0 ] || [ "$CFG_SIZE" -gt 65536 ] 2>/dev/null; then
|
|
bad gitconfig "size=${CFG_SIZE}B suspicious-directives=${HOOKCFG} (expected a few hundred B, 0)"
|
|
dex "grep -inE 'packObjectsHook|core\\.hooksPath|sshCommand' $GCFG | head -5"
|
|
if [ "$FIX" = 1 ]; then
|
|
# gitea rewrites a clean global gitconfig at startup, so quarantine + restart is
|
|
# both the repair and the proof: hand-editing it would only paper over the entry.
|
|
dex "cp $GCFG /data/gitea/home/.gitconfig.quarantine.$TS 2>/dev/null || cp $GCFG $GCFG.quarantine.$TS"
|
|
dex "rm -f $GCFG"
|
|
$D restart "$CONTAINER" >/dev/null 2>&1
|
|
sleep 12
|
|
NEW=$(dex "wc -c < $GCFG 2>/dev/null" | tr -d '\r ')
|
|
NEWHOOK=$(dex "grep -icE 'packObjectsHook|core\\.hooksPath' $GCFG 2>/dev/null" | tr -d '\r ')
|
|
[ -n "$NEWHOOK" ] || NEWHOOK=0
|
|
if [ "$NEWHOOK" = 0 ] && [ -n "$NEW" ]; then
|
|
fixed gitconfig "quarantined to .gitconfig.quarantine.$TS, gitea rebuilt a clean ${NEW}B one"
|
|
else
|
|
echo "FAIL gitconfig rebuild did not produce a clean file (size=$NEW hooks=$NEWHOOK)"
|
|
fi
|
|
fi
|
|
else
|
|
note gitconfig "${CFG_SIZE}B, no hook directives"
|
|
fi
|
|
|
|
# ---------- D. tampered repo hooks ----------
|
|
TAMPER=$(dex "grep -rlE '/tmp/[A-Za-z0-9_]+' /data/git/repositories 2>/dev/null | head -20")
|
|
TAMPER=$(echo "$TAMPER" | tr -d '\r' | sed '/^$/d')
|
|
if [ -n "$TAMPER" ]; then
|
|
bad hooks "hook files referencing /tmp:"
|
|
echo "$TAMPER" | sed 's/^/ /'
|
|
if [ "$FIX" = 1 ]; then
|
|
for h in $TAMPER; do
|
|
base=$(basename "$h")
|
|
tpl="/usr/share/git-core/templates/hooks/$base"
|
|
if dex "[ -f $tpl ]" >/dev/null 2>&1; then
|
|
dex "cp $tpl $h && chmod 755 $h"
|
|
else
|
|
dex "grep -v '/tmp/' $h > $h.clean && mv $h.clean $h && chmod 755 $h"
|
|
fi
|
|
done
|
|
LEFT=$(dex "grep -rlE '/tmp/[A-Za-z0-9_]+' /data/git/repositories 2>/dev/null | head -5" | tr -d '\r' | sed '/^$/d')
|
|
[ -z "$LEFT" ] && fixed hooks "restored from git templates" || echo "FAIL hooks still tampered: $LEFT"
|
|
fi
|
|
else
|
|
note hooks "no hook references /tmp"
|
|
fi
|
|
|
|
# ---------- E. dropped payloads ----------
|
|
PAYLOAD=$(dex "ls /tmp/giteahook* /tmp/col_* 2>/dev/null | head -20" | tr -d '\r' | sed '/^$/d')
|
|
if [ -n "$PAYLOAD" ]; then
|
|
bad payload "executable drops in /tmp:"
|
|
echo "$PAYLOAD" | sed 's/^/ /'
|
|
[ "$FIX" = 1 ] && { dex "rm -f /tmp/giteahook* /tmp/col_*" >/dev/null 2>&1; fixed payload "removed"; }
|
|
else
|
|
note payload "none"
|
|
fi
|
|
|
|
# ---------- F. git binary not shimmed ----------
|
|
# A leftover diagnostic wrapper (git.real present) silently changes every git call.
|
|
if dex "[ -e /usr/bin/git.real ]" >/dev/null 2>&1; then
|
|
bad gitbin "/usr/bin/git.real exists - /usr/bin/git is a wrapper, not the real binary"
|
|
if [ "$FIX" = 1 ]; then
|
|
dex "cp /usr/bin/git.real /usr/bin/git && rm -f /usr/bin/git.real" >/dev/null 2>&1
|
|
fixed gitbin "real git restored"
|
|
fi
|
|
else
|
|
note gitbin "$(dex 'git --version' | tr -d '\r')"
|
|
fi
|
|
|
|
# ---------- G. pack generation (the ladder's decisive rung) ----------
|
|
# Must use --stateless-rpc: that is the code path HTTP takes. A `clone file://` proves
|
|
# nothing here, which is exactly how this fault stayed misdiagnosed for a week.
|
|
if [ -z "$REPO" ]; then
|
|
REPO=$(dex 'ls -d /data/git/repositories/*/*.git 2>/dev/null | head -1' | tr -d '\r')
|
|
fi
|
|
if [ -n "$REPO" ]; then
|
|
SHA=$(dex "git --git-dir=$REPO rev-parse HEAD 2>/dev/null" | tr -d '\r')
|
|
if [ -n "$SHA" ]; then
|
|
BYTES=$(dex "printf '0032want $SHA\n00000009done\n' | git upload-pack --stateless-rpc $REPO 2>/dev/null | wc -c" | tr -d '\r ')
|
|
[ -n "$BYTES" ] || BYTES=0
|
|
if [ "$BYTES" -lt 512 ] 2>/dev/null; then
|
|
bad packgen "upload-pack --stateless-rpc produced only ${BYTES}B for $REPO (a real pack is KBs+)"
|
|
echo " -> git itself is being told not to work; check the gitconfig directives above"
|
|
else
|
|
note packgen "${BYTES}B pack from $REPO"
|
|
fi
|
|
else
|
|
bad packgen "cannot resolve HEAD of $REPO"
|
|
fi
|
|
else
|
|
bad packgen "no repository found under /data/git/repositories"
|
|
fi
|
|
|
|
# ---------- H. end-to-end clone inside the container ----------
|
|
# Bypasses frp, any reverse proxy and docker port forwarding: a failure here is gitea's.
|
|
PUB=$(dex 'ls -d /data/git/repositories/*/*.git 2>/dev/null | head -5' | tr -d '\r')
|
|
CLONE_OK=0
|
|
CLONE_MSG="no public repo reachable anonymously"
|
|
for r in $PUB; do
|
|
owner=$(basename "$(dirname "$r")")
|
|
name=$(basename "$r" .git)
|
|
out=$(dex "rm -rf /tmp/tsdoc; GIT_TERMINAL_PROMPT=0 git clone -q --filter=tree:0 --depth 1 http://localhost:3000/$owner/$name.git /tmp/tsdoc 2>&1; ls /tmp/tsdoc/.git >/dev/null 2>&1 && echo TSDOC_OK; rm -rf /tmp/tsdoc")
|
|
case "$out" in
|
|
*TSDOC_OK*) CLONE_OK=1; CLONE_MSG="$owner/$name"; break ;;
|
|
*"could not read Username"*|*"Authentication"*) continue ;;
|
|
*) CLONE_MSG="$owner/$name: $(echo "$out" | tr -d '\r' | tr '\n' ' ' | cut -c1-160)" ;;
|
|
esac
|
|
done
|
|
[ "$CLONE_OK" = 1 ] && note httpclone "$CLONE_MSG" || bad httpclone "$CLONE_MSG"
|
|
|
|
# ---------- verdict ----------
|
|
echo "=== verdict ==="
|
|
if [ "$BROKEN" = 0 ]; then
|
|
echo "RESULT healthy"; exit 0
|
|
fi
|
|
if [ -n "$REPAIRED" ]; then
|
|
echo "RESULT repaired:$REPAIRED (caller must re-verify from the client side)"; exit 0
|
|
fi
|
|
echo "RESULT still-broken (run with --fix to apply the reversible repairs)"; exit 1
|