# tools-sync: verify the gitea transport from this machine, and when it is broken, log # into the server and repair it. # # Why this exists: on 2026-08-25 the server started handing out empty packs. push kept # working, `ls-remote` kept working, only the pack plane was dead - so every sync run # "succeeded" while three commits sat stranded on one machine for eight days. Checking # the transport is cheap; discovering the outage a week late is not. # # .\gitea_doctor.ps1 # check only, report what is wrong # .\gitea_doctor.ps1 -Fix # check, then repair on the server, then re-verify # # Exit: 0 = transport healthy at the end of the run, 1 = still broken. param( [string]$EnvFile = "$env:USERPROFILE\skills_env.md", [string]$RepoUrl, # Log in and repair, instead of only reporting. [switch]$Fix, # Also allow repairs that can lose data (container recreate, repo surgery). The # server-side script asks for this separately; nothing here needs it by default. [switch]$AllowDestructive, [int]$SshTimeout = 15 ) $ErrorActionPreference = "Stop" function Fail([string]$Msg) { Write-Host "FAIL: $Msg"; exit 1 } # ---------- environment ---------- if (-not (Test-Path $EnvFile)) { Fail "$EnvFile not found. Copy references\skills_env_template.md there first." } $envText = Get-Content $EnvFile -Raw function Get-EnvKey([string]$Key) { $m = [regex]::Match($envText, "(?m)^\s*$([regex]::Escape($Key))\s*=\s*(.+?)\s*$") if (-not $m.Success) { return "" } $v = $m.Groups[1].Value.Trim() if ($v -match "^<.*>$") { return "" } # unfilled placeholder return $v } if (-not $RepoUrl) { $RepoUrl = Get-EnvKey "SKILLS_REPO_URL" } if (-not $RepoUrl) { Fail "SKILLS_REPO_URL missing from $EnvFile and no -RepoUrl given." } $sshHost = Get-EnvKey "GITEA_SSH_HOST" $sshPort = Get-EnvKey "GITEA_SSH_PORT" $sshHostA = Get-EnvKey "GITEA_SSH_HOST_ALT" $sshPortA = Get-EnvKey "GITEA_SSH_PORT_ALT" $sshUser = Get-EnvKey "GITEA_SSH_USER" $container = Get-EnvKey "GITEA_CONTAINER" $dockerBin = Get-EnvKey "GITEA_DOCKER_BIN" if (-not $sshPort) { $sshPort = "22" } if (-not $container) { $container = "gitea" } if (-not $dockerBin) { $dockerBin = "/usr/local/bin/docker" } # ---------- client-side transport check ---------- # Two planes, because they fail independently and the difference is the whole diagnosis: # refs = ls-remote / info-refs (kept working right through the 2026-09 outage) # pack = an actual fetch (this is what was dead) function Test-Transport { $result = [ordered]@{ Refs = $false; Pack = $false; Sha = ""; Detail = "" } $prev = $ErrorActionPreference; $ErrorActionPreference = "Continue" $refs = & git ls-remote $RepoUrl "refs/heads/*" 2>&1 | ForEach-Object { "$_" } $refsCode = $LASTEXITCODE $ErrorActionPreference = $prev if ($refsCode -eq 0 -and $refs) { $result.Refs = $true $result.Sha = (($refs -join "`n") -split "\s+")[0] } else { $result.Detail = ($refs -join " ") return $result } # tree:0 + depth 1 keeps this to a few KB; the outage failed even at that size, so a # tiny probe is a valid detector and costs nothing to run on every sync. $tmp = Join-Path ([IO.Path]::GetTempPath()) ("gitea-doctor-" + [guid]::NewGuid().ToString("N")) $prev = $ErrorActionPreference; $ErrorActionPreference = "Continue" $out = & git -c core.askPass=true clone --quiet --bare --depth 1 --filter=tree:0 $RepoUrl $tmp 2>&1 | ForEach-Object { "$_" } $code = $LASTEXITCODE $ErrorActionPreference = $prev if (Test-Path $tmp) { Remove-Item -Recurse -Force $tmp -ErrorAction SilentlyContinue } if ($code -eq 0) { $result.Pack = $true } else { $result.Detail = ($out -join " ") } return $result } Write-Host "=== gitea transport check: $RepoUrl ===" $t = Test-Transport if ($t.Refs) { Write-Host " refs plane: OK (main-ish head $($t.Sha))" } else { Write-Host " refs plane: BROKEN - $($t.Detail)" } if ($t.Pack) { Write-Host " pack plane: OK" } else { Write-Host " pack plane: BROKEN - $($t.Detail)" } if ($t.Refs -and $t.Pack) { Write-Host "transport healthy"; exit 0 } # A dead refs plane too means the host or gitea is down/unreachable, not the empty-pack # fault - still worth logging in, the triage script starts by checking the container. if (-not $Fix) { Write-Host "" Write-Host "Transport is broken. Re-run with -Fix to log into $sshUser@$sshHost and repair," Write-Host "or see the skill's Phase 1.5 to do it by hand." exit 1 } # ---------- reach the server ---------- if (-not $sshHost -or -not $sshUser) { Fail "GITEA_SSH_HOST / GITEA_SSH_USER missing from $EnvFile - cannot reach the server." } function Invoke-Remote([string]$HostName, [string]$Port, [string]$LocalScript, [string[]]$ScriptArgs) { # Copy the script over rather than piping it into `sh -s`. Piping looks simpler but # PS 5.1 encodes native-process stdin its own way and prepends a UTF-8 BOM, which # lands on the shebang and makes sh report "#!/bin/sh: No such file or directory". # scp moves bytes verbatim, so what runs there is exactly the file that is here. $remote = "/tmp/tools-sync-triage.$PID.sh" $prev = $ErrorActionPreference; $ErrorActionPreference = "Continue" $common = @("-o", "BatchMode=yes", "-o", "ConnectTimeout=$SshTimeout", "-o", "StrictHostKeyChecking=accept-new") $cp = & scp @common "-P" $Port $LocalScript "${sshUser}@${HostName}:$remote" 2>&1 | ForEach-Object { "$_" } if ($LASTEXITCODE -ne 0) { $ErrorActionPreference = $prev return @{ Code = 255; Out = ($cp -join "`n") } } $cmd = "sh $remote " + ($ScriptArgs -join " ") + "; rc=`$?; rm -f $remote; exit `$rc" $out = & ssh @common "-p" $Port "$sshUser@$HostName" $cmd 2>&1 | ForEach-Object { "$_" } $code = $LASTEXITCODE $ErrorActionPreference = $prev return @{ Code = $code; Out = ($out -join "`n") } } $triage = Join-Path $PSScriptRoot "gitea_triage.sh" if (-not (Test-Path $triage)) { Fail "gitea_triage.sh missing next to this script." } # Normalise to LF and no BOM before shipping it: a CRLF or BOM on the shebang line makes # sh fail with a "No such file or directory" that names the interpreter, not the cause. $triageStaged = Join-Path ([IO.Path]::GetTempPath()) "tools-sync-triage-$PID.sh" $triageText = ((Get-Content $triage -Raw) -replace "`r`n", "`n") -replace "^", "" [IO.File]::WriteAllText($triageStaged, $triageText, (New-Object System.Text.UTF8Encoding($false))) $args = @("--container", $container, "--docker", $dockerBin, "--fix") if ($AllowDestructive) { $args += "--allow-destructive" } $targets = @(@{ H = $sshHost; P = $sshPort; Label = "direct" }) if ($sshHostA -and $sshPortA) { $targets += @{ H = $sshHostA; P = $sshPortA; Label = "fallback" } } $r = $null foreach ($tg in $targets) { Write-Host "" Write-Host "=== server triage via $($tg.Label): $sshUser@$($tg.H):$($tg.P) ===" $r = Invoke-Remote $tg.H $tg.P $triageStaged $args if ($r.Out -notmatch "Connection (timed out|refused)|Could not resolve|Permission denied|No route to host|lost connection") { break } Write-Host " unreachable: $(($r.Out -split "`n")[0])" $r = $null } Remove-Item $triageStaged -Force -ErrorAction SilentlyContinue if (-not $r) { Fail "could not reach the gitea host on any configured path (see GITEA_SSH_HOST / _ALT)." } Write-Host $r.Out # ---------- re-verify from here ---------- # The server saying "repaired" is a claim; the client completing a fetch is the evidence. Write-Host "" Write-Host "=== re-verifying transport from this machine ===" $t2 = Test-Transport if ($t2.Refs -and $t2.Pack) { Write-Host "transport healthy again - re-run the sync." exit 0 } if ($t2.Refs) { Write-Host " refs plane: OK" } else { Write-Host " refs plane: BROKEN - $($t2.Detail)" } if ($t2.Pack) { Write-Host " pack plane: OK" } else { Write-Host " pack plane: BROKEN - $($t2.Detail)" } Fail "server triage ran but the transport is still broken. Read the RESULT line above; if it says still-broken, the fault is outside the known repairs - escalate to the skill's Phase 1.5 ladder."