166 lines
8.0 KiB
PowerShell
166 lines
8.0 KiB
PowerShell
# tools-sync: verify the gitea transport from this machine, and when it is broken, log
|
||
# into the server and repair it.
|
||
#
|
||
# Why this exists: on 2026-08-25 the server started handing out empty packs. push kept
|
||
# working, `ls-remote` kept working, only the pack plane was dead - so every sync run
|
||
# "succeeded" while three commits sat stranded on one machine for eight days. Checking
|
||
# the transport is cheap; discovering the outage a week late is not.
|
||
#
|
||
# .\gitea_doctor.ps1 # check only, report what is wrong
|
||
# .\gitea_doctor.ps1 -Fix # check, then repair on the server, then re-verify
|
||
#
|
||
# Exit: 0 = transport healthy at the end of the run, 1 = still broken.
|
||
|
||
param(
|
||
[string]$EnvFile = "$env:USERPROFILE\skills_env.md",
|
||
[string]$RepoUrl,
|
||
# Log in and repair, instead of only reporting.
|
||
[switch]$Fix,
|
||
# Also allow repairs that can lose data (container recreate, repo surgery). The
|
||
# server-side script asks for this separately; nothing here needs it by default.
|
||
[switch]$AllowDestructive,
|
||
[int]$SshTimeout = 15
|
||
)
|
||
|
||
$ErrorActionPreference = "Stop"
|
||
|
||
function Fail([string]$Msg) { Write-Host "FAIL: $Msg"; exit 1 }
|
||
|
||
# ---------- environment ----------
|
||
if (-not (Test-Path $EnvFile)) { Fail "$EnvFile not found. Copy references\skills_env_template.md there first." }
|
||
$envText = Get-Content $EnvFile -Raw
|
||
function Get-EnvKey([string]$Key) {
|
||
$m = [regex]::Match($envText, "(?m)^\s*$([regex]::Escape($Key))\s*=\s*(.+?)\s*$")
|
||
if (-not $m.Success) { return "" }
|
||
$v = $m.Groups[1].Value.Trim()
|
||
if ($v -match "^<.*>$") { return "" } # unfilled placeholder
|
||
return $v
|
||
}
|
||
|
||
if (-not $RepoUrl) { $RepoUrl = Get-EnvKey "SKILLS_REPO_URL" }
|
||
if (-not $RepoUrl) { Fail "SKILLS_REPO_URL missing from $EnvFile and no -RepoUrl given." }
|
||
|
||
$sshHost = Get-EnvKey "GITEA_SSH_HOST"
|
||
$sshPort = Get-EnvKey "GITEA_SSH_PORT"
|
||
$sshHostA = Get-EnvKey "GITEA_SSH_HOST_ALT"
|
||
$sshPortA = Get-EnvKey "GITEA_SSH_PORT_ALT"
|
||
$sshUser = Get-EnvKey "GITEA_SSH_USER"
|
||
$container = Get-EnvKey "GITEA_CONTAINER"
|
||
$dockerBin = Get-EnvKey "GITEA_DOCKER_BIN"
|
||
if (-not $sshPort) { $sshPort = "22" }
|
||
if (-not $container) { $container = "gitea" }
|
||
if (-not $dockerBin) { $dockerBin = "/usr/local/bin/docker" }
|
||
|
||
# ---------- client-side transport check ----------
|
||
# Two planes, because they fail independently and the difference is the whole diagnosis:
|
||
# refs = ls-remote / info-refs (kept working right through the 2026-09 outage)
|
||
# pack = an actual fetch (this is what was dead)
|
||
function Test-Transport {
|
||
$result = [ordered]@{ Refs = $false; Pack = $false; Sha = ""; Detail = "" }
|
||
|
||
$prev = $ErrorActionPreference; $ErrorActionPreference = "Continue"
|
||
$refs = & git ls-remote $RepoUrl "refs/heads/*" 2>&1 | ForEach-Object { "$_" }
|
||
$refsCode = $LASTEXITCODE
|
||
$ErrorActionPreference = $prev
|
||
if ($refsCode -eq 0 -and $refs) {
|
||
$result.Refs = $true
|
||
$result.Sha = (($refs -join "`n") -split "\s+")[0]
|
||
} else {
|
||
$result.Detail = ($refs -join " ")
|
||
return $result
|
||
}
|
||
|
||
# tree:0 + depth 1 keeps this to a few KB; the outage failed even at that size, so a
|
||
# tiny probe is a valid detector and costs nothing to run on every sync.
|
||
$tmp = Join-Path ([IO.Path]::GetTempPath()) ("gitea-doctor-" + [guid]::NewGuid().ToString("N"))
|
||
$prev = $ErrorActionPreference; $ErrorActionPreference = "Continue"
|
||
$out = & git -c core.askPass=true clone --quiet --bare --depth 1 --filter=tree:0 $RepoUrl $tmp 2>&1 | ForEach-Object { "$_" }
|
||
$code = $LASTEXITCODE
|
||
$ErrorActionPreference = $prev
|
||
if (Test-Path $tmp) { Remove-Item -Recurse -Force $tmp -ErrorAction SilentlyContinue }
|
||
if ($code -eq 0) { $result.Pack = $true } else { $result.Detail = ($out -join " ") }
|
||
return $result
|
||
}
|
||
|
||
Write-Host "=== gitea transport check: $RepoUrl ==="
|
||
$t = Test-Transport
|
||
if ($t.Refs) { Write-Host " refs plane: OK (main-ish head $($t.Sha))" } else { Write-Host " refs plane: BROKEN - $($t.Detail)" }
|
||
if ($t.Pack) { Write-Host " pack plane: OK" } else { Write-Host " pack plane: BROKEN - $($t.Detail)" }
|
||
|
||
if ($t.Refs -and $t.Pack) { Write-Host "transport healthy"; exit 0 }
|
||
|
||
# A dead refs plane too means the host or gitea is down/unreachable, not the empty-pack
|
||
# fault - still worth logging in, the triage script starts by checking the container.
|
||
if (-not $Fix) {
|
||
Write-Host ""
|
||
Write-Host "Transport is broken. Re-run with -Fix to log into $sshUser@$sshHost and repair,"
|
||
Write-Host "or see the skill's Phase 1.5 to do it by hand."
|
||
exit 1
|
||
}
|
||
|
||
# ---------- reach the server ----------
|
||
if (-not $sshHost -or -not $sshUser) { Fail "GITEA_SSH_HOST / GITEA_SSH_USER missing from $EnvFile - cannot reach the server." }
|
||
|
||
function Invoke-Remote([string]$HostName, [string]$Port, [string]$LocalScript, [string[]]$ScriptArgs) {
|
||
# Copy the script over rather than piping it into `sh -s`. Piping looks simpler but
|
||
# PS 5.1 encodes native-process stdin its own way and prepends a UTF-8 BOM, which
|
||
# lands on the shebang and makes sh report "#!/bin/sh: No such file or directory".
|
||
# scp moves bytes verbatim, so what runs there is exactly the file that is here.
|
||
$remote = "/tmp/tools-sync-triage.$PID.sh"
|
||
$prev = $ErrorActionPreference; $ErrorActionPreference = "Continue"
|
||
$common = @("-o", "BatchMode=yes", "-o", "ConnectTimeout=$SshTimeout", "-o", "StrictHostKeyChecking=accept-new")
|
||
|
||
$cp = & scp @common "-P" $Port $LocalScript "${sshUser}@${HostName}:$remote" 2>&1 | ForEach-Object { "$_" }
|
||
if ($LASTEXITCODE -ne 0) {
|
||
$ErrorActionPreference = $prev
|
||
return @{ Code = 255; Out = ($cp -join "`n") }
|
||
}
|
||
|
||
$cmd = "sh $remote " + ($ScriptArgs -join " ") + "; rc=`$?; rm -f $remote; exit `$rc"
|
||
$out = & ssh @common "-p" $Port "$sshUser@$HostName" $cmd 2>&1 | ForEach-Object { "$_" }
|
||
$code = $LASTEXITCODE
|
||
$ErrorActionPreference = $prev
|
||
return @{ Code = $code; Out = ($out -join "`n") }
|
||
}
|
||
|
||
$triage = Join-Path $PSScriptRoot "gitea_triage.sh"
|
||
if (-not (Test-Path $triage)) { Fail "gitea_triage.sh missing next to this script." }
|
||
# Normalise to LF and no BOM before shipping it: a CRLF or BOM on the shebang line makes
|
||
# sh fail with a "No such file or directory" that names the interpreter, not the cause.
|
||
$triageStaged = Join-Path ([IO.Path]::GetTempPath()) "tools-sync-triage-$PID.sh"
|
||
$triageText = ((Get-Content $triage -Raw) -replace "`r`n", "`n") -replace "^", ""
|
||
[IO.File]::WriteAllText($triageStaged, $triageText, (New-Object System.Text.UTF8Encoding($false)))
|
||
|
||
$triageArgs = @("--container", $container, "--docker", $dockerBin, "--fix")
|
||
if ($AllowDestructive) { $triageArgs += "--allow-destructive" }
|
||
|
||
$targets = @(@{ H = $sshHost; P = $sshPort; Label = "direct" })
|
||
if ($sshHostA -and $sshPortA) { $targets += @{ H = $sshHostA; P = $sshPortA; Label = "fallback" } }
|
||
|
||
$r = $null
|
||
foreach ($tg in $targets) {
|
||
Write-Host ""
|
||
Write-Host "=== server triage via $($tg.Label): $sshUser@$($tg.H):$($tg.P) ==="
|
||
$r = Invoke-Remote $tg.H $tg.P $triageStaged $triageArgs
|
||
if ($r.Out -notmatch "Connection (timed out|refused)|Could not resolve|Permission denied|No route to host|lost connection") { break }
|
||
Write-Host " unreachable: $(($r.Out -split "`n")[0])"
|
||
$r = $null
|
||
}
|
||
Remove-Item $triageStaged -Force -ErrorAction SilentlyContinue
|
||
if (-not $r) { Fail "could not reach the gitea host on any configured path (see GITEA_SSH_HOST / _ALT)." }
|
||
|
||
Write-Host $r.Out
|
||
|
||
# ---------- re-verify from here ----------
|
||
# The server saying "repaired" is a claim; the client completing a fetch is the evidence.
|
||
Write-Host ""
|
||
Write-Host "=== re-verifying transport from this machine ==="
|
||
$t2 = Test-Transport
|
||
if ($t2.Refs -and $t2.Pack) {
|
||
Write-Host "transport healthy again - re-run the sync."
|
||
exit 0
|
||
}
|
||
if ($t2.Refs) { Write-Host " refs plane: OK" } else { Write-Host " refs plane: BROKEN - $($t2.Detail)" }
|
||
if ($t2.Pack) { Write-Host " pack plane: OK" } else { Write-Host " pack plane: BROKEN - $($t2.Detail)" }
|
||
Fail "server triage ran but the transport is still broken. Read the RESULT line above; if it says still-broken, the fault is outside the known repairs - escalate to the skill's Phase 1.5 ladder."
|