235 lines
12 KiB
PowerShell
235 lines
12 KiB
PowerShell
# tools-sync: verify the gitea transport from this machine, and when it is broken, log
|
|
# into the server and repair it.
|
|
#
|
|
# Why this exists: on 2026-08-25 the server started handing out empty packs. push kept
|
|
# working, `ls-remote` kept working, only the pack plane was dead - so every sync run
|
|
# "succeeded" while three commits sat stranded on one machine for eight days. Checking
|
|
# the transport is cheap; discovering the outage a week late is not.
|
|
#
|
|
# .\gitea_doctor.ps1 # check only, report what is wrong
|
|
# .\gitea_doctor.ps1 -Fix # check, then repair on the server, then re-verify
|
|
#
|
|
# Exit: 0 = transport healthy at the end of the run, 1 = still broken.
|
|
|
|
param(
|
|
[string]$EnvFile = "$env:USERPROFILE\skills_env.md",
|
|
[string]$RepoUrl,
|
|
# Log in and repair, instead of only reporting.
|
|
[switch]$Fix,
|
|
# Also allow repairs that can lose data (container recreate, repo surgery). The
|
|
# server-side script asks for this separately; nothing here needs it by default.
|
|
[switch]$AllowDestructive,
|
|
[int]$SshTimeout = 15
|
|
)
|
|
|
|
$ErrorActionPreference = "Stop"
|
|
|
|
function Fail([string]$Msg) { Write-Host "FAIL: $Msg"; exit 1 }
|
|
|
|
# ---------- environment ----------
|
|
if (-not (Test-Path $EnvFile)) { Fail "$EnvFile not found. Copy references\skills_env_template.md there first." }
|
|
$envText = Get-Content $EnvFile -Raw
|
|
function Get-EnvKey([string]$Key) {
|
|
$m = [regex]::Match($envText, "(?m)^\s*$([regex]::Escape($Key))\s*=\s*(.+?)\s*$")
|
|
if (-not $m.Success) { return "" }
|
|
$v = ($m.Groups[1].Value -replace '\s+#.*$', '').Trim() # drop inline comment
|
|
if ($v -match "^<.*>$") { return "" } # unfilled placeholder
|
|
return $v
|
|
}
|
|
|
|
if (-not $RepoUrl) { $RepoUrl = Get-EnvKey "SKILLS_REPO_URL" }
|
|
if (-not $RepoUrl) { Fail "SKILLS_REPO_URL missing from $EnvFile and no -RepoUrl given." }
|
|
|
|
$sshHost = Get-EnvKey "GITEA_SSH_HOST"
|
|
$sshPort = Get-EnvKey "GITEA_SSH_PORT"
|
|
$sshHostA = Get-EnvKey "GITEA_SSH_HOST_ALT"
|
|
$sshPortA = Get-EnvKey "GITEA_SSH_PORT_ALT"
|
|
$sshUser = Get-EnvKey "GITEA_SSH_USER"
|
|
$container = Get-EnvKey "GITEA_CONTAINER"
|
|
$dockerBin = Get-EnvKey "GITEA_DOCKER_BIN"
|
|
if (-not $sshPort) { $sshPort = "22" }
|
|
if (-not $container) { $container = "gitea" }
|
|
if (-not $dockerBin) { $dockerBin = "/usr/local/bin/docker" }
|
|
|
|
# ---------- client-side transport check ----------
|
|
# Two planes, because they fail independently and the difference is the whole diagnosis:
|
|
# refs = ls-remote / info-refs (kept working right through the 2026-09 outage)
|
|
# pack = an actual fetch (this is what was dead)
|
|
function Test-Transport {
|
|
$result = [ordered]@{ Refs = $false; Pack = $false; Sha = ""; Detail = "" }
|
|
|
|
$prev = $ErrorActionPreference; $ErrorActionPreference = "Continue"
|
|
$refs = & git ls-remote $RepoUrl "refs/heads/*" 2>&1 | ForEach-Object { "$_" }
|
|
$refsCode = $LASTEXITCODE
|
|
$ErrorActionPreference = $prev
|
|
if ($refsCode -eq 0 -and $refs) {
|
|
$result.Refs = $true
|
|
$result.Sha = (($refs -join "`n") -split "\s+")[0]
|
|
} else {
|
|
$result.Detail = ($refs -join " ")
|
|
return $result
|
|
}
|
|
|
|
# tree:0 + depth 1 keeps this to a few KB; the outage failed even at that size, so a
|
|
# tiny probe is a valid detector and costs nothing to run on every sync.
|
|
$tmp = Join-Path ([IO.Path]::GetTempPath()) ("gitea-doctor-" + [guid]::NewGuid().ToString("N"))
|
|
$prev = $ErrorActionPreference; $ErrorActionPreference = "Continue"
|
|
$out = & git -c core.askPass=true clone --quiet --bare --depth 1 --filter=tree:0 $RepoUrl $tmp 2>&1 | ForEach-Object { "$_" }
|
|
$code = $LASTEXITCODE
|
|
$ErrorActionPreference = $prev
|
|
$tempRoot = [IO.Path]::GetFullPath([IO.Path]::GetTempPath()).TrimEnd([IO.Path]::DirectorySeparatorChar)
|
|
$resolvedTmp = [IO.Path]::GetFullPath($tmp)
|
|
if (-not $resolvedTmp.StartsWith($tempRoot + [IO.Path]::DirectorySeparatorChar, [StringComparison]::OrdinalIgnoreCase) -or
|
|
-not ([IO.Path]::GetFileName($resolvedTmp) -match '^gitea-doctor-[0-9a-f]{32}$')) {
|
|
throw "unsafe diagnostic clone cleanup target: $resolvedTmp"
|
|
}
|
|
if (Test-Path -LiteralPath $resolvedTmp) {
|
|
Remove-Item -LiteralPath $resolvedTmp -Recurse -Force -ErrorAction SilentlyContinue
|
|
}
|
|
if ($code -eq 0) { $result.Pack = $true } else { $result.Detail = ($out -join " ") }
|
|
return $result
|
|
}
|
|
|
|
Write-Host "=== gitea transport check: $RepoUrl ==="
|
|
$t = Test-Transport
|
|
if ($t.Refs) { Write-Host " refs plane: OK (main-ish head $($t.Sha))" } else { Write-Host " refs plane: BROKEN - $($t.Detail)" }
|
|
if ($t.Pack) { Write-Host " pack plane: OK" } else { Write-Host " pack plane: BROKEN - $($t.Detail)" }
|
|
|
|
if ($t.Refs -and $t.Pack) { Write-Host "transport healthy"; exit 0 }
|
|
|
|
# A dead refs plane too means the host or gitea is down/unreachable, not the empty-pack
|
|
# fault - still worth logging in, the triage script starts by checking the container.
|
|
if (-not $Fix) {
|
|
Write-Host ""
|
|
Write-Host "Transport is broken. Re-run with -Fix to log into $sshUser@$sshHost and repair,"
|
|
Write-Host "or see the skill's Phase 1.5 to do it by hand."
|
|
exit 1
|
|
}
|
|
|
|
# ---------- reach the server ----------
|
|
if (-not $sshHost -or -not $sshUser) { Fail "GITEA_SSH_HOST / GITEA_SSH_USER missing from $EnvFile - cannot reach the server." }
|
|
|
|
function Invoke-Remote([string]$HostName, [string]$Port, [string]$LocalScript, [string[]]$ScriptArgs) {
|
|
# Copy the script over rather than piping it into `sh -s`. Piping looks simpler but
|
|
# PS 5.1 encodes native-process stdin its own way and prepends a UTF-8 BOM, which
|
|
# lands on the shebang and makes sh report "#!/bin/sh: No such file or directory".
|
|
# scp moves bytes verbatim, so what runs there is exactly the file that is here.
|
|
$remote = "/tmp/tools-sync-triage.$PID.sh"
|
|
$prev = $ErrorActionPreference; $ErrorActionPreference = "Continue"
|
|
$common = @("-o", "BatchMode=yes", "-o", "ConnectTimeout=$SshTimeout", "-o", "StrictHostKeyChecking=accept-new")
|
|
|
|
$cp = & scp @common "-P" $Port $LocalScript "${sshUser}@${HostName}:$remote" 2>&1 | ForEach-Object { "$_" }
|
|
if ($LASTEXITCODE -ne 0) {
|
|
$ErrorActionPreference = $prev
|
|
return @{ Code = 255; Out = ($cp -join "`n") }
|
|
}
|
|
|
|
$cmd = "sh $remote " + ($ScriptArgs -join " ") + "; rc=`$?; rm -f $remote; exit `$rc"
|
|
$out = & ssh @common "-p" $Port "$sshUser@$HostName" $cmd 2>&1 | ForEach-Object { "$_" }
|
|
$code = $LASTEXITCODE
|
|
$ErrorActionPreference = $prev
|
|
return @{ Code = $code; Out = ($out -join "`n") }
|
|
}
|
|
|
|
function Find-PuttyExe([string]$Name) {
|
|
$found = Get-Command $Name -ErrorAction SilentlyContinue
|
|
if ($found) { return $found.Source }
|
|
$candidate = Join-Path $env:ProgramFiles ("PuTTY\" + $Name)
|
|
if (Test-Path -LiteralPath $candidate) { return $candidate }
|
|
return ""
|
|
}
|
|
|
|
function Get-BmemGiteaPassword {
|
|
# A previously user-authorized machine/global record is a credential source,
|
|
# not a value to print or copy into any repository. Use it only when the
|
|
# key-based SSH path is unavailable on this machine.
|
|
$memoryFile = Join-Path $env:USERPROFILE "bmem.md"
|
|
if (-not (Test-Path -LiteralPath $memoryFile)) { return "" }
|
|
$memoryText = Get-Content -LiteralPath $memoryFile -Raw -Encoding UTF8
|
|
$entry = [regex]::Match($memoryText, '(?m)^\s*-\s*\[G-0009\].*$').Value
|
|
if (-not $entry) { return "" }
|
|
$pattern = 'SSH\s*\u767B\u5F55\s*=\s*\u7528\u6237\u540D\s*bowei\s*[\u3001,\uFF0C]\s*\u5BC6\u7801\s*([^;\uFF1B,\uFF0C\u3001\u3002\s]+)'
|
|
return [regex]::Match($entry, $pattern).Groups[1].Value
|
|
}
|
|
|
|
function Invoke-RemotePutty([string]$HostName, [string]$Port, [string]$LocalScript, [string[]]$ScriptArgs) {
|
|
$fingerprint = Get-EnvKey "GITEA_HOST_KEY_FP"
|
|
$password = Get-BmemGiteaPassword
|
|
$plink = Find-PuttyExe "plink.exe"
|
|
$pscp = Find-PuttyExe "pscp.exe"
|
|
if (-not $fingerprint -or -not $password -or -not $plink -or -not $pscp) {
|
|
return @{ Code = 255; Out = "PuTTY fallback lacks a pinned host fingerprint, previously authorized bmem credential, or PuTTY executables" }
|
|
}
|
|
|
|
$remote = "/tmp/tools-sync-triage.$PID.putty.sh"
|
|
$prev = $ErrorActionPreference; $ErrorActionPreference = "Continue"
|
|
$plinkArgs = @("-batch", "-ssh", "-hostkey", $fingerprint, "-P", $Port,
|
|
"-pw", $password, "$sshUser@$HostName")
|
|
$probe = & $plink @plinkArgs "test ! -e $remote" 2>&1 | ForEach-Object { "$_" }
|
|
if ($LASTEXITCODE -ne 0) {
|
|
$ErrorActionPreference = $prev
|
|
return @{ Code = 255; Out = "remote triage staging path is occupied or SSH authentication failed: " + ($probe -join " ") }
|
|
}
|
|
$cp = & $pscp -batch -hostkey $fingerprint -P $Port -pw $password $LocalScript "${sshUser}@${HostName}:$remote" 2>&1 | ForEach-Object { "$_" }
|
|
if ($LASTEXITCODE -ne 0) {
|
|
$ErrorActionPreference = $prev
|
|
return @{ Code = 255; Out = "PuTTY script transfer failed: " + ($cp -join " ") }
|
|
}
|
|
$cmd = "sh $remote " + ($ScriptArgs -join " ") + "; rc=`$?; rm -f $remote; exit `$rc"
|
|
$out = & $plink @plinkArgs $cmd 2>&1 | ForEach-Object { "$_" }
|
|
$code = $LASTEXITCODE
|
|
$ErrorActionPreference = $prev
|
|
return @{ Code = $code; Out = ($out -join "`n") }
|
|
}
|
|
|
|
$triage = Join-Path $PSScriptRoot "gitea_triage.sh"
|
|
if (-not (Test-Path $triage)) { Fail "gitea_triage.sh missing next to this script." }
|
|
# Normalise to LF and no BOM before shipping it: a CRLF or BOM on the shebang line makes
|
|
# sh fail with a "No such file or directory" that names the interpreter, not the cause.
|
|
$triageStaged = Join-Path ([IO.Path]::GetTempPath()) "tools-sync-triage-$PID.sh"
|
|
$triageText = ((Get-Content $triage -Raw) -replace "`r`n", "`n") -replace "^$([char]0xFEFF)", ""
|
|
[IO.File]::WriteAllText($triageStaged, $triageText, (New-Object System.Text.UTF8Encoding($false)))
|
|
|
|
$triageArgs = @("--container", $container, "--docker", $dockerBin, "--fix")
|
|
if ($AllowDestructive) { $triageArgs += "--allow-destructive" }
|
|
|
|
$targets = @(@{ H = $sshHost; P = $sshPort; Label = "direct" })
|
|
if ($sshHostA -and $sshPortA) { $targets += @{ H = $sshHostA; P = $sshPortA; Label = "fallback" } }
|
|
|
|
$r = $null
|
|
foreach ($tg in $targets) {
|
|
Write-Host ""
|
|
Write-Host "=== server triage via $($tg.Label): $sshUser@$($tg.H):$($tg.P) ==="
|
|
$r = Invoke-Remote $tg.H $tg.P $triageStaged $triageArgs
|
|
if ($r.Out -notmatch "Connection (timed out|refused)|Could not resolve|Permission denied|No route to host|lost connection") { break }
|
|
Write-Host " unreachable: $(($r.Out -split "`n")[0])"
|
|
$r = $null
|
|
}
|
|
if (-not $r) {
|
|
Write-Host " OpenSSH authentication unavailable; trying the pinned PuTTY credential path"
|
|
foreach ($tg in $targets) {
|
|
Write-Host "=== server triage via pinned PuTTY: $sshUser@$($tg.H):$($tg.P) ==="
|
|
$r = Invoke-RemotePutty $tg.H $tg.P $triageStaged $triageArgs
|
|
if ($r.Code -ne 255) { break }
|
|
Write-Host " unavailable: $(($r.Out -split "`n")[0])"
|
|
$r = $null
|
|
}
|
|
}
|
|
Remove-Item $triageStaged -Force -ErrorAction SilentlyContinue
|
|
if (-not $r) { Fail "could not reach the gitea host on any configured path (see GITEA_SSH_HOST / _ALT)." }
|
|
|
|
Write-Host $r.Out
|
|
|
|
# ---------- re-verify from here ----------
|
|
# The server saying "repaired" is a claim; the client completing a fetch is the evidence.
|
|
Write-Host ""
|
|
Write-Host "=== re-verifying transport from this machine ==="
|
|
$t2 = Test-Transport
|
|
if ($t2.Refs -and $t2.Pack) {
|
|
Write-Host "transport healthy again - re-run the sync."
|
|
exit 0
|
|
}
|
|
if ($t2.Refs) { Write-Host " refs plane: OK" } else { Write-Host " refs plane: BROKEN - $($t2.Detail)" }
|
|
if ($t2.Pack) { Write-Host " pack plane: OK" } else { Write-Host " pack plane: BROKEN - $($t2.Detail)" }
|
|
Fail "server triage ran but the transport is still broken. Read the RESULT line above; if it says still-broken, the fault is outside the known repairs - escalate to the skill's Phase 1.5 ladder."
|