fix: handle BC table locks with proper retry and backoff
BC returns 500 with "being updated in a transaction done by another session" when tables are locked by other users - this is normal during business hours. Previously the script gave up after 3 quick retries. Changes: - Invoke-BCApi: increased retries to 10, table lock errors get longer waits (30s + 15s per attempt, up to 120s) vs generic errors (10s) - Export-EntityData: retries the whole entity up to 5 times on table lock with 60s/120s/180s/240s waits between attempts - Export-DocumentWithLines: same entity-level retry, restarts the full document+lines export on table lock rather than giving up - Also handles 429 rate limiting with longer backoff (30s per attempt) Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -95,7 +95,7 @@ function Invoke-BCApi {
|
||||
param(
|
||||
[string]$Url,
|
||||
[int]$TimeoutSec = 120,
|
||||
[int]$MaxRetries = 3
|
||||
[int]$MaxRetries = 10
|
||||
)
|
||||
|
||||
for ($attempt = 1; $attempt -le $MaxRetries; $attempt++) {
|
||||
@@ -111,15 +111,39 @@ function Invoke-BCApi {
|
||||
}
|
||||
catch {
|
||||
$statusCode = $null
|
||||
$errorBody = ""
|
||||
if ($_.Exception.Response) {
|
||||
$statusCode = [int]$_.Exception.Response.StatusCode
|
||||
}
|
||||
if ($_.ErrorDetails.Message) {
|
||||
$errorBody = $_.ErrorDetails.Message
|
||||
}
|
||||
|
||||
# Table lock: BC returns 500 with "being updated in a transaction"
|
||||
$isTableLock = $errorBody -match "transaction done by another session|being updated in a transaction|deadlock|locked"
|
||||
# Rate limit
|
||||
$isThrottled = ($statusCode -eq 429)
|
||||
# Server error
|
||||
$isServerError = ($statusCode -ge 500 -and -not $isTableLock)
|
||||
# Timeout
|
||||
$isTimeout = ($_ -match "Timeout")
|
||||
|
||||
$isRetryable = $isTableLock -or $isThrottled -or $isServerError -or $isTimeout
|
||||
|
||||
# Retry on 429 (throttled) or 5xx (server error) or timeout
|
||||
$isRetryable = ($statusCode -eq 429) -or ($statusCode -ge 500) -or ($_ -match "Timeout")
|
||||
if ($isRetryable -and $attempt -lt $MaxRetries) {
|
||||
$wait = $attempt * 10
|
||||
if ($isTableLock) {
|
||||
# Table locks can last a while - wait longer with jitter
|
||||
$wait = [math]::Min(30 + ($attempt * 15), 120)
|
||||
Write-Log " Table lock detected (attempt $attempt/$MaxRetries), waiting ${wait}s..." "WARN"
|
||||
}
|
||||
elseif ($isThrottled) {
|
||||
$wait = [math]::Min(30 * $attempt, 300)
|
||||
Write-Log " Rate limited (attempt $attempt/$MaxRetries), waiting ${wait}s..." "WARN"
|
||||
}
|
||||
else {
|
||||
$wait = [math]::Min(10 * $attempt, 120)
|
||||
Write-Log " Request failed (attempt $attempt/$MaxRetries), retrying in ${wait}s..." "WARN"
|
||||
}
|
||||
Start-Sleep -Seconds $wait
|
||||
continue
|
||||
}
|
||||
@@ -166,7 +190,9 @@ function Export-EntityData {
|
||||
)
|
||||
|
||||
$entityUrl = "$baseUrl/companies($CompanyId)/$EntityName"
|
||||
$maxEntityRetries = 5
|
||||
|
||||
for ($entityAttempt = 1; $entityAttempt -le $maxEntityRetries; $entityAttempt++) {
|
||||
Write-Log " Exporting $EntityName..."
|
||||
|
||||
try {
|
||||
@@ -181,13 +207,26 @@ function Export-EntityData {
|
||||
return $count
|
||||
}
|
||||
catch {
|
||||
Write-Log " Failed to export ${EntityName}: $_" "WARN"
|
||||
$errorMsg = "$_"
|
||||
$isTableLock = $errorMsg -match "transaction done by another session|being updated in a transaction|deadlock|locked"
|
||||
|
||||
if ($isTableLock -and $entityAttempt -lt $maxEntityRetries) {
|
||||
$wait = 60 * $entityAttempt
|
||||
Write-Log " Table lock on $EntityName (attempt $entityAttempt/$maxEntityRetries), restarting in ${wait}s..." "WARN"
|
||||
Start-Sleep -Seconds $wait
|
||||
continue
|
||||
}
|
||||
|
||||
Write-Log " Failed to export ${EntityName}: $errorMsg" "WARN"
|
||||
$outputFile = Join-Path $OutputDir "$EntityName.json"
|
||||
"[]" | Out-File -FilePath $outputFile -Encoding utf8
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
function Export-DocumentWithLines {
|
||||
param(
|
||||
[string]$CompanyId,
|
||||
@@ -197,7 +236,11 @@ function Export-DocumentWithLines {
|
||||
[string]$OutputDir
|
||||
)
|
||||
|
||||
Write-Log " Exporting $DocumentEntity (headers)..."
|
||||
# Retry the entire entity export if it fails (e.g. table lock on first page)
|
||||
$maxEntityRetries = 5
|
||||
|
||||
for ($entityAttempt = 1; $entityAttempt -le $maxEntityRetries; $entityAttempt++) {
|
||||
Write-Log " Exporting $DocumentEntity (headers + lines)..."
|
||||
|
||||
$docFile = Join-Path $OutputDir "$DocumentEntity.jsonl"
|
||||
$lineFile = Join-Path $OutputDir "$LineEntity.jsonl"
|
||||
@@ -206,6 +249,7 @@ function Export-DocumentWithLines {
|
||||
|
||||
$docCount = 0
|
||||
$lineCount = 0
|
||||
$failed = $false
|
||||
|
||||
try {
|
||||
# Step 1: Fetch document headers page by page (no $expand)
|
||||
@@ -272,12 +316,25 @@ function Export-DocumentWithLines {
|
||||
return ($docCount + $lineCount)
|
||||
}
|
||||
catch {
|
||||
Write-Log " Failed to export ${DocumentEntity} at doc #$docCount : $_" "WARN"
|
||||
$errorMsg = "$_"
|
||||
$isTableLock = $errorMsg -match "transaction done by another session|being updated in a transaction|deadlock|locked"
|
||||
|
||||
if ($isTableLock -and $entityAttempt -lt $maxEntityRetries) {
|
||||
$wait = 60 * $entityAttempt
|
||||
Write-Log " Table lock on $DocumentEntity (attempt $entityAttempt/$maxEntityRetries), restarting in ${wait}s..." "WARN"
|
||||
Start-Sleep -Seconds $wait
|
||||
continue
|
||||
}
|
||||
|
||||
Write-Log " Failed to export ${DocumentEntity} at doc #$docCount : $errorMsg" "WARN"
|
||||
Write-Log " Partial data saved ($docCount docs, $lineCount lines)" "WARN"
|
||||
return ($docCount + $lineCount)
|
||||
}
|
||||
}
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
# Main execution
|
||||
try {
|
||||
Write-Log "========================================="
|
||||
|
||||
Reference in New Issue
Block a user