diff --git a/bc-export.ps1 b/bc-export.ps1 index 84f3ae4..d7706f1 100755 --- a/bc-export.ps1 +++ b/bc-export.ps1 @@ -95,7 +95,7 @@ function Invoke-BCApi { param( [string]$Url, [int]$TimeoutSec = 120, - [int]$MaxRetries = 3 + [int]$MaxRetries = 10 ) for ($attempt = 1; $attempt -le $MaxRetries; $attempt++) { @@ -111,15 +111,39 @@ function Invoke-BCApi { } catch { $statusCode = $null + $errorBody = "" if ($_.Exception.Response) { $statusCode = [int]$_.Exception.Response.StatusCode } + if ($_.ErrorDetails.Message) { + $errorBody = $_.ErrorDetails.Message + } + + # Table lock: BC returns 500 with "being updated in a transaction" + $isTableLock = $errorBody -match "transaction done by another session|being updated in a transaction|deadlock|locked" + # Rate limit + $isThrottled = ($statusCode -eq 429) + # Server error + $isServerError = ($statusCode -ge 500 -and -not $isTableLock) + # Timeout + $isTimeout = ($_ -match "Timeout") + + $isRetryable = $isTableLock -or $isThrottled -or $isServerError -or $isTimeout - # Retry on 429 (throttled) or 5xx (server error) or timeout - $isRetryable = ($statusCode -eq 429) -or ($statusCode -ge 500) -or ($_ -match "Timeout") if ($isRetryable -and $attempt -lt $MaxRetries) { - $wait = $attempt * 10 - Write-Log " Request failed (attempt $attempt/$MaxRetries), retrying in ${wait}s..." "WARN" + if ($isTableLock) { + # Table locks can last a while - wait longer with jitter + $wait = [math]::Min(30 + ($attempt * 15), 120) + Write-Log " Table lock detected (attempt $attempt/$MaxRetries), waiting ${wait}s..." "WARN" + } + elseif ($isThrottled) { + $wait = [math]::Min(30 * $attempt, 300) + Write-Log " Rate limited (attempt $attempt/$MaxRetries), waiting ${wait}s..." "WARN" + } + else { + $wait = [math]::Min(10 * $attempt, 120) + Write-Log " Request failed (attempt $attempt/$MaxRetries), retrying in ${wait}s..." "WARN" + } Start-Sleep -Seconds $wait continue } @@ -166,26 +190,41 @@ function Export-EntityData { ) $entityUrl = "$baseUrl/companies($CompanyId)/$EntityName" + $maxEntityRetries = 5 - Write-Log " Exporting $EntityName..." + for ($entityAttempt = 1; $entityAttempt -le $maxEntityRetries; $entityAttempt++) { + Write-Log " Exporting $EntityName..." - try { - $data = Get-BCData -Url $entityUrl - $count = 0 - if ($data) { $count = $data.Count } + try { + $data = Get-BCData -Url $entityUrl + $count = 0 + if ($data) { $count = $data.Count } - $outputFile = Join-Path $OutputDir "$EntityName.json" - $data | ConvertTo-Json -Depth 10 | Out-File -FilePath $outputFile -Encoding utf8 + $outputFile = Join-Path $OutputDir "$EntityName.json" + $data | ConvertTo-Json -Depth 10 | Out-File -FilePath $outputFile -Encoding utf8 - Write-Log " $EntityName : $count records" - return $count - } - catch { - Write-Log " Failed to export ${EntityName}: $_" "WARN" - $outputFile = Join-Path $OutputDir "$EntityName.json" - "[]" | Out-File -FilePath $outputFile -Encoding utf8 - return 0 + Write-Log " $EntityName : $count records" + return $count + } + catch { + $errorMsg = "$_" + $isTableLock = $errorMsg -match "transaction done by another session|being updated in a transaction|deadlock|locked" + + if ($isTableLock -and $entityAttempt -lt $maxEntityRetries) { + $wait = 60 * $entityAttempt + Write-Log " Table lock on $EntityName (attempt $entityAttempt/$maxEntityRetries), restarting in ${wait}s..." "WARN" + Start-Sleep -Seconds $wait + continue + } + + Write-Log " Failed to export ${EntityName}: $errorMsg" "WARN" + $outputFile = Join-Path $OutputDir "$EntityName.json" + "[]" | Out-File -FilePath $outputFile -Encoding utf8 + return 0 + } } + + return 0 } function Export-DocumentWithLines { @@ -197,85 +236,103 @@ function Export-DocumentWithLines { [string]$OutputDir ) - Write-Log " Exporting $DocumentEntity (headers)..." + # Retry the entire entity export if it fails (e.g. table lock on first page) + $maxEntityRetries = 5 - $docFile = Join-Path $OutputDir "$DocumentEntity.jsonl" - $lineFile = Join-Path $OutputDir "$LineEntity.jsonl" - [System.IO.File]::WriteAllText($docFile, "") - [System.IO.File]::WriteAllText($lineFile, "") + for ($entityAttempt = 1; $entityAttempt -le $maxEntityRetries; $entityAttempt++) { + Write-Log " Exporting $DocumentEntity (headers + lines)..." - $docCount = 0 - $lineCount = 0 + $docFile = Join-Path $OutputDir "$DocumentEntity.jsonl" + $lineFile = Join-Path $OutputDir "$LineEntity.jsonl" + [System.IO.File]::WriteAllText($docFile, "") + [System.IO.File]::WriteAllText($lineFile, "") - try { - # Step 1: Fetch document headers page by page (no $expand) - # BC API default page size is ~100, with @odata.nextLink for more - $currentUrl = "$baseUrl/companies($CompanyId)/$DocumentEntity" + $docCount = 0 + $lineCount = 0 + $failed = $false - while ($currentUrl) { - $response = Invoke-BCApi -Url $currentUrl + try { + # Step 1: Fetch document headers page by page (no $expand) + # BC API default page size is ~100, with @odata.nextLink for more + $currentUrl = "$baseUrl/companies($CompanyId)/$DocumentEntity" - if (-not $response.value -or $response.value.Count -eq 0) { - break - } + while ($currentUrl) { + $response = Invoke-BCApi -Url $currentUrl - # Step 2: For each document in this page, fetch its lines - foreach ($doc in $response.value) { - $docCount++ - $docId = $doc.id + if (-not $response.value -or $response.value.Count -eq 0) { + break + } - # Write document header to disk - $jsonLine = $doc | ConvertTo-Json -Depth 10 -Compress - [System.IO.File]::AppendAllText($docFile, $jsonLine + "`n") + # Step 2: For each document in this page, fetch its lines + foreach ($doc in $response.value) { + $docCount++ + $docId = $doc.id - # Fetch lines for this document - $linesUrl = "$baseUrl/companies($CompanyId)/$DocumentEntity($docId)/$LineEntity" - try { - $linesResponse = Invoke-BCApi -Url $linesUrl -TimeoutSec 60 - if ($linesResponse.value -and $linesResponse.value.Count -gt 0) { - foreach ($line in $linesResponse.value) { - $lineCount++ - $lineJson = $line | ConvertTo-Json -Depth 10 -Compress - [System.IO.File]::AppendAllText($lineFile, $lineJson + "`n") - } + # Write document header to disk + $jsonLine = $doc | ConvertTo-Json -Depth 10 -Compress + [System.IO.File]::AppendAllText($docFile, $jsonLine + "`n") - # Handle pagination within lines (unlikely but possible) - $nextLinesUrl = $linesResponse.'@odata.nextLink' - while ($nextLinesUrl) { - $moreLinesResponse = Invoke-BCApi -Url $nextLinesUrl -TimeoutSec 60 - if ($moreLinesResponse.value) { - foreach ($line in $moreLinesResponse.value) { - $lineCount++ - $lineJson = $line | ConvertTo-Json -Depth 10 -Compress - [System.IO.File]::AppendAllText($lineFile, $lineJson + "`n") - } + # Fetch lines for this document + $linesUrl = "$baseUrl/companies($CompanyId)/$DocumentEntity($docId)/$LineEntity" + try { + $linesResponse = Invoke-BCApi -Url $linesUrl -TimeoutSec 60 + if ($linesResponse.value -and $linesResponse.value.Count -gt 0) { + foreach ($line in $linesResponse.value) { + $lineCount++ + $lineJson = $line | ConvertTo-Json -Depth 10 -Compress + [System.IO.File]::AppendAllText($lineFile, $lineJson + "`n") + } + + # Handle pagination within lines (unlikely but possible) + $nextLinesUrl = $linesResponse.'@odata.nextLink' + while ($nextLinesUrl) { + $moreLinesResponse = Invoke-BCApi -Url $nextLinesUrl -TimeoutSec 60 + if ($moreLinesResponse.value) { + foreach ($line in $moreLinesResponse.value) { + $lineCount++ + $lineJson = $line | ConvertTo-Json -Depth 10 -Compress + [System.IO.File]::AppendAllText($lineFile, $lineJson + "`n") + } + } + $nextLinesUrl = $moreLinesResponse.'@odata.nextLink' } - $nextLinesUrl = $moreLinesResponse.'@odata.nextLink' } } - } - catch { - Write-Log " Warning: failed to fetch lines for $DocumentEntity $docId : $_" "WARN" + catch { + Write-Log " Warning: failed to fetch lines for $DocumentEntity $docId : $_" "WARN" + } + + # Progress every 100 documents + if ($docCount % 100 -eq 0) { + Write-Log " Progress: $docCount documents, $lineCount lines" + } } - # Progress every 100 documents - if ($docCount % 100 -eq 0) { - Write-Log " Progress: $docCount documents, $lineCount lines" - } + # Next page of documents + $currentUrl = $response.'@odata.nextLink' } - # Next page of documents - $currentUrl = $response.'@odata.nextLink' + Write-Log " $DocumentEntity : $docCount documents, $lineCount lines (complete)" + return ($docCount + $lineCount) } + catch { + $errorMsg = "$_" + $isTableLock = $errorMsg -match "transaction done by another session|being updated in a transaction|deadlock|locked" - Write-Log " $DocumentEntity : $docCount documents, $lineCount lines (complete)" - return ($docCount + $lineCount) - } - catch { - Write-Log " Failed to export ${DocumentEntity} at doc #$docCount : $_" "WARN" - Write-Log " Partial data saved ($docCount docs, $lineCount lines)" "WARN" - return ($docCount + $lineCount) + if ($isTableLock -and $entityAttempt -lt $maxEntityRetries) { + $wait = 60 * $entityAttempt + Write-Log " Table lock on $DocumentEntity (attempt $entityAttempt/$maxEntityRetries), restarting in ${wait}s..." "WARN" + Start-Sleep -Seconds $wait + continue + } + + Write-Log " Failed to export ${DocumentEntity} at doc #$docCount : $errorMsg" "WARN" + Write-Log " Partial data saved ($docCount docs, $lineCount lines)" "WARN" + return ($docCount + $lineCount) + } } + + return 0 } # Main execution