Malin
d08250a479
The $expand approach had two fatal problems:
1. $top=50 with $expand made BC treat it as a hard limit with no
@odata.nextLink, so only 50 docs were exported total
2. salesOrders with $expand timed out even at 50 docs when orders
have many lines
New approach: fetch document headers normally (BC paginates fine on
its own), then for each document fetch its lines separately via
/salesInvoices({id})/salesInvoiceLines. More API calls but each is
small, fast, and reliable.
Also added:
- Invoke-BCApi with retry logic (backoff on 429/5xx/timeout)
- Separate output files: headers in {entity}.jsonl, lines in
{lineEntity}.jsonl
- Partial data is preserved if export fails mid-way
- Progress logged every 100 documents
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
403 lines
13 KiB
PowerShell
Executable File
403 lines
13 KiB
PowerShell
Executable File
#!/usr/bin/env pwsh
|
|
#
|
|
# Business Central Data Export via BC API v2.0
|
|
# Authenticates to Azure AD and extracts critical business data as JSON
|
|
#
|
|
|
|
param(
|
|
[Parameter(Mandatory=$true)]
|
|
[string]$OutputPath
|
|
)
|
|
|
|
# Get configuration from environment variables
|
|
$tenantId = $env:AZURE_TENANT_ID
|
|
$clientId = $env:AZURE_CLIENT_ID
|
|
$clientSecret = $env:AZURE_CLIENT_SECRET
|
|
$environmentName = $env:BC_ENVIRONMENT_NAME
|
|
$bcCompanyName = $env:BC_COMPANY_NAME # optional: filter to specific company
|
|
|
|
$baseUrl = "https://api.businesscentral.dynamics.com/v2.0/$tenantId/$environmentName/api/v2.0"
|
|
|
|
# Standalone entities to extract
|
|
$entities = @(
|
|
"accounts",
|
|
"customers",
|
|
"vendors",
|
|
"items",
|
|
"generalLedgerEntries",
|
|
"bankAccounts",
|
|
"employees",
|
|
"dimensions",
|
|
"dimensionValues",
|
|
"currencies",
|
|
"paymentTerms",
|
|
"paymentMethods",
|
|
"journals",
|
|
"countriesRegions"
|
|
)
|
|
|
|
# Document entities with line items
|
|
# Lines cannot be queried standalone at the top level.
|
|
# We fetch document headers first, then fetch lines per document.
|
|
$documentEntities = @{
|
|
"salesInvoices" = "salesInvoiceLines"
|
|
"salesOrders" = "salesOrderLines"
|
|
"salesCreditMemos" = "salesCreditMemoLines"
|
|
"purchaseInvoices" = "purchaseInvoiceLines"
|
|
"purchaseOrders" = "purchaseOrderLines"
|
|
}
|
|
|
|
# Token management
|
|
$script:currentToken = $null
|
|
$script:tokenExpiry = [datetime]::MinValue
|
|
|
|
function Write-Log {
|
|
param([string]$Message, [string]$Level = "INFO")
|
|
$timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ss"
|
|
Write-Host "[$timestamp] [$Level] $Message"
|
|
}
|
|
|
|
function Get-AzureADToken {
|
|
Write-Log "Authenticating to Azure AD..."
|
|
|
|
$tokenUrl = "https://login.microsoftonline.com/$tenantId/oauth2/v2.0/token"
|
|
|
|
$body = @{
|
|
client_id = $clientId
|
|
client_secret = $clientSecret
|
|
scope = "https://api.businesscentral.dynamics.com/.default"
|
|
grant_type = "client_credentials"
|
|
}
|
|
|
|
try {
|
|
$response = Invoke-RestMethod -Uri $tokenUrl -Method Post -Body $body -ContentType "application/x-www-form-urlencoded"
|
|
$script:currentToken = $response.access_token
|
|
# Refresh 5 minutes before actual expiry (tokens typically last 60-90 min)
|
|
$script:tokenExpiry = (Get-Date).AddSeconds($response.expires_in - 300)
|
|
Write-Log "Successfully authenticated (token valid for $($response.expires_in)s)"
|
|
return $script:currentToken
|
|
}
|
|
catch {
|
|
Write-Log "Failed to authenticate: $_" "ERROR"
|
|
throw
|
|
}
|
|
}
|
|
|
|
function Get-ValidToken {
|
|
if ($null -eq $script:currentToken -or (Get-Date) -ge $script:tokenExpiry) {
|
|
Write-Log "Token expired or missing, refreshing..."
|
|
Get-AzureADToken | Out-Null
|
|
}
|
|
return $script:currentToken
|
|
}
|
|
|
|
function Invoke-BCApi {
|
|
param(
|
|
[string]$Url,
|
|
[int]$TimeoutSec = 120,
|
|
[int]$MaxRetries = 3
|
|
)
|
|
|
|
for ($attempt = 1; $attempt -le $MaxRetries; $attempt++) {
|
|
$token = Get-ValidToken
|
|
$headers = @{
|
|
"Authorization" = "Bearer $token"
|
|
"Accept" = "application/json"
|
|
}
|
|
|
|
try {
|
|
$response = Invoke-RestMethod -Uri $Url -Method Get -Headers $headers -TimeoutSec $TimeoutSec
|
|
return $response
|
|
}
|
|
catch {
|
|
$statusCode = $null
|
|
if ($_.Exception.Response) {
|
|
$statusCode = [int]$_.Exception.Response.StatusCode
|
|
}
|
|
|
|
# Retry on 429 (throttled) or 5xx (server error) or timeout
|
|
$isRetryable = ($statusCode -eq 429) -or ($statusCode -ge 500) -or ($_ -match "Timeout")
|
|
if ($isRetryable -and $attempt -lt $MaxRetries) {
|
|
$wait = $attempt * 10
|
|
Write-Log " Request failed (attempt $attempt/$MaxRetries), retrying in ${wait}s..." "WARN"
|
|
Start-Sleep -Seconds $wait
|
|
continue
|
|
}
|
|
throw
|
|
}
|
|
}
|
|
}
|
|
|
|
function Get-BCData {
|
|
param(
|
|
[string]$Url
|
|
)
|
|
|
|
$allRecords = @()
|
|
$currentUrl = $Url
|
|
|
|
while ($currentUrl) {
|
|
$response = Invoke-BCApi -Url $currentUrl
|
|
|
|
if ($response.value) {
|
|
$allRecords += $response.value
|
|
}
|
|
|
|
$currentUrl = $response.'@odata.nextLink'
|
|
}
|
|
|
|
return $allRecords
|
|
}
|
|
|
|
function Get-Companies {
|
|
Write-Log "Fetching companies..."
|
|
$companiesUrl = "$baseUrl/companies"
|
|
$companies = Get-BCData -Url $companiesUrl
|
|
Write-Log "Found $($companies.Count) company/companies"
|
|
return $companies
|
|
}
|
|
|
|
function Export-EntityData {
|
|
param(
|
|
[string]$CompanyId,
|
|
[string]$CompanyName,
|
|
[string]$EntityName,
|
|
[string]$OutputDir
|
|
)
|
|
|
|
$entityUrl = "$baseUrl/companies($CompanyId)/$EntityName"
|
|
|
|
Write-Log " Exporting $EntityName..."
|
|
|
|
try {
|
|
$data = Get-BCData -Url $entityUrl
|
|
$count = 0
|
|
if ($data) { $count = $data.Count }
|
|
|
|
$outputFile = Join-Path $OutputDir "$EntityName.json"
|
|
$data | ConvertTo-Json -Depth 10 | Out-File -FilePath $outputFile -Encoding utf8
|
|
|
|
Write-Log " $EntityName : $count records"
|
|
return $count
|
|
}
|
|
catch {
|
|
Write-Log " Failed to export ${EntityName}: $_" "WARN"
|
|
$outputFile = Join-Path $OutputDir "$EntityName.json"
|
|
"[]" | Out-File -FilePath $outputFile -Encoding utf8
|
|
return 0
|
|
}
|
|
}
|
|
|
|
function Export-DocumentWithLines {
|
|
param(
|
|
[string]$CompanyId,
|
|
[string]$CompanyName,
|
|
[string]$DocumentEntity,
|
|
[string]$LineEntity,
|
|
[string]$OutputDir
|
|
)
|
|
|
|
Write-Log " Exporting $DocumentEntity (headers)..."
|
|
|
|
$docFile = Join-Path $OutputDir "$DocumentEntity.jsonl"
|
|
$lineFile = Join-Path $OutputDir "$LineEntity.jsonl"
|
|
[System.IO.File]::WriteAllText($docFile, "")
|
|
[System.IO.File]::WriteAllText($lineFile, "")
|
|
|
|
$docCount = 0
|
|
$lineCount = 0
|
|
|
|
try {
|
|
# Step 1: Fetch document headers page by page (no $expand)
|
|
# BC API default page size is ~100, with @odata.nextLink for more
|
|
$currentUrl = "$baseUrl/companies($CompanyId)/$DocumentEntity"
|
|
|
|
while ($currentUrl) {
|
|
$response = Invoke-BCApi -Url $currentUrl
|
|
|
|
if (-not $response.value -or $response.value.Count -eq 0) {
|
|
break
|
|
}
|
|
|
|
# Step 2: For each document in this page, fetch its lines
|
|
foreach ($doc in $response.value) {
|
|
$docCount++
|
|
$docId = $doc.id
|
|
|
|
# Write document header to disk
|
|
$jsonLine = $doc | ConvertTo-Json -Depth 10 -Compress
|
|
[System.IO.File]::AppendAllText($docFile, $jsonLine + "`n")
|
|
|
|
# Fetch lines for this document
|
|
$linesUrl = "$baseUrl/companies($CompanyId)/$DocumentEntity($docId)/$LineEntity"
|
|
try {
|
|
$linesResponse = Invoke-BCApi -Url $linesUrl -TimeoutSec 60
|
|
if ($linesResponse.value -and $linesResponse.value.Count -gt 0) {
|
|
foreach ($line in $linesResponse.value) {
|
|
$lineCount++
|
|
$lineJson = $line | ConvertTo-Json -Depth 10 -Compress
|
|
[System.IO.File]::AppendAllText($lineFile, $lineJson + "`n")
|
|
}
|
|
|
|
# Handle pagination within lines (unlikely but possible)
|
|
$nextLinesUrl = $linesResponse.'@odata.nextLink'
|
|
while ($nextLinesUrl) {
|
|
$moreLinesResponse = Invoke-BCApi -Url $nextLinesUrl -TimeoutSec 60
|
|
if ($moreLinesResponse.value) {
|
|
foreach ($line in $moreLinesResponse.value) {
|
|
$lineCount++
|
|
$lineJson = $line | ConvertTo-Json -Depth 10 -Compress
|
|
[System.IO.File]::AppendAllText($lineFile, $lineJson + "`n")
|
|
}
|
|
}
|
|
$nextLinesUrl = $moreLinesResponse.'@odata.nextLink'
|
|
}
|
|
}
|
|
}
|
|
catch {
|
|
Write-Log " Warning: failed to fetch lines for $DocumentEntity $docId : $_" "WARN"
|
|
}
|
|
|
|
# Progress every 100 documents
|
|
if ($docCount % 100 -eq 0) {
|
|
Write-Log " Progress: $docCount documents, $lineCount lines"
|
|
}
|
|
}
|
|
|
|
# Next page of documents
|
|
$currentUrl = $response.'@odata.nextLink'
|
|
}
|
|
|
|
Write-Log " $DocumentEntity : $docCount documents, $lineCount lines (complete)"
|
|
return ($docCount + $lineCount)
|
|
}
|
|
catch {
|
|
Write-Log " Failed to export ${DocumentEntity} at doc #$docCount : $_" "WARN"
|
|
Write-Log " Partial data saved ($docCount docs, $lineCount lines)" "WARN"
|
|
return ($docCount + $lineCount)
|
|
}
|
|
}
|
|
|
|
# Main execution
|
|
try {
|
|
Write-Log "========================================="
|
|
Write-Log "BC Data Export Script (API v2.0)"
|
|
Write-Log "========================================="
|
|
Write-Log "Environment: $environmentName"
|
|
Write-Log "Output Path: $OutputPath"
|
|
Write-Log "Entities to extract: $($entities.Count + $documentEntities.Count) ($($documentEntities.Count) with line items)"
|
|
|
|
# Create output directory
|
|
$exportDir = $OutputPath
|
|
if (-not (Test-Path $exportDir)) {
|
|
New-Item -ItemType Directory -Path $exportDir -Force | Out-Null
|
|
}
|
|
|
|
# Step 1: Get Azure AD token
|
|
Get-AzureADToken | Out-Null
|
|
|
|
# Step 2: Get companies
|
|
$companies = Get-Companies
|
|
|
|
if ($companies.Count -eq 0) {
|
|
Write-Log "No companies found in environment $environmentName" "ERROR"
|
|
exit 1
|
|
}
|
|
|
|
# Save companies list
|
|
$companies | ConvertTo-Json -Depth 10 | Out-File -FilePath (Join-Path $exportDir "companies.json") -Encoding utf8
|
|
|
|
# Filter to specific company if configured
|
|
$targetCompanies = $companies
|
|
if ($bcCompanyName) {
|
|
$targetCompanies = $companies | Where-Object { $_.name -eq $bcCompanyName -or $_.displayName -eq $bcCompanyName }
|
|
if ($targetCompanies.Count -eq 0) {
|
|
Write-Log "Company '$bcCompanyName' not found. Available: $($companies.name -join ', ')" "ERROR"
|
|
exit 1
|
|
}
|
|
Write-Log "Filtering to company: $bcCompanyName"
|
|
}
|
|
|
|
$totalRecords = 0
|
|
$totalEntities = 0
|
|
$failedEntities = @()
|
|
|
|
# Step 3: Export data for each company
|
|
foreach ($company in $targetCompanies) {
|
|
$companyName = $company.name
|
|
$companyId = $company.id
|
|
|
|
Write-Log "-----------------------------------------"
|
|
Write-Log "Exporting company: $companyName ($companyId)"
|
|
|
|
# Create company directory (sanitize name for filesystem)
|
|
$safeName = $companyName -replace '[\\/:*?"<>|]', '_'
|
|
$companyDir = Join-Path $exportDir $safeName
|
|
if (-not (Test-Path $companyDir)) {
|
|
New-Item -ItemType Directory -Path $companyDir -Force | Out-Null
|
|
}
|
|
|
|
# Export standalone entities
|
|
foreach ($entity in $entities) {
|
|
$count = Export-EntityData `
|
|
-CompanyId $companyId `
|
|
-CompanyName $companyName `
|
|
-EntityName $entity `
|
|
-OutputDir $companyDir
|
|
|
|
$totalRecords += $count
|
|
$totalEntities++
|
|
|
|
if ($count -eq 0) {
|
|
$failedEntities += "$companyName/$entity"
|
|
}
|
|
}
|
|
|
|
# Export document entities with their line items
|
|
foreach ($docEntity in $documentEntities.Keys) {
|
|
$lineEntity = $documentEntities[$docEntity]
|
|
|
|
$count = Export-DocumentWithLines `
|
|
-CompanyId $companyId `
|
|
-CompanyName $companyName `
|
|
-DocumentEntity $docEntity `
|
|
-LineEntity $lineEntity `
|
|
-OutputDir $companyDir
|
|
|
|
$totalRecords += $count
|
|
$totalEntities++
|
|
|
|
if ($count -eq 0) {
|
|
$failedEntities += "$companyName/$docEntity"
|
|
}
|
|
}
|
|
}
|
|
|
|
# Save export metadata
|
|
$metadata = @{
|
|
exportDate = (Get-Date -Format "yyyy-MM-dd HH:mm:ss UTC" -AsUTC)
|
|
environment = $environmentName
|
|
companies = @($targetCompanies | ForEach-Object { $_.name })
|
|
entitiesExported = $totalEntities
|
|
totalRecords = $totalRecords
|
|
failedEntities = $failedEntities
|
|
}
|
|
$metadata | ConvertTo-Json -Depth 5 | Out-File -FilePath (Join-Path $exportDir "export-metadata.json") -Encoding utf8
|
|
|
|
Write-Log "========================================="
|
|
Write-Log "Export completed"
|
|
Write-Log "Companies: $($targetCompanies.Count)"
|
|
Write-Log "Entities: $totalEntities"
|
|
Write-Log "Total records: $totalRecords"
|
|
if ($failedEntities.Count -gt 0) {
|
|
Write-Log "Failed/empty: $($failedEntities.Count) entities" "WARN"
|
|
}
|
|
Write-Log "========================================="
|
|
exit 0
|
|
}
|
|
catch {
|
|
Write-Log "Unexpected error: $_" "ERROR"
|
|
Write-Log "Stack trace: $($_.ScriptStackTrace)" "ERROR"
|
|
exit 1
|
|
}
|