testament/azure: major rewrite (#13246)
This commit features a major rewrite of Azure Pipelines integration, turning the spaghetti it originally was into something maintainable. Key changes: - No longer requires a ton of hooks into testament. - Results are now cached then bulk-uploaded to prevent throttling from Azure Pipelines, avoiding costly timeouts. - A low timeout is also employed to avoid inflated test time. - The integration is now documented.
This commit is contained in:
alaviss
Andreas Rumpf
parent
981ffc912e
commit
5124b2e575
2 changed files with 124 additions and 76 deletions
|
|
@ -6,90 +6,142 @@
|
|||
# Look at license.txt for more info.
|
||||
# All rights reserved.
|
||||
|
||||
import base64, json, httpclient, os, strutils
|
||||
import base64, json, httpclient, os, strutils, uri
|
||||
import specs
|
||||
|
||||
const
|
||||
ApiRuns = "/_apis/test/runs"
|
||||
ApiVersion = "?api-version=5.0"
|
||||
ApiResults = ApiRuns & "/$1/results"
|
||||
|
||||
var runId* = -1
|
||||
RunIdEnv = "TESTAMENT_AZURE_RUN_ID"
|
||||
CacheSize = 8 # How many results should be cached before uploading to
|
||||
# Azure Pipelines. This prevents throttling that might arise.
|
||||
|
||||
proc getAzureEnv(env: string): string =
|
||||
# Conversion rule at:
|
||||
# https://docs.microsoft.com/en-us/azure/devops/pipelines/process/variables#set-variables-in-pipeline
|
||||
env.toUpperAscii().replace('.', '_').getEnv
|
||||
|
||||
proc invokeRest(httpMethod: HttpMethod; api: string; body = ""): Response =
|
||||
let http = newHttpClient()
|
||||
defer: close http
|
||||
result = http.request(getAzureEnv("System.TeamFoundationCollectionUri") &
|
||||
getAzureEnv("System.TeamProjectId") & api & ApiVersion,
|
||||
httpMethod,
|
||||
$body,
|
||||
newHttpHeaders {
|
||||
"Accept": "application/json",
|
||||
"Authorization": "Basic " & encode(':' & getAzureEnv("System.AccessToken")),
|
||||
"Content-Type": "application/json"
|
||||
})
|
||||
if not result.code.is2xx:
|
||||
raise newException(HttpRequestError, "Server returned: " & result.body)
|
||||
template getRun(): string =
|
||||
## Get the test run attached to this instance
|
||||
getEnv(RunIdEnv)
|
||||
|
||||
proc finish*() {.noconv.} =
|
||||
if not isAzure or runId < 0:
|
||||
return
|
||||
template setRun(id: string) =
|
||||
## Attach a test run to this instance and its future children
|
||||
putEnv(RunIdEnv, id)
|
||||
|
||||
try:
|
||||
discard invokeRest(HttpPatch,
|
||||
ApiRuns & "/" & $runId,
|
||||
$ %* { "state": "Completed" })
|
||||
except:
|
||||
stderr.writeLine "##vso[task.logissue type=warning;]Unable to finalize Azure backend"
|
||||
stderr.writeLine getCurrentExceptionMsg()
|
||||
template delRun() =
|
||||
## Unattach the test run associtated with this instance and its future children
|
||||
delEnv(RunIdEnv)
|
||||
|
||||
runId = -1
|
||||
template warning(args: varargs[untyped]) =
|
||||
## Add a warning to the current task
|
||||
stderr.writeLine "##vso[task.logissue type=warning;]", args
|
||||
|
||||
# TODO: Only obtain a run id if tests are run
|
||||
# NOTE: We can't delete test runs with Azure's access token
|
||||
proc start*() =
|
||||
if not isAzure:
|
||||
return
|
||||
try:
|
||||
if runId < 0:
|
||||
runId = invokeRest(HttpPost,
|
||||
ApiRuns,
|
||||
$ %* {
|
||||
"automated": true,
|
||||
"build": { "id": getAzureEnv("Build.BuildId") },
|
||||
"buildPlatform": hostCPU,
|
||||
"controller": "nim-testament",
|
||||
"name": getAzureEnv("Agent.JobName")
|
||||
}).body.parseJson["id"].getInt(-1)
|
||||
except:
|
||||
stderr.writeLine "##vso[task.logissue type=warning;]Unable to initialize Azure backend"
|
||||
stderr.writeLine getCurrentExceptionMsg()
|
||||
let
|
||||
ownRun = not existsEnv RunIdEnv
|
||||
## Whether the test run is owned by this instance
|
||||
accessToken = getAzureEnv("System.AccessToken")
|
||||
## Access token to Azure Pipelines
|
||||
|
||||
var
|
||||
active = false ## Whether the backend should be activated
|
||||
requestBase: Uri ## Base URI for all API requests
|
||||
requestHeaders: HttpHeaders ## Headers required for all API requests
|
||||
results: JsonNode ## A cache for test results before uploading
|
||||
|
||||
proc request(api: string, httpMethod: HttpMethod, body = ""): Response {.inline.} =
|
||||
let client = newHttpClient(timeout = 3000)
|
||||
defer: close client
|
||||
result = client.request($(requestBase / api), httpMethod, body, requestHeaders)
|
||||
if result.code != Http200:
|
||||
raise newException(CatchableError, "Request failed")
|
||||
|
||||
proc init*() =
|
||||
## Initialize the Azure Pipelines backend.
|
||||
##
|
||||
## If an access token is provided and no test run is associated with the
|
||||
## current instance, this proc will create a test run named after the current
|
||||
## Azure Pipelines' job name, then associate it to the current testament
|
||||
## instance and its future children. Should this fail, the backend will be
|
||||
## disabled.
|
||||
if isAzure and accessToken.len > 0:
|
||||
active = true
|
||||
requestBase = parseUri(getAzureEnv("System.TeamFoundationCollectionUri")) /
|
||||
getAzureEnv("System.TeamProjectId") / "_apis" ? {"api-version": "5.0"}
|
||||
requestHeaders = newHttpHeaders {
|
||||
"Accept": "application/json",
|
||||
"Authorization": "Basic " & encode(':' & accessToken),
|
||||
"Content-Type": "application/json"
|
||||
}
|
||||
results = newJArray()
|
||||
if ownRun:
|
||||
try:
|
||||
let resp = request(
|
||||
"test/runs",
|
||||
HttpPost,
|
||||
$ %* {
|
||||
"automated": true,
|
||||
"build": { "id": getAzureEnv("Build.BuildId") },
|
||||
"buildPlatform": hostCPU,
|
||||
"controller": "nim-testament",
|
||||
"name": getAzureEnv("Agent.JobName")
|
||||
}
|
||||
)
|
||||
setRun $resp.body.parseJson["id"].getInt
|
||||
except:
|
||||
warning "Couldn't create test run for Azure Pipelines integration"
|
||||
# Set run id to empty to prevent child processes from trying to request
|
||||
# for yet another test run id, which wouldn't be shared with other
|
||||
# instances.
|
||||
setRun ""
|
||||
active = false
|
||||
elif getRun().len == 0:
|
||||
# Disable integration if there aren't any valid test run id
|
||||
active = false
|
||||
|
||||
proc uploadAndClear() =
|
||||
## Upload test results from cache to Azure Pipelines. Then clear the cache
|
||||
## after.
|
||||
if results.len > 0:
|
||||
try:
|
||||
discard request("test/runs/" & getRun() & "/results", HttpPost, $results)
|
||||
except:
|
||||
for i in results:
|
||||
warning "Couldn't log test result to Azure Pipelines: ",
|
||||
i["automatedTestName"], ", outcome: ", i["outcome"]
|
||||
results = newJArray()
|
||||
|
||||
proc finalize*() {.noconv.} =
|
||||
## Finalize the Azure Pipelines backend.
|
||||
##
|
||||
## If a test run has been associated and is owned by this instance, it will
|
||||
## be marked as complete.
|
||||
if active:
|
||||
if ownRun:
|
||||
uploadAndClear()
|
||||
try:
|
||||
discard request("test/runs/" & getRun(), HttpPatch,
|
||||
$ %* {"state": "Completed"})
|
||||
except:
|
||||
warning "Couldn't update test run ", getRun(), " on Azure Pipelines"
|
||||
delRun()
|
||||
|
||||
proc addTestResult*(name, category: string; durationInMs: int; errorMsg: string;
|
||||
outcome: TResultEnum) =
|
||||
if not isAzure or runId < 0:
|
||||
if not active:
|
||||
return
|
||||
|
||||
let outcome = case outcome
|
||||
of reSuccess: "Passed"
|
||||
of reDisabled, reJoined: "NotExecuted"
|
||||
else: "Failed"
|
||||
try:
|
||||
discard invokeRest(HttpPost,
|
||||
ApiResults % [$runId],
|
||||
$ %* [{
|
||||
"automatedTestName": name,
|
||||
"automatedTestStorage": category,
|
||||
"durationInMs": durationInMs,
|
||||
"errorMessage": errorMsg,
|
||||
"outcome": outcome,
|
||||
"testCaseTitle": name
|
||||
}])
|
||||
except:
|
||||
stderr.writeLine "##vso[task.logissue type=warning;]Unable to log test case: ",
|
||||
name, ", outcome: ", outcome
|
||||
stderr.writeLine getCurrentExceptionMsg()
|
||||
|
||||
results.add(%* {
|
||||
"automatedTestName": name,
|
||||
"automatedTestStorage": category,
|
||||
"durationInMs": durationInMs,
|
||||
"errorMessage": errorMsg,
|
||||
"outcome": outcome,
|
||||
"testCaseTitle": name
|
||||
})
|
||||
|
||||
if results.len > CacheSize:
|
||||
uploadAndClear()
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue