testament/azure: major rewrite (#13246)

This commit features a major rewrite of Azure Pipelines integration,
turning the spaghetti it originally was into something maintainable.

Key changes:
- No longer requires a ton of hooks into testament.
- Results are now cached then bulk-uploaded to prevent throttling from
  Azure Pipelines, avoiding costly timeouts.
- A low timeout is also employed to avoid inflated test time.
- The integration is now documented.

testament/azure.nim

@@ -6,90 +6,142 @@
 #    Look at license.txt for more info.
 #    All rights reserved.
 
-import base64, json, httpclient, os, strutils
+import base64, json, httpclient, os, strutils, uri
 import specs
 
 const
-  ApiRuns = "/_apis/test/runs"
+  RunIdEnv = "TESTAMENT_AZURE_RUN_ID"
-  ApiVersion = "?api-version=5.0"
+  CacheSize = 8 # How many results should be cached before uploading to
-  ApiResults = ApiRuns & "/$1/results"
+                # Azure Pipelines. This prevents throttling that might arise.
-
-var runId* = -1
 
 proc getAzureEnv(env: string): string =
   # Conversion rule at:
   # https://docs.microsoft.com/en-us/azure/devops/pipelines/process/variables#set-variables-in-pipeline
   env.toUpperAscii().replace('.', '_').getEnv
 
-proc invokeRest(httpMethod: HttpMethod; api: string; body = ""): Response =
+template getRun(): string =
-  let http = newHttpClient()
+  ## Get the test run attached to this instance
-  defer: close http
+  getEnv(RunIdEnv)
-  result = http.request(getAzureEnv("System.TeamFoundationCollectionUri") &
-                        getAzureEnv("System.TeamProjectId") & api & ApiVersion,
-                        httpMethod,
-                        $body,
-                        newHttpHeaders {
-                          "Accept": "application/json",
-                          "Authorization": "Basic " & encode(':' & getAzureEnv("System.AccessToken")),
-                          "Content-Type": "application/json"
-                        })
-  if not result.code.is2xx:
-    raise newException(HttpRequestError, "Server returned: " & result.body)
 
-proc finish*() {.noconv.} =
+template setRun(id: string) =
-  if not isAzure or runId < 0:
+  ## Attach a test run to this instance and its future children
-    return
+  putEnv(RunIdEnv, id)
 
-  try:
+template delRun() =
-    discard invokeRest(HttpPatch,
+  ## Unattach the test run associtated with this instance and its future children
-                       ApiRuns & "/" & $runId,
+  delEnv(RunIdEnv)
-                       $ %* { "state": "Completed" })
-  except:
-    stderr.writeLine "##vso[task.logissue type=warning;]Unable to finalize Azure backend"
-    stderr.writeLine getCurrentExceptionMsg()
 
-  runId = -1
+template warning(args: varargs[untyped]) =
+  ## Add a warning to the current task
+  stderr.writeLine "##vso[task.logissue type=warning;]", args
 
-# TODO: Only obtain a run id if tests are run
+let
-# NOTE: We can't delete test runs with Azure's access token
+  ownRun = not existsEnv RunIdEnv
-proc start*() =
+    ## Whether the test run is owned by this instance
-  if not isAzure:
+  accessToken = getAzureEnv("System.AccessToken")
-    return
+    ## Access token to Azure Pipelines
-  try:
+
-    if runId < 0:
+var
-      runId = invokeRest(HttpPost,
+  active = false ## Whether the backend should be activated
-                         ApiRuns,
+  requestBase: Uri ## Base URI for all API requests
-                         $ %* {
+  requestHeaders: HttpHeaders ## Headers required for all API requests
-                           "automated": true,
+  results: JsonNode ## A cache for test results before uploading
-                           "build": { "id": getAzureEnv("Build.BuildId") },
+
-                           "buildPlatform": hostCPU,
+proc request(api: string, httpMethod: HttpMethod, body = ""): Response {.inline.} =
-                           "controller": "nim-testament",
+  let client = newHttpClient(timeout = 3000)
-                           "name": getAzureEnv("Agent.JobName")
+  defer: close client
-                         }).body.parseJson["id"].getInt(-1)
+  result = client.request($(requestBase / api), httpMethod, body, requestHeaders)
-  except:
+  if result.code != Http200:
-    stderr.writeLine "##vso[task.logissue type=warning;]Unable to initialize Azure backend"
+    raise newException(CatchableError, "Request failed")
-    stderr.writeLine getCurrentExceptionMsg()
+
+proc init*() =
+  ## Initialize the Azure Pipelines backend.
+  ##
+  ## If an access token is provided and no test run is associated with the
+  ## current instance, this proc will create a test run named after the current
+  ## Azure Pipelines' job name, then associate it to the current testament
+  ## instance and its future children. Should this fail, the backend will be
+  ## disabled.
+  if isAzure and accessToken.len > 0:
+    active = true
+    requestBase = parseUri(getAzureEnv("System.TeamFoundationCollectionUri")) /
+      getAzureEnv("System.TeamProjectId") / "_apis" ? {"api-version": "5.0"}
+    requestHeaders = newHttpHeaders {
+      "Accept": "application/json",
+      "Authorization": "Basic " & encode(':' & accessToken),
+      "Content-Type": "application/json"
+    }
+    results = newJArray()
+    if ownRun:
+      try:
+        let resp = request(
+          "test/runs",
+          HttpPost,
+          $ %* {
+            "automated": true,
+            "build": { "id": getAzureEnv("Build.BuildId") },
+            "buildPlatform": hostCPU,
+            "controller": "nim-testament",
+            "name": getAzureEnv("Agent.JobName")
+          }
+        )
+        setRun $resp.body.parseJson["id"].getInt
+      except:
+        warning "Couldn't create test run for Azure Pipelines integration"
+        # Set run id to empty to prevent child processes from trying to request
+        # for yet another test run id, which wouldn't be shared with other
+        # instances.
+        setRun ""
+        active = false
+    elif getRun().len == 0:
+      # Disable integration if there aren't any valid test run id
+      active = false
+
+proc uploadAndClear() =
+  ## Upload test results from cache to Azure Pipelines. Then clear the cache
+  ## after.
+  if results.len > 0:
+    try:
+      discard request("test/runs/" & getRun() & "/results", HttpPost, $results)
+    except:
+      for i in results:
+        warning "Couldn't log test result to Azure Pipelines: ",
+          i["automatedTestName"], ", outcome: ", i["outcome"]
+    results = newJArray()
+
+proc finalize*() {.noconv.} =
+  ## Finalize the Azure Pipelines backend.
+  ##
+  ## If a test run has been associated and is owned by this instance, it will
+  ## be marked as complete.
+  if active:
+    if ownRun:
+      uploadAndClear()
+      try:
+        discard request("test/runs/" & getRun(), HttpPatch,
+                        $ %* {"state": "Completed"})
+      except:
+        warning "Couldn't update test run ", getRun(), " on Azure Pipelines"
+      delRun()
 
 proc addTestResult*(name, category: string; durationInMs: int; errorMsg: string;
                     outcome: TResultEnum) =
-  if not isAzure or runId < 0:
+  if not active:
     return
+
   let outcome = case outcome
                 of reSuccess: "Passed"
                 of reDisabled, reJoined: "NotExecuted"
                 else: "Failed"
-  try:
+
-    discard invokeRest(HttpPost,
+  results.add(%* {
-                       ApiResults % [$runId],
+      "automatedTestName": name,
-                       $ %* [{
+      "automatedTestStorage": category,
-                         "automatedTestName": name,
+      "durationInMs": durationInMs,
-                         "automatedTestStorage": category,
+      "errorMessage": errorMsg,
-                         "durationInMs": durationInMs,
+      "outcome": outcome,
-                         "errorMessage": errorMsg,
+      "testCaseTitle": name
-                         "outcome": outcome,
+  })
-                         "testCaseTitle": name
+
-                       }])
+  if results.len > CacheSize:
-  except:
+    uploadAndClear()
-    stderr.writeLine "##vso[task.logissue type=warning;]Unable to log test case: ",
-                     name, ", outcome: ", outcome
-    stderr.writeLine getCurrentExceptionMsg()

testament/testament.nim

@@ -47,6 +47,9 @@ Options:
   --backendLogging:on|off   Disable or enable backend logging. By default turned on.
   --megatest:on|off         Enable or disable megatest. Default is on.
   --skipFrom:file           Read tests to skip from `file` - one test per line, # comments ignored
+
+On Azure Pipelines, testament will also publish test results via Azure Pipelines' Test Management API
+provided that System.AccessToken is made available via the environment variable SYSTEM_ACCESSTOKEN.
 """ % resultsFile
 
 type
@@ -606,6 +609,7 @@ proc main() =
   os.putEnv "NIMTEST_COLOR", "never"
   os.putEnv "NIMTEST_OUTPUT_LVL", "PRINT_FAILURES"
 
+  azure.init()
   backend.open()
   var optPrintResults = false
   var optFailing = false
@@ -656,8 +660,6 @@ proc main() =
         quit Usage
     of "skipfrom":
       skipFrom = p.val.string
-    of "azurerunid":
-      runId = p.val.parseInt
     else:
       quit Usage
     p.next()
@@ -670,7 +672,6 @@ proc main() =
   of "all":
     #processCategory(r, Category"megatest", p.cmdLineRest.string, testsDir, runJoinableTests = false)
 
-    azure.start()
     var myself = quoteShell(findExe("testament" / "testament"))
     if targetsStr.len > 0:
       myself &= " " & quoteShell("--targets:" & targetsStr)
@@ -679,8 +680,6 @@ proc main() =
 
     if skipFrom.len > 0:
       myself &= " " & quoteShell("--skipFrom:" & skipFrom)
-    if isAzure:
-      myself &= " " & quoteShell("--azureRunId:" & $runId)
 
     var cats: seq[string]
     let rest = if p.cmdLineRest.string.len > 0: " " & p.cmdLineRest.string else: ""
@@ -706,16 +705,14 @@ proc main() =
         progressStatus(i)
         processCategory(r, Category(cati), p.cmdLineRest.string, testsDir, runJoinableTests = false)
     else:
-      addQuitProc azure.finish
+      addQuitProc azure.finalize
       quit osproc.execProcesses(cmds, {poEchoCmd, poStdErrToStdOut, poUsePath, poParentStreams}, beforeRunEvent = progressStatus)
   of "c", "cat", "category":
-    azure.start()
     skips = loadSkipFrom(skipFrom)
     var cat = Category(p.key)
     p.next
     processCategory(r, cat, p.cmdLineRest.string, testsDir, runJoinableTests = true)
   of "pcat":
-    azure.start()
     skips = loadSkipFrom(skipFrom)
     # 'pcat' is used for running a category in parallel. Currently the only
     # difference is that we don't want to run joinable tests here as they
@@ -730,7 +727,6 @@ proc main() =
     p.next
     processPattern(r, pattern, p.cmdLineRest.string, simulate)
   of "r", "run":
-    azure.start()
     # at least one directory is required in the path, to use as a category name
     let pathParts = split(p.key.string, {DirSep, AltSep})
     # "stdlib/nre/captures.nim" -> "stdlib" + "nre/captures.nim"
@@ -745,8 +741,8 @@ proc main() =
   if optPrintResults:
     if action == "html": openDefaultBrowser(resultsFile)
     else: echo r, r.data
+  azure.finalize()
   backend.close()
-  if isMainProcess: azure.finish()
   var failed = r.total - r.passed - r.skipped
   if failed != 0:
     echo "FAILURE! total: ", r.total, " passed: ", r.passed, " skipped: ",