Run the library testing on a pull request (#307)

.CI/Jenkinsfile

@@ -37,6 +37,12 @@ pipeline {
     booleanParam(name: 'generateSymbolicJacobian', defaultValue: false, description: 'master branch, with --generateSymbolicJacobian (ryzen-5950x-1). This is an experimental job that does not run on a fixed schedule.')
     booleanParam(name: 'wasm_jit', defaultValue: false, description: 'master branch, with --simCodeTarget=wasm-jit (ryzen-5950x-2). This is an experimental job that does not run on a fixed schedule.')
     booleanParam(name: 'heavy_tests', defaultValue: false, description: 'master branch, runs one test at a time. That is, no parallel launching of tests. omc will use multiple threads for each test (-n=1 is not set unlike the other regression tests.), (ryzen-5950x-1). This is an experimental job that does not run on a fixed schedule.')
+
+    string(name: 'pull_request', defaultValue: '', description: 'Test an OpenModelica pull request rather than a branch: its number, e.g. 16354. omc is built from refs/pull/<N>/merge - the pull request as it would land - the results fill a pr-<N> table, and the report compares them against the newest run of pull_request_baseline. Left empty, nothing of this runs.')
+    string(name: 'pull_request_baseline', defaultValue: 'master', description: 'The branch a pull request is compared against. Its newest run has to come from pull_request_node, or the report compares the machines as much as the pull request.')
+    string(name: 'pull_request_config', defaultValue: 'configs/conf.json', description: 'What a pull request run tests. A full run takes days, so a smaller configuration file is often the better question to ask.')
+    choice(name: 'pull_request_node', choices: ['ryzen-5950x-1', 'ryzen-5950x-2-1', 'ryzen-9950x'], description: 'The machine a pull request runs on. The default is the one that produces the master runs it is compared against.')
+    booleanParam(name: 'drop_stale_pull_request_tables', defaultValue: false, description: 'Drop the pr-<N> tables of pull requests that have been merged or closed, and of those tested more than 60 days ago. The reports published for them are not touched.')
   }
   environment {
     LC_ALL = 'C.UTF-8'
@@ -115,6 +121,31 @@ pipeline {
         }
       }
 
+      stage('pull request') {
+        agent {
+          node {
+            label "${params.pull_request_node}"
+            customWorkspace 'ws/OpenModelicaLibraryTestingWork'
+          }
+        }
+        options { skipDefaultCheckout() }
+        when {
+          beforeAgent true
+          expression { params.pull_request.trim() }
+        }
+        steps {
+          script {
+            if (!(params.pull_request.trim() ==~ /[0-9]+/)) {
+              error "pull_request is a pull request number; got '${params.pull_request}'"
+            }
+          }
+          // One build of omc per pull request is kept, as for a branch, and they
+          // accumulate: a pull request is tested once and never again.
+          sh 'find "$HOME/saved_omc" -maxdepth 1 -name "pr-*" -type d -mtime +14 -exec rm -rf {} ";" || true'
+          runRegressiontest("pr-${params.pull_request.trim()}", "pr-${params.pull_request.trim()}", '', '', false, '', '', false, false, 0, params.pull_request_config)
+        }
+      }
+
       stage('newInst-newBackend') {
         agent {
           node {
@@ -549,6 +580,61 @@ pipeline {
         sshPublisher(publishers: [sshPublisherDesc(configName: 'LibraryTestingReports', transfers: [sshTransfer(sourceFiles: 'overview*.html,history/**')])])
       }
     }
+
+    stage('drop stale pull request tables') {
+      agent {
+        dockerfile {
+          label 'linux'
+          dir '.CI/build-dep'
+          customWorkspace 'ws/OpenModelicaLibraryTestingReport'
+        }
+      }
+      when {
+        beforeAgent true
+        expression { params.drop_stale_pull_request_tables }
+      }
+      environment {
+        PGPASSFILE = credentials('omdb-pgpass')
+      }
+      steps {
+        sh './drop-pr-tables.py --yes'
+      }
+    }
+
+    /* A pull request is not reported on by the stage above: that one compares a
+     * branch against its own previous run, which a pull request has none of, and
+     * regenerating every overview for a job that tested one thing would take
+     * longer than the report itself. */
+    stage('pull request report') {
+      agent {
+        dockerfile {
+          label 'linux'
+          dir '.CI/build-dep'
+          customWorkspace 'ws/OpenModelicaLibraryTestingReport'
+        }
+      }
+      when {
+        beforeAgent true
+        expression { params.pull_request.trim() }
+      }
+      environment {
+        PYTHONIOENCODING = 'utf-8'
+        PGPASSFILE = credentials('omdb-pgpass')
+      }
+      steps {
+        script {
+          if (!(params.pull_request.trim() ==~ /[0-9]+/)) {
+            error "pull_request is a pull request number; got '${params.pull_request}'"
+          }
+        }
+        sh 'rm -rf history'
+        sh "./pr-report.py '${params.pull_request.trim()}' --baseline='${params.pull_request_baseline.trim()}'"
+        // The summary to comment on the pull request with, in the build log
+        // until there is a token to post it with.
+        sh 'cat history/pr-*/00_comment.md'
+        sshPublisher(publishers: [sshPublisherDesc(configName: 'LibraryTestingReports', transfers: [sshTransfer(sourceFiles: 'history/**')])])
+      }
+    }
   }
 }
 def omsimulatorHash() {
@@ -808,6 +894,23 @@ def runRegressiontest(branch, name, extraFlags, omsHash, omcompiler, extrasimfla
     }
   }
 
+  // A pull request is fetched from GitHub itself: refs/pull/* is not on the
+  // read-only mirror the rest of the job clones. The merge ref rather than the
+  // head one, since the question is what happens once it is merged, and it is
+  // checked out detached so that nothing is left behind for the next run of the
+  // same workspace to trip over.
+  def pullRequest = branch.startsWith('pr-') && branch.substring(3).isInteger() ? branch.substring(3) : ''
+  def checkoutRef = pullRequest ? """
+    if ! git fetch --force https://github.com/OpenModelica/OpenModelica.git refs/pull/${pullRequest}/merge; then
+      echo "Could not fetch refs/pull/${pullRequest}/merge: either there is no such pull request, or GitHub cannot merge it into its base branch."
+      exit 1
+    fi
+    git checkout -f --detach FETCH_HEAD || exit 1
+    git fetch --tags --force || exit 1
+""" : """
+    git reset --hard && git checkout -f "${branch}" && (git rev-parse --verify "tags/${branch}"  || (git reset --hard "origin/${branch}" && git pull)) && git fetch --tags --force || exit 1
+"""
+
   OMCPATH = "${omcompiler ? '../' : './'}OMCompiler"
 
   // The build runs in OMCompiler, one level below the cmake source tree.
@@ -883,7 +986,7 @@ def runRegressiontest(branch, name, extraFlags, omsHash, omcompiler, extrasimfla
   cd ${OMCPATH}
 
   if ! test -f ~/saved_omc/${name}/.nogit; then
-    git reset --hard && git checkout -f "${branch}" && (git rev-parse --verify "tags/${branch}"  || (git reset --hard "origin/${branch}" && git pull)) && git fetch --tags --force || exit 1
+    ${checkoutRef}
     git submodule update --init --recursive --force || (rm -rf * && git reset --hard && git submodule update --init --recursive --force) || exit 1
     git submodule foreach --recursive  "git fetch --tags --force && git reset --hard && git clean -fdxq -e /git -e /svn" || exit 1
     git clean -fdxq || exit 1

README.md

@@ -255,6 +255,58 @@ A tool that is a Python package rather than a command line needs a small driver
 script that takes the arguments its entry passes, simulates, writes the result
 file and exits non-zero when it fails; the entry then points `command` at it.
 
+### Testing a pull request
+
+A branch is tested against its own previous run, which says what broke *after* a
+change was merged. A pull request can be tested before that, against the newest
+run of `master`.
+
+In Jenkins, set the **`pull_request`** parameter to the pull request number and
+start the job; `pull_request_baseline`, `pull_request_config` and
+`pull_request_node` say what it is compared against, what it tests and where.
+None of the branch jobs run unless their own parameter is ticked as well.
+
+By hand it is two steps. The compiler is built from the merge ref - the pull
+request as it would land, not the branch on its own - and the run fills a
+`pr-<N>` table like any other branch:
+
+```bash
+git fetch --force https://github.com/OpenModelica/OpenModelica.git refs/pull/<N>/merge
+git checkout -f --detach FETCH_HEAD
+# build omc, then
+./test.py --branch=pr-<N> configs/conf.json
+```
+
+and the report compares that run against the newest run of `master`:
+
+```bash
+./pr-report.py <N>                 # --baseline=master by default
+```
+
+It writes `history/pr-<N>/<baseline run>..<pull request run>.html`, the same
+kind of page as the nightly regression reports, next to `00_comment.md`, a
+summary to comment on the pull request with. Both are published with the other
+reports.
+
+Two things make a difference mean something other than "the pull request did
+this", and the report says so when they apply: **the machine**, since two runs
+produced on different hardware compare the hardware as much as the change, and
+**the libraries**, since two runs that tested different library versions, or
+verified against different reference files, differ for reasons of their own. The
+baseline is also the newest `master` run rather than the commit the pull request
+is based on, so a difference can come from anything merged since it was
+branched - a reason to rebase before believing a surprising result.
+
+A full run takes days, so testing every pull request this way is not the idea;
+point `--branch=pr-<N>` at a smaller configuration file when the question is
+narrower.
+
+The tables accumulate, about 19500 rows each. `drop-pr-tables.py` drops the ones
+whose pull request has been merged or closed, and those tested more than
+`--older-than` days ago, together with the rows their runs left in the other
+tables; it lists them and does nothing unless it is given `--yes`. The reports
+published for them are not touched.
+
 ### Generate HTML results
 
   ```bash

drop-pr-tables.py

@@ -0,0 +1,97 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+"""Drop the result tables of pull requests that have been dealt with.
+
+Testing a pull request fills a pr-<N> table like any other branch, about 19500
+rows and a few megabytes, and unlike a branch it is never written to again once
+the pull request is merged or closed. This drops those tables, and the rows the
+runs left in the other tables, for pull requests that are no longer open or that
+were tested long enough ago not to matter.
+
+Nothing is dropped without --yes; without it the tables are only listed.
+"""
+
+import argparse, datetime, json, os, re, time, urllib.error, urllib.request
+import resultsdb
+
+parser = argparse.ArgumentParser(description='OpenModelica library testing pull request cleanup')
+parser.add_argument('--repo', default="OpenModelica/OpenModelica", help='the repository the pull requests belong to')
+parser.add_argument('--older-than', type=int, default=60, metavar='DAYS',
+                    help='also drop a pull request still open whose newest run is older than this (0: never)')
+parser.add_argument('--yes', action='store_true', help='actually drop them')
+resultsdb.addArgument(parser)
+args = parser.parse_args()
+
+prTableRe = re.compile(r"^pr-([0-9]+)$")
+
+db = resultsdb.connect(args.db)
+cursor = db.cursor()
+
+
+def pullRequestState(number):
+  """"open", "closed", "merged", or why we could not tell."""
+  url = "https://api.github.com/repos/%s/pulls/%s" % (args.repo, number)
+  request = urllib.request.Request(url, headers={"Accept": "application/vnd.github+json"})
+  # The rate limit is 60 requests an hour without one, which is enough for the
+  # handful of tables this looks at, but a token raises it and costs nothing.
+  token = os.environ.get("GITHUB_TOKEN")
+  if token:
+    request.add_header("Authorization", "Bearer %s" % token)
+  try:
+    pr = json.loads(urllib.request.urlopen(request).read().decode("utf-8"))
+  except urllib.error.HTTPError as e:
+    return "no answer from GitHub (%s)" % e
+  except Exception as e:
+    return "no answer from GitHub (%s)" % e
+  if pr.get("merged_at"):
+    return "merged"
+  return pr.get("state") or "unknown"
+
+
+def newestRun(table):
+  (date,) = cursor.execute("SELECT max(date) FROM %s" % db.quote(table)).fetchone()
+  return date
+
+
+def drop(table, branch):
+  """The table of a run, and everything the run wrote about itself elsewhere."""
+  cursor.execute("DROP TABLE %s" % db.quote(table))
+  for other in ["omcversion", "libversion", "history", "job_claim"]:
+    if db.tableExists(other):
+      cursor.execute("DELETE FROM %s WHERE branch=?" % db.quote(other), (branch,))
+  db.commit()
+
+
+tables = sorted((t for t in db.tables() if prTableRe.match(t)),
+                key=lambda t: int(prTableRe.match(t).group(1)))
+if not tables:
+  raise SystemExit("No pull request tables in this database")
+
+dropping = []
+for table in tables:
+  number = prTableRe.match(table).group(1)
+  date = newestRun(table)
+  age = (time.time() - date) / 86400.0 if date else 0
+  state = pullRequestState(number)
+  stale = args.older_than and date and age > args.older_than
+  why = None
+  if state in ("merged", "closed"):
+    why = "%s, tested %d days ago" % (state, age)
+  elif stale:
+    why = "still %s, but tested %d days ago" % (state, age)
+  print("%-12s %-8s %s%s" % (table, state,
+                             "last run %s" % datetime.datetime.fromtimestamp(date) if date else "no runs",
+                             ", dropping: %s" % why if why else ""))
+  if why:
+    dropping.append((table, why))
+
+if not dropping:
+  raise SystemExit(0)
+if not args.yes:
+  print("\n%d tables would be dropped; pass --yes to drop them" % len(dropping))
+  raise SystemExit(0)
+for (table, why) in dropping:
+  print("Dropping %s (%s)" % (table, why))
+  drop(table, table)
+print("The reports and files published for them are not touched; they are on the web server.")