From 07e531b2fe23b35b988c7ce7fad489465b81606f Mon Sep 17 00:00:00 2001 From: Adrian Pop Date: Mon, 24 Aug 2026 16:07:02 +0200 Subject: [PATCH] Run the library testing on a pull request (#307) The library testing runs against a branch and says what broke after a change was merged. This tests a pull request before that, and compares it against master. The compiler is built from refs/pull//merge - the pull request as it would land, not the branch on its own - fetched from GitHub itself, since refs/pull/* is not on the read-only mirror the job clones. It is checked out detached, so nothing is left in the shared workspace for the next run to trip over. The run fills a pr- table like any other branch, which the shared database of #295 makes cheap: the claim is (branch, libname), so it cannot collide with a master run, and the results carry their own omcversion, so nothing else can reuse them by mistake. What did not exist is the comparison. all-reports.py reports a branch against its own previous run, which is exactly what a pull request must not do, and its query reads one table. pr-report.py takes the newest run of pr- and the newest run of the baseline branch and compares them with the same rule and the same thresholds: the phase each model reached, and what each phase cost. Per library it compares the newest run each side has of it, because a run does not necessarily hold every library - one whose version, compiler and configuration were tested before keeps the results of the run that produced them. It writes the report next to the nightly ones, in history/pr-/...html, and a summary to comment on the pull request with beside it. Two things would make a difference mean something other than "the pull request did this", and the report says so when they apply: - the machine, since two runs produced on different hardware compare the hardware as much as the change. The parameter defaults to the node that produces the master runs, and the report names both machines, which #320 records per library; - the libraries, since two runs that tested different library versions, or verified against different reference files, differ for reasons of their own. The models one run has and the other does not are counted and listed rather than quietly left out of the comparison, since a library that failed to load looks like nothing at all otherwise. In Jenkins it is the pull_request parameter, with pull_request_baseline, pull_request_config and pull_request_node beside it. A full run takes days, so this is on demand and takes a configuration file: testing every pull request is not the idea. The tables accumulate, roughly 19500 rows each, and unlike a branch a pull request is tested once and never again. drop-pr-tables.py drops the tables of pull requests that have been merged or closed, and of those tested more than --older-than days ago, together with the rows their runs left in the other tables; it lists them and does nothing unless it is given --yes. The reports published for them are not touched. It is the drop_stale_pull_request_tables parameter in Jenkins, and the same stage keeps the per-pull-request omc builds on the test node from piling up. --- Generated by Claude Code. --- .CI/Jenkinsfile | 105 +++++++++++++- README.md | 52 +++++++ drop-pr-tables.py | 97 +++++++++++++ pr-report.py | 346 ++++++++++++++++++++++++++++++++++++++++++++++ pr.html.tpl | 64 +++++++++ resultsdb.py | 12 ++ 6 files changed, 675 insertions(+), 1 deletion(-) create mode 100755 drop-pr-tables.py create mode 100755 pr-report.py create mode 100644 pr.html.tpl diff --git a/.CI/Jenkinsfile b/.CI/Jenkinsfile index c12abc4..21ebae7 100644 --- a/.CI/Jenkinsfile +++ b/.CI/Jenkinsfile @@ -37,6 +37,12 @@ pipeline { booleanParam(name: 'generateSymbolicJacobian', defaultValue: false, description: 'master branch, with --generateSymbolicJacobian (ryzen-5950x-1). This is an experimental job that does not run on a fixed schedule.') booleanParam(name: 'wasm_jit', defaultValue: false, description: 'master branch, with --simCodeTarget=wasm-jit (ryzen-5950x-2). This is an experimental job that does not run on a fixed schedule.') booleanParam(name: 'heavy_tests', defaultValue: false, description: 'master branch, runs one test at a time. That is, no parallel launching of tests. omc will use multiple threads for each test (-n=1 is not set unlike the other regression tests.), (ryzen-5950x-1). This is an experimental job that does not run on a fixed schedule.') + + string(name: 'pull_request', defaultValue: '', description: 'Test an OpenModelica pull request rather than a branch: its number, e.g. 16354. omc is built from refs/pull//merge - the pull request as it would land - the results fill a pr- table, and the report compares them against the newest run of pull_request_baseline. Left empty, nothing of this runs.') + string(name: 'pull_request_baseline', defaultValue: 'master', description: 'The branch a pull request is compared against. Its newest run has to come from pull_request_node, or the report compares the machines as much as the pull request.') + string(name: 'pull_request_config', defaultValue: 'configs/conf.json', description: 'What a pull request run tests. A full run takes days, so a smaller configuration file is often the better question to ask.') + choice(name: 'pull_request_node', choices: ['ryzen-5950x-1', 'ryzen-5950x-2-1', 'ryzen-9950x'], description: 'The machine a pull request runs on. The default is the one that produces the master runs it is compared against.') + booleanParam(name: 'drop_stale_pull_request_tables', defaultValue: false, description: 'Drop the pr- tables of pull requests that have been merged or closed, and of those tested more than 60 days ago. The reports published for them are not touched.') } environment { LC_ALL = 'C.UTF-8' @@ -115,6 +121,31 @@ pipeline { } } + stage('pull request') { + agent { + node { + label "${params.pull_request_node}" + customWorkspace 'ws/OpenModelicaLibraryTestingWork' + } + } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { params.pull_request.trim() } + } + steps { + script { + if (!(params.pull_request.trim() ==~ /[0-9]+/)) { + error "pull_request is a pull request number; got '${params.pull_request}'" + } + } + // One build of omc per pull request is kept, as for a branch, and they + // accumulate: a pull request is tested once and never again. + sh 'find "$HOME/saved_omc" -maxdepth 1 -name "pr-*" -type d -mtime +14 -exec rm -rf {} ";" || true' + runRegressiontest("pr-${params.pull_request.trim()}", "pr-${params.pull_request.trim()}", '', '', false, '', '', false, false, 0, params.pull_request_config) + } + } + stage('newInst-newBackend') { agent { node { @@ -549,6 +580,61 @@ pipeline { sshPublisher(publishers: [sshPublisherDesc(configName: 'LibraryTestingReports', transfers: [sshTransfer(sourceFiles: 'overview*.html,history/**')])]) } } + + stage('drop stale pull request tables') { + agent { + dockerfile { + label 'linux' + dir '.CI/build-dep' + customWorkspace 'ws/OpenModelicaLibraryTestingReport' + } + } + when { + beforeAgent true + expression { params.drop_stale_pull_request_tables } + } + environment { + PGPASSFILE = credentials('omdb-pgpass') + } + steps { + sh './drop-pr-tables.py --yes' + } + } + + /* A pull request is not reported on by the stage above: that one compares a + * branch against its own previous run, which a pull request has none of, and + * regenerating every overview for a job that tested one thing would take + * longer than the report itself. */ + stage('pull request report') { + agent { + dockerfile { + label 'linux' + dir '.CI/build-dep' + customWorkspace 'ws/OpenModelicaLibraryTestingReport' + } + } + when { + beforeAgent true + expression { params.pull_request.trim() } + } + environment { + PYTHONIOENCODING = 'utf-8' + PGPASSFILE = credentials('omdb-pgpass') + } + steps { + script { + if (!(params.pull_request.trim() ==~ /[0-9]+/)) { + error "pull_request is a pull request number; got '${params.pull_request}'" + } + } + sh 'rm -rf history' + sh "./pr-report.py '${params.pull_request.trim()}' --baseline='${params.pull_request_baseline.trim()}'" + // The summary to comment on the pull request with, in the build log + // until there is a token to post it with. + sh 'cat history/pr-*/00_comment.md' + sshPublisher(publishers: [sshPublisherDesc(configName: 'LibraryTestingReports', transfers: [sshTransfer(sourceFiles: 'history/**')])]) + } + } } } def omsimulatorHash() { @@ -808,6 +894,23 @@ def runRegressiontest(branch, name, extraFlags, omsHash, omcompiler, extrasimfla } } + // A pull request is fetched from GitHub itself: refs/pull/* is not on the + // read-only mirror the rest of the job clones. The merge ref rather than the + // head one, since the question is what happens once it is merged, and it is + // checked out detached so that nothing is left behind for the next run of the + // same workspace to trip over. + def pullRequest = branch.startsWith('pr-') && branch.substring(3).isInteger() ? branch.substring(3) : '' + def checkoutRef = pullRequest ? """ + if ! git fetch --force https://github.com/OpenModelica/OpenModelica.git refs/pull/${pullRequest}/merge; then + echo "Could not fetch refs/pull/${pullRequest}/merge: either there is no such pull request, or GitHub cannot merge it into its base branch." + exit 1 + fi + git checkout -f --detach FETCH_HEAD || exit 1 + git fetch --tags --force || exit 1 +""" : """ + git reset --hard && git checkout -f "${branch}" && (git rev-parse --verify "tags/${branch}" || (git reset --hard "origin/${branch}" && git pull)) && git fetch --tags --force || exit 1 +""" + OMCPATH = "${omcompiler ? '../' : './'}OMCompiler" // The build runs in OMCompiler, one level below the cmake source tree. @@ -883,7 +986,7 @@ def runRegressiontest(branch, name, extraFlags, omsHash, omcompiler, extrasimfla cd ${OMCPATH} if ! test -f ~/saved_omc/${name}/.nogit; then - git reset --hard && git checkout -f "${branch}" && (git rev-parse --verify "tags/${branch}" || (git reset --hard "origin/${branch}" && git pull)) && git fetch --tags --force || exit 1 + ${checkoutRef} git submodule update --init --recursive --force || (rm -rf * && git reset --hard && git submodule update --init --recursive --force) || exit 1 git submodule foreach --recursive "git fetch --tags --force && git reset --hard && git clean -fdxq -e /git -e /svn" || exit 1 git clean -fdxq || exit 1 diff --git a/README.md b/README.md index 51faab0..7bc562f 100644 --- a/README.md +++ b/README.md @@ -255,6 +255,58 @@ A tool that is a Python package rather than a command line needs a small driver script that takes the arguments its entry passes, simulates, writes the result file and exits non-zero when it fails; the entry then points `command` at it. +### Testing a pull request + +A branch is tested against its own previous run, which says what broke *after* a +change was merged. A pull request can be tested before that, against the newest +run of `master`. + +In Jenkins, set the **`pull_request`** parameter to the pull request number and +start the job; `pull_request_baseline`, `pull_request_config` and +`pull_request_node` say what it is compared against, what it tests and where. +None of the branch jobs run unless their own parameter is ticked as well. + +By hand it is two steps. The compiler is built from the merge ref - the pull +request as it would land, not the branch on its own - and the run fills a +`pr-` table like any other branch: + +```bash +git fetch --force https://github.com/OpenModelica/OpenModelica.git refs/pull//merge +git checkout -f --detach FETCH_HEAD +# build omc, then +./test.py --branch=pr- configs/conf.json +``` + +and the report compares that run against the newest run of `master`: + +```bash +./pr-report.py # --baseline=master by default +``` + +It writes `history/pr-/...html`, the same +kind of page as the nightly regression reports, next to `00_comment.md`, a +summary to comment on the pull request with. Both are published with the other +reports. + +Two things make a difference mean something other than "the pull request did +this", and the report says so when they apply: **the machine**, since two runs +produced on different hardware compare the hardware as much as the change, and +**the libraries**, since two runs that tested different library versions, or +verified against different reference files, differ for reasons of their own. The +baseline is also the newest `master` run rather than the commit the pull request +is based on, so a difference can come from anything merged since it was +branched - a reason to rebase before believing a surprising result. + +A full run takes days, so testing every pull request this way is not the idea; +point `--branch=pr-` at a smaller configuration file when the question is +narrower. + +The tables accumulate, about 19500 rows each. `drop-pr-tables.py` drops the ones +whose pull request has been merged or closed, and those tested more than +`--older-than` days ago, together with the rows their runs left in the other +tables; it lists them and does nothing unless it is given `--yes`. The reports +published for them are not touched. + ### Generate HTML results ```bash diff --git a/drop-pr-tables.py b/drop-pr-tables.py new file mode 100755 index 0000000..73e7c8d --- /dev/null +++ b/drop-pr-tables.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +"""Drop the result tables of pull requests that have been dealt with. + +Testing a pull request fills a pr- table like any other branch, about 19500 +rows and a few megabytes, and unlike a branch it is never written to again once +the pull request is merged or closed. This drops those tables, and the rows the +runs left in the other tables, for pull requests that are no longer open or that +were tested long enough ago not to matter. + +Nothing is dropped without --yes; without it the tables are only listed. +""" + +import argparse, datetime, json, os, re, time, urllib.error, urllib.request +import resultsdb + +parser = argparse.ArgumentParser(description='OpenModelica library testing pull request cleanup') +parser.add_argument('--repo', default="OpenModelica/OpenModelica", help='the repository the pull requests belong to') +parser.add_argument('--older-than', type=int, default=60, metavar='DAYS', + help='also drop a pull request still open whose newest run is older than this (0: never)') +parser.add_argument('--yes', action='store_true', help='actually drop them') +resultsdb.addArgument(parser) +args = parser.parse_args() + +prTableRe = re.compile(r"^pr-([0-9]+)$") + +db = resultsdb.connect(args.db) +cursor = db.cursor() + + +def pullRequestState(number): + """"open", "closed", "merged", or why we could not tell.""" + url = "https://api.github.com/repos/%s/pulls/%s" % (args.repo, number) + request = urllib.request.Request(url, headers={"Accept": "application/vnd.github+json"}) + # The rate limit is 60 requests an hour without one, which is enough for the + # handful of tables this looks at, but a token raises it and costs nothing. + token = os.environ.get("GITHUB_TOKEN") + if token: + request.add_header("Authorization", "Bearer %s" % token) + try: + pr = json.loads(urllib.request.urlopen(request).read().decode("utf-8")) + except urllib.error.HTTPError as e: + return "no answer from GitHub (%s)" % e + except Exception as e: + return "no answer from GitHub (%s)" % e + if pr.get("merged_at"): + return "merged" + return pr.get("state") or "unknown" + + +def newestRun(table): + (date,) = cursor.execute("SELECT max(date) FROM %s" % db.quote(table)).fetchone() + return date + + +def drop(table, branch): + """The table of a run, and everything the run wrote about itself elsewhere.""" + cursor.execute("DROP TABLE %s" % db.quote(table)) + for other in ["omcversion", "libversion", "history", "job_claim"]: + if db.tableExists(other): + cursor.execute("DELETE FROM %s WHERE branch=?" % db.quote(other), (branch,)) + db.commit() + + +tables = sorted((t for t in db.tables() if prTableRe.match(t)), + key=lambda t: int(prTableRe.match(t).group(1))) +if not tables: + raise SystemExit("No pull request tables in this database") + +dropping = [] +for table in tables: + number = prTableRe.match(table).group(1) + date = newestRun(table) + age = (time.time() - date) / 86400.0 if date else 0 + state = pullRequestState(number) + stale = args.older_than and date and age > args.older_than + why = None + if state in ("merged", "closed"): + why = "%s, tested %d days ago" % (state, age) + elif stale: + why = "still %s, but tested %d days ago" % (state, age) + print("%-12s %-8s %s%s" % (table, state, + "last run %s" % datetime.datetime.fromtimestamp(date) if date else "no runs", + ", dropping: %s" % why if why else "")) + if why: + dropping.append((table, why)) + +if not dropping: + raise SystemExit(0) +if not args.yes: + print("\n%d tables would be dropped; pass --yes to drop them" % len(dropping)) + raise SystemExit(0) +for (table, why) in dropping: + print("Dropping %s (%s)" % (table, why)) + drop(table, table) +print("The reports and files published for them are not touched; they are on the web server.") diff --git a/pr-report.py b/pr-report.py new file mode 100755 index 0000000..b6abd57 --- /dev/null +++ b/pr-report.py @@ -0,0 +1,346 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +"""Compare a pull request run against a run of another branch. + +all-reports.py reports a branch against its own previous run, which is what a +pull request must not do: pr- has no previous run, and the question is not +"what changed since yesterday" but "what does this pull request change against +master". The comparison itself is the same one - the phase a model reached and +what each phase cost - only the two runs it is given come from two branches. +""" + +import argparse, codecs, datetime, html, os, re, time +import shared, resultsdb +from omcommon import friendlyStr, multiple_replace + +parser = argparse.ArgumentParser(description='OpenModelica library testing pull request report') +parser.add_argument('pullrequest', help='the pull request number, or its branch name pr-') +parser.add_argument('--baseline', default="master", help='the branch the pull request is compared against') +parser.add_argument('--date', type=int, default=0, help='the pull request run to report on (default: its newest)') +parser.add_argument('--baselinedate', type=int, default=0, help='the baseline run to compare against (default: its newest)') +parser.add_argument('--baseurl', default="http://libraries.openmodelica.org/branches") +parser.add_argument('--historyurl', default="http://libraries.openmodelica.org/branches/history") +parser.add_argument('--historypath', default="history") +parser.add_argument('--githuburl', default="https://github.com/OpenModelica/OpenModelica") +parser.add_argument('--markdown', default="", help='where to write the summary to comment on the pull request with (default: //00_comment.md)') +resultsdb.addArgument(parser) +args = parser.parse_args() + +os.environ['TZ'] = 'Europe/Stockholm' +time.tzset() + +# The same thresholds all-reports.py uses, so that a change reported here means +# what it means in the nightly reports. +timeMinPhase = 4 # Need to have completed code generation to report performance regressions +timeRel = 1.7 # Minimum 1.7x time is registered as a performance regression +timeAbs = 10 # Ignore performance regressions for times <10s... + +PHASES = [(1,"frontend"),(2,"backend"),(3,"simcode"),(4,"templates"),(5,"compile"),(6,"simulate")] + +m = re.match(r"^(?:pr-)?([0-9]+)$", args.pullrequest.strip()) +if not m: + raise SystemExit("Expected a pull request number or a pr- branch name, got '%s'" % args.pullrequest) +pr = m.group(1) +branch = "pr-%s" % pr +baseline = args.baseline.split("/")[-1] +prurl = "%s/pull/%s" % (args.githuburl, pr) + +db = resultsdb.connect(args.db) +cursor = db.cursor() + +for tbl in [branch, baseline]: + if not db.tableExists(tbl): + raise SystemExit("No results for '%s'; run test.py --branch=%s first" % (tbl, tbl)) + db.createDateIndex(tbl) + + +def dateStr(dint): + return str(datetime.datetime.fromtimestamp(dint).strftime('%Y-%m-%d %H:%M:%S')) + +def newestRun(table, upto=0): + """The newest run of a table, or the newest at or before upto.""" + if upto: + (d,) = cursor.execute("SELECT max(date) FROM %s WHERE date<=?" % db.quote(table), (upto,)).fetchone() + else: + (d,) = cursor.execute("SELECT max(date) FROM %s" % db.quote(table)).fetchone() + return d + +def libraries(table, upto): + return set(l for (l,) in cursor.execute( + "SELECT DISTINCT libname FROM %s WHERE date<=?" % db.quote(table), (upto,))) + +def libraryRun(table, libname, upto): + """The newest run of one library at or before upto. + + A run does not necessarily hold every library: test.py skips a library whose + version, compiler and configuration were tested before, so its results stay at + the date of the run that produced them. Comparing two runs therefore means + comparing, per library, the newest run each of them has of it. + """ + (d,) = cursor.execute("SELECT max(date) FROM %s WHERE date<=? AND libname=?" + % db.quote(table), (upto, libname)).fetchone() + return d + +# A database written before the machine was recorded, #320, has no host column. +hostColumn = "host" if "host" in db.columns("libversion") else "NULL" + +def libraryVersions(table, libname, date): + """(libversion, confighash, host) of a library in a run.""" + row = cursor.execute( + "SELECT libversion,confighash,%s FROM libversion WHERE %s AND date=? AND libname=?" + % (hostColumn, db.likeNoCase("branch")), (table, date, libname)).fetchone() + return row if row else (None, None, None) + +def omcVersion(table, date): + row = cursor.execute("SELECT omcversion FROM omcversion WHERE %s AND date=?" + % db.likeNoCase("branch"), (table, date)).fetchone() + return row[0].strip() if row and row[0] else "unknown" + +def models(table, libname, date): + return set(mod for (mod,) in cursor.execute( + "SELECT model FROM %s WHERE date=? AND libname=?" % db.quote(table), (date, libname))) + +def changedModels(table1, date1, table2, date2, libnames): + """The models whose phase or timings differ between the two runs. + + One query per set of libraries that share a pair of dates, as all-reports.py + does. The values of both runs are aggregated into one string per column, + ordered by which run they come from rather than by their date: a pull request + run is normally the newer of the two, but need not be. + """ + concat = ",".join(db.groupConcat(c, "ord") for c in + ["finalphase","frontend","backend","simcode","templates","compile","simulate"]) + cols = "model,libname,finalphase,frontend,backend,simcode,templates,compile,simulate" + inlibs = ",".join("'%s'" % libname for libname in sorted(libnames)) + query = """SELECT model,libname,%s FROM + (SELECT * FROM + (SELECT %s,0 AS ord FROM %s WHERE date=? AND libname IN (%s) + UNION ALL + SELECT %s,1 AS ord FROM %s WHERE date=? AND libname IN (%s)) AS runs + ORDER BY ord) AS phases + GROUP BY model,libname HAVING + (MIN(finalphase) <> MAX(finalphase)) OR + (MIN(finalphase) >= ? AND ( + (MAX(frontend) > ?*MIN(frontend) AND MAX(frontend) > ?) OR + (MAX(backend) > ?*MIN(backend) AND MAX(backend) > ?) OR + (MAX(simcode) > ?*MIN(simcode) AND MAX(simcode) > ?) OR + (MAX(templates) > ?*MIN(templates) AND MAX(templates) > ?) OR + (MAX(compile) > ?*MIN(compile) AND MAX(compile) > ?) OR + (MAX(simulate) > ?*MIN(simulate) AND MAX(simulate) > ?))) + """ % (concat, cols, db.quote(table1), inlibs, cols, db.quote(table2), inlibs) + cursor.execute(query, (date1, date2, timeMinPhase, + timeRel, timeAbs, timeRel, timeAbs, timeRel, timeAbs, + timeRel, timeAbs, timeRel, 2*timeAbs, timeRel, timeAbs)) + return cursor.fetchall() + + +prdate = newestRun(branch, args.date) +if not prdate: + raise SystemExit("No results for %s%s" % (branch, " at or before %d" % args.date if args.date else "")) +basedate = newestRun(baseline, args.baselinedate) +if not basedate: + raise SystemExit("No results for %s%s" % (baseline, " at or before %d" % args.baselinedate if args.baselinedate else "")) + +prlibs = libraries(branch, prdate) +baselibs = libraries(baseline, basedate) +libnames = sorted(prlibs & baselibs) +if not libnames: + raise SystemExit("%s and %s have no library in common" % (branch, baseline)) + +# Per library the newest run each side has of it, grouped by the pair of dates +# so that one query covers every library that shares it. +groups = {} +prlibdates = {} +baselibdates = {} +for libname in libnames: + d1 = libraryRun(baseline, libname, basedate) + d2 = libraryRun(branch, libname, prdate) + baselibdates[libname] = d1 + prlibdates[libname] = d2 + groups.setdefault((d1, d2), []).append(libname) + +changes = [] +for ((d1, d2), libs) in sorted(groups.items()): + changes += changedModels(baseline, d1, branch, d2, libs) +changes = sorted(changes, key=lambda x: (x[1], x[0])) + +# Models one of the runs has and the other does not: a library that grew a model, +# or one whose test did not run at all. They cannot be compared, but leaving them +# out of the report without saying so would hide a library that failed to load. +onlyPr = [] +onlyBaseline = [] +numCompared = 0 +for libname in libnames: + inpr = models(branch, libname, prlibdates[libname]) + inbase = models(baseline, libname, baselibdates[libname]) + onlyPr += [(libname, mod) for mod in sorted(inpr - inbase)] + onlyBaseline += [(libname, mod) for mod in sorted(inbase - inpr)] + numCompared += len(inpr & inbase) + + +def modelLink(table, libname, modelname, extension, text): + return '%s' % ( + args.baseurl, table, libname, libname, modelname, extension, html.escape(text)) + +def libraryLink(table, libname): + return '%s' % (args.baseurl, table, libname, libname, html.escape(libname)) + +def classify(group, times): + """(colour, message) for a model, in the terms all-reports.py uses.""" + (phase1, phase2) = [int(i) for i in group.split(",")] + if phase2 != phase1: + better = phase2 > phase1 + return ("better" if better else "warning", + "%s → %s" % (shared.finalphaseName(phase1), shared.finalphaseName(phase2)), + "improved" if better else "regression") + msgs = [] + colour = None + for ((phase, name), values) in zip(PHASES, times): + (t1, t2) = [float(d) for d in values.split(",")] + limit = 2*timeAbs if name == "compile" else timeAbs + if t2 > timeRel*t1 and t2 > limit: + colour = "warningPerformance" + msgs.append("%s performance %s → %s" % (shared.finalphaseName(phase), friendlyStr(t1), friendlyStr(t2))) + elif t1 > timeRel*t2 and t1 > limit: + if colour is None: + colour = "betterPerformance" + msgs.append("%s performance %s → %s" % (shared.finalphaseName(phase), friendlyStr(t1), friendlyStr(t2))) + if colour is None: + # Both runs are below timeMinPhase, or the difference is under the threshold + # in the direction the query did not test for. + return (None, "", None) + return (colour, " ".join(msgs), + "performance improved" if colour == "betterPerformance" else "performance regression") + +counts = {"improved": 0, "regression": 0, "performance improved": 0, "performance regression": 0} +rows = [] +markdownrows = [] +for (model, libname, group, frontend, backend, simcode, templates, compile, simulate) in changes: + (colour, msg, kind) = classify(group, [frontend, backend, simcode, templates, compile, simulate]) + if kind is None: + continue + counts[kind] += 1 + rows.append('%s%s %s %s%s' + % (libraryLink(branch, libname), + modelLink(branch, libname, model, "err", model), + modelLink(branch, libname, model, "sim", "(sim)"), + modelLink(baseline, libname, model, "err", "(%s)" % baseline), + colour, msg)) + markdownrows.append((libname, model, msg.replace("→", "->"))) + +# What makes a difference mean something other than "the pull request did this". +caveats = [] +prhosts = set() +basehosts = set() +libchanges = [] +for libname in libnames: + (lv1, lh1, host1) = libraryVersions(baseline, libname, baselibdates[libname]) + (lv2, lh2, host2) = libraryVersions(branch, libname, prlibdates[libname]) + prhosts.add(host2 or "unknown") + basehosts.add(host1 or "unknown") + if (lv1 or "").strip() != (lv2 or "").strip(): + libchanges.append("%sVersion %s in %s, %s in %s" + % (libraryLink(branch, libname), html.escape((lv1 or "").strip()), baseline, + html.escape((lv2 or "").strip()), branch)) + elif lh1 != lh2: + libchanges.append("%sConfiguration hash (OMC settings, the testing script or a " + "reference file changed)" % libraryLink(branch, libname)) + +if prhosts != basehosts: + caveats.append("The two runs were produced on different machines (%s against %s), so the timings " + "compare the hardware as much as the pull request. The phases a model reaches are " + "still comparable." % (", ".join(sorted(prhosts)), ", ".join(sorted(basehosts)))) +if libchanges: + caveats.append("%d of the %d libraries were not tested in the same version, or not with the same " + "configuration and reference files, in the two runs; see Library Changes below." + % (len(libchanges), len(libnames))) +if prlibs - baselibs: + caveats.append("%d libraries of the pull request run have no counterpart in the baseline run: %s." + % (len(prlibs - baselibs), ", ".join(sorted(prlibs - baselibs)))) +if baselibs - prlibs: + caveats.append("%d libraries of the baseline run were not tested by the pull request run: %s." + % (len(baselibs - prlibs), ", ".join(sorted(baselibs - prlibs)))) +# Always true, and in the report itself rather than among the caveats. +note = ("The baseline is the newest run of %s, not the commit the pull request is based on, so a " + "difference can also come from something merged into %s since the pull request was " + "branched." % (baseline, baseline)) + +reportname = "%s..%s.html" % (dateStr(basedate), dateStr(prdate)) +historydir = os.path.join(args.historypath, branch) +os.makedirs(historydir, exist_ok=True) + +with open("pr.html.tpl") as fin: + tpl = fin.read() +tpl = multiple_replace(tpl, + ("#PRURL#", prurl), + ("#PR#", pr), + ("#BRANCH#", branch), + ("#BASELINE#", html.escape(baseline)), + ("#CAVEATS#", "\n".join('

%s

' % c for c in caveats)), + ("#DATE1#", dateStr(basedate)), + ("#DATE2#", dateStr(prdate)), + ("#OMCVERSION1#", html.escape(omcVersion(baseline, basedate))), + ("#OMCVERSION2#", html.escape(omcVersion(branch, prdate))), + ("#HOST1#", html.escape(", ".join(sorted(basehosts)))), + ("#HOST2#", html.escape(", ".join(sorted(prhosts)))), + ("#NUMCOMPARED#", str(numCompared)), + ("#NUMIMPROVE#", str(counts["improved"])), + ("#NUMREGRESSION#", str(counts["regression"])), + ("#NUMPERFIMPROVE#", str(counts["performance improved"])), + ("#NUMPERFREGRESSION#", str(counts["performance regression"])), + ("#NUMONLYPR#", str(len(onlyPr))), + ("#NUMONLYBASELINE#", str(len(onlyBaseline))), + ("#LIBCHANGES#", "\n".join(libchanges)), + ("#MODELCHANGES#", "\n".join(rows)), + ("#MODELSONLYINONE#", "\n".join( + '%s%s%s' % (html.escape(libname), html.escape(model), where) + for (where, lst) in [("only in %s" % branch, onlyPr), ("only in %s" % baseline, onlyBaseline)] + for (libname, model) in lst)), +) +with codecs.open(os.path.join(historydir, reportname), "w", encoding="utf-8") as fout: + fout.write(tpl) + +# The index the history directory of every branch has, so that the report is +# reachable from the server without knowing its name. +reporturl = "%s/%s/%s" % (args.historyurl, branch, reportname.replace(" ", "%20")) +summary = ("%d improved, %d regressions; performance %d improved, %d regressions" + % (counts["improved"], counts["regression"], + counts["performance improved"], counts["performance regression"])) +indexname = os.path.join(historydir, "00_history.html") +index = [] +if os.path.exists(indexname): + with codecs.open(indexname, "r", encoding="utf-8") as fin: + # A report of the same two runs is one that has just been overwritten, so + # its line in the index is replaced rather than repeated. + index = [line for line in fin.read().splitlines() if line.strip() and reporturl not in line] +entry = '

%s against %s %s %s

' % (reporturl, branch, baseline, reportname, summary) +with codecs.open(indexname, "w", encoding="utf-8") as fout: + fout.write("".join(line + "\n" for line in index + [entry])) + +markdown = ["## Library testing for [#%s](%s) against `%s`" % (pr, prurl, baseline), "", + "| | Branch | Run | Compiler | Machine |", + "| --- | --- | --- | --- | --- |", + "| Baseline | `%s` | %s | %s | %s |" % (baseline, dateStr(basedate), omcVersion(baseline, basedate), ", ".join(sorted(basehosts))), + "| Pull request | `%s` | %s | %s | %s |" % (branch, dateStr(prdate), omcVersion(branch, prdate), ", ".join(sorted(prhosts))), + "", + "%d models compared, **%d improved, %d regressions**, performance %d improved, %d regressions." + % (numCompared, counts["improved"], counts["regression"], + counts["performance improved"], counts["performance regression"]), + "", "[Full report](%s)" % reporturl, ""] +if markdownrows: + markdown += ["
%d models affected" % len(markdownrows), "", + "| Library | Model | Change |", "| --- | --- | --- |"] + markdown += ["| %s | %s | %s |" % (libname, model, msg) for (libname, model, msg) in markdownrows] + markdown += ["", "
", ""] +markdown += ["
Caveats", ""] +markdown += ["- %s" % c.replace("→", "->") for c in caveats + [note]] +markdown += ["", "
", "", "---", "Generated by the OpenModelica library testing"] +markdownname = args.markdown or os.path.join(historydir, "00_comment.md") +with codecs.open(markdownname, "w", encoding="utf-8") as fout: + fout.write("\n".join(markdown) + "\n") + +print("%s: %s" % (branch, summary)) +print("Report: %s" % os.path.join(historydir, reportname)) +print("Comment: %s" % markdownname) +print("Published as %s" % reporturl) diff --git a/pr.html.tpl b/pr.html.tpl new file mode 100644 index 0000000..34a5670 --- /dev/null +++ b/pr.html.tpl @@ -0,0 +1,64 @@ + + + + + OpenModelica pull request #PR# against #BASELINE# + + + +

OpenModelica pull request #PR# against #BASELINE#

+ +#CAVEATS# +

The baseline is the newest run of #BASELINE#, not the commit the pull request is +based on, so a difference can also come from something merged into #BASELINE# since +the pull request was branched.

+ +

The two runs

+ + + + + +
BranchRunCompilerMachine
Baseline#BASELINE##DATE1##OMCVERSION1##HOST1#
Pull request#BRANCH##DATE2##OMCVERSION2##HOST2#
+ +

Summary

+ + + + + + + + + +
Models compared#NUMCOMPARED#
Number of Improvements#NUMIMPROVE#
Number of Regressions#NUMREGRESSION#
Number of Performance Improvements#NUMPERFIMPROVE#
Number of Performance Regressions#NUMPERFREGRESSION#
Models only in the pull request run#NUMONLYPR#
Models only in the baseline run#NUMONLYBASELINE#
+ +

Library Changes

+ + +#LIBCHANGES# +
LibraryChange
+ +

Models Affected

+ + +#MODELCHANGES# +
LibraryModelChange
+ +

Models Only in One of the Runs

+ + +#MODELSONLYINONE# +
LibraryModelWhere
+ + + diff --git a/resultsdb.py b/resultsdb.py index e709038..92773ab 100644 --- a/resultsdb.py +++ b/resultsdb.py @@ -163,6 +163,10 @@ def claimedBy(self, branch, libname, libversion, omcversion, confighash): def release(self): """Mark the claims of this run as finished.""" + def columns(self, table): + """The columns a table has, for the ones an older database may not have.""" + raise NotImplementedError + def vacuum(self): self.conn.execute("VACUUM") @@ -266,6 +270,9 @@ def addLibversionHost(self, cursor): def tables(self): return [t for (t,) in self.conn.execute("SELECT name FROM sqlite_master WHERE type='table'")] + def columns(self, table): + return [r[1] for r in self.conn.execute("PRAGMA table_info(%s)" % self.quote(table))] + def tableExists(self, name): return self.conn.execute( "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (name,)).fetchone() is not None @@ -437,6 +444,11 @@ def tables(self): return [t for (t,) in self.execute( "SELECT tablename FROM pg_tables WHERE schemaname=current_schema()")] + def columns(self, table): + return [c for (c,) in self.execute( + "SELECT column_name FROM information_schema.columns " + "WHERE table_schema=current_schema() AND table_name=?", (table,))] + def tableExists(self, name): return self.execute("SELECT 1 FROM pg_tables WHERE schemaname=current_schema() AND tablename=?", (name,)).fetchone() is not None