diff --git a/.CI/Jenkinsfile b/.CI/Jenkinsfile index c12abc4..21ebae7 100644 --- a/.CI/Jenkinsfile +++ b/.CI/Jenkinsfile @@ -37,6 +37,12 @@ pipeline { booleanParam(name: 'generateSymbolicJacobian', defaultValue: false, description: 'master branch, with --generateSymbolicJacobian (ryzen-5950x-1). This is an experimental job that does not run on a fixed schedule.') booleanParam(name: 'wasm_jit', defaultValue: false, description: 'master branch, with --simCodeTarget=wasm-jit (ryzen-5950x-2). This is an experimental job that does not run on a fixed schedule.') booleanParam(name: 'heavy_tests', defaultValue: false, description: 'master branch, runs one test at a time. That is, no parallel launching of tests. omc will use multiple threads for each test (-n=1 is not set unlike the other regression tests.), (ryzen-5950x-1). This is an experimental job that does not run on a fixed schedule.') + + string(name: 'pull_request', defaultValue: '', description: 'Test an OpenModelica pull request rather than a branch: its number, e.g. 16354. omc is built from refs/pull//merge - the pull request as it would land - the results fill a pr- table, and the report compares them against the newest run of pull_request_baseline. Left empty, nothing of this runs.') + string(name: 'pull_request_baseline', defaultValue: 'master', description: 'The branch a pull request is compared against. Its newest run has to come from pull_request_node, or the report compares the machines as much as the pull request.') + string(name: 'pull_request_config', defaultValue: 'configs/conf.json', description: 'What a pull request run tests. A full run takes days, so a smaller configuration file is often the better question to ask.') + choice(name: 'pull_request_node', choices: ['ryzen-5950x-1', 'ryzen-5950x-2-1', 'ryzen-9950x'], description: 'The machine a pull request runs on. The default is the one that produces the master runs it is compared against.') + booleanParam(name: 'drop_stale_pull_request_tables', defaultValue: false, description: 'Drop the pr- tables of pull requests that have been merged or closed, and of those tested more than 60 days ago. The reports published for them are not touched.') } environment { LC_ALL = 'C.UTF-8' @@ -115,6 +121,31 @@ pipeline { } } + stage('pull request') { + agent { + node { + label "${params.pull_request_node}" + customWorkspace 'ws/OpenModelicaLibraryTestingWork' + } + } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { params.pull_request.trim() } + } + steps { + script { + if (!(params.pull_request.trim() ==~ /[0-9]+/)) { + error "pull_request is a pull request number; got '${params.pull_request}'" + } + } + // One build of omc per pull request is kept, as for a branch, and they + // accumulate: a pull request is tested once and never again. + sh 'find "$HOME/saved_omc" -maxdepth 1 -name "pr-*" -type d -mtime +14 -exec rm -rf {} ";" || true' + runRegressiontest("pr-${params.pull_request.trim()}", "pr-${params.pull_request.trim()}", '', '', false, '', '', false, false, 0, params.pull_request_config) + } + } + stage('newInst-newBackend') { agent { node { @@ -549,6 +580,61 @@ pipeline { sshPublisher(publishers: [sshPublisherDesc(configName: 'LibraryTestingReports', transfers: [sshTransfer(sourceFiles: 'overview*.html,history/**')])]) } } + + stage('drop stale pull request tables') { + agent { + dockerfile { + label 'linux' + dir '.CI/build-dep' + customWorkspace 'ws/OpenModelicaLibraryTestingReport' + } + } + when { + beforeAgent true + expression { params.drop_stale_pull_request_tables } + } + environment { + PGPASSFILE = credentials('omdb-pgpass') + } + steps { + sh './drop-pr-tables.py --yes' + } + } + + /* A pull request is not reported on by the stage above: that one compares a + * branch against its own previous run, which a pull request has none of, and + * regenerating every overview for a job that tested one thing would take + * longer than the report itself. */ + stage('pull request report') { + agent { + dockerfile { + label 'linux' + dir '.CI/build-dep' + customWorkspace 'ws/OpenModelicaLibraryTestingReport' + } + } + when { + beforeAgent true + expression { params.pull_request.trim() } + } + environment { + PYTHONIOENCODING = 'utf-8' + PGPASSFILE = credentials('omdb-pgpass') + } + steps { + script { + if (!(params.pull_request.trim() ==~ /[0-9]+/)) { + error "pull_request is a pull request number; got '${params.pull_request}'" + } + } + sh 'rm -rf history' + sh "./pr-report.py '${params.pull_request.trim()}' --baseline='${params.pull_request_baseline.trim()}'" + // The summary to comment on the pull request with, in the build log + // until there is a token to post it with. + sh 'cat history/pr-*/00_comment.md' + sshPublisher(publishers: [sshPublisherDesc(configName: 'LibraryTestingReports', transfers: [sshTransfer(sourceFiles: 'history/**')])]) + } + } } } def omsimulatorHash() { @@ -808,6 +894,23 @@ def runRegressiontest(branch, name, extraFlags, omsHash, omcompiler, extrasimfla } } + // A pull request is fetched from GitHub itself: refs/pull/* is not on the + // read-only mirror the rest of the job clones. The merge ref rather than the + // head one, since the question is what happens once it is merged, and it is + // checked out detached so that nothing is left behind for the next run of the + // same workspace to trip over. + def pullRequest = branch.startsWith('pr-') && branch.substring(3).isInteger() ? branch.substring(3) : '' + def checkoutRef = pullRequest ? """ + if ! git fetch --force https://github.com/OpenModelica/OpenModelica.git refs/pull/${pullRequest}/merge; then + echo "Could not fetch refs/pull/${pullRequest}/merge: either there is no such pull request, or GitHub cannot merge it into its base branch." + exit 1 + fi + git checkout -f --detach FETCH_HEAD || exit 1 + git fetch --tags --force || exit 1 +""" : """ + git reset --hard && git checkout -f "${branch}" && (git rev-parse --verify "tags/${branch}" || (git reset --hard "origin/${branch}" && git pull)) && git fetch --tags --force || exit 1 +""" + OMCPATH = "${omcompiler ? '../' : './'}OMCompiler" // The build runs in OMCompiler, one level below the cmake source tree. @@ -883,7 +986,7 @@ def runRegressiontest(branch, name, extraFlags, omsHash, omcompiler, extrasimfla cd ${OMCPATH} if ! test -f ~/saved_omc/${name}/.nogit; then - git reset --hard && git checkout -f "${branch}" && (git rev-parse --verify "tags/${branch}" || (git reset --hard "origin/${branch}" && git pull)) && git fetch --tags --force || exit 1 + ${checkoutRef} git submodule update --init --recursive --force || (rm -rf * && git reset --hard && git submodule update --init --recursive --force) || exit 1 git submodule foreach --recursive "git fetch --tags --force && git reset --hard && git clean -fdxq -e /git -e /svn" || exit 1 git clean -fdxq || exit 1 diff --git a/README.md b/README.md index 51faab0..7bc562f 100644 --- a/README.md +++ b/README.md @@ -255,6 +255,58 @@ A tool that is a Python package rather than a command line needs a small driver script that takes the arguments its entry passes, simulates, writes the result file and exits non-zero when it fails; the entry then points `command` at it. +### Testing a pull request + +A branch is tested against its own previous run, which says what broke *after* a +change was merged. A pull request can be tested before that, against the newest +run of `master`. + +In Jenkins, set the **`pull_request`** parameter to the pull request number and +start the job; `pull_request_baseline`, `pull_request_config` and +`pull_request_node` say what it is compared against, what it tests and where. +None of the branch jobs run unless their own parameter is ticked as well. + +By hand it is two steps. The compiler is built from the merge ref - the pull +request as it would land, not the branch on its own - and the run fills a +`pr-` table like any other branch: + +```bash +git fetch --force https://github.com/OpenModelica/OpenModelica.git refs/pull//merge +git checkout -f --detach FETCH_HEAD +# build omc, then +./test.py --branch=pr- configs/conf.json +``` + +and the report compares that run against the newest run of `master`: + +```bash +./pr-report.py # --baseline=master by default +``` + +It writes `history/pr-/...html`, the same +kind of page as the nightly regression reports, next to `00_comment.md`, a +summary to comment on the pull request with. Both are published with the other +reports. + +Two things make a difference mean something other than "the pull request did +this", and the report says so when they apply: **the machine**, since two runs +produced on different hardware compare the hardware as much as the change, and +**the libraries**, since two runs that tested different library versions, or +verified against different reference files, differ for reasons of their own. The +baseline is also the newest `master` run rather than the commit the pull request +is based on, so a difference can come from anything merged since it was +branched - a reason to rebase before believing a surprising result. + +A full run takes days, so testing every pull request this way is not the idea; +point `--branch=pr-` at a smaller configuration file when the question is +narrower. + +The tables accumulate, about 19500 rows each. `drop-pr-tables.py` drops the ones +whose pull request has been merged or closed, and those tested more than +`--older-than` days ago, together with the rows their runs left in the other +tables; it lists them and does nothing unless it is given `--yes`. The reports +published for them are not touched. + ### Generate HTML results ```bash diff --git a/drop-pr-tables.py b/drop-pr-tables.py new file mode 100755 index 0000000..73e7c8d --- /dev/null +++ b/drop-pr-tables.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +"""Drop the result tables of pull requests that have been dealt with. + +Testing a pull request fills a pr- table like any other branch, about 19500 +rows and a few megabytes, and unlike a branch it is never written to again once +the pull request is merged or closed. This drops those tables, and the rows the +runs left in the other tables, for pull requests that are no longer open or that +were tested long enough ago not to matter. + +Nothing is dropped without --yes; without it the tables are only listed. +""" + +import argparse, datetime, json, os, re, time, urllib.error, urllib.request +import resultsdb + +parser = argparse.ArgumentParser(description='OpenModelica library testing pull request cleanup') +parser.add_argument('--repo', default="OpenModelica/OpenModelica", help='the repository the pull requests belong to') +parser.add_argument('--older-than', type=int, default=60, metavar='DAYS', + help='also drop a pull request still open whose newest run is older than this (0: never)') +parser.add_argument('--yes', action='store_true', help='actually drop them') +resultsdb.addArgument(parser) +args = parser.parse_args() + +prTableRe = re.compile(r"^pr-([0-9]+)$") + +db = resultsdb.connect(args.db) +cursor = db.cursor() + + +def pullRequestState(number): + """"open", "closed", "merged", or why we could not tell.""" + url = "https://api.github.com/repos/%s/pulls/%s" % (args.repo, number) + request = urllib.request.Request(url, headers={"Accept": "application/vnd.github+json"}) + # The rate limit is 60 requests an hour without one, which is enough for the + # handful of tables this looks at, but a token raises it and costs nothing. + token = os.environ.get("GITHUB_TOKEN") + if token: + request.add_header("Authorization", "Bearer %s" % token) + try: + pr = json.loads(urllib.request.urlopen(request).read().decode("utf-8")) + except urllib.error.HTTPError as e: + return "no answer from GitHub (%s)" % e + except Exception as e: + return "no answer from GitHub (%s)" % e + if pr.get("merged_at"): + return "merged" + return pr.get("state") or "unknown" + + +def newestRun(table): + (date,) = cursor.execute("SELECT max(date) FROM %s" % db.quote(table)).fetchone() + return date + + +def drop(table, branch): + """The table of a run, and everything the run wrote about itself elsewhere.""" + cursor.execute("DROP TABLE %s" % db.quote(table)) + for other in ["omcversion", "libversion", "history", "job_claim"]: + if db.tableExists(other): + cursor.execute("DELETE FROM %s WHERE branch=?" % db.quote(other), (branch,)) + db.commit() + + +tables = sorted((t for t in db.tables() if prTableRe.match(t)), + key=lambda t: int(prTableRe.match(t).group(1))) +if not tables: + raise SystemExit("No pull request tables in this database") + +dropping = [] +for table in tables: + number = prTableRe.match(table).group(1) + date = newestRun(table) + age = (time.time() - date) / 86400.0 if date else 0 + state = pullRequestState(number) + stale = args.older_than and date and age > args.older_than + why = None + if state in ("merged", "closed"): + why = "%s, tested %d days ago" % (state, age) + elif stale: + why = "still %s, but tested %d days ago" % (state, age) + print("%-12s %-8s %s%s" % (table, state, + "last run %s" % datetime.datetime.fromtimestamp(date) if date else "no runs", + ", dropping: %s" % why if why else "")) + if why: + dropping.append((table, why)) + +if not dropping: + raise SystemExit(0) +if not args.yes: + print("\n%d tables would be dropped; pass --yes to drop them" % len(dropping)) + raise SystemExit(0) +for (table, why) in dropping: + print("Dropping %s (%s)" % (table, why)) + drop(table, table) +print("The reports and files published for them are not touched; they are on the web server.") diff --git a/pr-report.py b/pr-report.py new file mode 100755 index 0000000..b6abd57 --- /dev/null +++ b/pr-report.py @@ -0,0 +1,346 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +"""Compare a pull request run against a run of another branch. + +all-reports.py reports a branch against its own previous run, which is what a +pull request must not do: pr- has no previous run, and the question is not +"what changed since yesterday" but "what does this pull request change against +master". The comparison itself is the same one - the phase a model reached and +what each phase cost - only the two runs it is given come from two branches. +""" + +import argparse, codecs, datetime, html, os, re, time +import shared, resultsdb +from omcommon import friendlyStr, multiple_replace + +parser = argparse.ArgumentParser(description='OpenModelica library testing pull request report') +parser.add_argument('pullrequest', help='the pull request number, or its branch name pr-') +parser.add_argument('--baseline', default="master", help='the branch the pull request is compared against') +parser.add_argument('--date', type=int, default=0, help='the pull request run to report on (default: its newest)') +parser.add_argument('--baselinedate', type=int, default=0, help='the baseline run to compare against (default: its newest)') +parser.add_argument('--baseurl', default="http://libraries.openmodelica.org/branches") +parser.add_argument('--historyurl', default="http://libraries.openmodelica.org/branches/history") +parser.add_argument('--historypath', default="history") +parser.add_argument('--githuburl', default="https://github.com/OpenModelica/OpenModelica") +parser.add_argument('--markdown', default="", help='where to write the summary to comment on the pull request with (default: //00_comment.md)') +resultsdb.addArgument(parser) +args = parser.parse_args() + +os.environ['TZ'] = 'Europe/Stockholm' +time.tzset() + +# The same thresholds all-reports.py uses, so that a change reported here means +# what it means in the nightly reports. +timeMinPhase = 4 # Need to have completed code generation to report performance regressions +timeRel = 1.7 # Minimum 1.7x time is registered as a performance regression +timeAbs = 10 # Ignore performance regressions for times <10s... + +PHASES = [(1,"frontend"),(2,"backend"),(3,"simcode"),(4,"templates"),(5,"compile"),(6,"simulate")] + +m = re.match(r"^(?:pr-)?([0-9]+)$", args.pullrequest.strip()) +if not m: + raise SystemExit("Expected a pull request number or a pr- branch name, got '%s'" % args.pullrequest) +pr = m.group(1) +branch = "pr-%s" % pr +baseline = args.baseline.split("/")[-1] +prurl = "%s/pull/%s" % (args.githuburl, pr) + +db = resultsdb.connect(args.db) +cursor = db.cursor() + +for tbl in [branch, baseline]: + if not db.tableExists(tbl): + raise SystemExit("No results for '%s'; run test.py --branch=%s first" % (tbl, tbl)) + db.createDateIndex(tbl) + + +def dateStr(dint): + return str(datetime.datetime.fromtimestamp(dint).strftime('%Y-%m-%d %H:%M:%S')) + +def newestRun(table, upto=0): + """The newest run of a table, or the newest at or before upto.""" + if upto: + (d,) = cursor.execute("SELECT max(date) FROM %s WHERE date<=?" % db.quote(table), (upto,)).fetchone() + else: + (d,) = cursor.execute("SELECT max(date) FROM %s" % db.quote(table)).fetchone() + return d + +def libraries(table, upto): + return set(l for (l,) in cursor.execute( + "SELECT DISTINCT libname FROM %s WHERE date<=?" % db.quote(table), (upto,))) + +def libraryRun(table, libname, upto): + """The newest run of one library at or before upto. + + A run does not necessarily hold every library: test.py skips a library whose + version, compiler and configuration were tested before, so its results stay at + the date of the run that produced them. Comparing two runs therefore means + comparing, per library, the newest run each of them has of it. + """ + (d,) = cursor.execute("SELECT max(date) FROM %s WHERE date<=? AND libname=?" + % db.quote(table), (upto, libname)).fetchone() + return d + +# A database written before the machine was recorded, #320, has no host column. +hostColumn = "host" if "host" in db.columns("libversion") else "NULL" + +def libraryVersions(table, libname, date): + """(libversion, confighash, host) of a library in a run.""" + row = cursor.execute( + "SELECT libversion,confighash,%s FROM libversion WHERE %s AND date=? AND libname=?" + % (hostColumn, db.likeNoCase("branch")), (table, date, libname)).fetchone() + return row if row else (None, None, None) + +def omcVersion(table, date): + row = cursor.execute("SELECT omcversion FROM omcversion WHERE %s AND date=?" + % db.likeNoCase("branch"), (table, date)).fetchone() + return row[0].strip() if row and row[0] else "unknown" + +def models(table, libname, date): + return set(mod for (mod,) in cursor.execute( + "SELECT model FROM %s WHERE date=? AND libname=?" % db.quote(table), (date, libname))) + +def changedModels(table1, date1, table2, date2, libnames): + """The models whose phase or timings differ between the two runs. + + One query per set of libraries that share a pair of dates, as all-reports.py + does. The values of both runs are aggregated into one string per column, + ordered by which run they come from rather than by their date: a pull request + run is normally the newer of the two, but need not be. + """ + concat = ",".join(db.groupConcat(c, "ord") for c in + ["finalphase","frontend","backend","simcode","templates","compile","simulate"]) + cols = "model,libname,finalphase,frontend,backend,simcode,templates,compile,simulate" + inlibs = ",".join("'%s'" % libname for libname in sorted(libnames)) + query = """SELECT model,libname,%s FROM + (SELECT * FROM + (SELECT %s,0 AS ord FROM %s WHERE date=? AND libname IN (%s) + UNION ALL + SELECT %s,1 AS ord FROM %s WHERE date=? AND libname IN (%s)) AS runs + ORDER BY ord) AS phases + GROUP BY model,libname HAVING + (MIN(finalphase) <> MAX(finalphase)) OR + (MIN(finalphase) >= ? AND ( + (MAX(frontend) > ?*MIN(frontend) AND MAX(frontend) > ?) OR + (MAX(backend) > ?*MIN(backend) AND MAX(backend) > ?) OR + (MAX(simcode) > ?*MIN(simcode) AND MAX(simcode) > ?) OR + (MAX(templates) > ?*MIN(templates) AND MAX(templates) > ?) OR + (MAX(compile) > ?*MIN(compile) AND MAX(compile) > ?) OR + (MAX(simulate) > ?*MIN(simulate) AND MAX(simulate) > ?))) + """ % (concat, cols, db.quote(table1), inlibs, cols, db.quote(table2), inlibs) + cursor.execute(query, (date1, date2, timeMinPhase, + timeRel, timeAbs, timeRel, timeAbs, timeRel, timeAbs, + timeRel, timeAbs, timeRel, 2*timeAbs, timeRel, timeAbs)) + return cursor.fetchall() + + +prdate = newestRun(branch, args.date) +if not prdate: + raise SystemExit("No results for %s%s" % (branch, " at or before %d" % args.date if args.date else "")) +basedate = newestRun(baseline, args.baselinedate) +if not basedate: + raise SystemExit("No results for %s%s" % (baseline, " at or before %d" % args.baselinedate if args.baselinedate else "")) + +prlibs = libraries(branch, prdate) +baselibs = libraries(baseline, basedate) +libnames = sorted(prlibs & baselibs) +if not libnames: + raise SystemExit("%s and %s have no library in common" % (branch, baseline)) + +# Per library the newest run each side has of it, grouped by the pair of dates +# so that one query covers every library that shares it. +groups = {} +prlibdates = {} +baselibdates = {} +for libname in libnames: + d1 = libraryRun(baseline, libname, basedate) + d2 = libraryRun(branch, libname, prdate) + baselibdates[libname] = d1 + prlibdates[libname] = d2 + groups.setdefault((d1, d2), []).append(libname) + +changes = [] +for ((d1, d2), libs) in sorted(groups.items()): + changes += changedModels(baseline, d1, branch, d2, libs) +changes = sorted(changes, key=lambda x: (x[1], x[0])) + +# Models one of the runs has and the other does not: a library that grew a model, +# or one whose test did not run at all. They cannot be compared, but leaving them +# out of the report without saying so would hide a library that failed to load. +onlyPr = [] +onlyBaseline = [] +numCompared = 0 +for libname in libnames: + inpr = models(branch, libname, prlibdates[libname]) + inbase = models(baseline, libname, baselibdates[libname]) + onlyPr += [(libname, mod) for mod in sorted(inpr - inbase)] + onlyBaseline += [(libname, mod) for mod in sorted(inbase - inpr)] + numCompared += len(inpr & inbase) + + +def modelLink(table, libname, modelname, extension, text): + return '%s' % ( + args.baseurl, table, libname, libname, modelname, extension, html.escape(text)) + +def libraryLink(table, libname): + return '%s' % (args.baseurl, table, libname, libname, html.escape(libname)) + +def classify(group, times): + """(colour, message) for a model, in the terms all-reports.py uses.""" + (phase1, phase2) = [int(i) for i in group.split(",")] + if phase2 != phase1: + better = phase2 > phase1 + return ("better" if better else "warning", + "%s → %s" % (shared.finalphaseName(phase1), shared.finalphaseName(phase2)), + "improved" if better else "regression") + msgs = [] + colour = None + for ((phase, name), values) in zip(PHASES, times): + (t1, t2) = [float(d) for d in values.split(",")] + limit = 2*timeAbs if name == "compile" else timeAbs + if t2 > timeRel*t1 and t2 > limit: + colour = "warningPerformance" + msgs.append("%s performance %s → %s" % (shared.finalphaseName(phase), friendlyStr(t1), friendlyStr(t2))) + elif t1 > timeRel*t2 and t1 > limit: + if colour is None: + colour = "betterPerformance" + msgs.append("%s performance %s → %s" % (shared.finalphaseName(phase), friendlyStr(t1), friendlyStr(t2))) + if colour is None: + # Both runs are below timeMinPhase, or the difference is under the threshold + # in the direction the query did not test for. + return (None, "", None) + return (colour, " ".join(msgs), + "performance improved" if colour == "betterPerformance" else "performance regression") + +counts = {"improved": 0, "regression": 0, "performance improved": 0, "performance regression": 0} +rows = [] +markdownrows = [] +for (model, libname, group, frontend, backend, simcode, templates, compile, simulate) in changes: + (colour, msg, kind) = classify(group, [frontend, backend, simcode, templates, compile, simulate]) + if kind is None: + continue + counts[kind] += 1 + rows.append('%s%s %s %s%s' + % (libraryLink(branch, libname), + modelLink(branch, libname, model, "err", model), + modelLink(branch, libname, model, "sim", "(sim)"), + modelLink(baseline, libname, model, "err", "(%s)" % baseline), + colour, msg)) + markdownrows.append((libname, model, msg.replace("→", "->"))) + +# What makes a difference mean something other than "the pull request did this". +caveats = [] +prhosts = set() +basehosts = set() +libchanges = [] +for libname in libnames: + (lv1, lh1, host1) = libraryVersions(baseline, libname, baselibdates[libname]) + (lv2, lh2, host2) = libraryVersions(branch, libname, prlibdates[libname]) + prhosts.add(host2 or "unknown") + basehosts.add(host1 or "unknown") + if (lv1 or "").strip() != (lv2 or "").strip(): + libchanges.append("%sVersion %s in %s, %s in %s" + % (libraryLink(branch, libname), html.escape((lv1 or "").strip()), baseline, + html.escape((lv2 or "").strip()), branch)) + elif lh1 != lh2: + libchanges.append("%sConfiguration hash (OMC settings, the testing script or a " + "reference file changed)" % libraryLink(branch, libname)) + +if prhosts != basehosts: + caveats.append("The two runs were produced on different machines (%s against %s), so the timings " + "compare the hardware as much as the pull request. The phases a model reaches are " + "still comparable." % (", ".join(sorted(prhosts)), ", ".join(sorted(basehosts)))) +if libchanges: + caveats.append("%d of the %d libraries were not tested in the same version, or not with the same " + "configuration and reference files, in the two runs; see Library Changes below." + % (len(libchanges), len(libnames))) +if prlibs - baselibs: + caveats.append("%d libraries of the pull request run have no counterpart in the baseline run: %s." + % (len(prlibs - baselibs), ", ".join(sorted(prlibs - baselibs)))) +if baselibs - prlibs: + caveats.append("%d libraries of the baseline run were not tested by the pull request run: %s." + % (len(baselibs - prlibs), ", ".join(sorted(baselibs - prlibs)))) +# Always true, and in the report itself rather than among the caveats. +note = ("The baseline is the newest run of %s, not the commit the pull request is based on, so a " + "difference can also come from something merged into %s since the pull request was " + "branched." % (baseline, baseline)) + +reportname = "%s..%s.html" % (dateStr(basedate), dateStr(prdate)) +historydir = os.path.join(args.historypath, branch) +os.makedirs(historydir, exist_ok=True) + +with open("pr.html.tpl") as fin: + tpl = fin.read() +tpl = multiple_replace(tpl, + ("#PRURL#", prurl), + ("#PR#", pr), + ("#BRANCH#", branch), + ("#BASELINE#", html.escape(baseline)), + ("#CAVEATS#", "\n".join('

%s

' % c for c in caveats)), + ("#DATE1#", dateStr(basedate)), + ("#DATE2#", dateStr(prdate)), + ("#OMCVERSION1#", html.escape(omcVersion(baseline, basedate))), + ("#OMCVERSION2#", html.escape(omcVersion(branch, prdate))), + ("#HOST1#", html.escape(", ".join(sorted(basehosts)))), + ("#HOST2#", html.escape(", ".join(sorted(prhosts)))), + ("#NUMCOMPARED#", str(numCompared)), + ("#NUMIMPROVE#", str(counts["improved"])), + ("#NUMREGRESSION#", str(counts["regression"])), + ("#NUMPERFIMPROVE#", str(counts["performance improved"])), + ("#NUMPERFREGRESSION#", str(counts["performance regression"])), + ("#NUMONLYPR#", str(len(onlyPr))), + ("#NUMONLYBASELINE#", str(len(onlyBaseline))), + ("#LIBCHANGES#", "\n".join(libchanges)), + ("#MODELCHANGES#", "\n".join(rows)), + ("#MODELSONLYINONE#", "\n".join( + '%s%s%s' % (html.escape(libname), html.escape(model), where) + for (where, lst) in [("only in %s" % branch, onlyPr), ("only in %s" % baseline, onlyBaseline)] + for (libname, model) in lst)), +) +with codecs.open(os.path.join(historydir, reportname), "w", encoding="utf-8") as fout: + fout.write(tpl) + +# The index the history directory of every branch has, so that the report is +# reachable from the server without knowing its name. +reporturl = "%s/%s/%s" % (args.historyurl, branch, reportname.replace(" ", "%20")) +summary = ("%d improved, %d regressions; performance %d improved, %d regressions" + % (counts["improved"], counts["regression"], + counts["performance improved"], counts["performance regression"])) +indexname = os.path.join(historydir, "00_history.html") +index = [] +if os.path.exists(indexname): + with codecs.open(indexname, "r", encoding="utf-8") as fin: + # A report of the same two runs is one that has just been overwritten, so + # its line in the index is replaced rather than repeated. + index = [line for line in fin.read().splitlines() if line.strip() and reporturl not in line] +entry = '

%s against %s %s %s

' % (reporturl, branch, baseline, reportname, summary) +with codecs.open(indexname, "w", encoding="utf-8") as fout: + fout.write("".join(line + "\n" for line in index + [entry])) + +markdown = ["## Library testing for [#%s](%s) against `%s`" % (pr, prurl, baseline), "", + "| | Branch | Run | Compiler | Machine |", + "| --- | --- | --- | --- | --- |", + "| Baseline | `%s` | %s | %s | %s |" % (baseline, dateStr(basedate), omcVersion(baseline, basedate), ", ".join(sorted(basehosts))), + "| Pull request | `%s` | %s | %s | %s |" % (branch, dateStr(prdate), omcVersion(branch, prdate), ", ".join(sorted(prhosts))), + "", + "%d models compared, **%d improved, %d regressions**, performance %d improved, %d regressions." + % (numCompared, counts["improved"], counts["regression"], + counts["performance improved"], counts["performance regression"]), + "", "[Full report](%s)" % reporturl, ""] +if markdownrows: + markdown += ["
%d models affected" % len(markdownrows), "", + "| Library | Model | Change |", "| --- | --- | --- |"] + markdown += ["| %s | %s | %s |" % (libname, model, msg) for (libname, model, msg) in markdownrows] + markdown += ["", "
", ""] +markdown += ["
Caveats", ""] +markdown += ["- %s" % c.replace("→", "->") for c in caveats + [note]] +markdown += ["", "
", "", "---", "Generated by the OpenModelica library testing"] +markdownname = args.markdown or os.path.join(historydir, "00_comment.md") +with codecs.open(markdownname, "w", encoding="utf-8") as fout: + fout.write("\n".join(markdown) + "\n") + +print("%s: %s" % (branch, summary)) +print("Report: %s" % os.path.join(historydir, reportname)) +print("Comment: %s" % markdownname) +print("Published as %s" % reporturl) diff --git a/pr.html.tpl b/pr.html.tpl new file mode 100644 index 0000000..34a5670 --- /dev/null +++ b/pr.html.tpl @@ -0,0 +1,64 @@ + + + + + OpenModelica pull request #PR# against #BASELINE# + + + +

OpenModelica pull request #PR# against #BASELINE#

+ +#CAVEATS# +

The baseline is the newest run of #BASELINE#, not the commit the pull request is +based on, so a difference can also come from something merged into #BASELINE# since +the pull request was branched.

+ +

The two runs

+ + + + + +
BranchRunCompilerMachine
Baseline#BASELINE##DATE1##OMCVERSION1##HOST1#
Pull request#BRANCH##DATE2##OMCVERSION2##HOST2#
+ +

Summary

+ + + + + + + + + +
Models compared#NUMCOMPARED#
Number of Improvements#NUMIMPROVE#
Number of Regressions#NUMREGRESSION#
Number of Performance Improvements#NUMPERFIMPROVE#
Number of Performance Regressions#NUMPERFREGRESSION#
Models only in the pull request run#NUMONLYPR#
Models only in the baseline run#NUMONLYBASELINE#
+ +

Library Changes

+ + +#LIBCHANGES# +
LibraryChange
+ +

Models Affected

+ + +#MODELCHANGES# +
LibraryModelChange
+ +

Models Only in One of the Runs

+ + +#MODELSONLYINONE# +
LibraryModelWhere
+ + + diff --git a/resultsdb.py b/resultsdb.py index e709038..92773ab 100644 --- a/resultsdb.py +++ b/resultsdb.py @@ -163,6 +163,10 @@ def claimedBy(self, branch, libname, libversion, omcversion, confighash): def release(self): """Mark the claims of this run as finished.""" + def columns(self, table): + """The columns a table has, for the ones an older database may not have.""" + raise NotImplementedError + def vacuum(self): self.conn.execute("VACUUM") @@ -266,6 +270,9 @@ def addLibversionHost(self, cursor): def tables(self): return [t for (t,) in self.conn.execute("SELECT name FROM sqlite_master WHERE type='table'")] + def columns(self, table): + return [r[1] for r in self.conn.execute("PRAGMA table_info(%s)" % self.quote(table))] + def tableExists(self, name): return self.conn.execute( "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (name,)).fetchone() is not None @@ -437,6 +444,11 @@ def tables(self): return [t for (t,) in self.execute( "SELECT tablename FROM pg_tables WHERE schemaname=current_schema()")] + def columns(self, table): + return [c for (c,) in self.execute( + "SELECT column_name FROM information_schema.columns " + "WHERE table_schema=current_schema() AND table_name=?", (table,))] + def tableExists(self, name): return self.execute("SELECT 1 FROM pg_tables WHERE schemaname=current_schema() AND tablename=?", (name,)).fetchone() is not None