diff --git a/.CI/Jenkinsfile b/.CI/Jenkinsfile index c12abc4..4ba59bc 100644 --- a/.CI/Jenkinsfile +++ b/.CI/Jenkinsfile @@ -458,9 +458,6 @@ pipeline { PGPASSFILE = credentials('omdb-pgpass') } steps { - //sshagent (credentials: ['Hudson-SSH-Key']) { - // createInitialHistoryFilesOnRemote() - //} sh 'rm -rf *.html history' sh ''' if ! test -d OpenModelica; then @@ -578,26 +575,6 @@ def installLibraries(boolean removePackageOrder, boolean conversionScript, name, } -def createInitialHistoryFilesOnRemote() { - sh """ -HISTORY_DIRECTORY='/var/www/libraries.openmodelica.org/branches/history/' -SEPARATOR='/' -HISTORY_FILE='00_history.html' - -for b in ${env.GITBRANCHES} ${env.GITBRANCHES_FMI} ${env.GITBRANCHES_NEWINST} ${env.GITBRANCHES_DAE} ${env.GITBRANCHES_CPP} ${env.GITBRANCHES_WASM_JIT}; do - BRANCH=`echo \${b} | cut -d '/' -f2` - FILE="\${HISTORY_DIRECTORY}\${BRANCH}\${SEPARATOR}\${HISTORY_FILE}" -ssh hudson@build.openmodelica.org << EOF -if [ ! -f \${FILE} ]; then - echo "Creating initial history: library.openmodelica.org: \${FILE}" - mkdir -p \${HISTORY_DIRECTORY}\${BRANCH} - touch \${FILE} -fi -EOF -done -""" -} - /** * `docker run` flags capping the container's cgroup at 90% of the node's RAM. test.py limits one * test at a time, not the sum of the parallel ones. --memory-swap has to repeat the limit; docker diff --git a/all-plots.py b/all-plots.py index e7c8026..b990c9c 100755 --- a/all-plots.py +++ b/all-plots.py @@ -26,7 +26,7 @@ libs = {} -import cgi, time, datetime +import time, datetime from omcommon import friendlyStr, multiple_replace db = resultsdb.connect(args.db) diff --git a/all-reports.py b/all-reports.py index c100d42..66f1443 100755 --- a/all-reports.py +++ b/all-reports.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -import urllib.request +import urllib.request, urllib.error, urllib.parse import codecs import sys, argparse, subprocess, os, time import simplejson as json @@ -45,10 +45,11 @@ libs = {} -import cgi, time, datetime +import time, datetime from omcommon import friendlyStr, multiple_replace db = resultsdb.connect(args.db) +db.createHistoryTable() cursor = db.cursor() def dateStr(dint): @@ -68,6 +69,160 @@ def libraryLink(branch, libname): def modelLink(libname, modelname, extension, text): return '%s' % (baseurl,branch,libname,libname,modelname,extension,text) +# 00_history.html, the index of the reports generated for a branch, lives on the +# web server next to them, and used to be the only record of what had already +# been reported: all-reports.py read it back over HTTP and skipped the branch +# when it could not, because starting from an empty index would have published a +# history with only today's report in it. A branch that had never been published +# had no index either, so its first run reported nothing until the file had been +# created on the server by hand. +# +# The [history] table now holds the same list, one row per report, so the index +# is a rendering of the database rather than the record itself. What the server +# has is still read - it is where the reports generated before the table existed +# are, and they are copied into it - but it is no longer needed to decide what +# has been reported, and a file that is missing, unreadable or out of date is +# rebuilt from the table instead of truncating the history. +# +# That leaves one case with nothing to go on: no rows in the table and no index +# to read, which is either a genuinely new branch or a server that is unwell. +# Two things have to agree before it is taken for a new branch, and anything +# else leaves it alone until a human has looked at it: +# +# - the history root is served, so the site is up and the index is a missing +# file rather than a broken deployment or something else answering; +# - the database holds at most FIRSTREPORT runs of the branch, so the report +# about to be generated is its first and none can have been lost. +FIRSTREPORT = 2 + +entryRe = re.compile(r'^

[^/"]+)">[^<]* ' + r'(?P\d+) improved, (?P\d+) regressions; ' + r'performance (?P\d+) improved, ' + r'(?P\d+) regressions

$') +fnameRe = re.compile(r'^(.+)[.][.](.+)[.]html$') + +def epochOf(datestr): + return int(time.mktime(datetime.datetime.strptime(datestr, "%Y-%m-%d %H:%M:%S").timetuple())) + +def parseIndex(text): + """The reports an index lists, and whatever else it holds, kept verbatim.""" + entries = [] + preamble = [] + for line in text.splitlines(): + if not line.strip(): + continue + m = entryRe.match(line.strip()) + dates = fnameRe.match(urllib.parse.unquote(m.group("fname"))) if m else None + try: + (d1, d2) = (epochOf(dates.group(1)), epochOf(dates.group(2))) if dates else (None, None) + except ValueError: + (d1, d2) = (None, None) + if d1 is None: + preamble.append(line) + continue + entries.append((d1, d2, urllib.parse.unquote(m.group("fname")), + int(m.group("improved")), int(m.group("regressions")), + int(m.group("perfimproved")), int(m.group("perfregressions")))) + return (entries, preamble) + +def renderEntry(branch, entry): + """One line of an index; the same line the report generator has always written.""" + (d1, d2, fname, improved, regressions, perfimproved, perfregressions) = entry + return ('

%s %s %d improved, %d regressions; ' + 'performance %d improved, %d regressions

' + % (historyurl, branch, fname.replace(" ", "%20"), branch, fname, + improved, regressions, perfimproved, perfregressions)) + +def renderIndex(branch, entries, preamble): + return "".join(line + "\n" for line in preamble + [renderEntry(branch, e) for e in entries]) + +def storedEntries(branch): + """The reports the database has for a branch, oldest first.""" + return [tuple(row) for row in cursor.execute( + "SELECT date1,date2,fname,improved,regressions,perfimproved,perfregressions " + "FROM history WHERE %s ORDER BY date1,date2" % db.likeNoCase("branch"), (branch,))] + +def storeEntries(branch, entries): + for entry in entries: + cursor.execute("INSERT INTO history (branch,date1,date2,fname,improved,regressions," + "perfimproved,perfregressions) VALUES (?,?,?,?,?,?,?,?)", (branch,) + entry) + db.commit() + +historyRootServed = None + +def historyRootIsServed(): + """Is the server actually serving the history tree right now?""" + global historyRootServed + if historyRootServed is None: + url = historyurl if historyurl.endswith("/") else historyurl + "/" + try: + urllib.request.urlopen(url).read() + historyRootServed = True + except Exception as e: + print("%s could not be read (%s)" % (url, e)) + historyRootServed = False + return historyRootServed + +def readPublishedIndex(branch): + """The index published for a branch, or None when it could not be read.""" + url = "%s/%s/00_history.html" % (historyurl, branch) + try: + return urllib.request.urlopen(url).read().decode('utf-8') + except urllib.error.HTTPError as e: + if e.code == 404: + print("%s is not there" % url) + else: + print("%s failed to open: %s" % (url, e)) + return None + except Exception as e: + print("%s failed to open: %s" % (url, e)) + return None + +def historyOf(branch, nruns): + """What has been reported for a branch: its entries, anything else its index + holds, and the index as published, or None to leave the branch alone. + + The database decides; the published index is read to fill it in with the + reports that predate it, and to say whether the two are in step. + """ + historyindex = "history/%s/00_history.html" % branch + if os.path.exists(historyindex): + # Already started in this workspace, either by an earlier invocation or + # because the branch was named twice; that copy is the newer one. + with codecs.open(historyindex, "r", encoding="utf-8") as fin: + published = fin.read() + else: + published = readPublishedIndex(branch) + stored = storedEntries(branch) + if published is None: + if stored: + print("Rebuilding the index of %s from the %d reports in the database" + % (branch, len(stored))) + return (stored, [], None) + if not historyRootIsServed(): + print("Neither the database nor the history root knows about %s; leaving it alone" % branch) + return None + if nruns > FIRSTREPORT: + print("%s has no index and no reports in the database although it has %d runs; " + "leaving it alone rather than publishing a history with only the newest " + "report in it" % (branch, nruns)) + return None + print("Starting a new history for %s" % branch) + return ([], [], None) + (entries, preamble) = parseIndex(published) + known = set((e[0], e[1]) for e in stored) + missing = [e for e in entries if (e[0], e[1]) not in known] + if missing: + print("Copying %d reports of %s from its index into the database" % (len(missing), branch)) + storeEntries(branch, missing) + stored = sorted(stored + missing) + inindex = set((e[0], e[1]) for e in entries) + onlyStored = [e for e in stored if (e[0], e[1]) not in inindex] + if onlyStored: + print("%d reports of %s are in the database but not in its index, which is " + "rewritten with all of them" % (len(onlyStored), branch)) + return (stored, preamble, published) + missing_branches = [] emails_to_send = {} for branch in branches: @@ -91,24 +246,22 @@ def modelLink(libname, modelname, extension, text): cursor.execute("SELECT date,omcversion FROM omcversion WHERE %s ORDER BY date ASC" % db.likeNoCase("branch"), (branch,)) entries = cursor.fetchall() n = len(entries) - urlToOpen = "%s/%s/00_history.html" % (historyurl, branch) - try: - urlContents = urllib.request.urlopen(urlToOpen).read().decode('utf-8') - except: - print(urlToOpen + " failed to open") + historydir = "history/%s" % branch + historyindex = "%s/00_history.html" % historydir + history = historyOf(branch, n) + if history is None: missing_branches.append(branch) continue - - generated = False + (reports, preamble, published) = history + reported = set((d1, d2) for (d1, d2, _, _, _, _, _) in reports) for i in range(1,n): d1 = entries[i-1][0] d2 = entries[i][0] - fname = "history/%s/%s..%s.html" % (branch,dateStr(d1),dateStr(d2)) - if fname.replace(" ","%20") in urlContents or fname in urlContents: + if (d1, d2) in reported: continue + fname = "history/%s/%s..%s.html" % (branch,dateStr(d1),dateStr(d2)) print("Generate %s" % fname) - generated = True v1 = getTagOrVersion(entries[i-1][1]) v2 = getTagOrVersion(entries[i][1]) thirdPartyChanged = "" @@ -245,7 +398,9 @@ def modelLink(libname, modelname, extension, text): libstrs.append("%sConfiguration hash (OMC settings or the testing script changed)" % libraryLink(branch, libname)) tpl = tpl.replace("#LIBCHANGES#","\n".join(libstrs)).replace("#NUMLIBS#",str(len(libstrs))) - email_summary_html = '

%s %s %d improved, %d regressions; performance %d improved, %d regressions

' % (historyurl, branch, os.path.basename(fname).replace(" ","%20"), branch, os.path.basename(fname),numImproved,numRegression,numPerformanceImproved,numPerformanceRegression) + entry = (d1, d2, os.path.basename(fname), numImproved, numRegression, + numPerformanceImproved, numPerformanceRegression) + email_summary_html = renderEntry(branch, entry) email_summary_plain = '%s/%s/%s: %d improved, %d regressions; performance %d improved, %d regressions

' % (historyurl, branch, os.path.basename(fname).replace(" ","%20"), numImproved, numRegression, numPerformanceImproved, numPerformanceRegression) if sum([numImproved,numRegression,numPerformanceImproved,numPerformanceRegression])>0: for email in emails_current: @@ -253,16 +408,21 @@ def modelLink(libname, modelname, extension, text): emails_to_send[email] = {"plain":[],"html":[]} emails_to_send[email]["plain"].append(email_summary_plain) emails_to_send[email]["html"].append(email_summary_html) - if not os.path.exists(os.path.dirname(fname)): - os.makedirs(os.path.dirname(fname)) + os.makedirs(historydir, exist_ok=True) with codecs.open(fname, "w", encoding="utf-8") as fout: fout.write(tpl) - if not os.path.exists(os.path.dirname("history/%s" % branch)): - os.makedirs("history/%s" % branch) - urlContents = urlContents + email_summary_html + "\n" - if generated: - with open("history/%s/00_history.html" % branch, "w") as fout: - fout.write(urlContents) + storeEntries(branch, [entry]) + reports = sorted(reports + [entry]) + + # The index is written whenever it does not already say what the database + # says, which covers the reports generated just now, an index that went + # missing or lost entries, and a branch that has none at all: publishing the + # directory is what creates it on the server. + index = renderIndex(branch, reports, preamble) + if published is None or index != published: + os.makedirs(historydir, exist_ok=True) + with codecs.open(historyindex, "w", encoding="utf-8") as fout: + fout.write(index) if not doemail: # We are done diff --git a/clean-dates.py b/clean-dates.py index 09b831e..1f7e08e 100755 --- a/clean-dates.py +++ b/clean-dates.py @@ -34,7 +34,7 @@ db = resultsdb.connect(args.db) cursor = db.cursor() -tables = db.tables() +tables = [tbl for tbl in db.tables() if tbl not in resultsdb.NON_RESULT_TABLES] for tbl in tables: cursor.execute("DELETE FROM %s WHERE date?" % db.quote(tbl), (stopTime.timestamp(),startTime.timestamp())) db.commit() diff --git a/doc/README.md b/doc/README.md index 5d908a6..578692f 100644 --- a/doc/README.md +++ b/doc/README.md @@ -155,6 +155,45 @@ WHERE libversion=? AND libname=? AND branch=? AND omcversion=? AND confighash=? and skips the library when the exact same combination of library version, OMC version and configuration was already tested. +### `history` + +The regression reports `all-reports.py` has generated, one row per pair of runs +of a branch: + +```sql +CREATE TABLE IF NOT EXISTS history ( + branch text NOT NULL, -- branch/configuration name = result table name + date1 integer NOT NULL, -- the older of the two runs compared + date2 integer NOT NULL, -- the newer one + fname text, -- the report file, "...html" + improved integer, -- models that reached a later phase than before + regressions integer, -- models that reached an earlier one + perfimproved integer, -- models that got faster by more than the threshold + perfregressions integer, -- models that got slower + PRIMARY KEY (branch, date1, date2) +) +``` + +The reports are published at +`libraries.openmodelica.org/branches/history//`, with an index, +`00_history.html`, listing one line per report. That index used to be the only +record of what had already been reported: `all-reports.py` read it back over +HTTP and skipped the branch when it could not, since starting from an empty one +would have published a history with only the newest report in it. + +This table holds the same list, so the index is a rendering of the database +rather than the record itself. A branch that has no index yet gets one - which +is what a per-pull-request branch needs, [#307][307] - and an index that is +missing, unreadable or has lost entries is rebuilt from here rather than +truncated. The published index is still read: it is where the reports generated +before the table existed are, and they are copied into it the first time a +branch is reported on. + +The table is created on demand by `all-reports.py`, and holds no results, so +`clean-dates.py` leaves it alone (`resultsdb.NON_RESULT_TABLES`). + +[307]: https://github.com/OpenModelica/OpenModelicaLibraryTesting/issues/307 + ### `datelookup_` (obsolete) `datelookup_(date, runDate, libname, branch)` was a cache mapping every @@ -185,15 +224,16 @@ separate files - whichever file is copied back last wins. ## Housekeeping scripts - `clean-dates.py --start --stop`: `DELETE FROM [] WHERE date?` - over every table, then `VACUUM`. Removes a range of bad runs. + over every table that holds results, then `VACUUM`. Removes a range of bad + runs. `history` and `job_claim` are skipped; they have no `date` column. - `clean-empty-omcversion-dates.py`: drops `omcversion` rows whose date has no result rows in the corresponding branch table. ## PostgreSQL layout The PostgreSQL database is a **mirror** of the sqlite3 one: the same tables with -the same names and columns, one table per branch plus `omcversion` and -`libversion`. That way the test scripts can push new results to the network +the same names and columns, one table per branch plus `omcversion`, +`libversion` and `history`. That way the test scripts can push new results to the network database with the same statements they use today, and the report scripts need no query rewriting beyond the sqlite `[name]` / PostgreSQL `"name"` quoting. diff --git a/resultsdb.py b/resultsdb.py index e709038..f632bba 100644 --- a/resultsdb.py +++ b/resultsdb.py @@ -42,6 +42,23 @@ SQLITE_TYPES = {"bigint": "integer", "int": "integer", "real": "real", "text": "text"} POSTGRES_TYPES = {"bigint": "bigint", "int": "integer", "real": "double precision", "text": "text"} +# The regression reports all-reports.py has generated, one row per pair of runs +# of a branch. The index published beside them, 00_history.html, lists the same +# reports, so it can be rebuilt from here when the published one is missing, +# unreadable or out of date - and a run that cannot read it back from the web +# server no longer has to choose between skipping the branch and publishing a +# history with only today's report in it. +HISTORY_COLUMNS = [ + ("branch", "text"), ("date1", "bigint"), ("date2", "bigint"), ("fname", "text"), + ("improved", "int"), ("regressions", "int"), + ("perfimproved", "int"), ("perfregressions", "int"), +] +HISTORY_KEY = ["branch", "date1", "date2"] + +# The tables that hold no results of a test run, and that the housekeeping +# scripts must not treat as one: they have no date column to clean up by. +NON_RESULT_TABLES = ["history", "job_claim"] + # What identifies a row, so that two machines writing the same shared table # cannot store the same result twice. Mirrors sqlite2postgres.py. KEYS = { @@ -145,6 +162,14 @@ def insertIgnore(self): """The clause that makes an INSERT skip a row that is already there.""" return "" + def createHistoryTable(self): + """The table of generated reports, created by whoever needs it first.""" + cols = ", ".join("%s %s%s" % (c, self.types[t], " NOT NULL" if c in HISTORY_KEY else "") + for (c, t) in HISTORY_COLUMNS) + self.execute("CREATE TABLE IF NOT EXISTS history (%s, PRIMARY KEY (%s))" + % (cols, ", ".join(HISTORY_KEY))) + self.commit() + def createDateIndex(self, branch): """The index the report queries need; test.py drops it before a run.""" self.execute("CREATE INDEX IF NOT EXISTS %s ON %s (date)" @@ -207,6 +232,7 @@ class _Sqlite(_Db): """The per-machine sqlite3 file the testing has always used.""" name = "sqlite3" + types = SQLITE_TYPES def __init__(self, path): self.conn = sqlite3.connect(path) @@ -292,6 +318,7 @@ class _Postgres(_Db): """The shared database several test machines write to at the same time.""" name = "postgresql" + types = POSTGRES_TYPES def __init__(self, url): try: