Keep the index of the generated reports in the database (#307)

.CI/Jenkinsfile

@@ -458,9 +458,6 @@ pipeline {
         PGPASSFILE = credentials('omdb-pgpass')
       }
       steps {
-        //sshagent (credentials: ['Hudson-SSH-Key']) {
-        //  createInitialHistoryFilesOnRemote()
-        //}
         sh 'rm -rf *.html history'
         sh '''
         if ! test -d OpenModelica; then
@@ -578,26 +575,6 @@ def installLibraries(boolean removePackageOrder, boolean conversionScript, name,
 }
 
 
-def createInitialHistoryFilesOnRemote() {
-  sh """
-HISTORY_DIRECTORY='/var/www/libraries.openmodelica.org/branches/history/'
-SEPARATOR='/'
-HISTORY_FILE='00_history.html'
-
-for b in ${env.GITBRANCHES} ${env.GITBRANCHES_FMI} ${env.GITBRANCHES_NEWINST} ${env.GITBRANCHES_DAE} ${env.GITBRANCHES_CPP} ${env.GITBRANCHES_WASM_JIT}; do
- BRANCH=`echo \${b} | cut -d '/' -f2`
- FILE="\${HISTORY_DIRECTORY}\${BRANCH}\${SEPARATOR}\${HISTORY_FILE}"
-ssh hudson@build.openmodelica.org << EOF
-if [ ! -f \${FILE} ]; then
-  echo "Creating initial history: library.openmodelica.org: \${FILE}"
-  mkdir -p \${HISTORY_DIRECTORY}\${BRANCH}
-  touch \${FILE}
-fi
-EOF
-done
-"""
-}
-
 /**
   * `docker run` flags capping the container's cgroup at 90% of the node's RAM. test.py limits one
   * test at a time, not the sum of the parallel ones. --memory-swap has to repeat the limit; docker

all-plots.py

@@ -26,7 +26,7 @@
 
 libs = {}
 
-import cgi, time, datetime
+import time, datetime
 from omcommon import friendlyStr, multiple_replace
 
 db = resultsdb.connect(args.db)

all-reports.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
-import urllib.request
+import urllib.request, urllib.error, urllib.parse
 import codecs
 import sys, argparse, subprocess, os, time
 import simplejson as json
@@ -45,10 +45,11 @@
 
 libs = {}
 
-import cgi, time, datetime
+import time, datetime
 from omcommon import friendlyStr, multiple_replace
 
 db = resultsdb.connect(args.db)
+db.createHistoryTable()
 cursor = db.cursor()
 
 def dateStr(dint):
@@ -68,6 +69,160 @@ def libraryLink(branch, libname):
 def modelLink(libname, modelname, extension, text):
   return '<a href="%s/%s/%s/files/%s_%s.%s">%s</a>' % (baseurl,branch,libname,libname,modelname,extension,text)
 
+# 00_history.html, the index of the reports generated for a branch, lives on the
+# web server next to them, and used to be the only record of what had already
+# been reported: all-reports.py read it back over HTTP and skipped the branch
+# when it could not, because starting from an empty index would have published a
+# history with only today's report in it. A branch that had never been published
+# had no index either, so its first run reported nothing until the file had been
+# created on the server by hand.
+#
+# The [history] table now holds the same list, one row per report, so the index
+# is a rendering of the database rather than the record itself. What the server
+# has is still read - it is where the reports generated before the table existed
+# are, and they are copied into it - but it is no longer needed to decide what
+# has been reported, and a file that is missing, unreadable or out of date is
+# rebuilt from the table instead of truncating the history.
+#
+# That leaves one case with nothing to go on: no rows in the table and no index
+# to read, which is either a genuinely new branch or a server that is unwell.
+# Two things have to agree before it is taken for a new branch, and anything
+# else leaves it alone until a human has looked at it:
+#
+#  - the history root is served, so the site is up and the index is a missing
+#    file rather than a broken deployment or something else answering;
+#  - the database holds at most FIRSTREPORT runs of the branch, so the report
+#    about to be generated is its first and none can have been lost.
+FIRSTREPORT = 2
+
+entryRe = re.compile(r'^<p><a href="[^"]*/(?P<fname>[^/"]+)">[^<]*</a> '
+                     r'(?P<improved>\d+) improved, (?P<regressions>\d+) regressions; '
+                     r'performance (?P<perfimproved>\d+) improved, '
+                     r'(?P<perfregressions>\d+) regressions</p>$')
+fnameRe = re.compile(r'^(.+)[.][.](.+)[.]html$')
+
+def epochOf(datestr):
+  return int(time.mktime(datetime.datetime.strptime(datestr, "%Y-%m-%d %H:%M:%S").timetuple()))
+
+def parseIndex(text):
+  """The reports an index lists, and whatever else it holds, kept verbatim."""
+  entries = []
+  preamble = []
+  for line in text.splitlines():
+    if not line.strip():
+      continue
+    m = entryRe.match(line.strip())
+    dates = fnameRe.match(urllib.parse.unquote(m.group("fname"))) if m else None
+    try:
+      (d1, d2) = (epochOf(dates.group(1)), epochOf(dates.group(2))) if dates else (None, None)
+    except ValueError:
+      (d1, d2) = (None, None)
+    if d1 is None:
+      preamble.append(line)
+      continue
+    entries.append((d1, d2, urllib.parse.unquote(m.group("fname")),
+                    int(m.group("improved")), int(m.group("regressions")),
+                    int(m.group("perfimproved")), int(m.group("perfregressions"))))
+  return (entries, preamble)
+
+def renderEntry(branch, entry):
+  """One line of an index; the same line the report generator has always written."""
+  (d1, d2, fname, improved, regressions, perfimproved, perfregressions) = entry
+  return ('<p><a href="%s/%s/%s">%s %s</a> %d improved, %d regressions; '
+          'performance %d improved, %d regressions</p>'
+          % (historyurl, branch, fname.replace(" ", "%20"), branch, fname,
+             improved, regressions, perfimproved, perfregressions))
+
+def renderIndex(branch, entries, preamble):
+  return "".join(line + "\n" for line in preamble + [renderEntry(branch, e) for e in entries])
+
+def storedEntries(branch):
+  """The reports the database has for a branch, oldest first."""
+  return [tuple(row) for row in cursor.execute(
+      "SELECT date1,date2,fname,improved,regressions,perfimproved,perfregressions "
+      "FROM history WHERE %s ORDER BY date1,date2" % db.likeNoCase("branch"), (branch,))]
+
+def storeEntries(branch, entries):
+  for entry in entries:
+    cursor.execute("INSERT INTO history (branch,date1,date2,fname,improved,regressions,"
+                   "perfimproved,perfregressions) VALUES (?,?,?,?,?,?,?,?)", (branch,) + entry)
+  db.commit()
+
+historyRootServed = None
+
+def historyRootIsServed():
+  """Is the server actually serving the history tree right now?"""
+  global historyRootServed
+  if historyRootServed is None:
+    url = historyurl if historyurl.endswith("/") else historyurl + "/"
+    try:
+      urllib.request.urlopen(url).read()
+      historyRootServed = True
+    except Exception as e:
+      print("%s could not be read (%s)" % (url, e))
+      historyRootServed = False
+  return historyRootServed
+
+def readPublishedIndex(branch):
+  """The index published for a branch, or None when it could not be read."""
+  url = "%s/%s/00_history.html" % (historyurl, branch)
+  try:
+    return urllib.request.urlopen(url).read().decode('utf-8')
+  except urllib.error.HTTPError as e:
+    if e.code == 404:
+      print("%s is not there" % url)
+    else:
+      print("%s failed to open: %s" % (url, e))
+    return None
+  except Exception as e:
+    print("%s failed to open: %s" % (url, e))
+    return None
+
+def historyOf(branch, nruns):
+  """What has been reported for a branch: its entries, anything else its index
+  holds, and the index as published, or None to leave the branch alone.
+
+  The database decides; the published index is read to fill it in with the
+  reports that predate it, and to say whether the two are in step.
+  """
+  historyindex = "history/%s/00_history.html" % branch
+  if os.path.exists(historyindex):
+    # Already started in this workspace, either by an earlier invocation or
+    # because the branch was named twice; that copy is the newer one.
+    with codecs.open(historyindex, "r", encoding="utf-8") as fin:
+      published = fin.read()
+  else:
+    published = readPublishedIndex(branch)
+  stored = storedEntries(branch)
+  if published is None:
+    if stored:
+      print("Rebuilding the index of %s from the %d reports in the database"
+            % (branch, len(stored)))
+      return (stored, [], None)
+    if not historyRootIsServed():
+      print("Neither the database nor the history root knows about %s; leaving it alone" % branch)
+      return None
+    if nruns > FIRSTREPORT:
+      print("%s has no index and no reports in the database although it has %d runs; "
+            "leaving it alone rather than publishing a history with only the newest "
+            "report in it" % (branch, nruns))
+      return None
+    print("Starting a new history for %s" % branch)
+    return ([], [], None)
+  (entries, preamble) = parseIndex(published)
+  known = set((e[0], e[1]) for e in stored)
+  missing = [e for e in entries if (e[0], e[1]) not in known]
+  if missing:
+    print("Copying %d reports of %s from its index into the database" % (len(missing), branch))
+    storeEntries(branch, missing)
+    stored = sorted(stored + missing)
+  inindex = set((e[0], e[1]) for e in entries)
+  onlyStored = [e for e in stored if (e[0], e[1]) not in inindex]
+  if onlyStored:
+    print("%d reports of %s are in the database but not in its index, which is "
+          "rewritten with all of them" % (len(onlyStored), branch))
+  return (stored, preamble, published)
+
 missing_branches = []
 emails_to_send = {}
 for branch in branches:
@@ -91,24 +246,22 @@ def modelLink(libname, modelname, extension, text):
   cursor.execute("SELECT date,omcversion FROM omcversion WHERE %s ORDER BY date ASC" % db.likeNoCase("branch"), (branch,))
   entries = cursor.fetchall()
   n = len(entries)
-  urlToOpen = "%s/%s/00_history.html" % (historyurl, branch)
-  try:
-    urlContents = urllib.request.urlopen(urlToOpen).read().decode('utf-8')
-  except:
-    print(urlToOpen + " failed to open")
+  historydir = "history/%s" % branch
+  historyindex = "%s/00_history.html" % historydir
+  history = historyOf(branch, n)
+  if history is None:
     missing_branches.append(branch)
     continue
-
-  generated = False
+  (reports, preamble, published) = history
+  reported = set((d1, d2) for (d1, d2, _, _, _, _, _) in reports)
 
   for i in range(1,n):
     d1 = entries[i-1][0]
     d2 = entries[i][0]
-    fname = "history/%s/%s..%s.html" % (branch,dateStr(d1),dateStr(d2))
-    if fname.replace(" ","%20") in urlContents or fname in urlContents:
+    if (d1, d2) in reported:
       continue
+    fname = "history/%s/%s..%s.html" % (branch,dateStr(d1),dateStr(d2))
     print("Generate %s" % fname)
-    generated = True
     v1 = getTagOrVersion(entries[i-1][1])
     v2 = getTagOrVersion(entries[i][1])
     thirdPartyChanged = ""
@@ -245,24 +398,31 @@ def modelLink(libname, modelname, extension, text):
         libstrs.append("<tr><td>%s</td><td>Configuration hash (OMC settings or the testing script changed)</td></tr>" % libraryLink(branch, libname))
     tpl = tpl.replace("#LIBCHANGES#","\n".join(libstrs)).replace("#NUMLIBS#",str(len(libstrs)))
 
-    email_summary_html = '<p><a href="%s/%s/%s">%s %s</a> %d improved, %d regressions; performance %d improved, %d regressions</p>' % (historyurl, branch, os.path.basename(fname).replace(" ","%20"), branch, os.path.basename(fname),numImproved,numRegression,numPerformanceImproved,numPerformanceRegression)
+    entry = (d1, d2, os.path.basename(fname), numImproved, numRegression,
+             numPerformanceImproved, numPerformanceRegression)
+    email_summary_html = renderEntry(branch, entry)
     email_summary_plain = '%s/%s/%s: %d improved, %d regressions; performance %d improved, %d regressions</p>' % (historyurl, branch, os.path.basename(fname).replace(" ","%20"), numImproved, numRegression, numPerformanceImproved, numPerformanceRegression)
     if sum([numImproved,numRegression,numPerformanceImproved,numPerformanceRegression])>0:
       for email in emails_current:
         if email not in emails_to_send:
           emails_to_send[email] = {"plain":[],"html":[]}
         emails_to_send[email]["plain"].append(email_summary_plain)
         emails_to_send[email]["html"].append(email_summary_html)
-    if not os.path.exists(os.path.dirname(fname)):
-      os.makedirs(os.path.dirname(fname))
+    os.makedirs(historydir, exist_ok=True)
     with codecs.open(fname, "w", encoding="utf-8") as fout:
       fout.write(tpl)
-    if not os.path.exists(os.path.dirname("history/%s" % branch)):
-      os.makedirs("history/%s" % branch)
-    urlContents = urlContents + email_summary_html + "\n"
-  if generated:
-    with open("history/%s/00_history.html" % branch, "w") as fout:
-      fout.write(urlContents)
+    storeEntries(branch, [entry])
+    reports = sorted(reports + [entry])
+
+  # The index is written whenever it does not already say what the database
+  # says, which covers the reports generated just now, an index that went
+  # missing or lost entries, and a branch that has none at all: publishing the
+  # directory is what creates it on the server.
+  index = renderIndex(branch, reports, preamble)
+  if published is None or index != published:
+    os.makedirs(historydir, exist_ok=True)
+    with codecs.open(historyindex, "w", encoding="utf-8") as fout:
+      fout.write(index)
 
 if not doemail:
   # We are done

clean-dates.py

@@ -34,7 +34,7 @@
 db = resultsdb.connect(args.db)
 cursor = db.cursor()
 
-tables = db.tables()
+tables = [tbl for tbl in db.tables() if tbl not in resultsdb.NON_RESULT_TABLES]
 for tbl in tables:
   cursor.execute("DELETE FROM %s WHERE date<? AND date>?" % db.quote(tbl), (stopTime.timestamp(),startTime.timestamp()))
 db.commit()

doc/README.md

@@ -155,6 +155,45 @@ WHERE libversion=? AND libname=? AND branch=? AND omcversion=? AND confighash=?
 and skips the library when the exact same combination of library version, OMC
 version and configuration was already tested.
 
+### `history`
+
+The regression reports `all-reports.py` has generated, one row per pair of runs
+of a branch:
+
+```sql
+CREATE TABLE IF NOT EXISTS history (
+  branch          text    NOT NULL,  -- branch/configuration name = result table name
+  date1           integer NOT NULL,  -- the older of the two runs compared
+  date2           integer NOT NULL,  -- the newer one
+  fname           text,              -- the report file, "<date1>..<date2>.html"
+  improved        integer,           -- models that reached a later phase than before
+  regressions     integer,           -- models that reached an earlier one
+  perfimproved    integer,           -- models that got faster by more than the threshold
+  perfregressions integer,           -- models that got slower
+  PRIMARY KEY (branch, date1, date2)
+)
+```
+
+The reports are published at
+`libraries.openmodelica.org/branches/history/<branch>/`, with an index,
+`00_history.html`, listing one line per report. That index used to be the only
+record of what had already been reported: `all-reports.py` read it back over
+HTTP and skipped the branch when it could not, since starting from an empty one
+would have published a history with only the newest report in it.
+
+This table holds the same list, so the index is a rendering of the database
+rather than the record itself. A branch that has no index yet gets one - which
+is what a per-pull-request branch needs, [#307][307] - and an index that is
+missing, unreadable or has lost entries is rebuilt from here rather than
+truncated. The published index is still read: it is where the reports generated
+before the table existed are, and they are copied into it the first time a
+branch is reported on.
+
+The table is created on demand by `all-reports.py`, and holds no results, so
+`clean-dates.py` leaves it alone (`resultsdb.NON_RESULT_TABLES`).
+
+[307]: https://github.com/OpenModelica/OpenModelicaLibraryTesting/issues/307
+
 ### `datelookup_<branch>` (obsolete)
 
 `datelookup_<branch>(date, runDate, libname, branch)` was a cache mapping every
@@ -185,15 +224,16 @@ separate files - whichever file is copied back last wins.
 ## Housekeeping scripts
 
 - `clean-dates.py --start --stop`: `DELETE FROM [<tbl>] WHERE date<? AND date>?`
-  over every table, then `VACUUM`. Removes a range of bad runs.
+  over every table that holds results, then `VACUUM`. Removes a range of bad
+  runs. `history` and `job_claim` are skipped; they have no `date` column.
 - `clean-empty-omcversion-dates.py`: drops `omcversion` rows whose date has no
   result rows in the corresponding branch table.
 
 ## PostgreSQL layout
 
 The PostgreSQL database is a **mirror** of the sqlite3 one: the same tables with
-the same names and columns, one table per branch plus `omcversion` and
-`libversion`. That way the test scripts can push new results to the network
+the same names and columns, one table per branch plus `omcversion`,
+`libversion` and `history`. That way the test scripts can push new results to the network
 database with the same statements they use today, and the report scripts need no
 query rewriting beyond the sqlite `[name]` / PostgreSQL `"name"` quoting.