Mark models a library no longer has as removed

all-plots.py

@@ -114,11 +114,11 @@ def plotLibrary(branch, libname, xs, total, frontend,backend,simcode,template,co
 
   db.createDateIndex(branch)
   libs = {}
-  for (date,libname,total,frontend,backend,simcode,template,compile,simulate,verify) in cursor.execute("""SELECT date,libname,COUNT(finalphase),%s
+  for (date,libname,total,frontend,backend,simcode,template,compile,simulate,verify) in cursor.execute("""SELECT date,libname,%s
     FROM %s
     GROUP BY date,libname
     ORDER BY libname,date ASC
-""" % (",".join(db.countIf("finalphase>=%d" % i) for i in range(1,8)), db.quote(branch))):
+""" % (",".join(db.countIf("finalphase>=%d" % i) for i in range(0,8)), db.quote(branch))):
     if libname not in libs:
       libs[libname] = ([],[],[],[],[],[],[],[],[])
     libs[libname][0].append(datetime.datetime.fromtimestamp(date))

all-reports.py

@@ -328,6 +328,7 @@ def historyOf(branch, nruns):
       query = ("""SELECT model,libname,%s FROM
     (SELECT model,libname,date,finalphase,frontend,backend,simcode,templates,compile,simulate FROM %%s WHERE date IN (?,?) AND libname IN (%%s) ORDER BY date) AS phases
   GROUP BY model,libname HAVING""" % concat + """
+    MIN(finalphase) >= 0 AND (
     (MIN(finalphase) <> MAX(finalphase)) OR
     ((MIN(finalphase) >= ?) AND
       (MAX(frontend) > ?*MIN(frontend) AND MAX(frontend) > ?) OR
@@ -336,7 +337,7 @@ def historyOf(branch, nruns):
       (MAX(templates) > ?*MIN(templates) AND MAX(templates) > ?) OR
       (MAX(compile) > ?*MIN(compile) AND MAX(compile) > ?) OR
       (MAX(simulate) > ?*MIN(simulate) AND MAX(simulate) > ?)
-    )
+    ))
   """) % (db.quote(branch),",".join(["'%s'" % libname for libname in startdates[d1lib]]))
       cursor.execute(query, (d1lib,d2,timeMinPhase,timeRel,timeAbs,timeRel,timeAbs,timeRel,timeAbs,timeRel,timeAbs,timeRel,2*timeAbs,timeRel,timeAbs))
       regressions += cursor.fetchall()

doc/README.md

@@ -98,6 +98,7 @@ Notes on the values, which are produced by `testmodel.py` and written by
 
 | value | name | meaning |
 | --- | --- | --- |
+| -1 | Removed | the library no longer has that model |
 | 0 | Failed | the front end did not finish |
 | 1 | FrontEnd | front end ok, back end failed |
 | 2 | BackEnd | back end ok, SimCode failed |
@@ -108,7 +109,18 @@ Notes on the values, which are produced by `testmodel.py` and written by
 | 7 | Verify | the result matches the reference file |
 
 Reports count models per phase with `WHERE finalphase >= i`, so the columns of
-the HTML tables are cumulative.
+the HTML tables are cumulative, and phase -1 falls outside all of them.
+
+A run writes a phase -1 row for every model the previous run of that library had
+and it no longer finds, so that a model dropped from a library stops being
+reported once the run that lost it is the newest one. Without it a library whose
+models all lose their `experiment` annotation - Physiomodel did in 2021 - keeps
+a newest run in this table from years ago, and is reported forever with the
+models of that run. The rows are written once, not at every run: the previous
+run of an empty library is the one holding the removals, which have no models
+left to remove. A library that fails to load has no models either, which is why
+`test.py` refuses to run at all when a `loadModel` fails rather than treating it
+as a library that lost every model.
 
 ### `omcversion`
 

pr-report.py

@@ -102,7 +102,8 @@ def omcVersion(table, date):
 
 def models(table, libname, date):
   return set(mod for (mod,) in cursor.execute(
-      "SELECT model FROM %s WHERE date=? AND libname=?" % db.quote(table), (date, libname)))
+      "SELECT model FROM %s WHERE date=? AND libname=? AND finalphase>=0" % db.quote(table),
+      (date, libname)))
 
 def changedModels(table1, date1, table2, date2, libnames):
   """The models whose phase or timings differ between the two runs.
@@ -123,14 +124,15 @@ def changedModels(table1, date1, table2, date2, libnames):
        SELECT %s,1 AS ord FROM %s WHERE date=? AND libname IN (%s)) AS runs
      ORDER BY ord) AS phases
   GROUP BY model,libname HAVING
+    MIN(finalphase) >= 0 AND (
     (MIN(finalphase) <> MAX(finalphase)) OR
     (MIN(finalphase) >= ? AND (
       (MAX(frontend) > ?*MIN(frontend) AND MAX(frontend) > ?) OR
       (MAX(backend) > ?*MIN(backend) AND MAX(backend) > ?) OR
       (MAX(simcode) > ?*MIN(simcode) AND MAX(simcode) > ?) OR
       (MAX(templates) > ?*MIN(templates) AND MAX(templates) > ?) OR
       (MAX(compile) > ?*MIN(compile) AND MAX(compile) > ?) OR
-      (MAX(simulate) > ?*MIN(simulate) AND MAX(simulate) > ?)))
+      (MAX(simulate) > ?*MIN(simulate) AND MAX(simulate) > ?))))
   """ % (concat, cols, db.quote(table1), inlibs, cols, db.quote(table2), inlibs)
   cursor.execute(query, (date1, date2, timeMinPhase,
                          timeRel, timeAbs, timeRel, timeAbs, timeRel, timeAbs,

report.py

@@ -70,7 +70,7 @@
       dates[branch][libname] = 0
       continue
     dates[branch][libname] = v[0]
-    for x in cursor.execute("SELECT model FROM %s WHERE libname=? AND date=?" % db.quote(branch), (libname,v[0])):
+    for x in cursor.execute("SELECT model FROM %s WHERE libname=? AND date=? AND finalphase>=0" % db.quote(branch), (libname,v[0])):
       if libname not in libs:
         libs[libname] = set()
       libs[libname].add(x[0])

shared.py

@@ -73,7 +73,12 @@ def libname(library, conf):
     return library+"_"+conf["libraryVersionNameForTests"] if conf["libraryVersionNameForTests"] else library
   return library+("_"+conf["libraryVersion"] if conf["libraryVersion"]!="default" else "")+(("_" + conf["configExtraName"]) if "configExtraName" in conf else "")
 
+# A model the run no longer found in its library; the reports ask for >= 0.
+DELETED_PHASE = -1
+
 def finalphaseName(finalphase):
+  if finalphase == DELETED_PHASE:
+    return "Removed"
   return ("Failed","FrontEnd","BackEnd","SimCode","Templates","Compile","Simulate","Verify")[finalphase]
 
 def getReferenceFileName(conf):

test.py

@@ -594,8 +594,11 @@ def hashReferenceFiles(s):
 
 stats_by_libname = {}
 skipped_libs = {}
+# A library that did not load cannot be told apart from one with no models left.
+failedToLoad = []
 tests=[]
 for (library,conf) in configs:
+  loadFailed = False
   # Only when asked, so a normal run's confighash is unchanged
   if args.nobuildmodel:
     conf["noBuildModel"] = True
@@ -657,6 +660,7 @@ def hashReferenceFiles(s):
           print("Failed to run command %s: %s" % (command,omc.sendExpression('OpenModelica.Scripting.getErrorString()')))
         except:
           print("Failed to run command %s OpenModelica.Scripting.getErrorString() failed..." % command)
+        loadFailed = True
     librariesToLoad = []
   else:
     librariesToLoad = [[library,conf["libraryVersion"]]] + conf.get("extraLibraries", [])
@@ -683,6 +687,11 @@ def hashReferenceFiles(s):
         print("Failed to load library %s %s: %s" % (library,versions,omc.sendExpression('OpenModelica.Scripting.getErrorString()')))
       except:
         print("Failed to load library %s %s. OpenModelica.Scripting.getErrorString() failed..." % (library,conf["libraryVersion"]))
+      loadFailed = True
+  if loadFailed:
+    failedToLoad.append(shared.libname(library, conf))
+    continue
+
   # adrpo: do not sort the top level names as sometimes that loads a bad MSL version
   # conf["loadFiles"] = sorted(omc.sendExpression("{getSourceFile(cl) for cl in getClassNames()}"))
   conf["loadFiles"] = omc.sendExpression("{getSourceFile(cl) for cl in getClassNames()}")
@@ -775,6 +784,10 @@ def hashReferenceFiles(s):
 except:
   pass
 
+if failedToLoad:
+  db.release()
+  raise SystemExit("Failed to load: %s" % ", ".join(failedToLoad))
+
 print("Checked which libraries to run")
 sys.stdout.flush()
 
@@ -879,7 +892,7 @@ def expectedExec(c):
   (model,lib,libName,name,data) = c
   if "expectedExec" in data:
     return data["expectedExec"]
-  cursor.execute("SELECT exectime FROM %s WHERE libname = ? AND model = ? ORDER BY date DESC LIMIT 1" % db.quote(primaryBranch), (libName,model))
+  cursor.execute("SELECT exectime FROM %s WHERE libname = ? AND model = ? AND finalphase >= 0 ORDER BY date DESC LIMIT 1" % db.quote(primaryBranch), (libName,model))
   v = cursor.fetchone()
   data["expectedExec"] = (v or (0.0,))[0]
   return data["expectedExec"]
@@ -906,7 +919,7 @@ def expectedExec(c):
 
 numberOfTests = len(tests)
 
-if numberOfTests==0:
+if numberOfTests==0 and not stats_by_libname:
   print("Everything already up to date. Not executing any tests.")
   sys.exit(0)
 
@@ -993,6 +1006,23 @@ def resultValues(model, libname, data, simulator=None):
     data.get("parsing") or 0.0
   )
 
+def removedModels(resultBranch, libname, tested):
+  """The models the previous run of that library had and this one no longer finds.
+
+  Once written, the removals are that library's newest rows, so a library that
+  stays empty is marked once rather than at every run.
+  """
+  previous = cursor.execute("""SELECT model FROM %s WHERE libname=? AND finalphase>=0
+      AND date=(SELECT MAX(date) FROM %s WHERE libname=? AND date<?)"""
+      % (db.quote(resultBranch), db.quote(resultBranch)),
+      (libname, libname, testRunStartTimeAsEpoch)).fetchall()
+  return sorted(set(model for (model,) in previous) - tested)
+
+def removedValues(model, libname):
+  """One row of a branch table saying the library no longer has that model."""
+  return (testRunStartTimeAsEpoch, libname, model,
+          0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0, 0, shared.DELETED_PHASE, 0.0)
+
 def cpu_name():
   if isWin:
     return processor()
@@ -1013,6 +1043,10 @@ def cpu_name():
 hostname = resultsdb.hostname()
 sysInfo = "%s: %s, %d GB RAM, %s%s" % (hostname, cpu_name(), int(math.ceil(psutil.virtual_memory().total / (1024.0**3))), ("Docker " + docker + " ") if docker else "", lsb_release)
 
+testedModels = dict((libname, set()) for libname in stats_by_libname)
+for (name,model,libname,data) in stats.values():
+  testedModels[libname].add(model)
+
 for (resultBranch, simulator) in resultBranches:
   db.createTables(resultBranch)
   for key in stats.keys():
@@ -1022,6 +1056,10 @@ def cpu_name():
     cursor.execute("INSERT INTO %s VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)%s" % (db.quote(resultBranch), db.insertIgnore()),
                    resultValues(model, libname, data, simulator))
   for libname in stats_by_libname.keys():
+    for model in removedModels(resultBranch, libname, testedModels[libname]):
+      cursor.execute("INSERT INTO %s VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)%s"
+                     % (db.quote(resultBranch), db.insertIgnore()),
+                     removedValues(model, libname))
     confighash = stats_by_libname[libname]["conf"]["confighash"]
     cursor.execute("INSERT INTO libversion VALUES (?,?,?,?,?,?,?)%s" % db.insertIgnore(), (testRunStartTimeAsEpoch, resultBranch, libname, stats_by_libname[libname]["conf"]["libraryLastChange"], confighash, hostname, sysInfo))
   cursor.execute("INSERT INTO omcversion VALUES (?,?,?)%s" % db.insertIgnore(), (testRunStartTimeAsEpoch, resultBranch, omc_version))