diff --git a/overview.html.tpl b/overview.html.tpl
index b62305c..e5188af 100644
--- a/overview.html.tpl
+++ b/overview.html.tpl
@@ -20,7 +20,7 @@
Tested branches
-| Branch | Version | Build time | Execution time | # Simulate | # Total |
+| Branch | Version | Build time | Execution time | Simulation time | # Simulate | # Verify | # Total |
#branches#
#entries#
diff --git a/report.py b/report.py
index 580d4c0..00e3568 100755
--- a/report.py
+++ b/report.py
@@ -38,7 +38,9 @@
nmodels = {}
nsimulate = {}
+nverify = {}
exectime = {}
+simtime = {}
missing_branches = []
for branch in branches:
@@ -77,7 +79,9 @@
branch_nmodels += 1
nmodels[branch] = branch_nmodels
nsimulate[branch] = 0
+ nverify[branch] = 0
exectime[branch] = 0.0
+ simtime[branch] = 0.0
entries = ""
# removing missing branches
@@ -131,6 +135,7 @@ def checkEqual(iterator):
entries += '%s%s | ' % (' class="warning"' if old_vs and old_vs[i]>vs[i] else (' class="better"' if old_vs and old_vs[i]%s' % diff_text) if diff_text else "")
entries += ""
nsimulate[branch] += vs[6]
+ nverify[branch] += vs[7]
if old_vs:
old_vs = [max(vs[i], old_vs[i]) for i in range(0, len(vs))]
else:
@@ -144,15 +149,17 @@ def checkEqual(iterator):
entries += '| %s | ' % (branch,lib,lib,branch)
entries += ("%s | %s | %s | %s | %s | %s | %s | %s | %s |
\n" % (friendlyStr(sums[0]),friendlyStr(sums[1]),friendlyStr(sums[2]),friendlyStr(sums[3]),friendlyStr(sums[4]),friendlyStr(sums[5]),friendlyStr(sums[6]),friendlyStr(sums[7]),friendlyStr(sums[8])))
exectime[branch] += sums[0]
+ simtime[branch] += sums[7]
entries += "\n"
# print(sorted(list(libs[lib])))
nummodels = sum(len(l) for l in libs.values())
-branches_lines = [("| %s | %s | %s | %s | %d | %d |
\n" % (html.escape(branch), html.escape(
+branches_lines = [("| %s | %s | %s | %s | %s | %d | %d | %d |
\n" % (html.escape(branch), html.escape(
(cursor.execute("SELECT omcversion FROM omcversion WHERE date=? AND branch=?", (max(dates[branch][lib] for lib in libnames),branch)).fetchone() or ["unknown"])[0]
- ), html.escape(dates_str[branch]), friendlyStr(exectime[branch]),
+ ), html.escape(dates_str[branch]), friendlyStr(exectime[branch]), friendlyStr(simtime[branch]),
" class=\"warning\"" if nummodels!=nmodels[branch] else "",
nsimulate[branch],
+ nverify[branch],
nmodels[branch]
)) for branch in branches]
template = open("overview.html.tpl").read()
diff --git a/test.py b/test.py
index e7f3600..32f397e 100755
--- a/test.py
+++ b/test.py
@@ -991,6 +991,26 @@ def phaseWithoutSimulator(data):
"""
return min(data.get("phase") or 0, BUILD_PHASE)
+def simulatedTime(simulated):
+ """The wall clock simulating and verifying once cost.
+
+ An older result file has no simwall and only says what the tool spent.
+ """
+ sim = simulated.get("simwall")
+ if sim is None:
+ sim = simulated.get("sim") or 0.0
+ return sim + ((simulated.get("diff") or {}).get("time") or 0.0)
+
+def runnerExecTime(data, simulated):
+ """The wall clock one runner of an FMU or of an artifact cost.
+
+ exectime is the whole run of the model, and all but the simulations is shared
+ by its runners, so each reports the shared part plus only its own simulation.
+ """
+ runners = [data] + list((data.get("simulators") or {}).values())
+ shared_time = (data.get("exectime") or 0.0) - sum(simulatedTime(r) for r in runners)
+ return max(0.0, shared_time) + simulatedTime(simulated)
+
def resultValues(model, libname, data, simulator=None):
"""One row of a branch table.
@@ -1005,7 +1025,7 @@ def resultValues(model, libname, data, simulator=None):
return (testRunStartTimeAsEpoch,
libname,
model,
- data.get("exectime") or 0.0,
+ runnerExecTime(data, simulated),
data.get("frontend") or 0.0,
data.get("backend") or 0.0,
data.get("simcode") or 0.0,
diff --git a/testmodel.py b/testmodel.py
index d236325..5ab34b9 100755
--- a/testmodel.py
+++ b/testmodel.py
@@ -236,6 +236,7 @@ def target(res):
"templates":None,
"build":None,
"sim":None,
+ "simwall":None, # Wall clock; sim is what the tool says it spent
"simcold":None,
"diff":None,
"phase":0,
@@ -779,9 +780,11 @@ def simElapsed():
else:
res = checkOutputTimeout("(rm -f %s.pipe ; mkfifo %s.pipe ; head -c 1048576 < %s.pipe >> %s & %s > %s.pipe 2>&1)" % (conf["fileName"],conf["fileName"],conf["fileName"],simFile,cmd,conf["fileName"]), conf["ulimitExe"], conf)
execstat["sim"] = simElapsed()
+ execstat["simwall"] = monotonic()-start
execstat["phase"] = 6
except TimeoutError as e:
execstat["sim"] = monotonic()-start
+ execstat["simwall"] = execstat["sim"]
# checkOutputTimeout raises TimeoutError for a command that fails as well as
# for one that runs out of time, so this covers both.
if len(runners) > 1:
@@ -917,7 +920,7 @@ def verifyAgainstReference(resFile, prefix, stat):
# results down with it and leaves the others alone - the first one included,
# see the TimeoutError handler above.
for (name, command) in runners[1:]:
- stat = {"sim": None, "diff": None, "phase": 5}
+ stat = {"sim": None, "simwall": None, "diff": None, "phase": 5}
simulators[name] = stat
simFileOther = os.path.abspath("../files/%s_%s.sim" % (conf["fileName"], name)).replace('\\','/')
other = resultFile(name)
@@ -926,16 +929,19 @@ def verifyAgainstReference(resFile, prefix, stat):
if useArtifact:
res = simulateArtifact(name, command, other, simFileOther)
stat["sim"] = res.get("timeSimulation") or (monotonic()-start)
+ stat["simwall"] = monotonic()-start
if not res.get("resultFile"):
writeResult()
continue
else:
simulateFmu(name, command, other, simFileOther)
stat["sim"] = monotonic()-start
+ stat["simwall"] = stat["sim"]
stat["phase"] = 6
verifyAgainstReference(other, artifactPrefix(name) + ".diff", stat)
except TimeoutError as e:
stat["sim"] = monotonic()-start
+ stat["simwall"] = stat["sim"]
with open(errFile, 'a+') as fp:
fp.write("%s timed out simulating the %s\n" % (name, "artifact" if useArtifact else "FMU"))
writeResult()