diff --git a/overview.html.tpl b/overview.html.tpl index b62305c..e5188af 100644 --- a/overview.html.tpl +++ b/overview.html.tpl @@ -20,7 +20,7 @@

Tested branches

- + #branches#
BranchVersionBuild timeExecution time# Simulate# Total
BranchVersionBuild timeExecution timeSimulation time# Simulate# Verify# Total
#entries# diff --git a/report.py b/report.py index 580d4c0..00e3568 100755 --- a/report.py +++ b/report.py @@ -38,7 +38,9 @@ nmodels = {} nsimulate = {} +nverify = {} exectime = {} +simtime = {} missing_branches = [] for branch in branches: @@ -77,7 +79,9 @@ branch_nmodels += 1 nmodels[branch] = branch_nmodels nsimulate[branch] = 0 + nverify[branch] = 0 exectime[branch] = 0.0 + simtime[branch] = 0.0 entries = "" # removing missing branches @@ -131,6 +135,7 @@ def checkEqual(iterator): entries += '%s%s' % (' class="warning"' if old_vs and old_vs[i]>vs[i] else (' class="better"' if old_vs and old_vs[i]%s' % diff_text) if diff_text else "") entries += "" nsimulate[branch] += vs[6] + nverify[branch] += vs[7] if old_vs: old_vs = [max(vs[i], old_vs[i]) for i in range(0, len(vs))] else: @@ -144,15 +149,17 @@ def checkEqual(iterator): entries += '%s' % (branch,lib,lib,branch) entries += ("%s%s%s%s%s%s%s%s%s\n" % (friendlyStr(sums[0]),friendlyStr(sums[1]),friendlyStr(sums[2]),friendlyStr(sums[3]),friendlyStr(sums[4]),friendlyStr(sums[5]),friendlyStr(sums[6]),friendlyStr(sums[7]),friendlyStr(sums[8]))) exectime[branch] += sums[0] + simtime[branch] += sums[7] entries += "\n" # print(sorted(list(libs[lib]))) nummodels = sum(len(l) for l in libs.values()) -branches_lines = [("%s%s%s%s%d%d\n" % (html.escape(branch), html.escape( +branches_lines = [("%s%s%s%s%s%d%d%d\n" % (html.escape(branch), html.escape( (cursor.execute("SELECT omcversion FROM omcversion WHERE date=? AND branch=?", (max(dates[branch][lib] for lib in libnames),branch)).fetchone() or ["unknown"])[0] - ), html.escape(dates_str[branch]), friendlyStr(exectime[branch]), + ), html.escape(dates_str[branch]), friendlyStr(exectime[branch]), friendlyStr(simtime[branch]), " class=\"warning\"" if nummodels!=nmodels[branch] else "", nsimulate[branch], + nverify[branch], nmodels[branch] )) for branch in branches] template = open("overview.html.tpl").read() diff --git a/test.py b/test.py index e7f3600..32f397e 100755 --- a/test.py +++ b/test.py @@ -991,6 +991,26 @@ def phaseWithoutSimulator(data): """ return min(data.get("phase") or 0, BUILD_PHASE) +def simulatedTime(simulated): + """The wall clock simulating and verifying once cost. + + An older result file has no simwall and only says what the tool spent. + """ + sim = simulated.get("simwall") + if sim is None: + sim = simulated.get("sim") or 0.0 + return sim + ((simulated.get("diff") or {}).get("time") or 0.0) + +def runnerExecTime(data, simulated): + """The wall clock one runner of an FMU or of an artifact cost. + + exectime is the whole run of the model, and all but the simulations is shared + by its runners, so each reports the shared part plus only its own simulation. + """ + runners = [data] + list((data.get("simulators") or {}).values()) + shared_time = (data.get("exectime") or 0.0) - sum(simulatedTime(r) for r in runners) + return max(0.0, shared_time) + simulatedTime(simulated) + def resultValues(model, libname, data, simulator=None): """One row of a branch table. @@ -1005,7 +1025,7 @@ def resultValues(model, libname, data, simulator=None): return (testRunStartTimeAsEpoch, libname, model, - data.get("exectime") or 0.0, + runnerExecTime(data, simulated), data.get("frontend") or 0.0, data.get("backend") or 0.0, data.get("simcode") or 0.0, diff --git a/testmodel.py b/testmodel.py index d236325..5ab34b9 100755 --- a/testmodel.py +++ b/testmodel.py @@ -236,6 +236,7 @@ def target(res): "templates":None, "build":None, "sim":None, + "simwall":None, # Wall clock; sim is what the tool says it spent "simcold":None, "diff":None, "phase":0, @@ -779,9 +780,11 @@ def simElapsed(): else: res = checkOutputTimeout("(rm -f %s.pipe ; mkfifo %s.pipe ; head -c 1048576 < %s.pipe >> %s & %s > %s.pipe 2>&1)" % (conf["fileName"],conf["fileName"],conf["fileName"],simFile,cmd,conf["fileName"]), conf["ulimitExe"], conf) execstat["sim"] = simElapsed() + execstat["simwall"] = monotonic()-start execstat["phase"] = 6 except TimeoutError as e: execstat["sim"] = monotonic()-start + execstat["simwall"] = execstat["sim"] # checkOutputTimeout raises TimeoutError for a command that fails as well as # for one that runs out of time, so this covers both. if len(runners) > 1: @@ -917,7 +920,7 @@ def verifyAgainstReference(resFile, prefix, stat): # results down with it and leaves the others alone - the first one included, # see the TimeoutError handler above. for (name, command) in runners[1:]: - stat = {"sim": None, "diff": None, "phase": 5} + stat = {"sim": None, "simwall": None, "diff": None, "phase": 5} simulators[name] = stat simFileOther = os.path.abspath("../files/%s_%s.sim" % (conf["fileName"], name)).replace('\\','/') other = resultFile(name) @@ -926,16 +929,19 @@ def verifyAgainstReference(resFile, prefix, stat): if useArtifact: res = simulateArtifact(name, command, other, simFileOther) stat["sim"] = res.get("timeSimulation") or (monotonic()-start) + stat["simwall"] = monotonic()-start if not res.get("resultFile"): writeResult() continue else: simulateFmu(name, command, other, simFileOther) stat["sim"] = monotonic()-start + stat["simwall"] = stat["sim"] stat["phase"] = 6 verifyAgainstReference(other, artifactPrefix(name) + ".diff", stat) except TimeoutError as e: stat["sim"] = monotonic()-start + stat["simwall"] = stat["sim"] with open(errFile, 'a+') as fp: fp.write("%s timed out simulating the %s\n" % (name, "artifact" if useArtifact else "FMU")) writeResult()