Add option --with-score to summary.py to automatically classify speedups

Legend for new column is:
* FASTER - strong speedup
* faster - spedup is detected but it is unreliable
* <empty> - no change in speed
* slower - slowdown is detected but it is unreliable
* SLOWER - strong slowdown

modules/ts/misc/summary.py

@@ -37,7 +37,8 @@ if __name__ == "__main__":
     parser.add_option("", "--module", dest="module", default=None, metavar="NAME", help="module prefix for test names")
     parser.add_option("", "--columns", dest="columns", default=None, metavar="NAMES", help="comma-separated list of column aliases")
     parser.add_option("", "--no-relatives", action="store_false", dest="calc_relatives", default=True, help="do not output relative values")
-    parser.add_option("", "--with-cycles-reduction", action="store_true", dest="calc_cr", default=False, help="alos output cycle reduction percentages")
+    parser.add_option("", "--with-cycles-reduction", action="store_true", dest="calc_cr", default=False, help="output cycle reduction percentages")
+    parser.add_option("", "--with-score", action="store_true", dest="calc_score", default=False, help="output automatic classification of speedups")
     parser.add_option("", "--show-all", action="store_true", dest="showall", default=False, help="also include empty and \"notrun\" lines")
     parser.add_option("", "--match", dest="match", default=None)
     parser.add_option("", "--match-replace", dest="match_replace", default="")
@@ -106,6 +107,7 @@ if __name__ == "__main__":
 
     # build table
     getter = metrix_table[options.metric][1]
+    getter_score = metrix_table["score"][1]
     if options.calc_relatives:
         getter_p = metrix_table[options.metric + "%"][1]
     if options.calc_cr:
@@ -129,6 +131,11 @@ if __name__ == "__main__":
         for set in metric_sets:
             tbl.newColumn(str(i) + "%", getSetName(set, i, options.columns) + "\nvs\n" + getSetName(test_sets[0], 0, options.columns) + "\n(x-factor)", align = "center", cssclass = "col_rel")
             i += 1
+    if options.calc_score:
+        i = 1
+        for set in metric_sets:
+            tbl.newColumn(str(i) + "S", getSetName(set, i, options.columns) + "\nvs\n" + getSetName(test_sets[0], 0, options.columns) + "\n(score)", align = "center", cssclass = "col_name")
+            i += 1
 
     # rows
     prevGroupName = None
@@ -157,6 +164,8 @@ if __name__ == "__main__":
                     tbl.newCell(str(i) + "%", "-")
                 if options.calc_cr and i > 0:
                     tbl.newCell(str(i) + "$", "-")
+                if options.calc_score and i > 0:
+                    tbl.newCell(str(i) + "$", "-")
             else:
                 status = case.get("status")
                 if status != "run":
@@ -167,6 +176,8 @@ if __name__ == "__main__":
                         tbl.newCell(str(i) + "%", "-", color = "red")
                     if options.calc_cr and i > 0:
                         tbl.newCell(str(i) + "$", "-", color = "red")
+                    if options.calc_score and i > 0:
+                        tbl.newCell(str(i) + "S", "-", color = "red")
                 else:
                     val = getter(case, cases[0], options.units)
                     if options.calc_relatives and i > 0 and val:
@@ -177,6 +188,10 @@ if __name__ == "__main__":
                         valcr = getter_cr(case, cases[0], options.units)
                     else:
                         valcr = None
+                    if options.calc_score and i > 0 and val:
+                        val_score = getter_score(case, cases[0], options.units)
+                    else:
+                        val_score = None
                     if not valp or i == 0:
                         color = None
                     elif valp > 1.05:
@@ -192,6 +207,8 @@ if __name__ == "__main__":
                         tbl.newCell(str(i) + "%", formatValue(valp, "%"), valp, color = color, bold = color)
                     if options.calc_cr and i > 0:
                         tbl.newCell(str(i) + "$", formatValue(valcr, "$"), valcr, color = color, bold = color)
+                    if options.calc_score and i > 0:
+                        tbl.newCell(str(i) + "S", formatValue(val_score, "S"), val_score, color = color, bold = color)
     if not needNewRow:
         tbl.trimLastRow()
 

modules/ts/misc/table_formatter.py

 #!/usr/bin/env python
 
-import sys, re, os.path, cgi, stat
+import sys, re, os.path, cgi, stat, math
 from optparse import OptionParser
 from color import getColorizer
 
@@ -627,6 +627,21 @@ def getCycleReduction(test, test0, metric):
         return None
     return (1.0-float(val)/val0)*100
 
+def getScore(test, test0, metric):
+    if not test or not test0:
+        return None
+    m0 = float(test.get("gmean", None))
+    m1 = float(test0.get("gmean", None))
+    if m0 == 0 or m1 == 0:
+        return None
+    s0 = float(test.get("gstddev", None))
+    s1 = float(test0.get("gstddev", None))
+    s = math.sqrt(s0*s0 + s1*s1)
+    m0 = math.log(m0)
+    m1 = math.log(m1)
+    if s == 0:
+        return None
+    return (m0-m1)/s
 
 metrix_table = \
 {
@@ -655,6 +670,8 @@ metrix_table = \
     "median$": ("Median (cycle reduction)", lambda test,test0,units: getCycleReduction(test, test0, "median")),
     "stddev$": ("Standard deviation (cycle reduction)", lambda test,test0,units: getCycleReduction(test, test0, "stddev")),
     "gstddev$": ("Standard deviation of Ln(time) (cycle reduction)", lambda test,test0,units: getCycleReduction(test, test0, "gstddev")),
+
+    "score": ("SCORE", lambda test,test0,units: getScore(test, test0, "gstddev")),
 }
 
 def formatValue(val, metric, units = None):
@@ -664,6 +681,18 @@ def formatValue(val, metric, units = None):
         return "%.2f" % val
     if metric.endswith("$"):
         return "%.2f%%" % val
+    if metric.endswith("S"):
+        if val > 3.5:
+            return "SLOWER"
+        if val < -3.5:
+            return "FASTER"
+        if val > -1.5 and val < 1.5:
+            return " "
+        if val < 0:
+            return "faster"
+        if val > 0:
+            return "slower"
+        #return "%.4f" % val
     return "%.3f %s" % (val, units)
 
 if __name__ == "__main__":