From 9e507063ca7813b0cca67dc158e2ced1c539d7d3 Mon Sep 17 00:00:00 2001 From: Brendan Dahl Date: Mon, 10 Dec 2012 10:19:02 -0800 Subject: [PATCH] Start of the stat utilities. --- test/driver.js | 17 ++- test/stats/results/.gitignore | 5 + test/stats/statcmp.py | 198 ++++++++++++++++++++++++++++++++++ test/test.py | 28 ++++- 4 files changed, 244 insertions(+), 4 deletions(-) create mode 100644 test/stats/results/.gitignore create mode 100644 test/stats/statcmp.py diff --git a/test/driver.js b/test/driver.js index 41ae3943a..e97e7c339 100644 --- a/test/driver.js +++ b/test/driver.js @@ -25,6 +25,7 @@ // https://github.com/mozilla/pdf.js/pull/764#issuecomment-2638944 // "firefox-bin: Fatal IO error 12 (Cannot allocate memory) on X server :1." // PDFJS.disableWorker = true; +PDFJS.enableStats = true; var appPath, browser, canvas, dummyCanvas, currentTaskIdx, manifest, stdout; var inFlightRequests = 0; @@ -45,6 +46,7 @@ function load() { browser = params.browser; var manifestFile = params.manifestFile; appPath = params.path; + var delay = params.delay || 0; canvas = document.createElement('canvas'); canvas.mozOpaque = true; @@ -67,7 +69,14 @@ function load() { nextTask(); } }; - r.send(null); + if (delay) { + log('\nDelaying for ' + delay + 'ms...\n'); + } + // When gathering the stats the numbers seem to be more reliable if the + // browser is given more time to startup. + setTimeout(function() { + r.send(null); + }, delay); } function cleanup() { @@ -110,6 +119,7 @@ function nextTask() { } var task = manifest[currentTaskIdx]; task.round = 0; + task.stats = {times: []}; log('Loading file "' + task.file + '"\n'); @@ -265,6 +275,8 @@ function nextPage(task, loadError) { }; var completeRender = (function(error) { page.destroy(); + task.stats = page.stats; + page.stats = new StatTimer(); snapshotCurrentPage(task, error); }); page.render(renderContext).then(function() { @@ -340,7 +352,8 @@ function sendTaskResult(snapshot, task, failure, result) { file: task.file, round: task.round, page: task.pageNum, - snapshot: snapshot + snapshot: snapshot, + stats: task.stats.times }); } diff --git a/test/stats/results/.gitignore b/test/stats/results/.gitignore new file mode 100644 index 000000000..76bedaeab --- /dev/null +++ b/test/stats/results/.gitignore @@ -0,0 +1,5 @@ +# Ignore everything in this directory +* +# Except this file +!.gitignore + diff --git a/test/stats/statcmp.py b/test/stats/statcmp.py new file mode 100644 index 000000000..9ce308e1c --- /dev/null +++ b/test/stats/statcmp.py @@ -0,0 +1,198 @@ +from numpy import * +from scipy import stats +import json, locale +from optparse import OptionParser + +VALID_GROUP_BYS = ['browser', 'pdf', 'page', 'round', 'stat'] +USAGE_EXAMPLE = "%prog BASELINE CURRENT" +class TestOptions(OptionParser): + def __init__(self, **kwargs): + OptionParser.__init__(self, **kwargs) + self.add_option("--groupBy", action="append", dest="groupBy", type="string", + help="How the statistics should grouped. Valid options: " + ', '.join(VALID_GROUP_BYS) + '.', default=[]) + + self.set_usage(USAGE_EXAMPLE) + + def verifyOptions(self, options, args): + if len(args) < 2: + self.error('There must be two comparison files arguments.') + # Veryify the group by options. + groupBy = [] + if not options.groupBy: + options.groupBy = ['browser,stat', 'browser,pdf,stat'] + for group in options.groupBy: + group = group.split(',') + for column in group: + if column not in VALID_GROUP_BYS: + self.error('Invalid group by option of "' + column + '"') + groupBy.append(group) + options.groupBy = groupBy + + return options + +## {{{ http://code.activestate.com/recipes/267662/ (r7) +import cStringIO,operator + +def indent(rows, hasHeader=False, headerChar='-', delim=' | ', justify='left', + separateRows=False, prefix='', postfix='', wrapfunc=lambda x:x): + """Indents a table by column. + - rows: A sequence of sequences of items, one sequence per row. + - hasHeader: True if the first row consists of the columns' names. + - headerChar: Character to be used for the row separator line + (if hasHeader==True or separateRows==True). + - delim: The column delimiter. + - justify: Determines how are data justified in their column. + Valid values are 'left','right' and 'center'. + - separateRows: True if rows are to be separated by a line + of 'headerChar's. + - prefix: A string prepended to each printed row. + - postfix: A string appended to each printed row. + - wrapfunc: A function f(text) for wrapping text; each element in + the table is first wrapped by this function.""" + # closure for breaking logical rows to physical, using wrapfunc + def rowWrapper(row): + newRows = [wrapfunc(str(item)).split('\n') for item in row] + return [[substr or '' for substr in item] for item in map(None,*newRows)] + # break each logical row into one or more physical ones + logicalRows = [rowWrapper(row) for row in rows] + # columns of physical rows + columns = map(None,*reduce(operator.add,logicalRows)) + # get the maximum of each column by the string length of its items + maxWidths = [max([len(str(item)) for item in column]) for column in columns] + rowSeparator = headerChar * (len(prefix) + len(postfix) + sum(maxWidths) + \ + len(delim)*(len(maxWidths)-1)) + # select the appropriate justify method + justify = {'center':str.center, 'right':str.rjust, 'left':str.ljust}[justify.lower()] + output=cStringIO.StringIO() + if separateRows: print >> output, rowSeparator + for physicalRows in logicalRows: + for row in physicalRows: + print >> output, \ + prefix \ + + delim.join([justify(str(item),width) for (item,width) in zip(row,maxWidths)]) \ + + postfix + if separateRows or hasHeader: print >> output, rowSeparator; hasHeader=False + return output.getvalue() + +# written by Mike Brown +# http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/148061 +def wrap_onspace(text, width): + """ + A word-wrap function that preserves existing line breaks + and most spaces in the text. Expects that existing line + breaks are posix newlines (\n). + """ + return reduce(lambda line, word, width=width: '%s%s%s' % + (line, + ' \n'[(len(line[line.rfind('\n')+1:]) + + len(word.split('\n',1)[0] + ) >= width)], + word), + text.split(' ') + ) + +import re +def wrap_onspace_strict(text, width): + """Similar to wrap_onspace, but enforces the width constraint: + words longer than width are split.""" + wordRegex = re.compile(r'\S{'+str(width)+r',}') + return wrap_onspace(wordRegex.sub(lambda m: wrap_always(m.group(),width),text),width) + +import math +def wrap_always(text, width): + """A simple word-wrap function that wraps text on exactly width characters. + It doesn't split the text in words.""" + return '\n'.join([ text[width*i:width*(i+1)] \ + for i in xrange(int(math.ceil(1.*len(text)/width))) ]) + +def formatTime(time): + return locale.format("%.*f", (0, time), True) + +# Group the stats by keys. We should really just stick these in a SQL database +# so we aren't reiventing the wheel. +def group(stats, groupBy): + vals = {} + for stat in stats: + key = [] + for group in groupBy: + key.append(stat[group]) + key = tuple(key) + if key not in vals: + vals[key] = [] + vals[key].append(stat['time']) + return vals; + + +def mean(l): + return array(l).mean() + + +# Take the somewhat normalized stats file and flatten it so there is a row for +# every recorded stat. +def flatten(stats): + rows = [] + for stat in stats: + for s in stat['stats']: + rows.append({ + 'browser': stat['browser'], + 'page': stat['page'], + 'pdf': stat['pdf'], + 'round': stat['round'], + 'stat': s['name'], + 'time': int(s['end']) - int(s['start']) + }) + return rows + +# Dump various stats in a table to compare the baseline and current results. +# T-test Refresher: +# If I understand t-test correctly, p is the probability that we'll observe +# another test that is as extreme as the current result assuming the null +# hypothesis is true. P is NOT the probability of the null hypothesis. +# The null hypothesis in this case is that the baseline and current results will +# be the same. It is generally accepted that you can reject the null hypothesis +# if the p-value is less than 0.05. So if p < 0.05 we can reject the results +# are the same which doesn't necessarily mean the results are faster/slower but +# it can be implied. +def stat(baseline, current, groupBy): + labels = groupBy + ['Baseline(ms)', 'Current(ms)', '+/-', '%', 'Result(P<.05)'] + baselineGroup = group(baseline, groupBy) + currentGroup = group(current, groupBy) + rows = [] + for key in baselineGroup: + t, p = stats.ttest_ind(baselineGroup[key], currentGroup[key], equal_var = False) + baseline = mean(baselineGroup[key]) + current = mean(currentGroup[key]) + speed = '' + if p < 0.05: + speed = 'faster' if current < baseline else 'slower' + row = list(key) + row += [ + formatTime(baseline), + formatTime(current), + formatTime(baseline - current), + round(100 * (1.0 * baseline - current) / baseline, 2), + speed + ] + rows.append(row) + rows.sort(key=lambda row: tuple(row[0:len(groupBy)])) + print indent([labels] + rows, hasHeader=True) + +def main(): + optionParser = TestOptions() + options, args = optionParser.parse_args() + options = optionParser.verifyOptions(options, args) + if options == None: + sys.exit(1) + + with open(args[0]) as baselineFile: + baseline = flatten(json.load(baselineFile)) + with open(args[1]) as currentFile: + current = flatten(json.load(currentFile)) + + for groupBy in options.groupBy: + print "-- Grouped By " + ', '.join(groupBy) + ' -- ' + stat(baseline, current, groupBy) + + +if __name__ == '__main__': + main() diff --git a/test/test.py b/test/test.py index 0575564d8..20de0529d 100644 --- a/test/test.py +++ b/test/test.py @@ -62,6 +62,10 @@ class TestOptions(OptionParser): help="Skips test PDFs downloading.", default=False) self.add_option("--ignoreDownloadErrors", action="store_true", dest="ignoreDownloadErrors", help="Ignores errors during test PDFs downloading.", default=False) + self.add_option("--statsFile", action="store", dest="statsFile", type="string", + help="The file where to store stats.", default=None) + self.add_option("--statsDelay", action="store", dest="statsDelay", type="int", + help="The amount of time in milliseconds the browser should wait before starting stats.", default=10000) self.set_usage(USAGE_EXAMPLE) def verifyOptions(self, options): @@ -75,6 +79,8 @@ class TestOptions(OptionParser): print "Warning: ignoring browser argument since manifest file was also supplied" if not options.browser and not options.browserManifestFile: print "Starting server on port %s." % options.port + if not options.statsFile: + options.statsDelay = 0 return options @@ -111,6 +117,8 @@ class State: numFBFFailures = 0 numLoadFailures = 0 eqLog = None + saveStats = False + stats = [ ] lastPost = { } class UnitTestState: @@ -323,6 +331,15 @@ class PDFTestHandler(TestHandlerBase): id, failure, round, page, snapshot = result['id'], result['failure'], result['round'], result['page'], result['snapshot'] taskResults = State.taskResults[browser][id] taskResults[round].append(Result(snapshot, failure, page)) + if State.saveStats: + stat = { + 'browser': browser, + 'pdf': id, + 'page': page, + 'round': round, + 'stats': result['stats'] + } + State.stats.append(stat) def isTaskDone(): numPages = result["numPages"] @@ -552,6 +569,8 @@ def setUp(options): taskResults.append([ ]) State.taskResults[b.name][id] = taskResults + if options.statsFile != None: + State.saveStats = True return testBrowsers def setUpUnitTests(options): @@ -572,6 +591,7 @@ def startBrowsers(browsers, options, path): host = 'http://%s:%s' % (SERVER_HOST, options.port) qs = '?browser='+ urllib.quote(b.name) +'&manifestFile='+ urllib.quote(options.manifestFile) qs += '&path=' + b.path + qs += '&delay=' + str(options.statsDelay) b.start(host + path + qs) def teardownBrowsers(browsers): @@ -689,7 +709,7 @@ def checkLoad(task, results, browser): print 'TEST-PASS | load test', task['id'], '| in', browser -def processResults(): +def processResults(options): print '' numFatalFailures = (State.numErrors + State.numFBFFailures) if 0 == State.numEqFailures and 0 == numFatalFailures: @@ -702,6 +722,10 @@ def processResults(): print ' different ref/snapshot:', State.numEqFailures if 0 < State.numFBFFailures: print ' different first/second rendering:', State.numFBFFailures + if options.statsFile != None: + with open(options.statsFile, 'w') as sf: + sf.write(json.dumps(State.stats, sort_keys=True, indent=4)) + print 'Wrote stats file: ' + options.statsFile def maybeUpdateRefImages(options, browser): @@ -752,7 +776,7 @@ def runTests(options, browsers): State.remaining[b] = 0 checkIfDone() time.sleep(1) - processResults() + processResults(options) finally: teardownBrowsers(browsers) t2 = time.time()