analysis.py

#!/usr/bin/python
from pymongo import MongoClient
from scipy import stats
import scipy
import numpy
import math
import sys
import argparse
import collections
import re
import os
from rpy2.robjects.packages import importr
import rpy2.robjects as ro
import rpy2




parser = argparse.ArgumentParser(description='Analyze test logs.')

parser.add_argument('--sortByFail', help='Sort the test cases by the number of failures.', action='store_true', required=False )
parser.add_argument('--getDuration', help='Get test execution time (in sec) across all tests.', action='store_true', required=False )
parser.add_argument('--getPassRate', help='Get test case pass rate for each build.', action='store_true', required=False )
parser.add_argument('--getBuildQuality', help='Get # of test pass/fail for each build.', action='store_true', required=False )
parser.add_argument('--prioritizeTests', help='Prioritize tests based on history. Tests that are likely to pass should be run first (they will be surprises if they fail).', action='store_true', required=False )
parser.add_argument('--support', help='Used for determining the support rate for prioritizing tests.', type=float, default=0.8, required=False )
parser.add_argument('--checkNewTestPerf', help='Check the performance of the newest test, using the execution time of prev 10 test runs (outlier removed). Returns the number of standard deviation away from previous runs.', action='store_true', required=False)
parser.add_argument('--checkNewTestPerfARIMA', help='Check the performance of the newest test using ARIMA model', action='store_true', required=False)
parser.add_argument('--perfTrend', help='Check the performance trend of each test.', action='store_true', required=False)
parser.add_argument('--testDeviation', help='Find tests with high deviations in execution time (unstable).', action='store_true', required=False )
parser.add_argument('--testStepDeviation', help='Input takes in a test id. Find steps in a test with high deviations in execution time (unstable).', type=str, required=False )
parser.add_argument('--testDeviationEntropy', help='Find tests with high deviations in execution time (unstable).', action='store_true', required=False )
parser.add_argument('--arimaModel', help='Examine ARIMA model (p, i, q, and drift), and execution time distribution.', action='store_true', required=False)
parser.add_argument('--examineAllHistory', help='Use 10 ... n points to build n - 10 ARIMA models and look for outliers in all historical builds.', action='store_true', required=False)
parser.add_argument('--compare', help='Compare two tests. Pass in two database names', type=str, required=False, nargs='+')
parser.add_argument('--windowSize', help='Number of most recent tests to consider when computing testDeviation', type=int, required=False, default=15)
parser.add_argument('--dbName', help='Name of the database which stores the test results', type=str, required=True)

parser.add_argument('--dst', help='Output folder for storing the graphs generated by the tool.', type=str, required=False, default="plots")

parser.add_argument('--test', help='Testing new features', action='store_true', required=False)


args = parser.parse_args()


conn = MongoClient()



db = conn[args.dbName]



# constants
PASS = 0
FAIL = 1
TIMEDOUT = 2


'''
R code for analysis
col.l <- colorRampPalette(c('green', 'yellow', 'orange', 'red'))(30)
levelplot(t(as.matrix(dat)), aspect="filled", col.regions = col.l)
'''


#d = dat[10,][!is.na(dat[10,])]
#plot(qcc(d, type="xbar.one"))



#################################################################
'''helper method for flatten an iterable'''
def flatten(iterable):
    results = []
    for i in iterable:
        if isinstance(i, collections.Iterable) and not isinstance(i, basestring):
            results.extend(flatten(i))
        else:
            results.append(i)
    return results
#################################################################


#################################################################
'''for sorting build numbers'''
def atoi(text):
    return int(text) if text.isdigit() else text

def natural_keys(text):
    '''
    alist.sort(key=natural_keys) sorts in human order
    http://nedbatchelder.com/blog/200712/human_sorting.html
    (See Toothy's implementation in the comments)
    '''
    return [ atoi(c) for c in re.split('(\d+)', text) ]
#################################################################


"""
Get the execution time of each test in each build (averaged across all runs)

R code for visualization:
dat = read.csv("testDuration.csv", row.names=1)
col.l <- colorRampPalette(c('green', 'yellow', 'orange', 'red'))(30)
levelplot(as.matrix(dat), aspect="fill", , col.regions = col.l)
"""
def findTestDuration():
    data = db.testStat.find({}, {'buildNumber':1})

    buildNumbers = list()
    for d in data:
        buildNumbers += d["buildNumber"]
        
    buildNumbers = sorted(list(set(buildNumbers)))

    testData = db.testStat.find({}, {'buildNumber': 1, 'duration': 1, 'passed': 1})
    buildTestDurations = {}


    ''' set up R code '''
    plot = importr('grDevices')
    graphics = importr('graphics')
    lattice = importr('lattice')

    ro.r('''
             plotExecutionTime <- function(fname) {
             require(lattice)
             dat = read.csv(fname, row.names=1)
             col.l <- colorRampPalette(c('green', 'yellow', 'orange', 'red'))(30)
            print(levelplot(as.matrix(dat), aspect="fill", col.regions = col.l))
        }
    ''')


    rPlotTime = ro.globalenv['plotExecutionTime']


    output = open('getDuration.csv', 'w')
    # change to folder which stores plots
    os.chdir(args.dst)

    allTestIDs = []
    for d in testData:
        testInfo = zip(d['buildNumber'], d['duration'], d['passed'])

        tid = d['_id']
        allTestIDs.append(tid)

        for build, duration, passed in testInfo:
            if build not in buildTestDurations:
                buildTestDurations[build] = {}
            if tid not in buildTestDurations[build]:
                buildTestDurations[build][tid] = []
            if passed:
                buildTestDurations[build][tid].append(duration)
            if passed == False: # test failed, so give it a NA
                buildTestDurations[build][tid].append(numpy.nan)

    output.write("build,"+",".join(allTestIDs)+"\n")
    allTestIDs = list(set(allTestIDs))
    builds = sorted(buildTestDurations.keys(), key = lambda x: natural_keys(x))

    #print "build,"+",".join(allTestIDs)
    count = 0
    for build in builds:
        testDurations = []
        for tid in allTestIDs:
            if tid in buildTestDurations[build]:
                time = numpy.array(buildTestDurations[build][tid])
                duration = time[~numpy.isnan(time)].mean()
                #duration = numpy.mean(buildTestDurations[build][tid])
                testDurations.append(duration)
            else: # the test was not available, so give it a NA
                testDurations.append(numpy.nan)
        
        #print build + "," + ",".join(map(str, testDurations)) # cast pass rate to string for printing
        count += 1
        output.write(build + "," + ",".join(map(str, testDurations)) + "\n") # cast pass rate to string for printing

    plot.pdf("getDuration.pdf")
    rPlotTime('../getDuration.csv')
    plot.dev_off()

    
        



"""
Sort the test cases by the number of times they fail
"""
def sortTestFail():
    #data = db.test.find({'duration' : {'$gt':0}}, {'duration' : 1, 'testName' : 0, 'testNumber': 1, '_id' : 0, 'testID': 1})
    tests = db.testStat.find({}, {'passed': 1, 'testName': 1})
    result = []

    testResultForBuild = {}
    numFailPerTest = []



    for test in tests:
        tid = test['_id']

        totalFailed = len(test['passed']) - sum(test['passed'])

        numFailPerTest.append((test['testName'], tid, totalFailed))


    for n, t, c in sorted(numFailPerTest, key=lambda x: x[2], reverse=True):
        print n, t, c
    


"""
Get the pass rate for each test case in each build
"""
def getPassRate():
    data = db.testStat.find({}, {'buildNumber':1})

    buildNumbers = list()
    # get unique build numbers
    for d in data:
        buildNumbers += d["buildNumber"]
    buildNumbers = list(set(buildNumbers))    
    buildNumbers = sorted(buildNumbers)

    testData = db.testStat.find({}, {'buildNumber': 1, 'passed':1})
    buildTestQuality = {}

    allTestIDs = []
    for d in testData:
        buildQuality = zip(d['buildNumber'], d['passed'])

        tid = d['_id']
        
        allTestIDs.append(tid)

        for build, passed in buildQuality:
            if build not in buildTestQuality:
                buildTestQuality[build] = {}

            if tid not in buildTestQuality[build]:
                buildTestQuality[build][tid] = [0, 0] # test pass/fail

            if passed:
                buildTestQuality[build][tid][PASS] += 1
            else:
                buildTestQuality[build][tid][FAIL] += 1

    allTestIDs = list(set(allTestIDs)) # remove duplicate test ids


    builds = sorted(buildTestQuality.keys(), key = lambda x: natural_keys(x))
    
    print "build,"+",".join(allTestIDs)
    for build in builds:
        passRate = []
        for tid in allTestIDs:
            if tid in buildTestQuality[build]:
                quality = buildTestQuality[build][tid]
                passRate.append(float(quality[PASS])/(quality[PASS]+quality[FAIL]))
            else:
                passRate.append('NA')
        print build + "," + ",".join(map(str, passRate)) # cast pass rate to string for printing
    

"""
Get the pass rate, # of passes, and # of fails for each build
"""
def getBuildQuality():
    #data = db.test.find({'duration' : {'$gt':0}}, {'duration' : 1, 'testName' : 0, 'testNumber': 1, '_id' : 0, 'testID': 1})
    data = db.testStat.find({}, {'buildNumber':1})


    buildNumbers = list()
    # get unique build numbers
    for d in data:
        buildNumbers += d["buildNumber"]
    buildNumbers = list(set(buildNumbers))    
    buildNumbers = sorted(buildNumbers, key = lambda x: natural_keys(x))


    testData = db.testStat.find({}, {'buildNumber': 1, 'passed': 1, 'timedout': 1})
    result = {}
    for d in testData:
        tmp = zip(d['buildNumber'], d['passed'], d['timedout'])

        tid = d['_id']

        test = {}
        for build, passed, timedout in tmp:
            if build not in result:
                result[build] = [0, 0, 0]
            if passed == True:
                result[build][PASS] += 1
            else:
                result[build][FAIL] += 1
            if timedout:
                result[build][TIMEDOUT] += 1

    print "build,numPass,numFail,passRate,numTimedout"
    builds = sorted(result.keys(), key = lambda x: natural_keys(x))
    for build in builds:
        passFail = result[build]
        print "%s,%s,%s,%s,%s" % (build, passFail[PASS], passFail[FAIL], float(passFail[PASS])/(passFail[PASS]+passFail[FAIL]), passFail[TIMEDOUT])
        #print passFail[2]




    

'''
Not complete yet
'''
# dat = read.csv("stdaway.csv", row.names=1, header=F)
def parameterAnalysis():
    testInfoDB = db.testStat.find({}, {'_id':1, 'testParams': 1, 'duration': 1, 'testName': 1, 'buildNumber': 1, 'passed': 1})
    testGroupByName = {}

    # first group the tests by test names in order to do parameter analysis
    # test with the same names are the same test with different parameters
    for test in testInfoDB:
        testName = test['testName']
        buildNumber = test['buildNumber']
        if testName not in testGroupByName:
            testGroupByName[testName] = []
        testGroupByName[testName].append(test)

    for testName, testGroup in testGroupByName.items():
        parameterDict = {}
        for test in testGroup:
            #TODO: need to skip failed tests


            # get the execution time for the set of parameters
            testTimeForParam = zip(test['testParams'], test['duration'])

            # group by build number
            # if the same build, then take average
            testTimeGroupedByBuild = zip(testTimeForParam, test['buildNumber'])
            testTimeGroupedByBuild = zip(testTimeGroupedByBuild, test['passed'])


        
            for testInfo in testTimeGroupedByBuild:#testTimeForParam:
                parameters = testInfo[0][0][0]
                duration = testInfo[0][0][1]
                buildNum = testInfo[0][1]
                passed = testInfo[1]
                if passed == False:
                    continue

                params = [p.split('=')[0] for p in parameters]
                paramValues = [p.split('=')[1] for p in parameters]

                # keep track of the parameters and their values
                for p, val in zip(params, paramValues):
                    if p not in parameterDict:
                        parameterDict[p] = []
                    parameterDict[p].append(val)

                # store the execution time of the test
                if 'duration' not in parameterDict:
                    parameterDict['duration'] = []
                parameterDict['duration'].append(duration)
                if 'buildNumber' not in parameterDict:
                    parameterDict['buildNumber'] = []
                parameterDict['buildNumber'].append(buildNum)



        paramNameList = []
        valList = []
        for param, vals in parameterDict.items():
            paramNameList.append(param)
            valList.append(vals)


        try:
            values = reduce(zip, valList)
        except TypeError:
            continue

        print ",".join(paramNameList)
        for v in values:
            print ",".join(map(str, flatten(v)))
        print '-------------------'



    '''
    for k, v in testTime.items():
        sortedTest = sorted(v, key = lambda x: x[0])
        #print sortedTest
        time = [x[1] for x in sortedTest]
    
        #print k+","+",".join(time)
        print k+","+str(numpy.var(time))
        if k == "9b1d2ce19c29d0f044c78b3226393e89":
            print k, time

        #print k, sortedTest
    f = db.test.find({"_id":111})
    '''



"""
Get the pass rate, # of passes, and # of fails for each build
"""
def prioritizedTests():
    testData = db.testStat.find({}, {'passed': 1, 'testRun': 1, 'testName': 1, 'buildNumber': 1})

    threshold = args.support

    buildNumbers = {}

    # look at previous n test runs
    for test in testData:

        testPassInBuild = zip(test['buildNumber'], test['passed'])

        # sort by build number
        testPassInBuild = list(set(sorted(testPassInBuild, key = lambda x: x[0])))

        passOrFail = [i[1] for i in testPassInBuild]

        if len(passOrFail) > args.windowSize:
            prev = passOrFail[-(args.windowSize+1):-1]
        else:
            prev = passOrFail[1:-1]
        last = passOrFail[-1]
        # if pass rate > support threshold (default 0.8)
        if sum(prev)/float(args.windowSize) > threshold:
            # these tests should run first
            #if last != True:
            print test['_id'], prev, last, test['testName']



'''
Finding outliers. sd is the number of standard deviation 
'''
def findOutliersMedian(data, sd = 2):
    d = numpy.abs(data - numpy.median(data))
    mdev = numpy.median(d)
    s = d/mdev if mdev else 0 # 
    return data[s<sd]

'''
Finding outliers. 
Return true if it is NOT an outlier.
'''
def findOutliersMean(data, sd=2):
    return abs(data - numpy.mean(data)) < sd * numpy.std(data)

'''
How many std away?
'''
def stdDiff(data):
    return abs(data - numpy.mean(data)) / numpy.std(data)

'''
How many MAD (median absolute deviation) away?
'''
def computeMAD(data):
    d = numpy.abs(data - numpy.median(data))
    #mdev = numpy.mean(d)/numpy.median(data)
    mdev = numpy.median(d)/numpy.median(data)
    #s = float(mdev - min(data))/(max(data)-min(data))
    #s = d/mdev if mdev else 0 # 
    return mdev


'''
How many deviations away for each of the execution time (including outliers)
'''
def stdAnalysis():
    allTests = db.testStat.find({}, {'_id':1, 'duration': 1, 'testName': 1, 'buildNumber': 1, 'testSteps': 1, 'passed': 1, 'testRun':1})

    for test in allTests:

        # only look at passed test cases
        passedTests = zip(test['passed'], test['duration'], test['testSteps'], test['buildNumber'])
        passedTests = filter(lambda x: x[0] == True, passedTests)

        testDuration = [duration for passed, duration, testSteps, buildNumber in passedTests]

        regressionIndex = [key for key, elem in enumerate(findOutliersMean(testDuration, 3).tolist()) if elem == False]

        testDuration = [duration for index, duration in enumerate(testDuration) if index not in regressionIndex]

        # how many std away are the execution time? on avearage
        result = test['_id']+ ',' + ','.join(map(str, stdDiff(testDuration)))
        for i in range(78-len(testDuration)):
            result+= 'NA,'
        print result
        #print test['_id']+ ',' + ','.join(map(str, stdDiff(testDuration)))

        #for regression in regressionIndex:
        #    print test['_id'], passedTests[regression][3]



'''
Check the performance of the newest test, and see if it is an outlier
'''
def checkNewTestPerformance():
    allTests = db.testStat.find({}, {'_id':1, 'duration': 1, 'testName': 1, 'buildNumber': 1, 'testSteps': 1, 'passed': 1, 'testRun':1})
    stdList = []
    timing = []



    output = open("checkNewTestPerformance.txt", "w")
    for test in allTests:

        # only look at passed test cases
        passedTests = zip(test['passed'], test['duration'], test['testSteps'], test['buildNumber'])

        passedTests = filter(lambda x: x[0] == True, passedTests)
        passedTests = sorted(passedTests, key = lambda x: natural_keys(x[3]))

        testDuration = [duration for passed, duration, testSteps, buildNumber in passedTests]

        outlierIndex = [key for key, elem in enumerate(findOutliersMean(testDuration, 3).tolist()) if elem == False]


        if len(testDuration) == 0: # test never passed
            continue

        newestTestTime = testDuration[-1]
        newestTestSteps = passedTests[-1][2]

        # remove outlier from the duration
        #prevTestTime = [time for index, time in enumerate(testDuration) if index not in outlierIndex][:-1]
        #prevTestSteps = [testInfo[2] for index, testInfo in enumerate(passedTests) if index not in outlierIndex][:-1]

        # get the test time and test step time
        prevTestTime = [time for index, time in enumerate(testDuration)][:-1]
        prevTestSteps = [testInfo[2] for index, testInfo in enumerate(passedTests) ][:-1]
        # version data, currently not used
        versions = [testInfo[3] for index, testInfo in enumerate(passedTests)]

        if len(prevTestTime) == 0:
            continue
        

        # only look at prev tests with a window size of 15
        # if we have less than 15, then we consider all
        if len(prevTestTime) > args.windowSize:
            prevTestTime = prevTestTime[-args.windowSize:]
        std = numpy.std(prevTestTime)
        # number of std away
        if std == 0:
            continue
        stdAway = abs(newestTestTime - numpy.mean(prevTestTime))/std
    
        stdList.append(std/numpy.mean(prevTestTime)*100)
        timing.append(numpy.mean(prevTestTime))


        # if it is an outlier
        if stdAway >= 3 and std != 0:
            diff = len(versions) - len(prevTestTime)
            #print sorted(versions, key = lambda x: natural_keys(x))[diff:]
            output.write('testName;stdAway;newestTestTime;prevTime;range\n')
            output.write( '%s;%s;%s;%s;%s\n' % (test['testName'], stdAway, newestTestTime, prevTestTime[-1], str(numpy.mean(prevTestTime)) +'+/-' + str(std)))
            output.write(",".join(map(str, prevTestTime)))
            output.write('\n')
            
            # stepwise timing difference between the newest test and previous test
            for prevStep, step in zip(prevTestSteps[-1], newestTestSteps):
                output.write( 'Diff:%s\n' %( step[0] - prevStep[0]))
                output.write(",".join(map(str, step)))
                output.write('\n')
                output.write(",".join(map(str, prevStep)))
                output.write('\n')
            
            prevTestSteps.append(newestTestSteps)
            # consider only past X tests (windowSize)
            checkStepwisePerfRegression(output, prevTestSteps[-args.windowSize:])
            output.write( '----------------------')
    output.close()
    


'''
Check which step in the newest test cause a performance regression
'''
def checkStepwisePerfRegression(output, allTestSteps):

    printed = False
    stepWiseResult = []
    # create an empty array to store the timing info for each step in prev tests
    for i in range(0, len(allTestSteps[0])):
        stepWiseResult.append([])

    for steps in allTestSteps:
        for i, step in enumerate(steps):
            stepWiseResult[i].append(step)

    for steps in stepWiseResult:
        timing = []
        for i, step in enumerate(steps):
            if i == 0:
                timing.append(step[1])
            timing.append(step[0])

        std = numpy.std(timing[1:-1])
        newestStepTime = timing[-1]
        stddiff = abs(newestStepTime - numpy.mean(timing[1:-1]))/std
        if stddiff > 2:
            if not printed:
                output.write( 'History of the execution time of the outlier step:\n')
                output.write( 'testName;stdAway;newestStepTime;prevStepTime\n')
                printed = True
            output.write( "%s;%s;%s;%s\n" % (timing[0], stddiff, newestStepTime, str(numpy.mean(timing[1:-1]))+'+/-'+str(std)))
            output.write(",".join(map(str, timing[1:])))
            output.write('\n')


'''
Using regression to find tests with an upward trend.
Plot moving averages for such tests.
'''
def findTestExecutionTrend():
    allTests = db.testStat.find({}, {'_id':1, 'duration': 1, 'testName': 1, 'buildNumber': 1, 'testSteps': 1, 'passed': 1, 'testRun':1})
    
    plot = importr('grDevices')
    graphics = importr('graphics')


    output = open("perfTrend.csv", 'w+')
    os.chdir(args.dst)
    output.write( 'testName;slope;pvalue;testDurations\n')
    for test in allTests:

        # only look at passed test cases
        passedTests = zip(test['passed'], test['duration'], test['testSteps'], test['buildNumber'])

        passedTests = filter(lambda x: x[0] == True, passedTests)
        passedTests = sorted(passedTests, key = lambda x: natural_keys(x[3]))

        if len(passedTests) > args.windowSize:
            passedTests = passedTests[-args.windowSize:] # pick only X number of newest tests

        testDuration = [duration for passed, duration, testSteps, buildNumber in passedTests]
        buildNumbers = [buildNumber for passed, duration, testSteps, buildNumber in passedTests]
        outlierIndex = [key for key, elem in enumerate(findOutliersMean(testDuration, 3).tolist()) if elem == False]

        # remove outliers
        testDuration = [duration for index, duration in enumerate(testDuration) if index not in outlierIndex]

        if len(testDuration) == 0: # test never passed
            continue

        movingAvg = calculateMovingAverage(testDuration, window=5).tolist()

        slope, intercept, r_value, p_value, std_err = stats.linregress(range(1, len(movingAvg)+1), movingAvg)
        #print slope, intercept, r_value, p_value, std_err
        if slope > 0 and p_value < 0.01:
            output.write("%s;%s;%s;%s\n"%(test['testName'], slope, p_value, testDuration))
            #movingAvg = calculateMovingAverage(testDuration, window=5).tolist()
            #print movingAvg
            plot.jpeg('movingAvg_%s_%s.jpg' % (slope, test['testName']))
            graphics.plot(ro.FloatVector(movingAvg), type='l', ylab="", main=test['testName'])
            plot.dev_off()



        
'''
not used in the script
'''
def intensity(values):
    result = [0]
    for p, n in zip(values, values[1:]):
        if n > p:
            result.append(result[-1]+1)
        else:
            if result[-1] == 0:
                result.append(0)
            else:
                result.append(result[-1]-1)
    return result




def calculateMovingAverage(values, window=3):
    weigths = numpy.repeat(1.0, window)/window
    movingAvg = numpy.convolve(values, weigths, 'valid')
    return movingAvg



'''
Get the p, i, q (ARIMA model) for each test
'''
def examineArimaModel():
    forecast = importr('forecast')
    stats = importr('stats')

    
    ro.r('''
            getPval <- function(model) {
            return((1-pnorm(abs(model$coef)/sqrt(diag(model$var.coef))))*2)
        }
    ''')

    ro.r('''
            getCoef <- function(model) {
            return(model$coef)
        }
    ''')


    rGetPval = ro.globalenv['getPval']
    rGetCoef = ro.globalenv['getCoef']

    allTests = db.testStat.find({}, {'_id':1, 'duration': 1, 'testName': 1, 'buildNumber': 1, 'testSteps': 1, 'passed': 1, 'testRun':1})
    stdList = []
    timing = []


    print "testName,arimaResult,COV,mean,std,numOutlier,perctOutlier,pval,coef,normPval"
    for test in allTests:

        # only look at passed test cases
        passedTests = zip(test['passed'], test['duration'], test['testSteps'], test['buildNumber'])

        passedTests = filter(lambda x: x[0] == True, passedTests)
        passedTests = sorted(passedTests, key = lambda x: natural_keys(x[3]))

        testDuration = [duration for passed, duration, testSteps, buildNumber in passedTests]


        outlierIndex = [index for index, elem in enumerate(findOutliersMean(testDuration, 3).tolist()) if elem == False]
        testDuration = [duration for index, duration in enumerate(passedTests) if index not in outlierIndex]


        if len(testDuration) == 0: # test never passed
            continue

        newestTestTime = testDuration[-1]
        newestTestSteps = passedTests[-1][2]

        # remove outlier from the duration
        allTestTime = [time for index, time in enumerate(testDuration) if index not in outlierIndex]
        #prevTestTime = [time for index, time in enumerate(testDuration)]#[:-1]
        versions = [testInfo[3] for index, testInfo in enumerate(passedTests)]

        if len(allTestTime) == 0:
            continue
        
        try:
            lambdaVal = forecast.BoxCox_lambda(ro.FloatVector(allTestTime))
            normalizedTimeRO = forecast.BoxCox(ro.FloatVector(allTestTime), lambdaVal)
            rTimeSeries = stats.ts(ro.FloatVector(normalizedTimeRO))
            arimaModel = forecast.auto_arima(rTimeSeries)
            arimaResult = re.findall('.*?ARIMA\(([0-9]+,[0-9]+,[0-9]+)\).*?', str(arimaModel))[0]
    
            pval = str(rGetPval(arimaModel)).replace('\n', '').strip()
            pval = re.sub('\s+', ' ', pval)

            coef = str(rGetCoef(arimaModel)).replace('\n', '').strip()
            coef = re.sub('\s+', ' ', coef)
        
            print "%s,%s,%s,%s,%s,%s,%s,%s,%s,%s" % (test['testName'], arimaResult.replace(',',' '), numpy.std(testDuration)/numpy.mean(testDuration), numpy.mean(testDuration), numpy.std(testDuration), len(outlierIndex), len(outlierIndex)/float(len(passedTests)), pval, coef, str(stats.shapiro_test(ro.FloatVector(allTestTime)).rx2('p.value')).split(']')[1].strip()) 
        except rpy2.rinterface.RRuntimeError:
            continue




'''
Build a model based on n to predict n+1.
If there are 11 executions, there will be 2 models.
Report if the actual vs predicted is an outlier.
'''
def arimaPredict():
    forecast = importr('forecast')
    stats = importr('stats')

    
    ro.r('''
            getPval <- function(model) {
            return((1-pnorm(abs(model$coef)/sqrt(diag(model$var.coef))))*2)
        }
    ''')

    ro.r('''
            getCoef <- function(model) {
            return(model$coef)
        }
    ''')


    rGetPval = ro.globalenv['getPval']
    rGetCoef = ro.globalenv['getCoef']

    allTests = db.testStat.find({}, {'_id':1, 'duration': 1, 'testName': 1, 'buildNumber': 1, 'testSteps': 1, 'passed': 1, 'testRun':1})
    stdList = []
    timing = []

    print 'testName;predictedCategory;arima;actual;predicted;origRunTime;prevTestSteps;curTestSteps;runID'
    for test in allTests:

        # only look at passed test cases
        passedTests = zip(test['passed'], test['duration'], test['testSteps'], test['buildNumber'])

        passedTests = filter(lambda x: x[0] == True, passedTests)
        passedTests = sorted(passedTests, key = lambda x: natural_keys(x[3]))

        testDuration = [duration for passed, duration, testSteps, buildNumber in passedTests]


        regressionIndex = [key for key, elem in enumerate(findOutliersMean(testDuration, 3).tolist()) if elem == False]


        if len(testDuration) == 0: # test never passed
            continue

        newestTestTime = testDuration[-1]
        newestTestSteps = passedTests[-1][2]

        # remove regression from the duration
        #prevTestTime = [time for index, time in enumerate(testDuration) if index not in regressionIndex][:-1]

        prevTestTime = [time for index, time in enumerate(testDuration)]#[:-1]
        if len(prevTestTime) == 0:
            continue
        
        if len(prevTestTime) <= args.windowSize:
            continue

        prevTestSteps = [testInfo[2] for index, testInfo in enumerate(passedTests)][:-1][-1]

        for i in range(args.windowSize, len(prevTestTime)):
            try:
                testTime = prevTestTime[0:i+1]
                lambdaVal = forecast.BoxCox_lambda(ro.FloatVector(testTime))
                #normalizedTimeRO = ro.FloatVector(testTime) #forecast.BoxCox(ro.FloatVector(testTime), lambdaVal)
                timeRO = ro.FloatVector(testTime)

                timeRO = forecast.BoxCox(ro.FloatVector(testTime), lambdaVal)
                rTimeSeries = stats.ts(ro.FloatVector(timeRO))
                arimaModel = forecast.auto_arima(rTimeSeries[0:-1]) # build model using first n
                arimaResult = re.findall('.*?ARIMA\(([0-9]+,[0-9]+,[0-9]+)\).*?', str(arimaModel))[0]

                # random walk
                if arimaResult == '0,0,0':
                    continue

                predictionResult = str(forecast.forecast(arimaModel, 1)).split('\n')[1]
                #print timeRO[-1], predictionResult, passedTests[i][3]
                predictionResult = re.split('\s+', predictionResult.strip())[2:]
                predictionResult = map(float, predictionResult)
                # lo 80, hi 80, lo 95, hi 95
                if timeRO[-1] >= predictionResult[3]:
                    #print test['testName'], 'high', arimaResult, timeRO[-1], predictionResult, passedTests[i][3]
                    print '%s;%s;%s;%s;%s;%s;%s;%s;%s' % (test['testName'], 'high', arimaResult, timeRO[-1], predictionResult, prevTestTime[0:i+1], prevTestSteps, newestTestSteps, passedTests[i][3])
                elif timeRO[-1] <= predictionResult[2]: 
                    #print test['testName'], 'low', arimaResult, timeRO[-1], predictionResult, passedTests[i][3]
                    print '%s;%s;%s;%s;%s;%s;%s;%s;%s' % (test['testName'], 'low', arimaResult, timeRO[-1], predictionResult, prevTestTime[0:i+1], prevTestSteps, newestTestSteps, passedTests[i][3])
    
            except rpy2.rinterface.RRuntimeError:
                continue


    



'''
Build a model based on n to predict the newest test.
Report if the actual vs predicted is an outlier.
'''
def checkNewTestPerfARIMA():
    forecast = importr('forecast')
    stats = importr('stats')
    timeSeries = importr('timeSeries')
    plot = importr('grDevices')
    graphics = importr('graphics')

    
    ro.r('''
            getPval <- function(model) {
            return((1-pnorm(abs(model$coef)/sqrt(diag(model$var.coef))))*2)
        }
    ''')

    ro.r('''
            getCoef <- function(model) {
            return(model$coef)
        }
    ''')


    rGetPval = ro.globalenv['getPval']
    rGetCoef = ro.globalenv['getCoef']

    allTests = db.testStat.find({}, {'_id':1, 'duration': 1, 'testName': 1, 'buildNumber': 1, 'testSteps': 1, 'passed': 1, 'testRun':1})
    stdList = []
    timing = []

    output = open('checkNewTestPerfARIMA.txt', 'w+')
    os.chdir(args.dst)

    output.write('testName;predictedCategory;arima;actual;predicted;origRunTime;prevTestSteps;curTestSteps;runID\n')
    for test in allTests:

        # only look at passed test cases
        passedTests = zip(test['passed'], test['duration'], test['testSteps'], test['buildNumber'])

        passedTests = filter(lambda x: x[0] == True, passedTests)
        passedTests = sorted(passedTests, key = lambda x: natural_keys(x[3]))

        testDuration = [duration for passed, duration, testSteps, buildNumber in passedTests]


        outlierIndex = [key for key, elem in enumerate(findOutliersMean(testDuration, 3).tolist()) if elem == False]


        if len(testDuration) == 0: # test never passed
            continue

        newestTestTime = testDuration[-1]
        newestTestSteps = passedTests[-1][2]

        # remove regression from the duration
        prevTestTime = [time for index, time in enumerate(testDuration[:-1]) if index not in outlierIndex]#[:-1]
        passedTest = [t for index, t in enumerate(passedTests[:-1]) if index not in outlierIndex]#[:-1]

        #prevTestTime = [time for index, time in enumerate(testDuration)]#[:-1]
        if len(prevTestTime) == 0:
            continue
        if len(prevTestTime) <= args.windowSize:
            continue

        prevTestStep = [testInfo[2] for index, testInfo in enumerate(passedTests)][:-1][-1]
        prevTestSteps = [testInfo[2] for index, testInfo in enumerate(passedTests)]
        try:
            prevTestTime.append(newestTestTime)
            testTime = prevTestTime
            lambdaVal = forecast.BoxCox_lambda(ro.FloatVector(testTime))
            timeRO = ro.FloatVector(testTime)

            timeRO = forecast.BoxCox(ro.FloatVector(testTime), lambdaVal)
            rTimeSeries = stats.ts(ro.FloatVector(timeRO))
            arimaModel = forecast.auto_arima(rTimeSeries[0:-1]) # build model using first n

            arimaResult = re.findall('.*?ARIMA\(([0-9]+,[0-9]+,[0-9]+)\).*?', str(arimaModel))[0]
            # random walk
            if arimaResult == '0,0,0':
                continue

            predictionResult = str(forecast.forecast(arimaModel, 1)).split('\n')[1]
            predictionResult = re.split('\s+', predictionResult.strip())[2:]
            predictionResult = map(float, predictionResult)
            # lo 80, hi 80, lo 95, hi 95
            if timeRO[-1] >= predictionResult[3]: # over hi 95
                # plotting predicted and actual graph
                plot.jpeg('arimaPredict_%s.jpg' % test['testName'])
                # predict future 5 builds
                graphics.plot(forecast.forecast(arimaModel, '5'))
                graphics.points(len(testTime), timeRO[-1])
                plot.dev_off()

                output.write('%s;%s;%s;%s;%s;%s;%s;%s;%s\n' % (test['testName'], 'high', arimaResult, timeRO[-1], predictionResult, prevTestTime, prevTestStep, newestTestSteps, passedTests[-1][3]))

                checkStepwisePerfRegression(output, prevTestSteps)
                output.write( '--------------------\n')

            '''
            elif timeRO[-1] <= predictionResult[2]: # lower than lo 95 
                # plotting predicted and actual graph
                plot.jpeg('%s.jpg' % test['testName'].split('.')[-1])
                graphics.plot(forecast.forecast(arimaModel, '5'))
                graphics.points(len(testTime), timeRO[-1])
                plot.dev_off()


                print '%s;%s;%s;%s;%s;%s;%s;%s;%s' % (test['testName'], 'low', arimaResult, timeRO[-1], predictionResult, prevTestTime[0:-1], prevTestStep, newestTestSteps, passedTests[-1][3])
                print 'History of the execution time of the outlier step:'
                checkStepwisePerfRegression(prevTestSteps)
                print '--------------------'
            '''
    
        except rpy2.rinterface.RRuntimeError:
            continue


'''
Compare the test execution time in the same build
'''
def compareTests(testsToCompare):
    # only give one db name, requires two
    if len(testsToCompare) < 2:
        parser.print_help()
        sys.exit(0)

    db1 = conn[testsToCompare[0]]
    db2 = conn[testsToCompare[1]]


    allTestsDB1 = db1.testStat.find({}, {'_id':1, 'duration': 1, 'testName': 1, 'buildNumber': 1, 'testSteps': 1, 'passed': 1, 'testRun':1})
    allTestsDB2 = db2.testStat.find({}, {'_id':1, 'duration': 1, 'testName': 1, 'buildNumber': 1, 'testSteps': 1, 'passed': 1, 'testRun':1})

    combinedTests = {} # combine test in two dbs by test id

    for test in allTestsDB1:
        if test['_id'] not in combinedTests:
            combinedTests[test['_id']] = []
        combinedTests[test['_id']].append(test)
    for test in allTestsDB2:
        if test['_id'] not in combinedTests:
            combinedTests[test['_id']] = []
        combinedTests[test['_id']].append(test)

    for tid, test in combinedTests.items():
        buildDict = {} # compare test according to their build
        if len(test) < 2: # the test is only executed in one DB
            continue
        # combine and sort all the builds in the two test
        buildNumbers = test[0]['buildNumber']
        buildNumbers += test[1]['buildNumber']
        buildNumbers = sorted(list(set(buildNumbers)))

        buildInDB1 = {}
        stepTiming1 = {}
        for build, duration, passed, steps in zip(test[0]['buildNumber'], test[0]['duration'], test[0]['passed'], test[0]['testSteps']):
            #if passed == False:
            #    continue
            #if build not in buildDict:
            #    buildDict[build] = [-1, -1] # store result for each test
            #buildDict[build][0] = duration

            if build not in buildInDB1:
                buildInDB1[build] = []
            if build not in stepTiming1:
                stepTiming1[build] = []

            buildInDB1[build].append(duration)
            stepTiming1[build].append(steps)



        buildInDB2 = {}
        stepTiming2 = {}
        for build, duration, passed, steps in zip(test[1]['buildNumber'], test[1]['duration'], test[1]['passed'], test[1]['testSteps']):
            #if passed == False:
            #    continue
            if build not in buildInDB2:
                buildInDB2[build] = []
            if build not in stepTiming2:
                stepTiming2[build] = []

            buildInDB2[build].append(duration)
            stepTiming2[build].append(steps)

        for build in buildNumbers:
            testDurationDB1 = -1
            testDurationDB2 = -1
            # take the average if a build has several runs
            if build in buildInDB1:
                testDurationDB1 = sum(buildInDB1[build])/len(buildInDB1[build])
            if build in buildInDB2:
                testDurationDB2 = sum(buildInDB2[build])/len(buildInDB2[build])
            #if testDurationDB1 < 0 or testDurationDB2 < 0:
            #    continue
            print tid, build, testDurationDB1, testDurationDB2
        



'''
Compute the median absolute deviation (MAD), and sort the tests
according to MAD. Only use the most recent tests.
'''
def findHighDeviationTests():
    allTests = db.testStat.find({}, {'_id':1, 'duration': 1, 'testName': 1, 'buildNumber': 1, 'testSteps': 1, 'passed': 1, 'testRun':1, 'timedout': 1})

    # store MAD for all tests
    testResults = []
    for test in allTests:
        if 'timedout' in test:
            passedTests = zip(test['passed'], test['duration'], test['testSteps'], test['buildNumber'], test['timedout'])
            # passedTests may contain timedout tests
            passedTests = filter(lambda x: x[0] == True or x[4] == True, passedTests)
            # sort by build
            passedTests = sorted(passedTests, key = lambda x: natural_keys(x[3]))
            testDuration = [duration for passed, duration, testSteps, buildNumber, timedout in passedTests]
            builds = [buildNumber for passed, duration, testSteps, buildNumber, timedout in passedTests]
        else:
            passedTests = zip(test['passed'], test['duration'], test['testSteps'], test['buildNumber'])
            # passedTests may contain timedout tests
            passedTests = filter(lambda x: x[0] == True, passedTests)
            # sort by build
            passedTests = sorted(passedTests, key = lambda x: natural_keys(x[3]))
            testDuration = [duration for passed, duration, testSteps, buildNumber in passedTests]
            builds = [buildNumber for passed, duration, testSteps, buildNumber in passedTests]

        # no passed nor timedout tests
        if len(testDuration) == 0 or len(testDuration) == 1:
            continue

        # comput MAD and normalized MAD for most recently args.windowSize tests
        if len(testDuration) >= args.windowSize:
            #print testDuration, testDuration[-args.windowSize:]
            #print min(testDuration), max(testDuration)
            mad = computeMAD(testDuration[-args.windowSize:])

            diff = (testDuration[-args.windowSize:] - numpy.median(testDuration[-args.windowSize:])).tolist()
            largest = builds[-args.windowSize:][diff.index(max(diff))]
            smallest = builds[-args.windowSize:][diff.index(min(diff))]

            if largest == smallest:
                continue

            #testResults.append((test['_id'], test['testName'], mad, numpy.mean(testDuration[-args.windowSize:]), numpy.std(testDuration[-args.windowSize:]), (smallest, largest), zip(builds[-args.windowSize:], testDuration[-args.windowSize:])))
            testResults.append((test['_id'], test['testName'], mad, numpy.mean(testDuration[-args.windowSize:]), numpy.std(testDuration[-args.windowSize:]), (smallest, largest), testDuration[-args.windowSize:]))
            #testResults.append((test['_id'], test['testName'], mad, numpy.mean(testDuration[-args.windowSize:]), numpy.std(testDuration[-args.windowSize:]), testDuration[-args.windowSize:]))
        else: # use all tests
            mad = computeMAD(testDuration)
            #diff = numpy.abs(testDuration - numpy.median(testDuration)).tolist()
            diff = (testDuration - numpy.median(testDuration)).tolist()
            largest = builds[diff.index(max(diff))]
            smallest = builds[diff.index(min(diff))]
            if largest == smallest:
                continue
            #testResults.append((test['_id'], test['testName'], mad, numpy.mean(testDuration), numpy.std(testDuration), (smallest, largest), zip(builds, testDuration)))
            testResults.append((test['_id'], test['testName'], mad, numpy.mean(testDuration), numpy.std(testDuration), (smallest, largest), testDuration))
            #testResults.append((test['_id'], test['testName'], mad, numpy.mean(testDuration), numpy.std(testDuration), testDuration))
    
    testResults = sorted(testResults, key = lambda x: x[2], reverse=True)
    mads = []
    covs = []
    output = open('testDeviation.csv', 'w')
    output.write( "testID;testName;mad;mean;std;smallLargeBuild;time\n")
    for t in testResults:
        output.write( "%s;%s;%s;%s+/-%s;%s;%s;%s\n" % (t[0], t[1], t[2], t[3], t[4], float(t[4])/t[3], t[5], t[6]))
        #mads.append(t[1])
        #covs.append(t[3]/t[2])
    #print numpy.corrcoef(mads, covs)


def findHighDeviationTestSteps(testID):
    test = db.testStat.find({'_id': testID}, {'_id':1, 'duration': 1, 'testName': 1, 'buildNumber': 1, 'testSteps': 1, 'passed': 1, 'testRun':1, 'timedout': 1})



    for t in test:
        stepTiming = {}
        for steps in t['testSteps']:
            for step in steps:
                if step[1] not in stepTiming:
                    stepTiming[step[1]] = []
                stepTiming[step[1]].append(step[0])

    results = []
    for stepName, times in stepTiming.items():
        if len(times) > args.windowSize:
            mad = computeMAD(times[-args.windowSize:])
            mean = numpy.mean(times[-args.windowSize:])
            std = numpy.std(times[-args.windowSize:])
            results.append([stepName, mad, "%s+/-%s" % (mean, std), times[-args.windowSize:]])
        else:
            mad = computeMAD(times)
            mean = numpy.mean(times)
            std = numpy.std(times)
            results.append([stepName, mad, "%s+/-%s" % (mean, std), times])

    results = sorted(results, key=lambda x: x[1], reverse=True)

    print 'stepName;mad;std;times'
    for r in results:
        print "%s;%s;%s;%s" % (r[0], r[1], r[2], r[3])

    


'''
Compute the entropy, and sort the tests
according to entropy. Only use the most recent tests.
'''
def findHighDeviationTestsEntropy():
    allTests = db.testStat.find({}, {'_id':1, 'duration': 1, 'testName': 1, 'buildNumber': 1, 'testSteps': 1, 'passed': 1, 'testRun':1, 'timedout': 1})

    # store MAD for all tests
    testResults = []
    for test in allTests:
        if 'timedout' in test:
            passedTests = zip(test['passed'], test['duration'], test['testSteps'], test['buildNumber'], test['timedout'])
            # passedTests may contain timedout tests
            passedTests = filter(lambda x: x[0] == True or x[4] == True, passedTests)
            # sort by build
            passedTests = sorted(passedTests, key = lambda x: natural_keys(x[3]))
            testDuration = [duration for passed, duration, testSteps, buildNumber, timedout in passedTests]
            builds = [buildNumber for passed, duration, testSteps, buildNumber, timedout in passedTests]
        else:
            passedTests = zip(test['passed'], test['duration'], test['testSteps'], test['buildNumber'])
            # passedTests may contain timedout tests
            passedTests = filter(lambda x: x[0] == True, passedTests)
            # sort by build
            passedTests = sorted(passedTests, key = lambda x: natural_keys(x[3]))
            testDuration = [duration for passed, duration, testSteps, buildNumber in passedTests]
            builds = [buildNumber for passed, duration, testSteps, buildNumber in passedTests]

        # no passed nor timedout tests
        if len(testDuration) == 0 or len(testDuration) == 1:
            continue

        # comput MAD and normalized MAD for most recently args.windowSize tests
        if len(testDuration) >= args.windowSize:
            #print testDuration, testDuration[-args.windowSize:]
            #print min(testDuration), max(testDuration)
            entropy = computeEntropy(testDuration[-args.windowSize:])
            #testResults.append((test['testName'], entropy, numpy.mean(testDuration[-args.windowSize:]), numpy.std(testDuration[-args.windowSize:]),zip(builds[-args.windowSize:], testDuration[-args.windowSize:])))
            testResults.append((test['testName'], entropy, numpy.mean(testDuration[-args.windowSize:]), numpy.std(testDuration[-args.windowSize:]),testDuration[-args.windowSize:]))
        else: # use all tests
            entropy = computeEntropy(testDuration)
            #testResults.append((test['testName'], entropy, numpy.mean(testDuration), numpy.std(testDuration), zip(builds, testDuration)))
            testResults.append((test['testName'], entropy, numpy.mean(testDuration), numpy.std(testDuration), testDuration))
    
    testResults = sorted(testResults, key = lambda x: x[1])
    print "testName,entropy,mean,std,time"
    for t in testResults:
        print "%s;%s;%s+/-%s;%s,%s" % (t[0], t[1], t[2], t[3], float(t[3])/t[2], t[4])
    #print numpy.corrcoef(mads, covs)
    





def computeEntropy(duration):
    "Calculates the Shannon entropy of a distribution"

    # get probability
    s = sum(duration)
    prob = [d/s for d in duration if d > 0]
    # calculate the entropy
    entropy = - sum([ p * math.log(p) / math.log(2.0) for p in prob ])

    return entropy    



    

def test():
    allTests = db.testStat.find({}, {'_id':1, 'duration': 1, 'testName': 1, 'buildNumber': 1, 'testSteps': 1, 'passed': 1, 'testRun':1, 'timedout': 1})

    for test in allTests:
        #if 'testRandomStrings' in test['testName']:
        if 'TestSegmentingTokenizerBase' in test['testName']:
            print test['buildNumber']
            print test['duration']

def test2():
    plot = importr('grDevices')
    graphics = importr('graphics')

    
    ro.r('''
            getPval <- function(model) {
            return((1-pnorm(abs(model$coef)/sqrt(diag(model$var.coef))))*2)
        }
    ''')

    ro.r('''
            getCoef <- function(model) {
            return(model$coef)
        }
    ''')


    rGetPval = ro.globalenv['getPval']
    rGetCoef = ro.globalenv['getCoef']

    allTests = db.testStat.find({}, {'_id':1, 'duration': 1, 'testName': 1, 'buildNumber': 1, 'testSteps': 1, 'passed': 1, 'testRun':1})
    stdList = []
    timing = []


    allDurations = []
    for test in allTests:

        # only look at passed test cases
        passedTests = zip(test['passed'], test['duration'], test['testSteps'], test['buildNumber'])

        passedTests = filter(lambda x: x[0] == True, passedTests)
        passedTests = sorted(passedTests, key = lambda x: natural_keys(x[3]))

        testDuration = [duration for passed, duration, testSteps, buildNumber in passedTests]

        allDurations.append(testDuration)

    
    tt = []

    durations = []
    for t in allDurations:
        if len(t) > 0:
            durations.append(t)

    for t in sorted(durations, key = lambda x: float(max(x)-min(x))/numpy.median(x), reverse=True):
    #for t in sorted(durations, key = lambda x: numpy.median(x), reverse=True):
        tt.append(ro.FloatVector(t))

    plot.pdf("lucene.pdf" )
    graphics.boxplot(tt, horizontal=True, outline=False)
    plot.dev_off()
    sys.exit(0)

    count = 0
    for i in range(0, len(tt), 10):
        plot.pdf("test%s.pdf" % count)
        count+=1
        graphics.boxplot(tt[i:i+10], horizontal=True, outline=False)
        plot.dev_off()
    sys.exit(0)
    plot.pdf("test.pdf")
    graphics.boxplot(tt[1:10], horizontal=True, outline=False)
    plot.dev_off()
    plot.pdf("test1.pdf")
    graphics.boxplot(tt[10:20], horizontal=True, outline=False)
    plot.dev_off()
    plot.pdf("test2.pdf")
    graphics.boxplot(tt[20:30], horizontal=True, outline=False)
    plot.dev_off()
    plot.pdf("test3.pdf")
    graphics.boxplot(tt[30:100], horizontal=True, outline=False)
    plot.dev_off()
    plot.pdf("test4.pdf")
    graphics.boxplot(tt[100:300], horizontal=True, outline=False)
    plot.dev_off()




def main():

    if args.getDuration:
        findTestDuration()
    elif args.sortByFail:
        sortTestFail()
    elif args.getPassRate:
        getPassRate()
    elif args.getBuildQuality:
        getBuildQuality()
    elif args.prioritizeTests:
        prioritizedTests()
    elif args.checkNewTestPerf:
        checkNewTestPerformance()
    elif args.perfTrend:
        findTestExecutionTrend()
    elif args.arimaModel:
        examineArimaModel()
    elif args.examineAllHistory:
        arimaPredict()
    elif args.checkNewTestPerfARIMA:
        checkNewTestPerfARIMA()
    elif args.compare:
        compareTests(args.compare)
    elif args.testDeviation:
        findHighDeviationTests()
    elif args.testDeviationEntropy:
        findHighDeviationTestsEntropy()
    elif args.testStepDeviation:
        findHighDeviationTestSteps(args.testStepDeviation)
    elif args.test:
        test()
    else:
        parser.print_help()



if __name__ == '__main__':
    main()