PerformanceBaselineDataSet.java

package br.ufrgs.inf.prosoft.tigris.sampling;

import com.google.common.util.concurrent.AtomicDouble;
import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics;
import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
import org.apache.commons.math3.stat.inference.TestUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicLong;

public class PerformanceBaselineDataSet {

    Logger logger = LoggerFactory.getLogger(PerformanceBaselineDataSet.class);

    DescriptiveStatistics baselinesPerSecondToNormal = new DescriptiveStatistics(300);
    Map<Long, Map<String, SummaryStatistics>> baselinesPerSecondToResponseTimes = new ConcurrentHashMap<>();
    private static Map<String, SummaryStatistics> currentBaseline = new ConcurrentHashMap<>();


    DescriptiveStatistics monitoringPerSecondToNormal = new DescriptiveStatistics(300);
    Map<Long, Map<String, SummaryStatistics>> monitoringPerSecondToResponseTimes = new ConcurrentHashMap<>();
    Map<String, SummaryStatistics> currentMonitoring = new ConcurrentHashMap<>();

    public void addPerformanceBaselineItem(Granularity item, long executionTime) {
        SummaryStatistics currentBaselineForItem = currentBaseline.getOrDefault(item.name, new SummaryStatistics());
        currentBaselineForItem.addValue(executionTime);
        currentBaseline.put(item.name, currentBaselineForItem);
    }

    public void addMonitoringItem(Granularity item, long executionTime) {
        SummaryStatistics currentMonitoringForItem = currentMonitoring.getOrDefault(item.name, new SummaryStatistics());
        currentMonitoringForItem.addValue(executionTime);
        currentMonitoring.put(item.name, currentMonitoringForItem);
    }

    public void trackBaselinePerSecond(long reqsPerSecond) {
        if (reqsPerSecond == 0)
            return;

        baselinesPerSecondToNormal.addValue(reqsPerSecond);
        baselinesPerSecondToResponseTimes.put(reqsPerSecond, new ConcurrentHashMap<>(currentBaseline));
        currentBaseline.clear();
    }

    public void trackMonitoringPerSecond(long operationsPerSecond) {
        if (operationsPerSecond == 0)
            return;

        monitoringPerSecondToNormal.addValue(operationsPerSecond);
        monitoringPerSecondToResponseTimes.put(operationsPerSecond, new ConcurrentHashMap<>(currentMonitoring));
        currentMonitoring.clear();
    }

    public Map<String, SummaryStatistics> getBaselineNormal() {
        double[] sortedValues = baselinesPerSecondToNormal.getSortedValues();

        return baselinesPerSecondToResponseTimes
                .get((long)sortedValues[sortedValues.length / 2]);
    }

    public Map<String, SummaryStatistics> getMonitoringNormal() {
        double[] sortedValues = monitoringPerSecondToNormal.getSortedValues();

        return monitoringPerSecondToResponseTimes.get((long)sortedValues[sortedValues.length / 2]);
    }

    public boolean isBaselineUnderAveragePlusStd(AtomicDouble decrease) {
        if (baselinesPerSecondToNormal.getN() == 0) {
            return true;
        }

        double[] sortedValues = baselinesPerSecondToNormal.getSortedValues();
        Map<String, SummaryStatistics> normal = baselinesPerSecondToResponseTimes.get((long)sortedValues[sortedValues.length / 2]);

        AtomicLong failed = new AtomicLong();
        AtomicLong success = new AtomicLong();

        SummaryStatistics decreaseStats = new SummaryStatistics();
//        normal.forEach((normalName, normalStats) -> {
//            DescriptiveStatistics currentStats = currentBaseline.get(normalName);
//            //compare only those methods that have the same amount of traces
////            if (currentStats != null && currentStats.getN() > 2 && normalStats.getN() == currentStats.getN()
////                    && TestUtils.tTest(normalStats.getMean(),
////                    currentStats, 0.05)) {
////                success.getAndIncrement();
////            } else { // TODO: should we decrement when not available?
////                failed.getAndIncrement();
////                decreaseStats.addValue((currentStats.getMean() - normalStats.getMean()) / normalStats.getMean());
////            }
//
//            if (currentStats != null && normalStats.getN() > 0 && normalStats.getN() == currentStats.getN()) {
//                double threshold = normalStats.getMean() + normalStats.getStandardDeviation();
//                if (currentStats.getMean() < threshold) {
//                    success.getAndIncrement();
//                } else {
//                    decreaseStats.addValue((currentStats.getMean() - normalStats.getMean()) / normalStats.getMean());
//                    failed.getAndIncrement();
//                }
//            }
//        });
//
//        decrease.set(decreaseStats.getMax());
//        return success.get() > failed.get();



        DescriptiveStatistics normalStatsMedians = new DescriptiveStatistics();
        DescriptiveStatistics currentStatsMedians = new DescriptiveStatistics();

        //comparing with t-test
        normal.forEach((normalName, normalStats) -> {
            SummaryStatistics currentStats = currentBaseline.get(normalName);
            //compare only those methods that have the same amount of traces
            if (currentStats != null && currentStats.getN() > 2 && normalStats.getN() > 2) {
                normalStatsMedians.addValue(normalStats.getMean());
                currentStatsMedians.addValue(currentStats.getMean());

                double threshold = normalStats.getMean() + normalStats.getStandardDeviation();
                if (currentStats.getMean() > threshold) {
                    failed.getAndIncrement();
                } else {
                    decreaseStats.addValue((normalStats.getMean() - currentStats.getMean()) / currentStats.getMean());
                    success.getAndIncrement();
                }
            }
        });

        if (!Double.isNaN(decreaseStats.getMax())) {
            decrease.set(decreaseStats.getMax());
        } else {
            decrease.set(0);
        }

        if (normalStatsMedians.getN() < 2) {
            return true;
        }

        //for lusearch
        if (success.get() + failed.get() > 1000) {
            return success.get() > failed.get();
        }

        return //!TestUtils.pairedTTest(normalStatsMedians.getValues(), currentStatsMedians.getValues(), 0.05) ||
                failed.get() <= 2;
    }

    public boolean isMonitoringUnderAveragePlusStd(AtomicDouble increase) {
        if (monitoringPerSecondToNormal.getN() == 0) {
            return true;
        }

        double[] sortedValues = monitoringPerSecondToNormal.getSortedValues();
        Map<String, SummaryStatistics> normal =
                monitoringPerSecondToResponseTimes.get((long)sortedValues[sortedValues.length / 2]);

        AtomicLong failed = new AtomicLong();
        AtomicLong success = new AtomicLong();

        SummaryStatistics increaseStats = new SummaryStatistics();
//        normal.forEach((normalName, normalStats) -> {
//            DescriptiveStatistics currentStats = currentMonitoring.get(normalName);
//            //compare only those methods that have the same amount of traces
////            if (currentStats != null && currentStats.getN() > 2 && normalStats.getN() == currentStats.getN()
////                && TestUtils.tTest(normalStats.getMean(),
////                    currentStats, 0.05)) {
////                success.getAndIncrement();
////                increaseStats.addValue((normalStats.getMean() - currentStats.getMean()) / currentStats.getMean());
////            } else {
////                failed.getAndIncrement();
////            }
//
//            if (currentStats != null && normalStats.getN() > 0 && normalStats.getN() == currentStats.getN()) {
//                double threshold = normalStats.getMean() + normalStats.getStandardDeviation();
//                if (currentStats.getMean() > threshold) {
//                    failed.getAndIncrement();
//                } else if (currentStats.getMean() < normalStats.getMean()) {
//                    increaseStats.addValue((normalStats.getMean() - currentStats.getMean()) / currentStats.getMean());
//                    success.getAndIncrement();
//                }
//            }
//        });

//        if (!Double.isNaN(increaseStats.getMax())) {
//            increase.set(increaseStats.getMax());
//        } else {
//            increase.set(0);
//        }
//        return failed.get() <= 2;


        DescriptiveStatistics normalStatsMedians = new DescriptiveStatistics();
        DescriptiveStatistics currentStatsMedians = new DescriptiveStatistics();

        //comparing with t-test
        normal.forEach((normalName, normalStats) -> {
            SummaryStatistics currentStats = currentMonitoring.get(normalName);
            //compare only those methods that have the same amount of traces
            if (currentStats != null && currentStats.getN() > 2 && normalStats.getN() > 2) {
                normalStatsMedians.addValue(normalStats.getMean());
                currentStatsMedians.addValue(currentStats.getMean());

                double threshold = normalStats.getMean() + normalStats.getStandardDeviation();
                if (currentStats.getMean() > threshold) {
                    failed.getAndIncrement();
                } else {
                    increaseStats.addValue((normalStats.getMean() - currentStats.getMean()) / currentStats.getMean());
                    success.getAndIncrement();
                }
            }
        });

        if (!Double.isNaN(increaseStats.getMax())) {
            increase.set(increaseStats.getMax());
        } else {
            increase.set(0);
        }

        if (normalStatsMedians.getN() < 2) {
            return true;
        }

        //for lusearch
        if (success.get() + failed.get() > 1000) {
            return success.get() > failed.get();
        }

//        return (normalStatsMedians.getMean() + normalStatsMedians.getStandardDeviation()) > currentStatsMedians.getMean()
//                || TestUtils.pairedTTest(normalStatsMedians.getValues(), currentStatsMedians.getValues(), 0.05);

        // returns true iff mean difference is =/= 0, means are different
        return //!TestUtils.pairedTTest(normalStatsMedians.getValues(), currentStatsMedians.getValues(), 0.05) ||
                failed.get() <= 2;

        // To test the (2-sided) hypothesis <code>mean 1 = mean 2 </code> at
        //     * the 95%, use
        //     * <br><code>tTest(sampleStats1, sampleStats2, 0.05) </code>
//        return TestUtils.tTest(normalStatsMedians, currentStatsMedians, 0.05);

//                if (currentStats.getMean() <= threshold) {
//                    success.getAndIncrement();
//                } else {
//                    failed.getAndIncrement();
//                }
//            }
//        });
//
//        return success.get() > failed.get();
    }

//    public SummaryStatistics getNormalBaselineAverage() {
//        double[] sortedValues = baselinesPerSecondToNormal.getSortedValues();
//
//        SummaryStatistics averageStats = new SummaryStatistics();
//
//        Map<String, DescriptiveStatistics> methodToResponseTimes = baselinesPerSecondToResponseTimes.get(sortedValues[sortedValues.length / 2]);
//
//        methodToResponseTimes.forEach((name, stats) -> {
//            double[] values = stats.getValues();
//            for (int i = 0; i < values.length; i++) {
//                if (values.length == 5) {
//                    averageStats.addValue();
//                }
//            }
//        });
//
//        return (mean.get() / methodToResponseTimes.size());
//    }
}