Statistics.java

package br.ufrgs.inf.prosoft.tigris.sampling;

import br.ufrgs.inf.prosoft.tigris.utils.StatisticalTest;
import org.apache.commons.math3.ml.neuralnet.sofm.util.ExponentialDecayFunction;
import org.apache.commons.math3.random.EmpiricalDistribution;
import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics;
import org.junit.Assert;
import org.junit.Test;
import umontreal.ssj.gof.GofStat;
import umontreal.ssj.probdist.NormalDist;

import java.util.Arrays;

public class Statistics {

    public void test(double[] sample){
        DescriptiveStatistics frequencies = new DescriptiveStatistics();
        for(double d : sample){
            frequencies.addValue(d);
        }
        System.out.println("KS pvalue should be higher than significance level (0.05): "
                + ((StatisticalTest.isNormalDistribution(frequencies.getValues(), 0.5)) ? " yes" : " no"));

        Arrays.sort(sample);
        double[] adresult = GofStat.andersonDarling(sample, new NormalDist());
        System.out.println("Dstat: " + adresult[0] + " pvalue:" + adresult[1]);
    }

    @Test
    public void normalization(){

        double[] sample = {4.50e+1, 5.20e+1, 4.30e+1,4.90e+1,4.80e+1,5.40e+1,5.30e+1,4.90e+1,4.90e+1};
        test(sample);
//        double d_stat = TestUtils.kolmogorovSmirnovStatistic(unitNormal, sample);

        test(new double[]{2, 3, 3, 4, 3, 3, 2});
        test(new double[]{93.0, 103.0, 95.0, 101.0, 91.0, 105.0, 96.0, 94.0, 101.0});
        test(new double[]{26, 33, 65, 28, 34, 25, 55, 44, 50, 36, 26, 37, 43, 62, 35, 38, 45, 32, 28, 34});
        test(new double[]{20, 15, 26, 32, 18, 28, 35, 14, 26, 22, 17});
        test(new double[]{565, 116, 69, 78, 319, 501, 633, 655, 848, 1226, 1633, 2459, 3375, 4669, 6152, 7436, 9526, 12619, 12455, 7113, 2104, 241});
        test(new double[]{9.0964868007346, 9.05841203941304, 10.4561973049786, 11.4290519020928, 9.80406201402498, 8.00495612940127, 10.9560511244096, 10.1628617100704, 9.5404880778431, 8.31409730387256, 10.5850756498228, 8.94573786127721, 9.67884319773862, 10.2674043676357, 8.88672394683865, 12.3582450296069, 11.3323127708954, 8.98248290920661, 10.0195219450004, 10.2978738217737, 8.12677492317708, 8.89950454463237, 9.77696513465945, 10.514898975753, 10.672289112785, 10.694498534934, 10.1821924686095, 11.142573152688, 9.77887037458247, 11.0049010075665, 10.2154096657666, 9.85591747299326, 8.42526783048421, 9.81110530928476, 10.8197900519108, 10.3297258106185, 10.0227636883243, 11.2987975587751, 8.44486064977259, 11.5225576269198, 8.12224293685105, 10.0007857247433, 11.5138891839899, 9.3831006575966, 8.12476347171477, 9.09856941554296, 8.62142101694361, 9.46470997205525, 9.41790826247466, 8.48703712461685, 10.2715646747326, 9.33049825280273, 10.6699012172013, 9.18072916828602, 8.41649117636405, 9.33567958667778, 10.049876040196, 11.3127990745231, 11.6586896309607, 8.89584835227498, 9.70752934278868, 8.86246838225148, 10.2161331491117, 9.85382278972802, 9.05752306126498, 11.3505257827677, 8.44762586593948, 9.15864805742191, 9.57567977055168, 11.7688636824855, 10.2024057615904, 10.3104037114343, 10.1580229817262, 10.0680455332437, 9.91423812485741, 9.94504630463698, 8.52395458741078, 10.2521461941422, 10.1036308255006, 9.18956474594286, 9.96954494398682, 9.78104199181701, 9.92382325277581, 8.63597929467619, 9.21347593192941, 9.89855218854907, 11.0121033672495, 9.63264753881509, 10.6248219653335, 10.486189745896, 10.5244954307719, 8.53373914749441, 9.3983432649672, 11.0903843560089, 9.44459585712476, 10.0859955061661, 11.3387705764577, 9.84586163010232, 11.4741429769762, 10.441848372811, 9.55045713422817, 10.6267816577505, 10.4443547888383, 9.89379735185878, 9.72354469139489, 9.12066768498717, 10.0661011719204, 10.3323960556678, 11.505082984854, 10.7859172636304, 11.8287650643979, 10.4116817822928, 10.1463678500856, 9.25763259285018, 8.95707586672983, 10.2875032181518, 9.73858400501329, 8.70348547219759, 9.14762353313967, 9.82731108963226, 10.849859681053, 10.5423597494633, 8.58089709606474, 8.47543607758568, 9.15931239135863, 9.27560590704311, 10.5368346988892, 9.83172309690772, 9.16056782666665, 9.2012213506938, 9.66210982564607, 10.5190510051426, 8.62151894187291, 9.90524992792067, 10.138710665855, 9.30829258153867, 10.8290286694516, 12.1178308609245, 9.50553095064575, 9.95398394895215, 12.0245206821336, 10.2477642478284, 8.55456942605116, 11.6070473804305, 10.1562361056329, 9.43630469486672, 10.3170511798909, 9.01652208335814, 9.86807032119238, 10.8489404774946, 10.7826270556231, 10.0870496269544, 10.9833771671752, 10.117870221528, 10.5996195058974, 11.0329621602774, 10.4132314334195, 9.74158685858501, 9.48294737415442, 9.86849961215289, 9.91013610376275, 7.83468897764531, 10.514842063025, 11.4804302009497, 10.1196554587607, 10.39118899319, 9.62353860043821, 8.97417942332151, 9.35318610423095, 10.3556147738602, 11.9155451973464, 8.27736636882948, 9.32257441690336, 10.252826149316, 9.7535116214245, 10.9162334195511, 10.1678750592301, 10.9559458131311, 8.60102009638715, 9.2628210057096, 9.78873072091305, 8.35114511028973, 10.1637620813489, 9.46509546070039, 8.29088640463265, 11.1023522365736, 8.23070146296037, 9.21646018542163, 8.78784331889502, 10.2988093157657, 10.537263338189, 10.8732424762337, 10.0350589477381, 8.64559077573426, 9.82216826151432, 12.9407607539763, 9.58702405409422, 8.68540299123893, 8.90868676037665, 9.83588804978778, 9.45456213634599, 9.88659569326315, 9.46423529809919, 8.51323166422944, 9.06114471768126, 7.0763024483504, 11.3048132198129, 10.9310615911962, 10.243499074504, 12.2494232010316, 9.92165717330645, 11.1055496127494, 10.3824281515872, 10.9363315824212, 9.47366362946216, 10.5074206502971, 9.92574082866448, 9.98656454297378, 10.3434126760638, 9.16080972535168, 10.6397501919665, 8.88350130241863, 10.2657096522122, 9.29382344383002, 10.2707284719202, 10.6950720070684, 10.5671180659654, 8.58795244344749, 9.50269445287456, 10.5319945577608, 10.8884770667036, 10.2324575999004, 9.32891854565115, 8.9740321510626, 10.4126653257714, 9.98364553876901, 10.0039606824373, 11.373614936858, 8.33792326220959, 8.42278194538867, 8.92005049489281, 9.56511841862538, 10.6829314510294, 10.6412977717579, 11.1075024003369, 10.5483699986234, 10.3218235881083, 9.91439811222411, 10.0027837642839, 9.50565960381905});


        //https://github.com/umontreal-simul/ssj

        //http://jdistlib.sourceforge.net/
//        NormalityTest.xxx_statistic(double[] x);
//        NormalityTest.xxx_p_value(double stat, int df);
    }

    @Test
    public void testCalculatePercentile() {
        //given
        double[] values = new double[]{1,2,3,4,5};

        EmpiricalDistribution distribution = new EmpiricalDistribution(values.length);
        distribution.load(values);

        //when
        double percentile = distribution.cumulativeProbability(4);

        //then
        Assert.assertEquals(percentile, 0.8, 0.1);
    }

    @Test
    public void testQuantiles(){
        DescriptiveStatistics frequencies = new DescriptiveStatistics();
        double[] values = new double[]{7.1, 7.4, 7.5, 7.7, 7.8, 7.9};
        for(double d : values)
            frequencies.addValue(d);

        //then
        Assert.assertEquals(frequencies.getPercentile(25), 7.325, 0.1);
        Assert.assertEquals(frequencies.getPercentile(50), 7.6, 0.1);
        Assert.assertEquals(frequencies.getPercentile(75), 7.825, 0.1);
    }

    @Test
    public void test2Quantiles(){
        DescriptiveStatistics frequencies = new DescriptiveStatistics();
        double[] values = new double[]{1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,2,2,4};
        for(double d : values)
            frequencies.addValue(d);

        //then
        System.out.println(frequencies.getPercentile(25));
        System.out.println(frequencies.getPercentile(50));
        System.out.println(frequencies.getPercentile(75));
    }

    @Test
    public void testExponentialDecay(){
        ExponentialDecayFunction decay = new ExponentialDecayFunction(100, 0.01, 3600);
        System.out.println(decay.value(0));
        System.out.println(decay.value(1));
        System.out.println(decay.value(2));
        System.out.println(decay.value(3));
        System.out.println(decay.value(4));

    }

}