001 /* 002 * Licensed to the Apache Software Foundation (ASF) under one or more 003 * contributor license agreements. See the NOTICE file distributed with 004 * this work for additional information regarding copyright ownership. 005 * The ASF licenses this file to You under the Apache License, Version 2.0 006 * (the "License"); you may not use this file except in compliance with 007 * the License. You may obtain a copy of the License at 008 * 009 * http://www.apache.org/licenses/LICENSE-2.0 010 * 011 * Unless required by applicable law or agreed to in writing, software 012 * distributed under the License is distributed on an "AS IS" BASIS, 013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 014 * See the License for the specific language governing permissions and 015 * limitations under the License. 016 */ 017 package org.apache.commons.math.stat.correlation; 018 019 import org.apache.commons.math.TestUtils; 020 import org.apache.commons.math.linear.RealMatrix; 021 import org.apache.commons.math.linear.Array2DRowRealMatrix; 022 import org.apache.commons.math.stat.descriptive.moment.Variance; 023 024 import junit.framework.TestCase; 025 026 public class CovarianceTest extends TestCase { 027 028 protected final double[] longleyData = new double[] { 029 60323,83.0,234289,2356,1590,107608,1947, 030 61122,88.5,259426,2325,1456,108632,1948, 031 60171,88.2,258054,3682,1616,109773,1949, 032 61187,89.5,284599,3351,1650,110929,1950, 033 63221,96.2,328975,2099,3099,112075,1951, 034 63639,98.1,346999,1932,3594,113270,1952, 035 64989,99.0,365385,1870,3547,115094,1953, 036 63761,100.0,363112,3578,3350,116219,1954, 037 66019,101.2,397469,2904,3048,117388,1955, 038 67857,104.6,419180,2822,2857,118734,1956, 039 68169,108.4,442769,2936,2798,120445,1957, 040 66513,110.8,444546,4681,2637,121950,1958, 041 68655,112.6,482704,3813,2552,123366,1959, 042 69564,114.2,502601,3931,2514,125368,1960, 043 69331,115.7,518173,4806,2572,127852,1961, 044 70551,116.9,554894,4007,2827,130081,1962 045 }; 046 047 protected final double[] swissData = new double[] { 048 80.2,17.0,15,12,9.96, 049 83.1,45.1,6,9,84.84, 050 92.5,39.7,5,5,93.40, 051 85.8,36.5,12,7,33.77, 052 76.9,43.5,17,15,5.16, 053 76.1,35.3,9,7,90.57, 054 83.8,70.2,16,7,92.85, 055 92.4,67.8,14,8,97.16, 056 82.4,53.3,12,7,97.67, 057 82.9,45.2,16,13,91.38, 058 87.1,64.5,14,6,98.61, 059 64.1,62.0,21,12,8.52, 060 66.9,67.5,14,7,2.27, 061 68.9,60.7,19,12,4.43, 062 61.7,69.3,22,5,2.82, 063 68.3,72.6,18,2,24.20, 064 71.7,34.0,17,8,3.30, 065 55.7,19.4,26,28,12.11, 066 54.3,15.2,31,20,2.15, 067 65.1,73.0,19,9,2.84, 068 65.5,59.8,22,10,5.23, 069 65.0,55.1,14,3,4.52, 070 56.6,50.9,22,12,15.14, 071 57.4,54.1,20,6,4.20, 072 72.5,71.2,12,1,2.40, 073 74.2,58.1,14,8,5.23, 074 72.0,63.5,6,3,2.56, 075 60.5,60.8,16,10,7.72, 076 58.3,26.8,25,19,18.46, 077 65.4,49.5,15,8,6.10, 078 75.5,85.9,3,2,99.71, 079 69.3,84.9,7,6,99.68, 080 77.3,89.7,5,2,100.00, 081 70.5,78.2,12,6,98.96, 082 79.4,64.9,7,3,98.22, 083 65.0,75.9,9,9,99.06, 084 92.2,84.6,3,3,99.46, 085 79.3,63.1,13,13,96.83, 086 70.4,38.4,26,12,5.62, 087 65.7,7.7,29,11,13.79, 088 72.7,16.7,22,13,11.22, 089 64.4,17.6,35,32,16.92, 090 77.6,37.6,15,7,4.97, 091 67.6,18.7,25,7,8.65, 092 35.0,1.2,37,53,42.34, 093 44.7,46.6,16,29,50.43, 094 42.8,27.7,22,29,58.33 095 }; 096 097 098 /** 099 * Test Longley dataset against R. 100 * Data Source: J. Longley (1967) "An Appraisal of Least Squares 101 * Programs for the Electronic Computer from the Point of View of the User" 102 * Journal of the American Statistical Association, vol. 62. September, 103 * pp. 819-841. 104 * 105 * Data are from NIST: 106 * http://www.itl.nist.gov/div898/strd/lls/data/LINKS/DATA/Longley.dat 107 */ 108 public void testLongly() { 109 RealMatrix matrix = createRealMatrix(longleyData, 16, 7); 110 RealMatrix covarianceMatrix = new Covariance(matrix).getCovarianceMatrix(); 111 double[] rData = new double[] { 112 12333921.73333333246, 3.679666000000000e+04, 343330206.333333313, 113 1649102.666666666744, 1117681.066666666651, 23461965.733333334, 16240.93333333333248, 114 36796.66000000000, 1.164576250000000e+02, 1063604.115416667, 115 6258.666250000000, 3490.253750000000, 73503.000000000, 50.92333333333334, 116 343330206.33333331347, 1.063604115416667e+06, 9879353659.329166412, 117 56124369.854166664183, 30880428.345833335072, 685240944.600000024, 470977.90000000002328, 118 1649102.66666666674, 6.258666250000000e+03, 56124369.854166664, 119 873223.429166666698, -115378.762499999997, 4462741.533333333, 2973.03333333333330, 120 1117681.06666666665, 3.490253750000000e+03, 30880428.345833335, 121 -115378.762499999997, 484304.095833333326, 1764098.133333333, 1382.43333333333339, 122 23461965.73333333433, 7.350300000000000e+04, 685240944.600000024, 123 4462741.533333333209, 1764098.133333333302, 48387348.933333330, 32917.40000000000146, 124 16240.93333333333, 5.092333333333334e+01, 470977.900000000, 125 2973.033333333333, 1382.433333333333, 32917.40000000, 22.66666666666667 126 }; 127 128 TestUtils.assertEquals("covariance matrix", createRealMatrix(rData, 7, 7), covarianceMatrix, 10E-9); 129 130 } 131 132 /** 133 * Test R Swiss fertility dataset against R. 134 * Data Source: R datasets package 135 */ 136 public void testSwissFertility() { 137 RealMatrix matrix = createRealMatrix(swissData, 47, 5); 138 RealMatrix covarianceMatrix = new Covariance(matrix).getCovarianceMatrix(); 139 double[] rData = new double[] { 140 156.0424976873265, 100.1691489361702, -64.36692876965772, -79.7295097132285, 241.5632030527289, 141 100.169148936170251, 515.7994172062905, -124.39283071230344, -139.6574005550416, 379.9043755781684, 142 -64.3669287696577, -124.3928307123034, 63.64662349676226, 53.5758556891767, -190.5606105457909, 143 -79.7295097132285, -139.6574005550416, 53.57585568917669, 92.4560592044403, -61.6988297872340, 144 241.5632030527289, 379.9043755781684, -190.56061054579092, -61.6988297872340, 1739.2945371877890 145 }; 146 147 TestUtils.assertEquals("covariance matrix", createRealMatrix(rData, 5, 5), covarianceMatrix, 10E-13); 148 } 149 150 /** 151 * Constant column 152 */ 153 public void testConstant() { 154 double[] noVariance = new double[] {1, 1, 1, 1}; 155 double[] values = new double[] {1, 2, 3, 4}; 156 assertEquals(0d, new Covariance().covariance(noVariance, values, true), Double.MIN_VALUE); 157 assertEquals(0d, new Covariance().covariance(noVariance, noVariance, true), Double.MIN_VALUE); 158 } 159 160 161 /** 162 * Insufficient data 163 */ 164 public void testInsufficientData() { 165 double[] one = new double[] {1}; 166 double[] two = new double[] {2}; 167 try { 168 new Covariance().covariance(one, two, false); 169 fail("Expecting IllegalArgumentException"); 170 } catch (IllegalArgumentException ex) { 171 // Expected 172 } 173 RealMatrix matrix = new Array2DRowRealMatrix(new double[][] {{0},{1}}); 174 try { 175 new Covariance(matrix); 176 fail("Expecting IllegalArgumentException"); 177 } catch (IllegalArgumentException ex) { 178 // Expected 179 } 180 } 181 182 /** 183 * Verify that diagonal entries are consistent with Variance computation and matrix matches 184 * column-by-column covariances 185 */ 186 public void testConsistency() { 187 final RealMatrix matrix = createRealMatrix(swissData, 47, 5); 188 final RealMatrix covarianceMatrix = new Covariance(matrix).getCovarianceMatrix(); 189 190 // Variances on the diagonal 191 Variance variance = new Variance(); 192 for (int i = 0; i < 5; i++) { 193 assertEquals(variance.evaluate(matrix.getColumn(i)), covarianceMatrix.getEntry(i,i), 10E-14); 194 } 195 196 // Symmetry, column-consistency 197 assertEquals(covarianceMatrix.getEntry(2, 3), 198 new Covariance().covariance(matrix.getColumn(2), matrix.getColumn(3), true), 10E-14); 199 assertEquals(covarianceMatrix.getEntry(2, 3), covarianceMatrix.getEntry(3, 2), Double.MIN_VALUE); 200 201 // All columns same -> all entries = column variance 202 RealMatrix repeatedColumns = new Array2DRowRealMatrix(47, 3); 203 for (int i = 0; i < 3; i++) { 204 repeatedColumns.setColumnMatrix(i, matrix.getColumnMatrix(0)); 205 } 206 RealMatrix repeatedCovarianceMatrix = new Covariance(repeatedColumns).getCovarianceMatrix(); 207 double columnVariance = variance.evaluate(matrix.getColumn(0)); 208 for (int i = 0; i < 3; i++) { 209 for (int j = 0; j < 3; j++) { 210 assertEquals(columnVariance, repeatedCovarianceMatrix.getEntry(i, j), 10E-14); 211 } 212 } 213 214 // Check bias-correction defaults 215 double[][] data = matrix.getData(); 216 TestUtils.assertEquals("Covariances", 217 covarianceMatrix, new Covariance().computeCovarianceMatrix(data),Double.MIN_VALUE); 218 TestUtils.assertEquals("Covariances", 219 covarianceMatrix, new Covariance().computeCovarianceMatrix(data, true),Double.MIN_VALUE); 220 221 double[] x = data[0]; 222 double[] y = data[1]; 223 assertEquals(new Covariance().covariance(x, y), 224 new Covariance().covariance(x, y, true), Double.MIN_VALUE); 225 } 226 227 protected RealMatrix createRealMatrix(double[] data, int nRows, int nCols) { 228 double[][] matrixData = new double[nRows][nCols]; 229 int ptr = 0; 230 for (int i = 0; i < nRows; i++) { 231 System.arraycopy(data, ptr, matrixData[i], 0, nCols); 232 ptr += nCols; 233 } 234 return new Array2DRowRealMatrix(matrixData); 235 } 236 }