1 /***
2 * <copyright>
3 * Copyright 1997-2002 BBNT Solutions, LLC
4 * under sponsorship of the Defense Advanced Research Projects Agency (DARPA).
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the Cougaar Open Source License as published by
8 * DARPA on the Cougaar Open Source Website (www.cougaar.org).
9 *
10 * THE COUGAAR SOFTWARE AND ANY DERIVATIVE SUPPLIED BY LICENSOR IS
11 * PROVIDED 'AS IS' WITHOUT WARRANTIES OF ANY KIND, WHETHER EXPRESS OR
12 * IMPLIED, INCLUDING (BUT NOT LIMITED TO) ALL IMPLIED WARRANTIES OF
13 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, AND WITHOUT
14 * ANY WARRANTIES AS TO NON-INFRINGEMENT. IN NO EVENT SHALL COPYRIGHT
15 * HOLDER BE LIABLE FOR ANY DIRECT, SPECIAL, INDIRECT OR CONSEQUENTIAL
16 * DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE OF DATA OR PROFITS,
17 * TORTIOUS CONDUCT, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
18 * PERFORMANCE OF THE COUGAAR SOFTWARE.
19 * </copyright>
20 *
21 * Created on Aug 26, 2002
22 */
23 package test.net.sourceforge.pmd.stat;
24
25 import junit.framework.AssertionFailedError;
26 import junit.framework.TestCase;
27 import net.sourceforge.pmd.Report;
28 import net.sourceforge.pmd.Rule;
29 import net.sourceforge.pmd.RuleContext;
30 import net.sourceforge.pmd.ast.SimpleNode;
31 import net.sourceforge.pmd.stat.DataPoint;
32 import net.sourceforge.pmd.stat.Metric;
33 import net.sourceforge.pmd.stat.StatisticalRule;
34 import net.sourceforge.pmd.symboltable.SourceFileScope;
35
36 import java.util.ArrayList;
37 import java.util.Collections;
38 import java.util.Iterator;
39 import java.util.List;
40 import java.util.Random;
41
42 /***
43 * This class tests the Statistical Rules in PMD.
44 *
45 * The idea is, that we fill up 999 datapoints into
46 * the Stat Rule, and then throw random parameters
47 * at it.
48 *
49 * The three parameters which are checked are:
50 * sigma - # Sigmas over the mean.
51 * topscore - Only the top 5 or so items.
52 * minimum - Only things of score 10 or better
53 *
54 * When more than one parameter is lumped together, then
55 * we expect the one which would return the fewest to
56 * determine what gets sent back.
57 *
58 * So, we throw each collection of parameters, where each
59 * one is a different order into the system. We check the
60 * results off of what the smallest value should be.
61 *
62 * If you are going to work with StatisticalRule any, please
63 * bump the "NUM_TESTS" number up to something like 128. That
64 * way you are more likely to identify problems. It is set low
65 * now to make building and running tests easier (when we aren't
66 * touching the file.)
67 *
68 * Note also, that when verifying the Sigma, I wasn't quite able
69 * to determine how many results it would return (it would vary
70 * from -2 to 2 of what I expected.) That is what the delta
71 * parameter on the verify method takes. If you can figure it
72 * out exactly, (without stealing code from the StatRule) then
73 * feel free to change it and tighten the deltas.
74 */
75 public class StatisticalRuleTest extends TestCase {
76
77 private static final int POINTS = 100;
78
79 private DataPoint points[] = new DataPoint[POINTS];
80 private MockStatisticalRule IUT = null;
81 private String testName = null;
82 private Random random = new Random();
83
84 public static final double MAX_MINIMUM = POINTS;
85 public static final double NO_MINIMUM = -1.0;
86 public static final double MAX_SIGMA = 5.0;
87 public static final double NO_SIGMA = -1.0;
88 public static final int MIN_TOPSCORE = 0;
89 public static final int NO_TOPSCORE = -1;
90
91
92 public static final double MEAN = 49.5;
93 public static final double SIGMA = 29.0115;
94 public static final int NUM_TESTS = 1;
95
96 public static final double DELTA = 0.005;
97
98 public StatisticalRuleTest(String name) {
99 super(name);
100 this.testName = name;
101 }
102
103 public void setUp() {
104 IUT = new MockStatisticalRule();
105 if (testName.endsWith("0")) {
106 for (int i = 0; i < POINTS; i++) {
107 points[i] = new DataPoint();
108 points[i].setScore(1.0 * i);
109 SimpleNode s = new SimpleNode(1);
110 s.setScope(new SourceFileScope("foo"));
111 s.testingOnly__setBeginLine(i);
112 s.testingOnly__setBeginColumn(1);
113 points[i].setNode(s);
114 points[i].setMessage("DataPoint[" + Integer.toString(i) + "]");
115
116 IUT.addDataPoint(points[i]);
117 }
118 } else if (testName.endsWith("1")) {
119 for (int i = POINTS-1; i >= 0; i--) {
120 points[i] = new DataPoint();
121 points[i].setScore(1.0 * i);
122 SimpleNode s = new SimpleNode(1);
123 s.setScope(new SourceFileScope("foo"));
124 s.testingOnly__setBeginLine(i);
125 s.testingOnly__setBeginColumn(1);
126 points[i].setNode(s);
127 points[i].setMessage("DataPoint[" + Integer.toString(i) + "]");
128
129 IUT.addDataPoint(points[i]);
130 }
131 } else {
132 List lPoints = new ArrayList();
133 for (int i = 0; i < POINTS; i++) {
134 points[i] = new DataPoint();
135 points[i].setScore(1.0 * i);
136 SimpleNode s = new SimpleNode(1);
137 s.setScope(new SourceFileScope("foo"));
138 s.testingOnly__setBeginLine(i);
139 s.testingOnly__setBeginColumn(1);
140 s.testingOnly__setBeginColumn(1);
141 points[i].setNode(s);
142 points[i].setMessage("DataPoint[" + Integer.toString(i) + "]");
143
144 lPoints.add(points[i]);
145 }
146
147 Collections.shuffle(lPoints);
148 for (int i = 0; i < POINTS; i++) {
149 IUT.addDataPoint((DataPoint) lPoints.get(i));
150 }
151 }
152
153 }
154
155 /*** * This test verifies that the Stat rule creates a Metric,
156 * with the proper values.
157 */
158 public void testMetrics() throws Throwable {
159 Report report = makeReport(IUT);
160 Iterator metrics = report.metrics();
161
162 assertTrue(metrics.hasNext());
163 Object o = metrics.next();
164
165 assertTrue(o instanceof Metric);
166 Metric m = (Metric) o;
167
168 assertEquals("test.net.sourceforge.pmd.stat.MockStatisticalRule", m.getMetricName());
169
170 assertEquals(0.0, m.getLowValue(), 0.05);
171 assertEquals(POINTS -1.0, m.getHighValue(), 0.05);
172 assertEquals(MEAN, m.getAverage(), 0.05);
173 assertEquals(SIGMA, m.getStandardDeviation(), 0.05);
174 }
175
176 /***
177 * This returns a Random value for Sigma which will
178 * return some values.
179 */
180 public double randomSigma() {
181 return random.nextDouble() * 1.0;
182 }
183
184 /***
185 * This returns a Random value for Sigma which value
186 * is greater than the parameter.
187 */
188 public double randomSigma(int minimum) {
189 double minSigma = ((POINTS -1 - minimum) - MEAN) / SIGMA;
190
191 if ((minSigma <= 0) || (minSigma > 2))
192 return randomSigma();
193
194 return minSigma + (random.nextDouble() * (2 - minSigma));
195 }
196
197 /***
198 * This returns the expected number of results when
199 * the Sigma rating is the smallest.
200 */
201 public int expectedSigma(double sigma) {
202 long expectedMin = Math.round(MEAN + (sigma * SIGMA));
203
204 if (((POINTS -1) - expectedMin) < 0)
205 return 0;
206 return (POINTS -1) - (int) expectedMin;
207 }
208
209 /***
210 * This generates a random minimum value for testing.
211 */
212 public double randomMinimum() {
213 return random.nextDouble() * (POINTS -1);
214 }
215
216 /***
217 * This generates a random minimum value for which fewer
218 * results would be returned.
219 */
220 public double randomMinimum(int minimum) {
221 double diffTarget = 1.0 * (POINTS -1 - minimum);
222 return (random.nextDouble() * minimum) + diffTarget;
223 }
224
225 /***
226 * This returns the expected number of reports.
227 *
228 * If the Minimum comes in at 521.569 then we expect
229 * 522, 523, ... 999 will pass.
230 */
231 public int expectedMinimum(double minimum) {
232 Double d = new Double(minimum);
233 return POINTS -1 - d.intValue();
234 }
235
236 public void testExpectedMinimum() {
237 for (int i = 0; i < POINTS -1; i++) {
238 assertEquals("Integer Min", POINTS -1 - i, expectedMinimum(i * 1.0));
239 assertEquals("Double Min", POINTS -1 - i, expectedMinimum((i * 1.0) + 0.5));
240 }
241 }
242
243 /***
244 * This returns a random value for Top Score.
245 */
246 public int randomTopScore() {
247 return random.nextInt(POINTS -1);
248 }
249
250 /***
251 * This will return a random value for the Top Score
252 * which will return more than the minimum provided.
253 */
254 public int randomTopScore(double target) {
255 if (target < 0)
256 return 0;
257
258 return random.nextInt((new Double(target)).intValue());
259 }
260
261 /***
262 * This will return the expected number of results
263 * with the given Top Score.
264 */
265 public int expectedTopScore(int target) {
266 return target;
267 }
268
269
270 public void testSingleDatapoint() {
271 StatisticalRule IUT = new MockStatisticalRule();
272
273 DataPoint point = new DataPoint();
274 point.setScore(POINTS + 1.0);
275 SimpleNode s = new SimpleNode(1);
276 s.setScope(new SourceFileScope("foo"));
277 s.testingOnly__setBeginLine(POINTS + 1);
278 s.testingOnly__setBeginColumn(1);
279 point.setNode(s);
280 point.setMessage("SingleDataPoint");
281
282 IUT.addProperty("minimum", Integer.toString(POINTS));
283
284 IUT.addDataPoint(point);
285
286 Report report = makeReport(IUT);
287
288 assertEquals("Expecting only one result.", 1, report.size());
289 }
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304 public void testS() throws Throwable {
305 verifyResults(MAX_SIGMA, NO_MINIMUM, NO_TOPSCORE, 0, 2);
306
307 for (int i = 0; i < NUM_TESTS; i++) {
308 double sigma = randomSigma();
309 verifyResults(sigma, -1.0, -1, expectedSigma(sigma), 2);
310 }
311 }
312
313 public void testS1() throws Throwable {
314 testS();
315 }
316
317 public void testS2() throws Throwable {
318 testS();
319 }
320
321 public void testS3() throws Throwable {
322 testS();
323 }
324
325 public void testS4() throws Throwable {
326 testS();
327 }
328
329 public void testS5() throws Throwable {
330 testS();
331 }
332
333
334 public void testT() throws Throwable {
335 verifyResults(NO_SIGMA, NO_MINIMUM, MIN_TOPSCORE, 0, 0);
336
337 for (int i = 0; i < NUM_TESTS; i++) {
338 int topScore = randomTopScore();
339 verifyResults(-1.0, -1.0, topScore, expectedTopScore(topScore), 0);
340 }
341 }
342
343 public void testT1() throws Throwable {
344 testT();
345 }
346
347 public void testT2() throws Throwable {
348 testT();
349 }
350
351 public void testT3() throws Throwable {
352 testT();
353 }
354
355 public void testT4() throws Throwable {
356 testT();
357 }
358
359 public void testT5() throws Throwable {
360 testT();
361 }
362
363 public void testM() throws Throwable {
364 verifyResults(NO_SIGMA, MAX_MINIMUM, NO_TOPSCORE, 0, 0);
365
366 for (int i = 0; i < NUM_TESTS; i++) {
367 double minimum = randomMinimum();
368 verifyResults(-1.0, minimum, -1, expectedMinimum(minimum), 0);
369 }
370 }
371
372 public void testM1() throws Throwable {
373 testM();
374 }
375
376 public void testM2() throws Throwable {
377 testM();
378 }
379
380 public void testM3() throws Throwable {
381 testM();
382 }
383
384 public void testM4() throws Throwable {
385 testM();
386 }
387
388 public void testM5() throws Throwable {
389 testM();
390 }
391
392 public void testST() throws Throwable {
393 verifyResults(randomSigma(), NO_MINIMUM, MIN_TOPSCORE, 0, 0);
394
395 for (int i = 0; i < NUM_TESTS; i++) {
396 double sigma = randomSigma();
397 int topScore = randomTopScore(expectedSigma(sigma));
398
399 verifyResults(sigma, NO_MINIMUM, topScore, expectedTopScore(topScore), 0);
400 }
401 }
402
403 public void testST1() throws Throwable {
404 testST();
405 }
406
407 public void testST2() throws Throwable {
408 testST();
409 }
410
411 public void testST3() throws Throwable {
412 testST();
413 }
414
415 public void testST4() throws Throwable {
416 testST();
417 }
418
419 public void testST5() throws Throwable {
420 testST();
421 }
422
423 public void testTS() throws Throwable {
424 verifyResults(MAX_SIGMA, NO_MINIMUM, randomTopScore(), 0, 0);
425
426 for (int i = 0; i < NUM_TESTS; i++) {
427 int topScore = randomTopScore();
428 double sigma = randomSigma(expectedTopScore(topScore));
429
430 verifyResults(sigma, -1.0, topScore, expectedSigma(sigma), 2);
431 }
432 }
433
434 public void testTS1() throws Throwable {
435 testTS();
436 }
437
438 public void testTS2() throws Throwable {
439 testTS();
440 }
441
442 public void testTS3() throws Throwable {
443 testTS();
444 }
445
446 public void testTS4() throws Throwable {
447 testTS();
448 }
449
450 public void testTS5() throws Throwable {
451 testTS();
452 }
453
454 public void testSM() throws Throwable {
455 verifyResults(randomSigma(), MAX_MINIMUM, NO_TOPSCORE, 0, 0);
456 for (int i = 0; i < NUM_TESTS; i++) {
457 double sigma = randomSigma();
458 double minimum = randomMinimum(expectedSigma(sigma));
459
460 verifyResults(sigma, minimum, -1, expectedMinimum(minimum), 0);
461 }
462
463 }
464
465 public void testSM1() throws Throwable {
466 testSM();
467 }
468
469 public void testSM2() throws Throwable {
470 testSM();
471 }
472
473 public void testSM3() throws Throwable {
474 testSM();
475 }
476
477 public void testSM4() throws Throwable {
478 testSM();
479 }
480
481 public void testSM5() throws Throwable {
482 testSM();
483 }
484
485
486 public void testMS() throws Throwable {
487 verifyResults(MAX_SIGMA, randomMinimum(), NO_TOPSCORE, 0, 0);
488 for (int i = 0; i < NUM_TESTS; i++) {
489 double minimum = randomMinimum();
490 double sigma = randomSigma(expectedMinimum(minimum));
491
492 verifyResults(sigma, minimum, -1, expectedSigma(sigma), 2);
493 }
494 }
495
496 public void testMS1() throws Throwable {
497 testMS();
498 }
499
500 public void testMS2() throws Throwable {
501 testMS();
502 }
503
504 public void testMS3() throws Throwable {
505 testMS();
506 }
507
508 public void testMS4() throws Throwable {
509 testMS();
510 }
511
512 public void testMS5() throws Throwable {
513 testMS();
514 }
515
516
517 public void testTM() throws Throwable {
518 verifyResults(NO_SIGMA, MAX_MINIMUM, randomTopScore(), 0, 0);
519 for (int i = 0; i < NUM_TESTS; i++) {
520 int topScore = randomTopScore();
521 double minimum = randomMinimum(expectedTopScore(topScore));
522
523 verifyResults(NO_SIGMA, minimum, topScore, expectedMinimum(minimum), 0);
524 }
525 }
526
527 public void testTM1() throws Throwable {
528 testTM();
529 }
530
531 public void testTM2() throws Throwable {
532 testTM();
533 }
534
535 public void testTM3() throws Throwable {
536 testTM();
537 }
538
539 public void testTM4() throws Throwable {
540 testTM();
541 }
542
543 public void testTM5() throws Throwable {
544 testTM();
545 }
546
547
548 public void testMT() throws Throwable {
549 verifyResults(NO_SIGMA, randomMinimum(), MIN_TOPSCORE, 0, 0);
550 for (int i = 0; i < NUM_TESTS; i++) {
551 double minimum = randomMinimum();
552 int topScore = randomTopScore(expectedMinimum(minimum));
553
554 verifyResults(NO_SIGMA, minimum, topScore, expectedTopScore(topScore), 0);
555 }
556 }
557
558 public void testMT1() throws Throwable {
559 testMT();
560 }
561
562 public void testMT2() throws Throwable {
563 testMT();
564 }
565
566 public void testMT3() throws Throwable {
567 testMT();
568 }
569
570 public void testMT4() throws Throwable {
571 testMT();
572 }
573
574 public void testMT5() throws Throwable {
575 testMT();
576 }
577
578
579 public void testSTM() throws Throwable {
580 double sigma = randomSigma();
581 verifyResults(sigma, MAX_MINIMUM, randomTopScore(expectedSigma(sigma)), 0, 0);
582
583 for (int i = 0; i < NUM_TESTS; i++) {
584 sigma = randomSigma();
585 int topScore = randomTopScore(expectedSigma(sigma));
586 double minimum = randomMinimum(expectedTopScore(topScore));
587
588 verifyResults(sigma, minimum, topScore, expectedMinimum(minimum), 0);
589 }
590 }
591
592 public void testSTM1() throws Throwable {
593 testSTM();
594 }
595
596 public void testSTM2() throws Throwable {
597 testSTM();
598 }
599
600 public void testSTM3() throws Throwable {
601 testSTM();
602 }
603
604 public void testSTM4() throws Throwable {
605 testSTM();
606 }
607
608 public void testSTM5() throws Throwable {
609 testSTM();
610 }
611
612 public void testSMT() throws Throwable {
613 double sigma = randomSigma();
614 verifyResults(sigma, randomMinimum(expectedSigma(sigma)), MIN_TOPSCORE, 0, 0);
615
616 for (int i = 0; i < NUM_TESTS; i++) {
617 sigma = randomSigma();
618 double minimum = randomMinimum(expectedSigma(sigma));
619 int topScore = randomTopScore(expectedMinimum(minimum));
620
621 verifyResults(sigma, minimum, topScore, expectedTopScore(topScore), 0);
622 }
623 }
624
625 public void testSMT1() throws Throwable {
626 testSMT();
627 }
628
629 public void testSMT2() throws Throwable {
630 testSMT();
631 }
632
633 public void testSMT3() throws Throwable {
634 testSMT();
635 }
636
637 public void testSMT4() throws Throwable {
638 testSMT();
639 }
640
641 public void testSMT5() throws Throwable {
642 testSMT();
643 }
644
645 public void testTSM() throws Throwable {
646 int topScore = randomTopScore();
647 verifyResults(randomSigma(expectedTopScore(topScore)), MAX_MINIMUM, topScore, 0, 0);
648
649 for (int i = 0; i < NUM_TESTS; i++) {
650 topScore = randomTopScore();
651 double sigma = randomSigma(expectedTopScore(topScore));
652 double minimum = randomMinimum(expectedSigma(sigma));
653
654 verifyResults(sigma, minimum, topScore, expectedMinimum(minimum), 0);
655 }
656 }
657
658 public void testTSM1() throws Throwable {
659 testTSM();
660 }
661
662 public void testTSM2() throws Throwable {
663 testTSM();
664 }
665
666 public void testTSM3() throws Throwable {
667 testTSM();
668 }
669
670 public void testTSM4() throws Throwable {
671 testTSM();
672 }
673
674 public void testTSM5() throws Throwable {
675 testTSM();
676 }
677
678 public void testTMS() throws Throwable {
679 int topScore = randomTopScore();
680 verifyResults(MAX_SIGMA, randomMinimum(expectedTopScore(topScore)), topScore, 0, 0);
681
682 for (int i = 0; i < NUM_TESTS; i++) {
683 topScore = randomTopScore();
684 double minimum = randomMinimum(expectedTopScore(topScore));
685 double sigma = randomSigma(expectedMinimum(minimum));
686
687 verifyResults(sigma, minimum, topScore, expectedSigma(sigma), 2);
688 }
689 }
690
691 public void testTMS1() throws Throwable {
692 testTMS();
693 }
694
695 public void testTMS2() throws Throwable {
696 testTMS();
697 }
698
699 public void testTMS3() throws Throwable {
700 testTMS();
701 }
702
703 public void testTMS4() throws Throwable {
704 testTMS();
705 }
706
707 public void testTMS5() throws Throwable {
708 testTMS();
709 }
710
711 /***
712 * Verifies what happens when you pass these parameters
713 * into the thing. DELTA is the amount of error allowed.
714 * Usually DELTA is only used for Sigma, as we really can't
715 * calculate it exactly.
716 */
717
718 public void verifyResults(double sigma, double minimum, int topScore, int expected, int delta) {
719 try {
720 setUp();
721 if (sigma >= 0) {
722 IUT.addProperty("sigma", Double.toString(sigma));
723 }
724
725 if (minimum >= 0) {
726 IUT.addProperty("minimum", Double.toString(minimum));
727 }
728
729 if (topScore >= 0) {
730 IUT.addProperty("topscore", Integer.toString(topScore));
731 }
732
733 Report report = makeReport(IUT);
734 if (delta == 0) {
735 assertEquals("Unexpected number of results: sigma= " + Double.toString(sigma) + " min= " + Double.toString(minimum) + " topscore= " + Integer.toString(topScore), expected, report.size());
736 } else {
737 String assertStr = "Unexpected number of results: sigma= " + Double.toString(sigma) + " min= " + Double.toString(minimum) + " topscore= " + Integer.toString(topScore) + " expected= " + Integer.toString(expected) + " +/- " + Integer.toString(delta) + " actual-result= " + report.size();
738
739 assertTrue(assertStr, report.size() >= (expected - delta));
740 assertTrue(assertStr, report.size() <= (expected + delta));
741 }
742 } catch (AssertionFailedError afe) {
743 System.err.println("******** " + testName + " ***********");
744 if (sigma != NO_SIGMA) {
745 System.err.println("SIGMA: " + Double.toString(sigma) + " EXPECT: " + Integer.toString(expectedSigma(sigma)));
746 }
747
748 if (minimum != NO_MINIMUM) {
749 System.err.println("MIN: " + Double.toString(minimum) + " EXPECT: " + Integer.toString(expectedMinimum(minimum)));
750 }
751
752 if (topScore != NO_TOPSCORE) {
753 System.err.println("TOP: " + Integer.toString(topScore) + " EXPECT: " + Integer.toString(expectedTopScore(topScore)));
754 }
755
756 throw afe;
757
758 }
759 }
760
761 public Report makeReport(Rule IUT) {
762 List list = new ArrayList();
763 Report report = new Report();
764
765 RuleContext ctx = new RuleContext();
766 ctx.setReport(report);
767 ctx.setSourceCodeFilename(testName);
768
769 IUT.apply(list, ctx);
770
771 return report;
772 }
773 }