1   /***
2    * <copyright>
3    *  Copyright 1997-2002 BBNT Solutions, LLC
4    *  under sponsorship of the Defense Advanced Research Projects Agency (DARPA).
5    *
6    *  This program is free software; you can redistribute it and/or modify
7    *  it under the terms of the Cougaar Open Source License as published by
8    *  DARPA on the Cougaar Open Source Website (www.cougaar.org).
9    *
10   *  THE COUGAAR SOFTWARE AND ANY DERIVATIVE SUPPLIED BY LICENSOR IS
11   *  PROVIDED 'AS IS' WITHOUT WARRANTIES OF ANY KIND, WHETHER EXPRESS OR
12   *  IMPLIED, INCLUDING (BUT NOT LIMITED TO) ALL IMPLIED WARRANTIES OF
13   *  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, AND WITHOUT
14   *  ANY WARRANTIES AS TO NON-INFRINGEMENT.  IN NO EVENT SHALL COPYRIGHT
15   *  HOLDER BE LIABLE FOR ANY DIRECT, SPECIAL, INDIRECT OR CONSEQUENTIAL
16   *  DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE OF DATA OR PROFITS,
17   *  TORTIOUS CONDUCT, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
18   *  PERFORMANCE OF THE COUGAAR SOFTWARE.
19   * </copyright>
20   *
21   * Created on Aug 26, 2002
22   */
23  package test.net.sourceforge.pmd.stat;
24  
25  import junit.framework.AssertionFailedError;
26  import junit.framework.TestCase;
27  import net.sourceforge.pmd.Report;
28  import net.sourceforge.pmd.Rule;
29  import net.sourceforge.pmd.RuleContext;
30  import net.sourceforge.pmd.ast.SimpleNode;
31  import net.sourceforge.pmd.stat.DataPoint;
32  import net.sourceforge.pmd.stat.Metric;
33  import net.sourceforge.pmd.stat.StatisticalRule;
34  import net.sourceforge.pmd.symboltable.SourceFileScope;
35  
36  import java.util.ArrayList;
37  import java.util.Collections;
38  import java.util.Iterator;
39  import java.util.List;
40  import java.util.Random;
41  
42  /***
43   * This class tests the Statistical Rules in PMD.
44   *
45   * The idea is, that we fill up 999 datapoints into
46   * the Stat Rule, and then throw random parameters
47   * at it.
48   *
49   * The three parameters which are checked are:
50   * 		sigma - # Sigmas over the mean.
51   * 		topscore - Only the top 5 or so items.
52   * 		minimum - Only things of score 10 or better
53   *
54   * When more than one parameter is lumped together, then
55   * we expect the one which would return the fewest to
56   * determine what gets sent back.
57   *
58   * So, we throw each collection of parameters, where each
59   * one is a different order into the system.  We check the
60   * results off of what the smallest value should be.
61   *
62   * If you are going to work with StatisticalRule any, please
63   * bump the "NUM_TESTS" number up to something like 128.  That
64   * way you are more likely to identify problems.  It is set low
65   * now to make building and running tests easier (when we aren't
66   * touching the file.)
67   *
68   * Note also, that when verifying the Sigma, I wasn't quite able
69   * to determine how many results it would return (it would vary
70   * from -2 to 2 of what I expected.)  That is what the delta
71   * parameter on the verify method takes.  If you can figure it
72   * out exactly, (without stealing code from the StatRule) then
73   * feel free to change it and tighten the deltas.
74   */
75  public class StatisticalRuleTest extends TestCase {
76  
77      private static final int POINTS = 100;
78  
79      private DataPoint points[] = new DataPoint[POINTS];
80      private MockStatisticalRule IUT = null;
81      private String testName = null;
82      private Random random = new Random();
83  
84      public static final double MAX_MINIMUM = POINTS;
85      public static final double NO_MINIMUM = -1.0;
86      public static final double MAX_SIGMA = 5.0;
87      public static final double NO_SIGMA = -1.0;
88      public static final int MIN_TOPSCORE = 0;
89      public static final int NO_TOPSCORE = -1;
90  
91  
92      public static final double MEAN = 49.5;
93      public static final double SIGMA = 29.0115;
94      public static final int NUM_TESTS = 1;
95  
96      public static final double DELTA = 0.005;
97  
98      public StatisticalRuleTest(String name) {
99          super(name);
100         this.testName = name;
101     }
102 
103     public void setUp() {
104         IUT = new MockStatisticalRule();
105         if (testName.endsWith("0")) {
106             for (int i = 0; i < POINTS; i++) {
107                 points[i] = new DataPoint();
108                 points[i].setScore(1.0 * i);
109                 SimpleNode s = new SimpleNode(1);
110                 s.setScope(new SourceFileScope("foo"));
111                 s.testingOnly__setBeginLine(i);
112                 s.testingOnly__setBeginColumn(1);
113                 points[i].setNode(s);
114                 points[i].setMessage("DataPoint[" + Integer.toString(i) + "]");
115 
116                 IUT.addDataPoint(points[i]);
117             }
118         } else if (testName.endsWith("1")) {
119             for (int i = POINTS-1; i >= 0; i--) {
120                 points[i] = new DataPoint();
121                 points[i].setScore(1.0 * i);
122                 SimpleNode s = new SimpleNode(1);
123                 s.setScope(new SourceFileScope("foo"));
124                 s.testingOnly__setBeginLine(i);
125                 s.testingOnly__setBeginColumn(1);
126                 points[i].setNode(s);
127                 points[i].setMessage("DataPoint[" + Integer.toString(i) + "]");
128 
129                 IUT.addDataPoint(points[i]);
130             }
131         } else {
132             List lPoints = new ArrayList();
133             for (int i = 0; i < POINTS; i++) {
134                 points[i] = new DataPoint();
135                 points[i].setScore(1.0 * i);
136                 SimpleNode s = new SimpleNode(1);
137                 s.setScope(new SourceFileScope("foo"));
138                 s.testingOnly__setBeginLine(i);
139                 s.testingOnly__setBeginColumn(1);
140                 s.testingOnly__setBeginColumn(1);
141                 points[i].setNode(s);
142                 points[i].setMessage("DataPoint[" + Integer.toString(i) + "]");
143 
144                 lPoints.add(points[i]);
145             }
146 
147             Collections.shuffle(lPoints);
148             for (int i = 0; i < POINTS; i++) {
149                 IUT.addDataPoint((DataPoint) lPoints.get(i));
150             }
151         }
152 
153     }
154 
155     /***                           * This test verifies that the Stat rule creates a Metric,
156      * with the proper values.
157      */
158     public void testMetrics() throws Throwable {
159         Report report = makeReport(IUT);
160         Iterator metrics = report.metrics();
161 
162         assertTrue(metrics.hasNext());
163         Object o = metrics.next();
164 
165         assertTrue(o instanceof Metric);
166         Metric m = (Metric) o;
167 
168         assertEquals("test.net.sourceforge.pmd.stat.MockStatisticalRule", m.getMetricName());
169 
170         assertEquals(0.0, m.getLowValue(), 0.05);
171         assertEquals(POINTS -1.0, m.getHighValue(), 0.05);
172         assertEquals(MEAN, m.getAverage(), 0.05);
173         assertEquals(SIGMA, m.getStandardDeviation(), 0.05);
174     }
175 
176     /***
177      * This returns a Random value for Sigma which will
178      * return some values.
179      */
180     public double randomSigma() {
181         return random.nextDouble() * 1.0;
182     }
183 
184     /***
185      * This returns a Random value for Sigma which value
186      * is greater than the parameter.
187      */
188     public double randomSigma(int minimum) {
189         double minSigma = ((POINTS -1 - minimum) - MEAN) / SIGMA;
190 
191         if ((minSigma <= 0) || (minSigma > 2))
192             return randomSigma();
193 
194         return minSigma + (random.nextDouble() * (2 - minSigma));
195     }
196 
197     /***
198      * This returns the expected number of results when
199      * the Sigma rating is the smallest.
200      */
201     public int expectedSigma(double sigma) {
202         long expectedMin = Math.round(MEAN + (sigma * SIGMA));
203 
204         if (((POINTS -1) - expectedMin) < 0)
205             return 0;
206         return (POINTS -1) - (int) expectedMin;
207     }
208 
209     /***
210      * This generates a random minimum value for testing.
211      */
212     public double randomMinimum() {
213         return random.nextDouble() * (POINTS -1);
214     }
215 
216     /***
217      * This generates a random minimum value for which fewer
218      * results would be returned.
219      */
220     public double randomMinimum(int minimum) {
221         double diffTarget = 1.0 * (POINTS -1 - minimum);
222         return (random.nextDouble() * minimum) + diffTarget;
223     }
224 
225     /***
226      * This returns the expected number of reports.
227      *
228      * If the Minimum comes in at 521.569 then we expect
229      * 522, 523, ... 999 will pass.
230      */
231     public int expectedMinimum(double minimum) {
232         Double d = new Double(minimum);
233         return POINTS -1 - d.intValue();
234     }
235 
236     public void testExpectedMinimum() {
237         for (int i = 0; i < POINTS -1; i++) {
238             assertEquals("Integer Min", POINTS -1 - i, expectedMinimum(i * 1.0));
239             assertEquals("Double Min", POINTS -1 - i, expectedMinimum((i * 1.0) + 0.5));
240         }
241     }
242 
243     /***
244      * This returns a random value for Top Score.
245      */
246     public int randomTopScore() {
247         return random.nextInt(POINTS -1);
248     }
249 
250     /***
251      * This will return a random value for the Top Score
252      * which will return more than the minimum provided.
253      */
254     public int randomTopScore(double target) {
255         if (target < 0)
256             return 0;
257 
258         return random.nextInt((new Double(target)).intValue());
259     }
260 
261     /***
262      * This will return the expected number of results
263      * with the given Top Score.
264      */
265     public int expectedTopScore(int target) {
266         return target;
267     }
268 
269     // Test Single Datapoint
270     public void testSingleDatapoint() {
271         StatisticalRule IUT = new MockStatisticalRule();
272 
273         DataPoint point = new DataPoint();
274         point.setScore(POINTS + 1.0);
275         SimpleNode s = new SimpleNode(1);
276         s.setScope(new SourceFileScope("foo"));
277         s.testingOnly__setBeginLine(POINTS + 1);
278         s.testingOnly__setBeginColumn(1);
279         point.setNode(s);
280         point.setMessage("SingleDataPoint");
281 
282         IUT.addProperty("minimum", Integer.toString(POINTS));
283 
284         IUT.addDataPoint(point);
285 
286         Report report = makeReport(IUT);
287 
288         assertEquals("Expecting only one result.", 1, report.size());
289     }
290 
291     // Okay, we have three properties we need to
292     // test in Combination:
293     //  S = Sigma
294     //  T = Top Score
295     //  M = Minimum
296     //
297     // They are listed in decreasing order of what
298     // to expect.
299     //
300     // Thus testSM() should have the Sigma less than
301     // the minimum, so we expect the Minimum # of results.
302     //
303 
304     public void testS() throws Throwable {
305         verifyResults(MAX_SIGMA, NO_MINIMUM, NO_TOPSCORE, 0, 2);
306 
307         for (int i = 0; i < NUM_TESTS; i++) {
308             double sigma = randomSigma();
309             verifyResults(sigma, -1.0, -1, expectedSigma(sigma), 2);
310         }
311     }
312 
313     public void testS1() throws Throwable {
314         testS();
315     }
316 
317     public void testS2() throws Throwable {
318         testS();
319     }
320 
321     public void testS3() throws Throwable {
322         testS();
323     }
324 
325     public void testS4() throws Throwable {
326         testS();
327     }
328 
329     public void testS5() throws Throwable {
330         testS();
331     }
332 
333 
334     public void testT() throws Throwable {
335         verifyResults(NO_SIGMA, NO_MINIMUM, MIN_TOPSCORE, 0, 0);
336 
337         for (int i = 0; i < NUM_TESTS; i++) {
338             int topScore = randomTopScore();
339             verifyResults(-1.0, -1.0, topScore, expectedTopScore(topScore), 0);
340         }
341     }
342 
343     public void testT1() throws Throwable {
344         testT();
345     }
346 
347     public void testT2() throws Throwable {
348         testT();
349     }
350 
351     public void testT3() throws Throwable {
352         testT();
353     }
354 
355     public void testT4() throws Throwable {
356         testT();
357     }
358 
359     public void testT5() throws Throwable {
360         testT();
361     }
362 
363     public void testM() throws Throwable {
364         verifyResults(NO_SIGMA, MAX_MINIMUM, NO_TOPSCORE, 0, 0);
365 
366         for (int i = 0; i < NUM_TESTS; i++) {
367             double minimum = randomMinimum();
368             verifyResults(-1.0, minimum, -1, expectedMinimum(minimum), 0);
369         }
370     }
371 
372     public void testM1() throws Throwable {
373         testM();
374     }
375 
376     public void testM2() throws Throwable {
377         testM();
378     }
379 
380     public void testM3() throws Throwable {
381         testM();
382     }
383 
384     public void testM4() throws Throwable {
385         testM();
386     }
387 
388     public void testM5() throws Throwable {
389         testM();
390     }
391 
392     public void testST() throws Throwable {
393         verifyResults(randomSigma(), NO_MINIMUM, MIN_TOPSCORE, 0, 0);
394 
395         for (int i = 0; i < NUM_TESTS; i++) {
396             double sigma = randomSigma();
397             int topScore = randomTopScore(expectedSigma(sigma));
398 
399             verifyResults(sigma, NO_MINIMUM, topScore, expectedTopScore(topScore), 0);
400         }
401     }
402 
403     public void testST1() throws Throwable {
404         testST();
405     }
406 
407     public void testST2() throws Throwable {
408         testST();
409     }
410 
411     public void testST3() throws Throwable {
412         testST();
413     }
414 
415     public void testST4() throws Throwable {
416         testST();
417     }
418 
419     public void testST5() throws Throwable {
420         testST();
421     }
422 
423     public void testTS() throws Throwable {
424         verifyResults(MAX_SIGMA, NO_MINIMUM, randomTopScore(), 0, 0);
425 
426         for (int i = 0; i < NUM_TESTS; i++) {
427             int topScore = randomTopScore();
428             double sigma = randomSigma(expectedTopScore(topScore));
429 
430             verifyResults(sigma, -1.0, topScore, expectedSigma(sigma), 2);
431         }
432     }
433 
434     public void testTS1() throws Throwable {
435         testTS();
436     }
437 
438     public void testTS2() throws Throwable {
439         testTS();
440     }
441 
442     public void testTS3() throws Throwable {
443         testTS();
444     }
445 
446     public void testTS4() throws Throwable {
447         testTS();
448     }
449 
450     public void testTS5() throws Throwable {
451         testTS();
452     }
453 
454     public void testSM() throws Throwable {
455         verifyResults(randomSigma(), MAX_MINIMUM, NO_TOPSCORE, 0, 0);
456         for (int i = 0; i < NUM_TESTS; i++) {
457             double sigma = randomSigma();
458             double minimum = randomMinimum(expectedSigma(sigma));
459 
460             verifyResults(sigma, minimum, -1, expectedMinimum(minimum), 0);
461         }
462 
463     }
464 
465     public void testSM1() throws Throwable {
466         testSM();
467     }
468 
469     public void testSM2() throws Throwable {
470         testSM();
471     }
472 
473     public void testSM3() throws Throwable {
474         testSM();
475     }
476 
477     public void testSM4() throws Throwable {
478         testSM();
479     }
480 
481     public void testSM5() throws Throwable {
482         testSM();
483     }
484 
485 
486     public void testMS() throws Throwable {
487         verifyResults(MAX_SIGMA, randomMinimum(), NO_TOPSCORE, 0, 0);
488         for (int i = 0; i < NUM_TESTS; i++) {
489             double minimum = randomMinimum();
490             double sigma = randomSigma(expectedMinimum(minimum));
491 
492             verifyResults(sigma, minimum, -1, expectedSigma(sigma), 2);
493         }
494     }
495 
496     public void testMS1() throws Throwable {
497         testMS();
498     }
499 
500     public void testMS2() throws Throwable {
501         testMS();
502     }
503 
504     public void testMS3() throws Throwable {
505         testMS();
506     }
507 
508     public void testMS4() throws Throwable {
509         testMS();
510     }
511 
512     public void testMS5() throws Throwable {
513         testMS();
514     }
515 
516 
517     public void testTM() throws Throwable {
518         verifyResults(NO_SIGMA, MAX_MINIMUM, randomTopScore(), 0, 0);
519         for (int i = 0; i < NUM_TESTS; i++) {
520             int topScore = randomTopScore();
521             double minimum = randomMinimum(expectedTopScore(topScore));
522 
523             verifyResults(NO_SIGMA, minimum, topScore, expectedMinimum(minimum), 0);
524         }
525     }
526 
527     public void testTM1() throws Throwable {
528         testTM();
529     }
530 
531     public void testTM2() throws Throwable {
532         testTM();
533     }
534 
535     public void testTM3() throws Throwable {
536         testTM();
537     }
538 
539     public void testTM4() throws Throwable {
540         testTM();
541     }
542 
543     public void testTM5() throws Throwable {
544         testTM();
545     }
546 
547 
548     public void testMT() throws Throwable {
549         verifyResults(NO_SIGMA, randomMinimum(), MIN_TOPSCORE, 0, 0);
550         for (int i = 0; i < NUM_TESTS; i++) {
551             double minimum = randomMinimum();
552             int topScore = randomTopScore(expectedMinimum(minimum));
553 
554             verifyResults(NO_SIGMA, minimum, topScore, expectedTopScore(topScore), 0);
555         }
556     }
557 
558     public void testMT1() throws Throwable {
559         testMT();
560     }
561 
562     public void testMT2() throws Throwable {
563         testMT();
564     }
565 
566     public void testMT3() throws Throwable {
567         testMT();
568     }
569 
570     public void testMT4() throws Throwable {
571         testMT();
572     }
573 
574     public void testMT5() throws Throwable {
575         testMT();
576     }
577 
578 
579     public void testSTM() throws Throwable {
580         double sigma = randomSigma();
581         verifyResults(sigma, MAX_MINIMUM, randomTopScore(expectedSigma(sigma)), 0, 0);
582 
583         for (int i = 0; i < NUM_TESTS; i++) {
584             sigma = randomSigma();
585             int topScore = randomTopScore(expectedSigma(sigma));
586             double minimum = randomMinimum(expectedTopScore(topScore));
587 
588             verifyResults(sigma, minimum, topScore, expectedMinimum(minimum), 0);
589         }
590     }
591 
592     public void testSTM1() throws Throwable {
593         testSTM();
594     }
595 
596     public void testSTM2() throws Throwable {
597         testSTM();
598     }
599 
600     public void testSTM3() throws Throwable {
601         testSTM();
602     }
603 
604     public void testSTM4() throws Throwable {
605         testSTM();
606     }
607 
608     public void testSTM5() throws Throwable {
609         testSTM();
610     }
611 
612     public void testSMT() throws Throwable {
613         double sigma = randomSigma();
614         verifyResults(sigma, randomMinimum(expectedSigma(sigma)), MIN_TOPSCORE, 0, 0);
615 
616         for (int i = 0; i < NUM_TESTS; i++) {
617             sigma = randomSigma();
618             double minimum = randomMinimum(expectedSigma(sigma));
619             int topScore = randomTopScore(expectedMinimum(minimum));
620 
621             verifyResults(sigma, minimum, topScore, expectedTopScore(topScore), 0);
622         }
623     }
624 
625     public void testSMT1() throws Throwable {
626         testSMT();
627     }
628 
629     public void testSMT2() throws Throwable {
630         testSMT();
631     }
632 
633     public void testSMT3() throws Throwable {
634         testSMT();
635     }
636 
637     public void testSMT4() throws Throwable {
638         testSMT();
639     }
640 
641     public void testSMT5() throws Throwable {
642         testSMT();
643     }
644 
645     public void testTSM() throws Throwable {
646         int topScore = randomTopScore();
647         verifyResults(randomSigma(expectedTopScore(topScore)), MAX_MINIMUM, topScore, 0, 0);
648 
649         for (int i = 0; i < NUM_TESTS; i++) {
650             topScore = randomTopScore();
651             double sigma = randomSigma(expectedTopScore(topScore));
652             double minimum = randomMinimum(expectedSigma(sigma));
653 
654             verifyResults(sigma, minimum, topScore, expectedMinimum(minimum), 0);
655         }
656     }
657 
658     public void testTSM1() throws Throwable {
659         testTSM();
660     }
661 
662     public void testTSM2() throws Throwable {
663         testTSM();
664     }
665 
666     public void testTSM3() throws Throwable {
667         testTSM();
668     }
669 
670     public void testTSM4() throws Throwable {
671         testTSM();
672     }
673 
674     public void testTSM5() throws Throwable {
675         testTSM();
676     }
677 
678     public void testTMS() throws Throwable {
679         int topScore = randomTopScore();
680         verifyResults(MAX_SIGMA, randomMinimum(expectedTopScore(topScore)), topScore, 0, 0);
681 
682         for (int i = 0; i < NUM_TESTS; i++) {
683             topScore = randomTopScore();
684             double minimum = randomMinimum(expectedTopScore(topScore));
685             double sigma = randomSigma(expectedMinimum(minimum));
686 
687             verifyResults(sigma, minimum, topScore, expectedSigma(sigma), 2);
688         }
689     }
690 
691     public void testTMS1() throws Throwable {
692         testTMS();
693     }
694 
695     public void testTMS2() throws Throwable {
696         testTMS();
697     }
698 
699     public void testTMS3() throws Throwable {
700         testTMS();
701     }
702 
703     public void testTMS4() throws Throwable {
704         testTMS();
705     }
706 
707     public void testTMS5() throws Throwable {
708         testTMS();
709     }
710 
711     /***
712      * Verifies what happens when you pass these parameters
713      * into the thing.  DELTA is the amount of error allowed.
714      * Usually DELTA is only used for Sigma, as we really can't
715      * calculate it exactly.
716      */
717 
718     public void verifyResults(double sigma, double minimum, int topScore, int expected, int delta) {
719         try {
720             setUp();
721             if (sigma >= 0) {
722                 IUT.addProperty("sigma", Double.toString(sigma));
723             }
724 
725             if (minimum >= 0) {
726                 IUT.addProperty("minimum", Double.toString(minimum));
727             }
728 
729             if (topScore >= 0) {
730                 IUT.addProperty("topscore", Integer.toString(topScore));
731             }
732 
733             Report report = makeReport(IUT);
734             if (delta == 0) {
735                 assertEquals("Unexpected number of results: sigma= " + Double.toString(sigma) + " min= " + Double.toString(minimum) + " topscore= " + Integer.toString(topScore), expected, report.size());
736             } else {
737                 String assertStr = "Unexpected number of results: sigma= " + Double.toString(sigma) + " min= " + Double.toString(minimum) + " topscore= " + Integer.toString(topScore) + " expected= " + Integer.toString(expected) + " +/- " + Integer.toString(delta) + " actual-result= " + report.size();
738 
739                 assertTrue(assertStr, report.size() >= (expected - delta));
740                 assertTrue(assertStr, report.size() <= (expected + delta));
741             }
742         } catch (AssertionFailedError afe) {
743             System.err.println("******** " + testName + " ***********");
744             if (sigma != NO_SIGMA) {
745                 System.err.println("SIGMA: " + Double.toString(sigma) + " EXPECT: " + Integer.toString(expectedSigma(sigma)));
746             }
747 
748             if (minimum != NO_MINIMUM) {
749                 System.err.println("MIN: " + Double.toString(minimum) + " EXPECT: " + Integer.toString(expectedMinimum(minimum)));
750             }
751 
752             if (topScore != NO_TOPSCORE) {
753                 System.err.println("TOP: " + Integer.toString(topScore) + " EXPECT: " + Integer.toString(expectedTopScore(topScore)));
754             }
755 
756             throw afe;
757 
758         }
759     }
760 
761     public Report makeReport(Rule IUT) {
762         List list = new ArrayList();
763         Report report = new Report();
764 
765         RuleContext ctx = new RuleContext();
766         ctx.setReport(report);
767         ctx.setSourceCodeFilename(testName);
768 
769         IUT.apply(list, ctx);
770 
771         return report;
772     }
773 }