View Javadoc
1 /*** 2 * <copyright> 3 * Copyright 1997-2002 BBNT Solutions, LLC 4 * under sponsorship of the Defense Advanced Research Projects Agency (DARPA). 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the Cougaar Open Source License as published by 8 * DARPA on the Cougaar Open Source Website (www.cougaar.org). 9 * 10 * THE COUGAAR SOFTWARE AND ANY DERIVATIVE SUPPLIED BY LICENSOR IS 11 * PROVIDED 'AS IS' WITHOUT WARRANTIES OF ANY KIND, WHETHER EXPRESS OR 12 * IMPLIED, INCLUDING (BUT NOT LIMITED TO) ALL IMPLIED WARRANTIES OF 13 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, AND WITHOUT 14 * ANY WARRANTIES AS TO NON-INFRINGEMENT. IN NO EVENT SHALL COPYRIGHT 15 * HOLDER BE LIABLE FOR ANY DIRECT, SPECIAL, INDIRECT OR CONSEQUENTIAL 16 * DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE OF DATA OR PROFITS, 17 * TORTIOUS CONDUCT, ARISING OUT OF OR IN CONNECTION WITH THE USE OR 18 * PERFORMANCE OF THE COUGAAR SOFTWARE. 19 * </copyright> 20 * 21 * Created on Aug 26, 2002 22 */ 23 package test.net.sourceforge.pmd.stat; 24 25 import junit.framework.AssertionFailedError; 26 import junit.framework.TestCase; 27 import net.sourceforge.pmd.Report; 28 import net.sourceforge.pmd.Rule; 29 import net.sourceforge.pmd.RuleContext; 30 import net.sourceforge.pmd.stat.DataPoint; 31 import net.sourceforge.pmd.stat.Metric; 32 import net.sourceforge.pmd.stat.StatisticalRule; 33 34 import java.util.ArrayList; 35 import java.util.Collections; 36 import java.util.Iterator; 37 import java.util.List; 38 import java.util.Random; 39 40 /*** 41 * This class tests the Statistical Rules in PMD. 42 * 43 * The idea is, that we fill up 999 datapoints into 44 * the Stat Rule, and then throw random parameters 45 * at it. 46 * 47 * The three parameters which are checked are: 48 * sigma - # Sigmas over the mean. 49 * topscore - Only the top 5 or so items. 50 * minimum - Only things of score 10 or better 51 * 52 * When more than one parameter is lumped together, then 53 * we expect the one which would return the fewest to 54 * determine what gets sent back. 55 * 56 * So, we throw each collection of parameters, where each 57 * one is a different order into the system. We check the 58 * results off of what the smallest value should be. 59 * 60 * If you are going to work with StatisticalRule any, please 61 * bump the "NUM_TESTS" number up to something like 128. That 62 * way you are more likely to identify problems. It is set low 63 * now to make building and running tests easier (when we aren't 64 * touching the file.) 65 * 66 * Note also, that when verifying the Sigma, I wasn't quite able 67 * to determine how many results it would return (it would vary 68 * from -2 to 2 of what I expected.) That is what the delta 69 * parameter on the verify method takes. If you can figure it 70 * out exactly, (without stealing code from the StatRule) then 71 * feel free to change it and tighten the deltas. 72 */ 73 public class StatisticalRuleTest extends TestCase { 74 75 private static final int POINTS = 100; 76 77 private DataPoint points[] = new DataPoint[POINTS]; 78 private MockStatisticalRule IUT = null; 79 private String testName = null; 80 private Random random = new Random(); 81 82 public static final double MAX_MINIMUM = POINTS; 83 public static final double NO_MINIMUM = -1.0; 84 public static final double MAX_SIGMA = 5.0; 85 public static final double NO_SIGMA = -1.0; 86 public static final int MIN_TOPSCORE = 0; 87 public static final int NO_TOPSCORE = -1; 88 89 90 public static final double MEAN = 49.5; 91 public static final double SIGMA = 29.0115; 92 public static final int NUM_TESTS = 1; 93 94 public static final double DELTA = 0.005; 95 96 public StatisticalRuleTest(String name) { 97 super(name); 98 this.testName = name; 99 } 100 101 public void setUp() { 102 IUT = new MockStatisticalRule(); 103 if (testName.endsWith("0")) { 104 for (int i = 0; i < POINTS; i++) { 105 points[i] = new DataPoint(); 106 points[i].setScore(1.0 * i); 107 points[i].setLineNumber(i); 108 points[i].setMessage("DataPoint[" + Integer.toString(i) + "]"); 109 110 IUT.addDataPoint(points[i]); 111 } 112 } else if (testName.endsWith("1")) { 113 for (int i = POINTS-1; i >= 0; i--) { 114 points[i] = new DataPoint(); 115 points[i].setScore(1.0 * i); 116 points[i].setLineNumber(i); 117 points[i].setMessage("DataPoint[" + Integer.toString(i) + "]"); 118 119 IUT.addDataPoint(points[i]); 120 } 121 } else { 122 List lPoints = new ArrayList(); 123 for (int i = 0; i < POINTS; i++) { 124 DataPoint point = new DataPoint(); 125 point.setScore(1.0 * i); 126 point.setLineNumber(i); 127 point.setMessage("DataPoint[" + Integer.toString(i) + "]"); 128 129 lPoints.add(point); 130 } 131 132 Collections.shuffle(lPoints); 133 for (int i = 0; i < POINTS; i++) { 134 IUT.addDataPoint((DataPoint) lPoints.get(i)); 135 } 136 } 137 138 } 139 140 /*** 141 * This test verifies that the Stat rule creates a Metric, 142 * with the proper values. 143 */ 144 public void testMetrics() throws Throwable { 145 Report report = makeReport(IUT); 146 Iterator metrics = report.metrics(); 147 148 assertTrue(metrics.hasNext()); 149 Object o = metrics.next(); 150 151 assertTrue(o instanceof Metric); 152 Metric m = (Metric) o; 153 154 assertEquals("test.net.sourceforge.pmd.stat.MockStatisticalRule", m.getMetricName()); 155 156 assertEquals(0.0, m.getLowValue(), 0.05); 157 assertEquals(POINTS -1.0, m.getHighValue(), 0.05); 158 assertEquals(MEAN, m.getAverage(), 0.05); 159 assertEquals(SIGMA, m.getStandardDeviation(), 0.05); 160 } 161 162 /*** 163 * This returns a Random value for Sigma which will 164 * return some values. 165 */ 166 public double randomSigma() { 167 return random.nextDouble() * 1.0; 168 } 169 170 /*** 171 * This returns a Random value for Sigma which value 172 * is greater than the parameter. 173 */ 174 public double randomSigma(int minimum) { 175 double minSigma = ((POINTS -1 - minimum) - MEAN) / SIGMA; 176 177 if ((minSigma <= 0) || (minSigma > 2)) 178 return randomSigma(); 179 180 return minSigma + (random.nextDouble() * (2 - minSigma)); 181 } 182 183 /*** 184 * This returns the expected number of results when 185 * the Sigma rating is the smallest. 186 */ 187 public int expectedSigma(double sigma) { 188 long expectedMin = Math.round(MEAN + (sigma * SIGMA)); 189 190 if (((POINTS -1) - expectedMin) < 0) 191 return 0; 192 return (POINTS -1) - (int) expectedMin; 193 } 194 195 /*** 196 * This generates a random minimum value for testing. 197 */ 198 public double randomMinimum() { 199 return random.nextDouble() * (POINTS -1); 200 } 201 202 /*** 203 * This generates a random minimum value for which fewer 204 * results would be returned. 205 */ 206 public double randomMinimum(int minimum) { 207 double diffTarget = 1.0 * (POINTS -1 - minimum); 208 return (random.nextDouble() * minimum) + diffTarget; 209 } 210 211 /*** 212 * This returns the expected number of reports. 213 * 214 * If the Minimum comes in at 521.569 then we expect 215 * 522, 523, ... 999 will pass. 216 */ 217 public int expectedMinimum(double minimum) { 218 Double d = new Double(minimum); 219 return POINTS -1 - d.intValue(); 220 } 221 222 public void testExpectedMinimum() { 223 for (int i = 0; i < POINTS -1; i++) { 224 assertEquals("Integer Min", POINTS -1 - i, expectedMinimum(i * 1.0)); 225 assertEquals("Double Min", POINTS -1 - i, expectedMinimum((i * 1.0) + 0.5)); 226 } 227 } 228 229 /*** 230 * This returns a random value for Top Score. 231 */ 232 public int randomTopScore() { 233 return random.nextInt(POINTS -1); 234 } 235 236 /*** 237 * This will return a random value for the Top Score 238 * which will return more than the minimum provided. 239 */ 240 public int randomTopScore(double target) { 241 if (target < 0) 242 return 0; 243 244 return random.nextInt((new Double(target)).intValue()); 245 } 246 247 /*** 248 * This will return the expected number of results 249 * with the given Top Score. 250 */ 251 public int expectedTopScore(int target) { 252 return target; 253 } 254 255 // Test Single Datapoint 256 public void testSingleDatapoint() { 257 StatisticalRule IUT = new MockStatisticalRule(); 258 259 DataPoint point = new DataPoint(); 260 point.setScore(POINTS + 1.0); 261 point.setLineNumber(POINTS + 1); 262 point.setMessage("SingleDataPoint"); 263 264 IUT.addProperty("minimum", Integer.toString(POINTS)); 265 266 IUT.addDataPoint(point); 267 268 Report report = makeReport(IUT); 269 270 assertEquals("Expecting only one result.", 1, report.size()); 271 } 272 273 // Okay, we have three properties we need to 274 // test in Combination: 275 // S = Sigma 276 // T = Top Score 277 // M = Minimum 278 // 279 // They are listed in decreasing order of what 280 // to expect. 281 // 282 // Thus testSM() should have the Sigma less than 283 // the minimum, so we expect the Minimum # of results. 284 // 285 286 public void testS() throws Throwable { 287 verifyResults(MAX_SIGMA, NO_MINIMUM, NO_TOPSCORE, 0, 2); 288 289 for (int i = 0; i < NUM_TESTS; i++) { 290 double sigma = randomSigma(); 291 verifyResults(sigma, -1.0, -1, expectedSigma(sigma), 2); 292 } 293 } 294 295 public void testS1() throws Throwable { 296 testS(); 297 } 298 299 public void testS2() throws Throwable { 300 testS(); 301 } 302 303 public void testS3() throws Throwable { 304 testS(); 305 } 306 307 public void testS4() throws Throwable { 308 testS(); 309 } 310 311 public void testS5() throws Throwable { 312 testS(); 313 } 314 315 316 public void testT() throws Throwable { 317 verifyResults(NO_SIGMA, NO_MINIMUM, MIN_TOPSCORE, 0, 0); 318 319 for (int i = 0; i < NUM_TESTS; i++) { 320 int topScore = randomTopScore(); 321 verifyResults(-1.0, -1.0, topScore, expectedTopScore(topScore), 0); 322 } 323 } 324 325 public void testT1() throws Throwable { 326 testT(); 327 } 328 329 public void testT2() throws Throwable { 330 testT(); 331 } 332 333 public void testT3() throws Throwable { 334 testT(); 335 } 336 337 public void testT4() throws Throwable { 338 testT(); 339 } 340 341 public void testT5() throws Throwable { 342 testT(); 343 } 344 345 public void testM() throws Throwable { 346 verifyResults(NO_SIGMA, MAX_MINIMUM, NO_TOPSCORE, 0, 0); 347 348 for (int i = 0; i < NUM_TESTS; i++) { 349 double minimum = randomMinimum(); 350 verifyResults(-1.0, minimum, -1, expectedMinimum(minimum), 0); 351 } 352 } 353 354 public void testM1() throws Throwable { 355 testM(); 356 } 357 358 public void testM2() throws Throwable { 359 testM(); 360 } 361 362 public void testM3() throws Throwable { 363 testM(); 364 } 365 366 public void testM4() throws Throwable { 367 testM(); 368 } 369 370 public void testM5() throws Throwable { 371 testM(); 372 } 373 374 public void testST() throws Throwable { 375 verifyResults(randomSigma(), NO_MINIMUM, MIN_TOPSCORE, 0, 0); 376 377 for (int i = 0; i < NUM_TESTS; i++) { 378 double sigma = randomSigma(); 379 int topScore = randomTopScore(expectedSigma(sigma)); 380 381 verifyResults(sigma, NO_MINIMUM, topScore, expectedTopScore(topScore), 0); 382 } 383 } 384 385 public void testST1() throws Throwable { 386 testST(); 387 } 388 389 public void testST2() throws Throwable { 390 testST(); 391 } 392 393 public void testST3() throws Throwable { 394 testST(); 395 } 396 397 public void testST4() throws Throwable { 398 testST(); 399 } 400 401 public void testST5() throws Throwable { 402 testST(); 403 } 404 405 public void testTS() throws Throwable { 406 verifyResults(MAX_SIGMA, NO_MINIMUM, randomTopScore(), 0, 0); 407 408 for (int i = 0; i < NUM_TESTS; i++) { 409 int topScore = randomTopScore(); 410 double sigma = randomSigma(expectedTopScore(topScore)); 411 412 verifyResults(sigma, -1.0, topScore, expectedSigma(sigma), 2); 413 } 414 } 415 416 public void testTS1() throws Throwable { 417 testTS(); 418 } 419 420 public void testTS2() throws Throwable { 421 testTS(); 422 } 423 424 public void testTS3() throws Throwable { 425 testTS(); 426 } 427 428 public void testTS4() throws Throwable { 429 testTS(); 430 } 431 432 public void testTS5() throws Throwable { 433 testTS(); 434 } 435 436 public void testSM() throws Throwable { 437 verifyResults(randomSigma(), MAX_MINIMUM, NO_TOPSCORE, 0, 0); 438 for (int i = 0; i < NUM_TESTS; i++) { 439 double sigma = randomSigma(); 440 double minimum = randomMinimum(expectedSigma(sigma)); 441 442 verifyResults(sigma, minimum, -1, expectedMinimum(minimum), 0); 443 } 444 445 } 446 447 public void testSM1() throws Throwable { 448 testSM(); 449 } 450 451 public void testSM2() throws Throwable { 452 testSM(); 453 } 454 455 public void testSM3() throws Throwable { 456 testSM(); 457 } 458 459 public void testSM4() throws Throwable { 460 testSM(); 461 } 462 463 public void testSM5() throws Throwable { 464 testSM(); 465 } 466 467 468 public void testMS() throws Throwable { 469 verifyResults(MAX_SIGMA, randomMinimum(), NO_TOPSCORE, 0, 0); 470 for (int i = 0; i < NUM_TESTS; i++) { 471 double minimum = randomMinimum(); 472 double sigma = randomSigma(expectedMinimum(minimum)); 473 474 verifyResults(sigma, minimum, -1, expectedSigma(sigma), 2); 475 } 476 } 477 478 public void testMS1() throws Throwable { 479 testMS(); 480 } 481 482 public void testMS2() throws Throwable { 483 testMS(); 484 } 485 486 public void testMS3() throws Throwable { 487 testMS(); 488 } 489 490 public void testMS4() throws Throwable { 491 testMS(); 492 } 493 494 public void testMS5() throws Throwable { 495 testMS(); 496 } 497 498 499 public void testTM() throws Throwable { 500 verifyResults(NO_SIGMA, MAX_MINIMUM, randomTopScore(), 0, 0); 501 for (int i = 0; i < NUM_TESTS; i++) { 502 int topScore = randomTopScore(); 503 double minimum = randomMinimum(expectedTopScore(topScore)); 504 505 verifyResults(NO_SIGMA, minimum, topScore, expectedMinimum(minimum), 0); 506 } 507 } 508 509 public void testTM1() throws Throwable { 510 testTM(); 511 } 512 513 public void testTM2() throws Throwable { 514 testTM(); 515 } 516 517 public void testTM3() throws Throwable { 518 testTM(); 519 } 520 521 public void testTM4() throws Throwable { 522 testTM(); 523 } 524 525 public void testTM5() throws Throwable { 526 testTM(); 527 } 528 529 530 public void testMT() throws Throwable { 531 verifyResults(NO_SIGMA, randomMinimum(), MIN_TOPSCORE, 0, 0); 532 for (int i = 0; i < NUM_TESTS; i++) { 533 double minimum = randomMinimum(); 534 int topScore = randomTopScore(expectedMinimum(minimum)); 535 536 verifyResults(NO_SIGMA, minimum, topScore, expectedTopScore(topScore), 0); 537 } 538 } 539 540 public void testMT1() throws Throwable { 541 testMT(); 542 } 543 544 public void testMT2() throws Throwable { 545 testMT(); 546 } 547 548 public void testMT3() throws Throwable { 549 testMT(); 550 } 551 552 public void testMT4() throws Throwable { 553 testMT(); 554 } 555 556 public void testMT5() throws Throwable { 557 testMT(); 558 } 559 560 561 public void testSTM() throws Throwable { 562 double sigma = randomSigma(); 563 verifyResults(sigma, MAX_MINIMUM, randomTopScore(expectedSigma(sigma)), 0, 0); 564 565 for (int i = 0; i < NUM_TESTS; i++) { 566 sigma = randomSigma(); 567 int topScore = randomTopScore(expectedSigma(sigma)); 568 double minimum = randomMinimum(expectedTopScore(topScore)); 569 570 verifyResults(sigma, minimum, topScore, expectedMinimum(minimum), 0); 571 } 572 } 573 574 public void testSTM1() throws Throwable { 575 testSTM(); 576 } 577 578 public void testSTM2() throws Throwable { 579 testSTM(); 580 } 581 582 public void testSTM3() throws Throwable { 583 testSTM(); 584 } 585 586 public void testSTM4() throws Throwable { 587 testSTM(); 588 } 589 590 public void testSTM5() throws Throwable { 591 testSTM(); 592 } 593 594 public void testSMT() throws Throwable { 595 double sigma = randomSigma(); 596 verifyResults(sigma, randomMinimum(expectedSigma(sigma)), MIN_TOPSCORE, 0, 0); 597 598 for (int i = 0; i < NUM_TESTS; i++) { 599 sigma = randomSigma(); 600 double minimum = randomMinimum(expectedSigma(sigma)); 601 int topScore = randomTopScore(expectedMinimum(minimum)); 602 603 verifyResults(sigma, minimum, topScore, expectedTopScore(topScore), 0); 604 } 605 } 606 607 public void testSMT1() throws Throwable { 608 testSMT(); 609 } 610 611 public void testSMT2() throws Throwable { 612 testSMT(); 613 } 614 615 public void testSMT3() throws Throwable { 616 testSMT(); 617 } 618 619 public void testSMT4() throws Throwable { 620 testSMT(); 621 } 622 623 public void testSMT5() throws Throwable { 624 testSMT(); 625 } 626 627 public void testTSM() throws Throwable { 628 int topScore = randomTopScore(); 629 verifyResults(randomSigma(expectedTopScore(topScore)), MAX_MINIMUM, topScore, 0, 0); 630 631 for (int i = 0; i < NUM_TESTS; i++) { 632 topScore = randomTopScore(); 633 double sigma = randomSigma(expectedTopScore(topScore)); 634 double minimum = randomMinimum(expectedSigma(sigma)); 635 636 verifyResults(sigma, minimum, topScore, expectedMinimum(minimum), 0); 637 } 638 } 639 640 public void testTSM1() throws Throwable { 641 testTSM(); 642 } 643 644 public void testTSM2() throws Throwable { 645 testTSM(); 646 } 647 648 public void testTSM3() throws Throwable { 649 testTSM(); 650 } 651 652 public void testTSM4() throws Throwable { 653 testTSM(); 654 } 655 656 public void testTSM5() throws Throwable { 657 testTSM(); 658 } 659 660 public void testTMS() throws Throwable { 661 int topScore = randomTopScore(); 662 verifyResults(MAX_SIGMA, randomMinimum(expectedTopScore(topScore)), topScore, 0, 0); 663 664 for (int i = 0; i < NUM_TESTS; i++) { 665 topScore = randomTopScore(); 666 double minimum = randomMinimum(expectedTopScore(topScore)); 667 double sigma = randomSigma(expectedMinimum(minimum)); 668 669 verifyResults(sigma, minimum, topScore, expectedSigma(sigma), 2); 670 } 671 } 672 673 public void testTMS1() throws Throwable { 674 testTMS(); 675 } 676 677 public void testTMS2() throws Throwable { 678 testTMS(); 679 } 680 681 public void testTMS3() throws Throwable { 682 testTMS(); 683 } 684 685 public void testTMS4() throws Throwable { 686 testTMS(); 687 } 688 689 public void testTMS5() throws Throwable { 690 testTMS(); 691 } 692 693 /*** 694 * Verifies what happens when you pass these parameters 695 * into the thing. DELTA is the amount of error allowed. 696 * Usually DELTA is only used for Sigma, as we really can't 697 * calculate it exactly. 698 */ 699 700 public void verifyResults(double sigma, double minimum, int topScore, int expected, int delta) { 701 try { 702 setUp(); 703 if (sigma >= 0) { 704 IUT.addProperty("sigma", Double.toString(sigma)); 705 } 706 707 if (minimum >= 0) { 708 IUT.addProperty("minimum", Double.toString(minimum)); 709 } 710 711 if (topScore >= 0) { 712 IUT.addProperty("topscore", Integer.toString(topScore)); 713 } 714 715 Report report = makeReport(IUT); 716 if (delta == 0) { 717 assertEquals("Unexpected number of results: sigma= " + Double.toString(sigma) + " min= " + Double.toString(minimum) + " topscore= " + Integer.toString(topScore), expected, report.size()); 718 } else { 719 String assertStr = "Unexpected number of results: sigma= " + Double.toString(sigma) + " min= " + Double.toString(minimum) + " topscore= " + Integer.toString(topScore) + " expected= " + Integer.toString(expected) + " +/- " + Integer.toString(delta) + " actual-result= " + report.size(); 720 721 assertTrue(assertStr, report.size() >= (expected - delta)); 722 assertTrue(assertStr, report.size() <= (expected + delta)); 723 } 724 } catch (AssertionFailedError afe) { 725 System.err.println("******** " + testName + " ***********"); 726 if (sigma != NO_SIGMA) { 727 System.err.println("SIGMA: " + Double.toString(sigma) + " EXPECT: " + Integer.toString(expectedSigma(sigma))); 728 } 729 730 if (minimum != NO_MINIMUM) { 731 System.err.println("MIN: " + Double.toString(minimum) + " EXPECT: " + Integer.toString(expectedMinimum(minimum))); 732 } 733 734 if (topScore != NO_TOPSCORE) { 735 System.err.println("TOP: " + Integer.toString(topScore) + " EXPECT: " + Integer.toString(expectedTopScore(topScore))); 736 } 737 738 throw afe; 739 740 } 741 } 742 743 public Report makeReport(Rule IUT) { 744 List list = new ArrayList(); 745 Report report = new Report(); 746 747 RuleContext ctx = new RuleContext(); 748 ctx.setReport(report); 749 ctx.setSourceCodeFilename(testName); 750 751 IUT.apply(list, ctx); 752 753 return report; 754 } 755 }

This page was automatically generated by Maven