001/* 002 * (C) Copyright 2014-2015 Nuxeo SA (http://nuxeo.com/) and contributors. 003 * 004 * All rights reserved. This program and the accompanying materials 005 * are made available under the terms of the GNU Lesser General Public License 006 * (LGPL) version 2.1 which accompanies this distribution, and is available at 007 * http://www.gnu.org/licenses/lgpl-2.1.html 008 * 009 * This library is distributed in the hope that it will be useful, 010 * but WITHOUT ANY WARRANTY; without even the implied warranty of 011 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 012 * Lesser General Public License for more details. 013 * 014 * Contributors: 015 * slacoin, jcarsique 016 * 017 */ 018package org.nuxeo.runtime.test.runner; 019 020import java.lang.annotation.ElementType; 021import java.lang.annotation.Inherited; 022import java.lang.annotation.Retention; 023import java.lang.annotation.RetentionPolicy; 024import java.lang.annotation.Target; 025 026import javax.inject.Inject; 027 028import org.apache.commons.logging.Log; 029import org.apache.commons.logging.LogFactory; 030import org.apache.log4j.MDC; 031import org.junit.ClassRule; 032import org.junit.Ignore; 033import org.junit.Rule; 034import org.junit.internal.AssumptionViolatedException; 035import org.junit.rules.MethodRule; 036import org.junit.rules.TestRule; 037import org.junit.runner.Description; 038import org.junit.runner.notification.Failure; 039import org.junit.runner.notification.RunListener; 040import org.junit.runner.notification.RunNotifier; 041import org.junit.runners.model.FrameworkMethod; 042import org.junit.runners.model.Statement; 043 044/** 045 * Define execution rules for an annotated random bug. 046 * <p> 047 * Principle is to increase consistency on tests which have a random behavior. Such test is a headache because: 048 * <ul> 049 * <li>some developers may ask to ignore a random test since it's not reliable and produces useless noise most of the 050 * time,</li> 051 * <li>however, the test may still be useful in continuous integration for checking the non-random part of code it 052 * covers,</li> 053 * <li>and, after all, there's a random bug which should be fixed!</li> 054 * </ul> 055 * </p> 056 * <p> 057 * Compared to the @{@link Ignore} JUnit annotation, the advantage is to provide different behaviors for different use 058 * cases. The wanted behavior depending on whereas: 059 * <ul> 060 * <li>we are working on something else and don't want being bothered by an unreliable test,</li> 061 * <li>we are working on the covered code and want to be warned in case of regression,</li> 062 * <li>we are working on the random bug and want to reproduce it.</li> 063 * </ul> 064 * </p> 065 * That means that a random bug cannot be ignored. But must attempt to reproduce or hide its random aspect, depending on 066 * its execution context. For instance: <blockquote> 067 * 068 * <pre> 069 * <code> 070 * import org.nuxeo.runtime.test.runner.FeaturesRunner; 071 * import org.nuxeo.runtime.test.RandomBugRule; 072 * 073 * {@literal @}RunWith(FeaturesRunner.class) 074 * public class TestSample { 075 * public static final String NXP99999 = "Some comment or description"; 076 * 077 * {@literal @}Test 078 * {@literal @}RandomBugRule.Repeat(issue = NXP99999, onFailure=5, onSuccess=50) 079 * public void testWhichFailsSometimes() throws Exception { 080 * assertTrue(java.lang.Math.random() > 0.2); 081 * } 082 * }</code> 083 * </pre> 084 * 085 * </blockquote> 086 * <p> 087 * In the above example, the test fails sometimes. With the {@link RandomBug.Repeat} annotation, it will be repeated in 088 * case of failure up to 5 times until success. This is the default {@link Mode#RELAX} mode. In order to reproduce the 089 * bug, use the {@link Mode#STRICT} mode. It will be repeated in case of success up to 50 times until failure. In 090 * {@link Mode#BYPASS} mode, the test is ignored. 091 * </p> 092 * <p> 093 * You may also repeat a whole suite in the same way by annotating the class itself. You may want also want to skip some 094 * tests, then you can annotate them and set {@link Repeat#bypass()} to true. 095 * </p> 096 * 097 * @see Mode 098 * @since 5.9.5 099 */ 100public class RandomBug { 101 private static final Log log = LogFactory.getLog(RandomBug.class); 102 103 protected static final RandomBug self = new RandomBug(); 104 105 /** 106 * Repeat condition based on 107 * 108 * @see Mode 109 */ 110 @Retention(RetentionPolicy.RUNTIME) 111 @Target({ ElementType.METHOD, ElementType.TYPE }) 112 @Inherited 113 public @interface Repeat { 114 /** 115 * Reference in issue management system. Recommendation is to use a constant which name is the issue reference 116 * and value is a description or comment. 117 */ 118 String issue(); 119 120 /** 121 * Times to repeat until failure in case of success 122 */ 123 int onSuccess() default 30; 124 125 /** 126 * Times to repeat until success in case of failure 127 */ 128 int onFailure() default 10; 129 130 /** 131 * Bypass a method/suite .... 132 */ 133 boolean bypass() default false; 134 } 135 136 public static class Feature extends SimpleFeature { 137 @ClassRule 138 public static TestRule onClass() { 139 return self.onTest(); 140 } 141 142 @Rule 143 public MethodRule onMethod() { 144 return self.onMethod(); 145 } 146 } 147 148 public class RepeatRule implements TestRule, MethodRule { 149 @Inject 150 protected RunNotifier notifier; 151 152 public RepeatStatement statement; 153 154 @Override 155 public Statement apply(Statement base, Description description) { 156 final Repeat actual = description.getAnnotation(Repeat.class); 157 if (actual == null) { 158 return base; 159 } 160 return statement = onRepeat(actual, notifier, base, description); 161 } 162 163 @Override 164 public Statement apply(Statement base, FrameworkMethod method, Object fixtureTarget) { 165 final Repeat actual = method.getAnnotation(Repeat.class); 166 if (actual == null) { 167 return base; 168 } 169 Class<?> fixtureType = fixtureTarget.getClass(); 170 Description description = Description.createTestDescription(fixtureType, method.getName(), 171 method.getAnnotations()); 172 return statement = onRepeat(actual, notifier, base, description); 173 } 174 } 175 176 protected RepeatRule onTest() { 177 return new RepeatRule(); 178 } 179 180 protected RepeatRule onMethod() { 181 return new RepeatRule(); 182 } 183 184 public static final String MODE_PROPERTY = "nuxeo.tests.random.mode"; 185 186 /** 187 * <ul> 188 * <li>BYPASS: the test is ignored. Like with @{@link Ignore} JUnit annotation.</li> 189 * <li>STRICT: the test must fail. On success, the test is repeated until failure or the limit number of tries 190 * {@link Repeat#onSuccess()} is reached. If it does not fail during the tries, then the whole test class is marked 191 * as failed.</li> 192 * <li>RELAX: the test must succeed. On failure, the test is repeated until success or the limit number of tries 193 * {@link Repeat#onFailure()} is reached.</li> 194 * </ul> 195 * Could be set by the environment using the <em>nuxeo.tests.random.mode</em>T system property. 196 */ 197 public static enum Mode { 198 BYPASS, STRICT, RELAX 199 }; 200 201 /** 202 * The default mode if {@link #MODE_PROPERTY} is not set. 203 */ 204 public final Mode DEFAULT = Mode.RELAX; 205 206 protected Mode fetchMode() { 207 String mode = System.getProperty(MODE_PROPERTY, DEFAULT.name()); 208 return Mode.valueOf(mode.toUpperCase()); 209 } 210 211 protected abstract class RepeatStatement extends Statement { 212 protected final Repeat params; 213 214 protected final RunNotifier notifier; 215 216 protected boolean gotFailure; 217 218 protected final RunListener listener = new RunListener() { 219 @Override 220 public void testStarted(Description desc) throws Exception { 221 log.debug(displayName(desc) + " STARTED"); 222 }; 223 224 @Override 225 public void testFailure(Failure failure) throws Exception { 226 gotFailure = true; 227 log.debug(displayName(failure.getDescription()) + " FAILURE"); 228 log.trace(failure, failure.getException()); 229 } 230 231 @Override 232 public void testAssumptionFailure(Failure failure) { 233 log.debug(displayName(failure.getDescription()) + " ASSUMPTION FAILURE"); 234 log.trace(failure, failure.getException()); 235 } 236 237 @Override 238 public void testIgnored(Description desc) throws Exception { 239 log.debug(displayName(desc) + " IGNORED"); 240 }; 241 242 @Override 243 public void testFinished(Description desc) throws Exception { 244 log.debug(displayName(desc) + " FINISHED"); 245 }; 246 }; 247 248 protected final Statement base; 249 250 protected int serial; 251 252 protected Description description; 253 254 protected RepeatStatement(Repeat someParams, RunNotifier aNotifier, Statement aStatement, 255 Description aDescription) { 256 params = someParams; 257 notifier = aNotifier; 258 base = aStatement; 259 description = aDescription; 260 } 261 262 protected String displayName(Description desc) { 263 String displayName = desc.getClassName().substring(desc.getClassName().lastIndexOf(".") + 1); 264 if (desc.isTest()) { 265 displayName += "." + desc.getMethodName(); 266 } 267 return displayName; 268 } 269 270 protected void onEnter(int aSerial) { 271 MDC.put("fRepeat", serial = aSerial); 272 } 273 274 protected void onLeave() { 275 MDC.remove("fRepeat"); 276 } 277 278 @Override 279 public void evaluate() throws Throwable { 280 Error error = error(); 281 notifier.addListener(listener); 282 try { 283 log.debug(displayName(description) + " STARTED"); 284 for (int retry = 1; retry <= retryCount(); retry++) { 285 gotFailure = false; 286 onEnter(retry); 287 try { 288 log.debug(displayName(description) + " retry " + retry); 289 base.evaluate(); 290 } catch (AssumptionViolatedException cause) { 291 Throwable t = new Throwable("On retry " + retry).initCause(cause); 292 error.addSuppressed(t); 293 notifier.fireTestAssumptionFailed(new Failure(description, t)); 294 } catch (Throwable cause) { 295 // Repeat annotation is on method (else the Throwable is not raised up to here) 296 Throwable t = new Throwable("On retry " + retry).initCause(cause); 297 error.addSuppressed(t); 298 if (returnOnFailure()) { 299 notifier.fireTestFailure(new Failure(description, t)); 300 } else { 301 gotFailure = true; 302 log.debug(displayName(description) + " FAILURE SWALLOW"); 303 log.trace(t, t); 304 } 305 } finally { 306 onLeave(); 307 } 308 if (gotFailure && returnOnFailure()) { 309 log.debug(displayName(description) + " returnOnFailure"); 310 return; 311 } 312 if (!gotFailure && returnOnSuccess()) { 313 log.debug(displayName(description) + " returnOnSuccess"); 314 return; 315 } 316 } 317 } finally { 318 log.debug(displayName(description) + " FINISHED"); 319 notifier.removeListener(listener); 320 } 321 log.trace("throw " + error); 322 throw error; 323 } 324 325 protected abstract Error error(); 326 327 protected abstract int retryCount(); 328 329 protected abstract boolean returnOnSuccess(); 330 331 protected abstract boolean returnOnFailure(); 332 } 333 334 protected class RepeatOnFailure extends RepeatStatement { 335 protected String issue; 336 337 protected RepeatOnFailure(Repeat someParams, RunNotifier aNotifier, Statement aStatement, 338 Description description) { 339 super(someParams, aNotifier, aStatement, description); 340 } 341 342 @Override 343 protected Error error() { 344 return new AssertionError(String.format("No success after %d tries. Either the bug is not random " 345 + "or you should increase the 'onFailure' value.\n" + "Issue: %s", params.onFailure(), issue)); 346 } 347 348 @Override 349 protected int retryCount() { 350 return params.onFailure(); 351 } 352 353 @Override 354 protected boolean returnOnFailure() { 355 return false; 356 } 357 358 @Override 359 protected boolean returnOnSuccess() { 360 return true; 361 } 362 } 363 364 protected class RepeatOnSuccess extends RepeatStatement { 365 protected RepeatOnSuccess(Repeat someParams, RunNotifier aNotifier, Statement aStatement, 366 Description description) { 367 super(someParams, aNotifier, aStatement, description); 368 } 369 370 @Override 371 protected Error error() { 372 return new AssertionError(String.format("No failure after %d tries. Either the bug is fixed " 373 + "or you should increase the 'onSuccess' value.\n" + "Issue: %s", params.onSuccess(), 374 params.issue())); 375 } 376 377 @Override 378 protected boolean returnOnFailure() { 379 return true; 380 } 381 382 @Override 383 protected boolean returnOnSuccess() { 384 return false; 385 } 386 387 @Override 388 protected int retryCount() { 389 return params.onSuccess(); 390 } 391 } 392 393 protected class Bypass extends RepeatStatement { 394 public Bypass(Repeat someParams, RunNotifier aNotifier, Statement aStatement, Description description) { 395 super(someParams, aNotifier, aStatement, description); 396 } 397 398 @Override 399 public void evaluate() throws Throwable { 400 notifier.fireTestIgnored(description); 401 } 402 403 @Override 404 protected Error error() { 405 throw new UnsupportedOperationException(); 406 } 407 408 @Override 409 protected int retryCount() { 410 throw new UnsupportedOperationException(); 411 } 412 413 @Override 414 protected boolean returnOnSuccess() { 415 throw new UnsupportedOperationException(); 416 } 417 418 @Override 419 protected boolean returnOnFailure() { 420 return false; 421 } 422 } 423 424 protected RepeatStatement onRepeat(Repeat someParams, RunNotifier aNotifier, Statement aStatement, 425 Description description) { 426 if (someParams.bypass()) { 427 return new Bypass(someParams, aNotifier, aStatement, description); 428 } 429 switch (fetchMode()) { 430 case BYPASS: 431 return new Bypass(someParams, aNotifier, aStatement, description); 432 case STRICT: 433 return new RepeatOnSuccess(someParams, aNotifier, aStatement, description); 434 case RELAX: 435 return new RepeatOnFailure(someParams, aNotifier, aStatement, description); 436 } 437 throw new IllegalArgumentException("no such mode"); 438 } 439 440}