001/* 002 * (C) Copyright 2014-2018 Nuxeo (http://nuxeo.com/) and others. 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 * 016 * Contributors: 017 * slacoin, jcarsique 018 * 019 */ 020package org.nuxeo.runtime.test.runner; 021 022import java.lang.annotation.ElementType; 023import java.lang.annotation.Inherited; 024import java.lang.annotation.Retention; 025import java.lang.annotation.RetentionPolicy; 026import java.lang.annotation.Target; 027 028import javax.inject.Inject; 029 030import org.apache.commons.logging.Log; 031import org.apache.commons.logging.LogFactory; 032import org.apache.logging.log4j.ThreadContext; 033import org.junit.ClassRule; 034import org.junit.Ignore; 035import org.junit.Rule; 036import org.junit.internal.AssumptionViolatedException; 037import org.junit.rules.MethodRule; 038import org.junit.rules.TestRule; 039import org.junit.runner.Description; 040import org.junit.runner.notification.Failure; 041import org.junit.runner.notification.RunListener; 042import org.junit.runner.notification.RunNotifier; 043import org.junit.runners.model.FrameworkMethod; 044import org.junit.runners.model.Statement; 045 046/** 047 * Define execution rules for an annotated random bug. 048 * <p> 049 * Principle is to increase consistency on tests which have a random behavior. Such test is a headache because: 050 * <ul> 051 * <li>some developers may ask to ignore a random test since it's not reliable and produces useless noise most of the 052 * time,</li> 053 * <li>however, the test may still be useful in continuous integration for checking the non-random part of code it 054 * covers,</li> 055 * <li>and, after all, there's a random bug which should be fixed!</li> 056 * </ul> 057 * </p> 058 * <p> 059 * Compared to the @{@link Ignore} JUnit annotation, the advantage is to provide different behaviors for different use 060 * cases. The wanted behavior depending on whereas: 061 * <ul> 062 * <li>we are working on something else and don't want being bothered by an unreliable test,</li> 063 * <li>we are working on the covered code and want to be warned in case of regression,</li> 064 * <li>we are working on the random bug and want to reproduce it.</li> 065 * </ul> 066 * </p> 067 * That means that a random bug cannot be ignored. But must attempt to reproduce or hide its random aspect, depending on 068 * its execution context. For instance: <blockquote> 069 * 070 * <pre> 071 * <code> 072 * import org.nuxeo.runtime.test.runner.FeaturesRunner; 073 * import org.nuxeo.runtime.test.RandomBugRule; 074 * 075 * {@literal @}RunWith(FeaturesRunner.class) 076 * public class TestSample { 077 * public static final String NXP99999 = "Some comment or description"; 078 * 079 * {@literal @}Test 080 * {@literal @}RandomBug.Repeat(issue = NXP99999, onFailure=5, onSuccess=50) 081 * public void testWhichFailsSometimes() throws Exception { 082 * assertTrue(java.lang.Math.random() > 0.2); 083 * } 084 * }</code> 085 * </pre> 086 * 087 * </blockquote> 088 * <p> 089 * In the above example, the test fails sometimes. With the {@link RandomBug.Repeat} annotation, it will be repeated in 090 * case of failure up to 5 times until success. This is the default {@link Mode#RELAX} mode. In order to reproduce the 091 * bug, use the {@link Mode#STRICT} mode. It will be repeated in case of success up to 50 times until failure. In 092 * {@link Mode#BYPASS} mode, the test is ignored. 093 * </p> 094 * <p> 095 * You may also repeat a whole suite in the same way by annotating the class itself. You may want also want to skip some 096 * tests, then you can annotate them and set {@link Repeat#bypass()} to true. 097 * </p> 098 * 099 * @see Mode 100 * @since 5.9.5 101 */ 102public class RandomBug { 103 private static final Log log = LogFactory.getLog(RandomBug.class); 104 105 protected static final RandomBug self = new RandomBug(); 106 107 /** 108 * Repeat condition based on 109 * 110 * @see Mode 111 */ 112 @Retention(RetentionPolicy.RUNTIME) 113 @Target({ ElementType.METHOD, ElementType.TYPE }) 114 @Inherited 115 public @interface Repeat { 116 /** 117 * Reference in issue management system. Recommendation is to use a constant which name is the issue reference 118 * and value is a description or comment. 119 */ 120 String issue(); 121 122 /** 123 * Times to repeat until failure in case of success 124 */ 125 int onSuccess() default 30; 126 127 /** 128 * Times to repeat until success in case of failure 129 */ 130 int onFailure() default 10; 131 132 /** 133 * Bypass a method/suite .... 134 */ 135 boolean bypass() default false; 136 } 137 138 public static class Feature implements RunnerFeature { 139 @ClassRule 140 public static TestRule onClass() { 141 return self.onTest(); 142 } 143 144 @Rule 145 public MethodRule onMethod() { 146 return self.onMethod(); 147 } 148 } 149 150 public class RepeatRule implements TestRule, MethodRule { 151 @Inject 152 protected RunNotifier notifier; 153 154 @Inject 155 FeaturesRunner runner; 156 157 public RepeatStatement statement; 158 159 @Override 160 public Statement apply(Statement base, Description description) { 161 Repeat actual = runner.getConfig(Repeat.class); 162 if (actual.issue() == null) { 163 return base; 164 } 165 return statement = onRepeat(actual, notifier, base, description); 166 } 167 168 @Override 169 public Statement apply(Statement base, FrameworkMethod method, Object fixtureTarget) { 170 Repeat actual = method.getAnnotation(Repeat.class); 171 if (actual == null) { 172 return base; 173 } 174 Class<?> fixtureType = fixtureTarget.getClass(); 175 Description description = Description.createTestDescription(fixtureType, method.getName(), 176 method.getAnnotations()); 177 return statement = onRepeat(actual, notifier, base, description); 178 } 179 } 180 181 protected RepeatRule onTest() { 182 return new RepeatRule(); 183 } 184 185 protected RepeatRule onMethod() { 186 return new RepeatRule(); 187 } 188 189 public static final String MODE_PROPERTY = "nuxeo.tests.random.mode"; 190 191 /** 192 * <ul> 193 * <li>BYPASS: the test is ignored. Like with @{@link Ignore} JUnit annotation.</li> 194 * <li>STRICT: the test must fail. On success, the test is repeated until failure or the limit number of tries 195 * {@link Repeat#onSuccess()} is reached. If it does not fail during the tries, then the whole test class is marked 196 * as failed.</li> 197 * <li>RELAX: the test must succeed. On failure, the test is repeated until success or the limit number of tries 198 * {@link Repeat#onFailure()} is reached.</li> 199 * </ul> 200 * Could be set by the environment using the <em>nuxeo.tests.random.mode</em>T system property. 201 */ 202 public enum Mode { 203 BYPASS, STRICT, RELAX 204 } 205 206 /** 207 * The default mode if {@link #MODE_PROPERTY} is not set. 208 */ 209 public final Mode DEFAULT = Mode.RELAX; 210 211 protected Mode fetchMode() { 212 String mode = System.getProperty(MODE_PROPERTY, DEFAULT.name()); 213 return Mode.valueOf(mode.toUpperCase()); 214 } 215 216 protected abstract class RepeatStatement extends Statement { 217 protected final Repeat params; 218 219 protected final RunNotifier notifier; 220 221 protected boolean gotFailure; 222 223 protected final RunListener listener = new RunListener() { 224 @Override 225 public void testStarted(Description desc) { 226 log.debug(displayName(desc) + " STARTED"); 227 } 228 229 @Override 230 public void testFailure(Failure failure) { 231 gotFailure = true; 232 log.debug(displayName(failure.getDescription()) + " FAILURE"); 233 log.trace(failure, failure.getException()); 234 } 235 236 @Override 237 public void testAssumptionFailure(Failure failure) { 238 log.debug(displayName(failure.getDescription()) + " ASSUMPTION FAILURE"); 239 log.trace(failure, failure.getException()); 240 } 241 242 @Override 243 public void testIgnored(Description desc) { 244 log.debug(displayName(desc) + " IGNORED"); 245 } 246 247 @Override 248 public void testFinished(Description desc) { 249 log.debug(displayName(desc) + " FINISHED"); 250 } 251 }; 252 253 protected final Statement base; 254 255 protected int serial; 256 257 protected Description description; 258 259 protected RepeatStatement(Repeat someParams, RunNotifier aNotifier, Statement aStatement, 260 Description aDescription) { 261 params = someParams; 262 notifier = aNotifier; 263 base = aStatement; 264 description = aDescription; 265 } 266 267 protected String displayName(Description desc) { 268 String displayName = desc.getClassName().substring(desc.getClassName().lastIndexOf(".") + 1); 269 if (desc.isTest()) { 270 displayName += "." + desc.getMethodName(); 271 } 272 return displayName; 273 } 274 275 protected void onEnter(int serial) { 276 this.serial = serial; 277 ThreadContext.put("fRepeat", Integer.toString(serial)); 278 } 279 280 protected void onLeave() { 281 ThreadContext.remove("fRepeat"); 282 } 283 284 @Override 285 public void evaluate() { 286 Error error = error(); 287 notifier.addListener(listener); 288 try { 289 log.debug(displayName(description) + " STARTED"); 290 for (int retry = 1; retry <= retryCount(); retry++) { 291 gotFailure = false; 292 onEnter(retry); 293 try { 294 log.debug(displayName(description) + " retry " + retry); 295 base.evaluate(); 296 } catch (AssumptionViolatedException cause) { 297 Throwable t = new Throwable("On retry " + retry, cause); 298 error.addSuppressed(t); 299 notifier.fireTestAssumptionFailed(new Failure(description, t)); 300 } catch (Throwable cause) { 301 // Repeat annotation is on method (else the Throwable is not raised up to here) 302 Throwable t = new Throwable("On retry " + retry, cause); 303 error.addSuppressed(t); 304 if (returnOnFailure()) { 305 notifier.fireTestFailure(new Failure(description, t)); 306 } else { 307 gotFailure = true; 308 log.debug(displayName(description) + " FAILURE SWALLOW"); 309 log.trace(t, t); 310 } 311 } finally { 312 onLeave(); 313 } 314 if (gotFailure && returnOnFailure()) { 315 log.debug(displayName(description) + " returnOnFailure"); 316 return; 317 } 318 if (!gotFailure && returnOnSuccess()) { 319 log.debug(displayName(description) + " returnOnSuccess"); 320 return; 321 } 322 } 323 } finally { 324 log.debug(displayName(description) + " FINISHED"); 325 notifier.removeListener(listener); 326 } 327 log.trace("throw " + error); 328 throw error; 329 } 330 331 protected abstract Error error(); 332 333 protected abstract int retryCount(); 334 335 protected abstract boolean returnOnSuccess(); 336 337 protected abstract boolean returnOnFailure(); 338 } 339 340 protected class RepeatOnFailure extends RepeatStatement { 341 protected String issue; 342 343 protected RepeatOnFailure(Repeat someParams, RunNotifier aNotifier, Statement aStatement, 344 Description description) { 345 super(someParams, aNotifier, aStatement, description); 346 } 347 348 @Override 349 protected Error error() { 350 return new AssertionError(String.format( 351 "No success after %d tries. Either the bug is not random " 352 + "or you should increase the 'onFailure' value.\n" + "Issue: %s", 353 params.onFailure(), issue)); 354 } 355 356 @Override 357 protected int retryCount() { 358 return params.onFailure(); 359 } 360 361 @Override 362 protected boolean returnOnFailure() { 363 return false; 364 } 365 366 @Override 367 protected boolean returnOnSuccess() { 368 return true; 369 } 370 } 371 372 protected class RepeatOnSuccess extends RepeatStatement { 373 protected RepeatOnSuccess(Repeat someParams, RunNotifier aNotifier, Statement aStatement, 374 Description description) { 375 super(someParams, aNotifier, aStatement, description); 376 } 377 378 @Override 379 protected Error error() { 380 return new AssertionError(String.format( 381 "No failure after %d tries. Either the bug is fixed " 382 + "or you should increase the 'onSuccess' value.\n" + "Issue: %s", 383 params.onSuccess(), params.issue())); 384 } 385 386 @Override 387 protected boolean returnOnFailure() { 388 return true; 389 } 390 391 @Override 392 protected boolean returnOnSuccess() { 393 return false; 394 } 395 396 @Override 397 protected int retryCount() { 398 return params.onSuccess(); 399 } 400 } 401 402 protected class Bypass extends RepeatStatement { 403 public Bypass(Repeat someParams, RunNotifier aNotifier, Statement aStatement, Description description) { 404 super(someParams, aNotifier, aStatement, description); 405 } 406 407 @Override 408 public void evaluate() { 409 notifier.fireTestIgnored(description); 410 } 411 412 @Override 413 protected Error error() { 414 throw new UnsupportedOperationException(); 415 } 416 417 @Override 418 protected int retryCount() { 419 throw new UnsupportedOperationException(); 420 } 421 422 @Override 423 protected boolean returnOnSuccess() { 424 throw new UnsupportedOperationException(); 425 } 426 427 @Override 428 protected boolean returnOnFailure() { 429 return false; 430 } 431 } 432 433 protected RepeatStatement onRepeat(Repeat someParams, RunNotifier aNotifier, Statement aStatement, 434 Description description) { 435 if (someParams.bypass()) { 436 return new Bypass(someParams, aNotifier, aStatement, description); 437 } 438 switch (fetchMode()) { 439 case BYPASS: 440 return new Bypass(someParams, aNotifier, aStatement, description); 441 case STRICT: 442 return new RepeatOnSuccess(someParams, aNotifier, aStatement, description); 443 case RELAX: 444 return new RepeatOnFailure(someParams, aNotifier, aStatement, description); 445 } 446 throw new IllegalArgumentException("no such mode"); 447 } 448 449}