001/* 002 * (C) Copyright 2014-2018 Nuxeo (http://nuxeo.com/) and others. 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 * 016 * Contributors: 017 * slacoin, jcarsique 018 * 019 */ 020package org.nuxeo.runtime.test.runner; 021 022import java.lang.annotation.ElementType; 023import java.lang.annotation.Inherited; 024import java.lang.annotation.Retention; 025import java.lang.annotation.RetentionPolicy; 026import java.lang.annotation.Target; 027 028import javax.inject.Inject; 029 030import org.apache.commons.logging.Log; 031import org.apache.commons.logging.LogFactory; 032import org.apache.logging.log4j.ThreadContext; 033import org.junit.ClassRule; 034import org.junit.Ignore; 035import org.junit.Rule; 036import org.junit.internal.AssumptionViolatedException; 037import org.junit.rules.MethodRule; 038import org.junit.rules.TestRule; 039import org.junit.runner.Description; 040import org.junit.runner.notification.Failure; 041import org.junit.runner.notification.RunListener; 042import org.junit.runner.notification.RunNotifier; 043import org.junit.runners.model.FrameworkMethod; 044import org.junit.runners.model.Statement; 045 046/** 047 * Define execution rules for an annotated random bug. 048 * <p> 049 * Principle is to increase consistency on tests which have a random behavior. Such test is a headache because: 050 * <ul> 051 * <li>some developers may ask to ignore a random test since it's not reliable and produces useless noise most of the 052 * time,</li> 053 * <li>however, the test may still be useful in continuous integration for checking the non-random part of code it 054 * covers,</li> 055 * <li>and, after all, there's a random bug which should be fixed!</li> 056 * </ul> 057 * <p> 058 * Compared to the @{@link Ignore} JUnit annotation, the advantage is to provide different behaviors for different use 059 * cases. The wanted behavior depending on whereas: 060 * <ul> 061 * <li>we are working on something else and don't want being bothered by an unreliable test,</li> 062 * <li>we are working on the covered code and want to be warned in case of regression,</li> 063 * <li>we are working on the random bug and want to reproduce it.</li> 064 * </ul> 065 * <p> 066 * That means that a random bug cannot be ignored. But must attempt to reproduce or hide its random aspect, depending on 067 * its execution context. For instance: <blockquote> 068 * 069 * <pre> 070 * <code> 071 * import org.nuxeo.runtime.test.runner.FeaturesRunner; 072 * import org.nuxeo.runtime.test.RandomBugRule; 073 * 074 * {@literal @}RunWith(FeaturesRunner.class) 075 * public class TestSample { 076 * public static final String NXP99999 = "Some comment or description"; 077 * 078 * {@literal @}Test 079 * {@literal @}RandomBug.Repeat(issue = NXP99999, onFailure=5, onSuccess=50) 080 * public void testWhichFailsSometimes() throws Exception { 081 * assertTrue(java.lang.Math.random() > 0.2); 082 * } 083 * }</code> 084 * </pre> 085 * 086 * </blockquote> 087 * <p> 088 * In the above example, the test fails sometimes. With the {@link RandomBug.Repeat} annotation, it will be repeated in 089 * case of failure up to 5 times until success. This is the default {@link Mode#RELAX} mode. In order to reproduce the 090 * bug, use the {@link Mode#STRICT} mode. It will be repeated in case of success up to 50 times until failure. In 091 * {@link Mode#BYPASS} mode, the test is ignored. 092 * </p> 093 * <p> 094 * You may also repeat a whole suite in the same way by annotating the class itself. You may want also want to skip some 095 * tests, then you can annotate them and set {@link Repeat#bypass()} to true. 096 * </p> 097 * 098 * @see Mode 099 * @since 5.9.5 100 */ 101public class RandomBug { 102 private static final Log log = LogFactory.getLog(RandomBug.class); 103 104 protected static final RandomBug self = new RandomBug(); 105 106 /** 107 * Repeat condition based on 108 * 109 * @see Mode 110 */ 111 @Retention(RetentionPolicy.RUNTIME) 112 @Target({ ElementType.METHOD, ElementType.TYPE }) 113 @Inherited 114 public @interface Repeat { 115 /** 116 * Reference in issue management system. Recommendation is to use a constant which name is the issue reference 117 * and value is a description or comment. 118 */ 119 String issue(); 120 121 /** 122 * Times to repeat until failure in case of success 123 */ 124 int onSuccess() default 30; 125 126 /** 127 * Times to repeat until success in case of failure 128 */ 129 int onFailure() default 10; 130 131 /** 132 * Bypass a method/suite .... 133 */ 134 boolean bypass() default false; 135 } 136 137 public static class Feature implements RunnerFeature { 138 @ClassRule 139 public static TestRule onClass() { 140 return self.onTest(); 141 } 142 143 @Rule 144 public MethodRule onMethod() { 145 return self.onMethod(); 146 } 147 } 148 149 public class RepeatRule implements TestRule, MethodRule { 150 @Inject 151 protected RunNotifier notifier; 152 153 @Inject 154 FeaturesRunner runner; 155 156 public RepeatStatement statement; 157 158 @Override 159 public Statement apply(Statement base, Description description) { 160 Repeat actual = runner.getConfig(Repeat.class); 161 if (actual.issue() == null) { 162 return base; 163 } 164 return statement = onRepeat(actual, notifier, base, description); 165 } 166 167 @Override 168 public Statement apply(Statement base, FrameworkMethod method, Object fixtureTarget) { 169 Repeat actual = method.getAnnotation(Repeat.class); 170 if (actual == null) { 171 return base; 172 } 173 Class<?> fixtureType = fixtureTarget.getClass(); 174 Description description = Description.createTestDescription(fixtureType, method.getName(), 175 method.getAnnotations()); 176 return statement = onRepeat(actual, notifier, base, description); 177 } 178 } 179 180 protected RepeatRule onTest() { 181 return new RepeatRule(); 182 } 183 184 protected RepeatRule onMethod() { 185 return new RepeatRule(); 186 } 187 188 public static final String MODE_PROPERTY = "nuxeo.tests.random.mode"; 189 190 /** 191 * <ul> 192 * <li>BYPASS: the test is ignored. Like with @{@link Ignore} JUnit annotation.</li> 193 * <li>STRICT: the test must fail. On success, the test is repeated until failure or the limit number of tries 194 * {@link Repeat#onSuccess()} is reached. If it does not fail during the tries, then the whole test class is marked 195 * as failed.</li> 196 * <li>RELAX: the test must succeed. On failure, the test is repeated until success or the limit number of tries 197 * {@link Repeat#onFailure()} is reached.</li> 198 * </ul> 199 * Could be set by the environment using the <em>nuxeo.tests.random.mode</em>T system property. 200 */ 201 public enum Mode { 202 BYPASS, STRICT, RELAX 203 } 204 205 /** 206 * The default mode if {@link #MODE_PROPERTY} is not set. 207 */ 208 public final Mode DEFAULT = Mode.RELAX; 209 210 protected Mode fetchMode() { 211 String mode = System.getProperty(MODE_PROPERTY, DEFAULT.name()); 212 return Mode.valueOf(mode.toUpperCase()); 213 } 214 215 protected abstract class RepeatStatement extends Statement { 216 protected final Repeat params; 217 218 protected final RunNotifier notifier; 219 220 protected boolean gotFailure; 221 222 protected final RunListener listener = new RunListener() { 223 @Override 224 public void testStarted(Description desc) { 225 log.debug(displayName(desc) + " STARTED"); 226 } 227 228 @Override 229 public void testFailure(Failure failure) { 230 gotFailure = true; 231 log.debug(displayName(failure.getDescription()) + " FAILURE"); 232 log.trace(failure, failure.getException()); 233 } 234 235 @Override 236 public void testAssumptionFailure(Failure failure) { 237 log.debug(displayName(failure.getDescription()) + " ASSUMPTION FAILURE"); 238 log.trace(failure, failure.getException()); 239 } 240 241 @Override 242 public void testIgnored(Description desc) { 243 log.debug(displayName(desc) + " IGNORED"); 244 } 245 246 @Override 247 public void testFinished(Description desc) { 248 log.debug(displayName(desc) + " FINISHED"); 249 } 250 }; 251 252 protected final Statement base; 253 254 protected int serial; 255 256 protected Description description; 257 258 protected RepeatStatement(Repeat someParams, RunNotifier aNotifier, Statement aStatement, 259 Description aDescription) { 260 params = someParams; 261 notifier = aNotifier; 262 base = aStatement; 263 description = aDescription; 264 } 265 266 protected String displayName(Description desc) { 267 String displayName = desc.getClassName().substring(desc.getClassName().lastIndexOf(".") + 1); 268 if (desc.isTest()) { 269 displayName += "." + desc.getMethodName(); 270 } 271 return displayName; 272 } 273 274 protected void onEnter(int serial) { 275 this.serial = serial; 276 ThreadContext.put("fRepeat", Integer.toString(serial)); 277 } 278 279 protected void onLeave() { 280 ThreadContext.remove("fRepeat"); 281 } 282 283 @Override 284 public void evaluate() { 285 Error error = error(); 286 notifier.addListener(listener); 287 try { 288 log.debug(displayName(description) + " STARTED"); 289 for (int retry = 1; retry <= retryCount(); retry++) { 290 gotFailure = false; 291 onEnter(retry); 292 try { 293 log.debug(displayName(description) + " retry " + retry); 294 base.evaluate(); 295 } catch (AssumptionViolatedException cause) { 296 Throwable t = new Throwable("On retry " + retry, cause); 297 error.addSuppressed(t); 298 notifier.fireTestAssumptionFailed(new Failure(description, t)); 299 } catch (Throwable cause) { 300 // Repeat annotation is on method (else the Throwable is not raised up to here) 301 Throwable t = new Throwable("On retry " + retry, cause); 302 error.addSuppressed(t); 303 if (returnOnFailure()) { 304 notifier.fireTestFailure(new Failure(description, t)); 305 } else { 306 gotFailure = true; 307 log.debug(displayName(description) + " FAILURE SWALLOW"); 308 log.trace(t, t); 309 } 310 } finally { 311 onLeave(); 312 } 313 if (gotFailure && returnOnFailure()) { 314 log.debug(displayName(description) + " returnOnFailure"); 315 return; 316 } 317 if (!gotFailure && returnOnSuccess()) { 318 log.debug(displayName(description) + " returnOnSuccess"); 319 return; 320 } 321 } 322 } finally { 323 log.debug(displayName(description) + " FINISHED"); 324 notifier.removeListener(listener); 325 } 326 log.trace("throw " + error); 327 throw error; 328 } 329 330 protected abstract Error error(); 331 332 protected abstract int retryCount(); 333 334 protected abstract boolean returnOnSuccess(); 335 336 protected abstract boolean returnOnFailure(); 337 } 338 339 protected class RepeatOnFailure extends RepeatStatement { 340 protected String issue; 341 342 protected RepeatOnFailure(Repeat someParams, RunNotifier aNotifier, Statement aStatement, 343 Description description) { 344 super(someParams, aNotifier, aStatement, description); 345 } 346 347 @Override 348 protected Error error() { 349 return new AssertionError(String.format( 350 "No success after %d tries. Either the bug is not random " 351 + "or you should increase the 'onFailure' value.\n" + "Issue: %s", 352 params.onFailure(), issue)); 353 } 354 355 @Override 356 protected int retryCount() { 357 return params.onFailure(); 358 } 359 360 @Override 361 protected boolean returnOnFailure() { 362 return false; 363 } 364 365 @Override 366 protected boolean returnOnSuccess() { 367 return true; 368 } 369 } 370 371 protected class RepeatOnSuccess extends RepeatStatement { 372 protected RepeatOnSuccess(Repeat someParams, RunNotifier aNotifier, Statement aStatement, 373 Description description) { 374 super(someParams, aNotifier, aStatement, description); 375 } 376 377 @Override 378 protected Error error() { 379 return new AssertionError(String.format( 380 "No failure after %d tries. Either the bug is fixed " 381 + "or you should increase the 'onSuccess' value.\n" + "Issue: %s", 382 params.onSuccess(), params.issue())); 383 } 384 385 @Override 386 protected boolean returnOnFailure() { 387 return true; 388 } 389 390 @Override 391 protected boolean returnOnSuccess() { 392 return false; 393 } 394 395 @Override 396 protected int retryCount() { 397 return params.onSuccess(); 398 } 399 } 400 401 protected class Bypass extends RepeatStatement { 402 public Bypass(Repeat someParams, RunNotifier aNotifier, Statement aStatement, Description description) { 403 super(someParams, aNotifier, aStatement, description); 404 } 405 406 @Override 407 public void evaluate() { 408 notifier.fireTestIgnored(description); 409 } 410 411 @Override 412 protected Error error() { 413 throw new UnsupportedOperationException(); 414 } 415 416 @Override 417 protected int retryCount() { 418 throw new UnsupportedOperationException(); 419 } 420 421 @Override 422 protected boolean returnOnSuccess() { 423 throw new UnsupportedOperationException(); 424 } 425 426 @Override 427 protected boolean returnOnFailure() { 428 return false; 429 } 430 } 431 432 protected RepeatStatement onRepeat(Repeat someParams, RunNotifier aNotifier, Statement aStatement, 433 Description description) { 434 if (someParams.bypass()) { 435 return new Bypass(someParams, aNotifier, aStatement, description); 436 } 437 switch (fetchMode()) { 438 case BYPASS: 439 return new Bypass(someParams, aNotifier, aStatement, description); 440 case STRICT: 441 return new RepeatOnSuccess(someParams, aNotifier, aStatement, description); 442 case RELAX: 443 return new RepeatOnFailure(someParams, aNotifier, aStatement, description); 444 } 445 throw new IllegalArgumentException("no such mode"); 446 } 447 448}