001/*
002 * (C) Copyright 2014-2015 Nuxeo SA (http://nuxeo.com/) and others.
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 *     http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 *
016 * Contributors:
017 *     slacoin, jcarsique
018 *
019 */
020package org.nuxeo.runtime.test.runner;
021
022import java.lang.annotation.ElementType;
023import java.lang.annotation.Inherited;
024import java.lang.annotation.Retention;
025import java.lang.annotation.RetentionPolicy;
026import java.lang.annotation.Target;
027
028import javax.inject.Inject;
029
030import org.apache.commons.logging.Log;
031import org.apache.commons.logging.LogFactory;
032import org.apache.log4j.MDC;
033import org.junit.ClassRule;
034import org.junit.Ignore;
035import org.junit.Rule;
036import org.junit.internal.AssumptionViolatedException;
037import org.junit.rules.MethodRule;
038import org.junit.rules.TestRule;
039import org.junit.runner.Description;
040import org.junit.runner.notification.Failure;
041import org.junit.runner.notification.RunListener;
042import org.junit.runner.notification.RunNotifier;
043import org.junit.runners.model.FrameworkMethod;
044import org.junit.runners.model.Statement;
045
046/**
047 * Define execution rules for an annotated random bug.
048 * <p>
049 * Principle is to increase consistency on tests which have a random behavior. Such test is a headache because:
050 * <ul>
051 * <li>some developers may ask to ignore a random test since it's not reliable and produces useless noise most of the
052 * time,</li>
053 * <li>however, the test may still be useful in continuous integration for checking the non-random part of code it
054 * covers,</li>
055 * <li>and, after all, there's a random bug which should be fixed!</li>
056 * </ul>
057 * </p>
058 * <p>
059 * Compared to the @{@link Ignore} JUnit annotation, the advantage is to provide different behaviors for different use
060 * cases. The wanted behavior depending on whereas:
061 * <ul>
062 * <li>we are working on something else and don't want being bothered by an unreliable test,</li>
063 * <li>we are working on the covered code and want to be warned in case of regression,</li>
064 * <li>we are working on the random bug and want to reproduce it.</li>
065 * </ul>
066 * </p>
067 * That means that a random bug cannot be ignored. But must attempt to reproduce or hide its random aspect, depending on
068 * its execution context. For instance: <blockquote>
069 *
070 * <pre>
071 * <code>
072 * import org.nuxeo.runtime.test.runner.FeaturesRunner;
073 * import org.nuxeo.runtime.test.RandomBugRule;
074 *
075 * {@literal @}RunWith(FeaturesRunner.class)
076 * public class TestSample {
077 *     public static final String NXP99999 = "Some comment or description";
078 *
079 *     {@literal @}Test
080 *     {@literal @}RandomBug.Repeat(issue = NXP99999, onFailure=5, onSuccess=50)
081 *     public void testWhichFailsSometimes() throws Exception {
082 *         assertTrue(java.lang.Math.random() > 0.2);
083 *     }
084 * }</code>
085 * </pre>
086 *
087 * </blockquote>
088 * <p>
089 * In the above example, the test fails sometimes. With the {@link RandomBug.Repeat} annotation, it will be repeated in
090 * case of failure up to 5 times until success. This is the default {@link Mode#RELAX} mode. In order to reproduce the
091 * bug, use the {@link Mode#STRICT} mode. It will be repeated in case of success up to 50 times until failure. In
092 * {@link Mode#BYPASS} mode, the test is ignored.
093 * </p>
094 * <p>
095 * You may also repeat a whole suite in the same way by annotating the class itself. You may want also want to skip some
096 * tests, then you can annotate them and set {@link Repeat#bypass()} to true.
097 * </p>
098 *
099 * @see Mode
100 * @since 5.9.5
101 */
102public class RandomBug {
103    private static final Log log = LogFactory.getLog(RandomBug.class);
104
105    protected static final RandomBug self = new RandomBug();
106
107    /**
108     * Repeat condition based on
109     *
110     * @see Mode
111     */
112    @Retention(RetentionPolicy.RUNTIME)
113    @Target({ ElementType.METHOD, ElementType.TYPE })
114    @Inherited
115    public @interface Repeat {
116        /**
117         * Reference in issue management system. Recommendation is to use a constant which name is the issue reference
118         * and value is a description or comment.
119         */
120        String issue();
121
122        /**
123         * Times to repeat until failure in case of success
124         */
125        int onSuccess() default 30;
126
127        /**
128         * Times to repeat until success in case of failure
129         */
130        int onFailure() default 10;
131
132        /**
133         * Bypass a method/suite ....
134         */
135        boolean bypass() default false;
136    }
137
138    public static class Feature extends SimpleFeature {
139        @ClassRule
140        public static TestRule onClass() {
141            return self.onTest();
142        }
143
144        @Rule
145        public MethodRule onMethod() {
146            return self.onMethod();
147        }
148    }
149
150    public class RepeatRule implements TestRule, MethodRule {
151        @Inject
152        protected RunNotifier notifier;
153
154        @Inject
155        FeaturesRunner runner;
156
157        public RepeatStatement statement;
158
159        @Override
160        public Statement apply(Statement base, Description description) {
161            Repeat actual = runner.getConfig(Repeat.class);
162            if (actual.issue() == null) {
163                return base;
164            }
165            return statement = onRepeat(actual, notifier, base, description);
166        }
167
168        @Override
169        public Statement apply(Statement base, FrameworkMethod method, Object fixtureTarget) {
170            Repeat actual = method.getAnnotation(Repeat.class);
171            if (actual == null) {
172                return base;
173            }
174            Class<?> fixtureType = fixtureTarget.getClass();
175            Description description = Description.createTestDescription(fixtureType, method.getName(),
176                    method.getAnnotations());
177            return statement = onRepeat(actual, notifier, base, description);
178        }
179    }
180
181    protected RepeatRule onTest() {
182        return new RepeatRule();
183    }
184
185    protected RepeatRule onMethod() {
186        return new RepeatRule();
187    }
188
189    public static final String MODE_PROPERTY = "nuxeo.tests.random.mode";
190
191    /**
192     * <ul>
193     * <li>BYPASS: the test is ignored. Like with @{@link Ignore} JUnit annotation.</li>
194     * <li>STRICT: the test must fail. On success, the test is repeated until failure or the limit number of tries
195     * {@link Repeat#onSuccess()} is reached. If it does not fail during the tries, then the whole test class is marked
196     * as failed.</li>
197     * <li>RELAX: the test must succeed. On failure, the test is repeated until success or the limit number of tries
198     * {@link Repeat#onFailure()} is reached.</li>
199     * </ul>
200     * Could be set by the environment using the <em>nuxeo.tests.random.mode</em>T system property.
201     */
202    public static enum Mode {
203        BYPASS, STRICT, RELAX
204    };
205
206    /**
207     * The default mode if {@link #MODE_PROPERTY} is not set.
208     */
209    public final Mode DEFAULT = Mode.RELAX;
210
211    protected Mode fetchMode() {
212        String mode = System.getProperty(MODE_PROPERTY, DEFAULT.name());
213        return Mode.valueOf(mode.toUpperCase());
214    }
215
216    protected abstract class RepeatStatement extends Statement {
217        protected final Repeat params;
218
219        protected final RunNotifier notifier;
220
221        protected boolean gotFailure;
222
223        protected final RunListener listener = new RunListener() {
224            @Override
225            public void testStarted(Description desc) throws Exception {
226                log.debug(displayName(desc) + " STARTED");
227            };
228
229            @Override
230            public void testFailure(Failure failure) throws Exception {
231                gotFailure = true;
232                log.debug(displayName(failure.getDescription()) + " FAILURE");
233                log.trace(failure, failure.getException());
234            }
235
236            @Override
237            public void testAssumptionFailure(Failure failure) {
238                log.debug(displayName(failure.getDescription()) + " ASSUMPTION FAILURE");
239                log.trace(failure, failure.getException());
240            }
241
242            @Override
243            public void testIgnored(Description desc) throws Exception {
244                log.debug(displayName(desc) + " IGNORED");
245            };
246
247            @Override
248            public void testFinished(Description desc) throws Exception {
249                log.debug(displayName(desc) + " FINISHED");
250            };
251        };
252
253        protected final Statement base;
254
255        protected int serial;
256
257        protected Description description;
258
259        protected RepeatStatement(Repeat someParams, RunNotifier aNotifier, Statement aStatement,
260                Description aDescription) {
261            params = someParams;
262            notifier = aNotifier;
263            base = aStatement;
264            description = aDescription;
265        }
266
267        protected String displayName(Description desc) {
268            String displayName = desc.getClassName().substring(desc.getClassName().lastIndexOf(".") + 1);
269            if (desc.isTest()) {
270                displayName += "." + desc.getMethodName();
271            }
272            return displayName;
273        }
274
275        protected void onEnter(int aSerial) {
276            MDC.put("fRepeat", serial = aSerial);
277        }
278
279        protected void onLeave() {
280            MDC.remove("fRepeat");
281        }
282
283        @Override
284        public void evaluate() throws Throwable {
285            Error error = error();
286            notifier.addListener(listener);
287            try {
288                log.debug(displayName(description) + " STARTED");
289                for (int retry = 1; retry <= retryCount(); retry++) {
290                    gotFailure = false;
291                    onEnter(retry);
292                    try {
293                        log.debug(displayName(description) + " retry " + retry);
294                        base.evaluate();
295                    } catch (AssumptionViolatedException cause) {
296                        Throwable t = new Throwable("On retry " + retry).initCause(cause);
297                        error.addSuppressed(t);
298                        notifier.fireTestAssumptionFailed(new Failure(description, t));
299                    } catch (Throwable cause) {
300                        // Repeat annotation is on method (else the Throwable is not raised up to here)
301                        Throwable t = new Throwable("On retry " + retry).initCause(cause);
302                        error.addSuppressed(t);
303                        if (returnOnFailure()) {
304                            notifier.fireTestFailure(new Failure(description, t));
305                        } else {
306                            gotFailure = true;
307                            log.debug(displayName(description) + " FAILURE SWALLOW");
308                            log.trace(t, t);
309                        }
310                    } finally {
311                        onLeave();
312                    }
313                    if (gotFailure && returnOnFailure()) {
314                        log.debug(displayName(description) + " returnOnFailure");
315                        return;
316                    }
317                    if (!gotFailure && returnOnSuccess()) {
318                        log.debug(displayName(description) + " returnOnSuccess");
319                        return;
320                    }
321                }
322            } finally {
323                log.debug(displayName(description) + " FINISHED");
324                notifier.removeListener(listener);
325            }
326            log.trace("throw " + error);
327            throw error;
328        }
329
330        protected abstract Error error();
331
332        protected abstract int retryCount();
333
334        protected abstract boolean returnOnSuccess();
335
336        protected abstract boolean returnOnFailure();
337    }
338
339    protected class RepeatOnFailure extends RepeatStatement {
340        protected String issue;
341
342        protected RepeatOnFailure(Repeat someParams, RunNotifier aNotifier, Statement aStatement,
343                Description description) {
344            super(someParams, aNotifier, aStatement, description);
345        }
346
347        @Override
348        protected Error error() {
349            return new AssertionError(String.format("No success after %d tries. Either the bug is not random "
350                    + "or you should increase the 'onFailure' value.\n" + "Issue: %s", params.onFailure(), issue));
351        }
352
353        @Override
354        protected int retryCount() {
355            return params.onFailure();
356        }
357
358        @Override
359        protected boolean returnOnFailure() {
360            return false;
361        }
362
363        @Override
364        protected boolean returnOnSuccess() {
365            return true;
366        }
367    }
368
369    protected class RepeatOnSuccess extends RepeatStatement {
370        protected RepeatOnSuccess(Repeat someParams, RunNotifier aNotifier, Statement aStatement,
371                Description description) {
372            super(someParams, aNotifier, aStatement, description);
373        }
374
375        @Override
376        protected Error error() {
377            return new AssertionError(String.format("No failure after %d tries. Either the bug is fixed "
378                    + "or you should increase the 'onSuccess' value.\n" + "Issue: %s", params.onSuccess(),
379                    params.issue()));
380        }
381
382        @Override
383        protected boolean returnOnFailure() {
384            return true;
385        }
386
387        @Override
388        protected boolean returnOnSuccess() {
389            return false;
390        }
391
392        @Override
393        protected int retryCount() {
394            return params.onSuccess();
395        }
396    }
397
398    protected class Bypass extends RepeatStatement {
399        public Bypass(Repeat someParams, RunNotifier aNotifier, Statement aStatement, Description description) {
400            super(someParams, aNotifier, aStatement, description);
401        }
402
403        @Override
404        public void evaluate() throws Throwable {
405            notifier.fireTestIgnored(description);
406        }
407
408        @Override
409        protected Error error() {
410            throw new UnsupportedOperationException();
411        }
412
413        @Override
414        protected int retryCount() {
415            throw new UnsupportedOperationException();
416        }
417
418        @Override
419        protected boolean returnOnSuccess() {
420            throw new UnsupportedOperationException();
421        }
422
423        @Override
424        protected boolean returnOnFailure() {
425            return false;
426        }
427    }
428
429    protected RepeatStatement onRepeat(Repeat someParams, RunNotifier aNotifier, Statement aStatement,
430            Description description) {
431        if (someParams.bypass()) {
432            return new Bypass(someParams, aNotifier, aStatement, description);
433        }
434        switch (fetchMode()) {
435        case BYPASS:
436            return new Bypass(someParams, aNotifier, aStatement, description);
437        case STRICT:
438            return new RepeatOnSuccess(someParams, aNotifier, aStatement, description);
439        case RELAX:
440            return new RepeatOnFailure(someParams, aNotifier, aStatement, description);
441        }
442        throw new IllegalArgumentException("no such mode");
443    }
444
445}