001/*
002 * (C) Copyright 2014-2015 Nuxeo SA (http://nuxeo.com/) and contributors.
003 *
004 * All rights reserved. This program and the accompanying materials
005 * are made available under the terms of the GNU Lesser General Public License
006 * (LGPL) version 2.1 which accompanies this distribution, and is available at
007 * http://www.gnu.org/licenses/lgpl-2.1.html
008 *
009 * This library is distributed in the hope that it will be useful,
010 * but WITHOUT ANY WARRANTY; without even the implied warranty of
011 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
012 * Lesser General Public License for more details.
013 *
014 * Contributors:
015 *     slacoin, jcarsique
016 *
017 */
018package org.nuxeo.runtime.test.runner;
019
020import java.lang.annotation.ElementType;
021import java.lang.annotation.Inherited;
022import java.lang.annotation.Retention;
023import java.lang.annotation.RetentionPolicy;
024import java.lang.annotation.Target;
025
026import javax.inject.Inject;
027
028import org.apache.commons.logging.Log;
029import org.apache.commons.logging.LogFactory;
030import org.apache.log4j.MDC;
031import org.junit.ClassRule;
032import org.junit.Ignore;
033import org.junit.Rule;
034import org.junit.internal.AssumptionViolatedException;
035import org.junit.rules.MethodRule;
036import org.junit.rules.TestRule;
037import org.junit.runner.Description;
038import org.junit.runner.notification.Failure;
039import org.junit.runner.notification.RunListener;
040import org.junit.runner.notification.RunNotifier;
041import org.junit.runners.model.FrameworkMethod;
042import org.junit.runners.model.Statement;
043
044/**
045 * Define execution rules for an annotated random bug.
046 * <p>
047 * Principle is to increase consistency on tests which have a random behavior. Such test is a headache because:
048 * <ul>
049 * <li>some developers may ask to ignore a random test since it's not reliable and produces useless noise most of the
050 * time,</li>
051 * <li>however, the test may still be useful in continuous integration for checking the non-random part of code it
052 * covers,</li>
053 * <li>and, after all, there's a random bug which should be fixed!</li>
054 * </ul>
055 * </p>
056 * <p>
057 * Compared to the @{@link Ignore} JUnit annotation, the advantage is to provide different behaviors for different use
058 * cases. The wanted behavior depending on whereas:
059 * <ul>
060 * <li>we are working on something else and don't want being bothered by an unreliable test,</li>
061 * <li>we are working on the covered code and want to be warned in case of regression,</li>
062 * <li>we are working on the random bug and want to reproduce it.</li>
063 * </ul>
064 * </p>
065 * That means that a random bug cannot be ignored. But must attempt to reproduce or hide its random aspect, depending on
066 * its execution context. For instance: <blockquote>
067 *
068 * <pre>
069 * <code>
070 * import org.nuxeo.runtime.test.runner.FeaturesRunner;
071 * import org.nuxeo.runtime.test.RandomBugRule;
072 *
073 * {@literal @}RunWith(FeaturesRunner.class)
074 * public class TestSample {
075 *     public static final String NXP99999 = "Some comment or description";
076 *
077 *     {@literal @}Test
078 *     {@literal @}RandomBugRule.Repeat(issue = NXP99999, onFailure=5, onSuccess=50)
079 *     public void testWhichFailsSometimes() throws Exception {
080 *         assertTrue(java.lang.Math.random() > 0.2);
081 *     }
082 * }</code>
083 * </pre>
084 *
085 * </blockquote>
086 * <p>
087 * In the above example, the test fails sometimes. With the {@link RandomBug.Repeat} annotation, it will be repeated in
088 * case of failure up to 5 times until success. This is the default {@link Mode#RELAX} mode. In order to reproduce the
089 * bug, use the {@link Mode#STRICT} mode. It will be repeated in case of success up to 50 times until failure. In
090 * {@link Mode#BYPASS} mode, the test is ignored.
091 * </p>
092 * <p>
093 * You may also repeat a whole suite in the same way by annotating the class itself. You may want also want to skip some
094 * tests, then you can annotate them and set {@link Repeat#bypass()} to true.
095 * </p>
096 *
097 * @see Mode
098 * @since 5.9.5
099 */
100public class RandomBug {
101    private static final Log log = LogFactory.getLog(RandomBug.class);
102
103    protected static final RandomBug self = new RandomBug();
104
105    /**
106     * Repeat condition based on
107     *
108     * @see Mode
109     */
110    @Retention(RetentionPolicy.RUNTIME)
111    @Target({ ElementType.METHOD, ElementType.TYPE })
112    @Inherited
113    public @interface Repeat {
114        /**
115         * Reference in issue management system. Recommendation is to use a constant which name is the issue reference
116         * and value is a description or comment.
117         */
118        String issue();
119
120        /**
121         * Times to repeat until failure in case of success
122         */
123        int onSuccess() default 30;
124
125        /**
126         * Times to repeat until success in case of failure
127         */
128        int onFailure() default 10;
129
130        /**
131         * Bypass a method/suite ....
132         */
133        boolean bypass() default false;
134    }
135
136    public static class Feature extends SimpleFeature {
137        @ClassRule
138        public static TestRule onClass() {
139            return self.onTest();
140        }
141
142        @Rule
143        public MethodRule onMethod() {
144            return self.onMethod();
145        }
146    }
147
148    public class RepeatRule implements TestRule, MethodRule {
149        @Inject
150        protected RunNotifier notifier;
151
152        public RepeatStatement statement;
153
154        @Override
155        public Statement apply(Statement base, Description description) {
156            final Repeat actual = description.getAnnotation(Repeat.class);
157            if (actual == null) {
158                return base;
159            }
160            return statement = onRepeat(actual, notifier, base, description);
161        }
162
163        @Override
164        public Statement apply(Statement base, FrameworkMethod method, Object fixtureTarget) {
165            final Repeat actual = method.getAnnotation(Repeat.class);
166            if (actual == null) {
167                return base;
168            }
169            Class<?> fixtureType = fixtureTarget.getClass();
170            Description description = Description.createTestDescription(fixtureType, method.getName(),
171                    method.getAnnotations());
172            return statement = onRepeat(actual, notifier, base, description);
173        }
174    }
175
176    protected RepeatRule onTest() {
177        return new RepeatRule();
178    }
179
180    protected RepeatRule onMethod() {
181        return new RepeatRule();
182    }
183
184    public static final String MODE_PROPERTY = "nuxeo.tests.random.mode";
185
186    /**
187     * <ul>
188     * <li>BYPASS: the test is ignored. Like with @{@link Ignore} JUnit annotation.</li>
189     * <li>STRICT: the test must fail. On success, the test is repeated until failure or the limit number of tries
190     * {@link Repeat#onSuccess()} is reached. If it does not fail during the tries, then the whole test class is marked
191     * as failed.</li>
192     * <li>RELAX: the test must succeed. On failure, the test is repeated until success or the limit number of tries
193     * {@link Repeat#onFailure()} is reached.</li>
194     * </ul>
195     * Could be set by the environment using the <em>nuxeo.tests.random.mode</em>T system property.
196     */
197    public static enum Mode {
198        BYPASS, STRICT, RELAX
199    };
200
201    /**
202     * The default mode if {@link #MODE_PROPERTY} is not set.
203     */
204    public final Mode DEFAULT = Mode.RELAX;
205
206    protected Mode fetchMode() {
207        String mode = System.getProperty(MODE_PROPERTY, DEFAULT.name());
208        return Mode.valueOf(mode.toUpperCase());
209    }
210
211    protected abstract class RepeatStatement extends Statement {
212        protected final Repeat params;
213
214        protected final RunNotifier notifier;
215
216        protected boolean gotFailure;
217
218        protected final RunListener listener = new RunListener() {
219            @Override
220            public void testStarted(Description desc) throws Exception {
221                log.debug(displayName(desc) + " STARTED");
222            };
223
224            @Override
225            public void testFailure(Failure failure) throws Exception {
226                gotFailure = true;
227                log.debug(displayName(failure.getDescription()) + " FAILURE");
228                log.trace(failure, failure.getException());
229            }
230
231            @Override
232            public void testAssumptionFailure(Failure failure) {
233                log.debug(displayName(failure.getDescription()) + " ASSUMPTION FAILURE");
234                log.trace(failure, failure.getException());
235            }
236
237            @Override
238            public void testIgnored(Description desc) throws Exception {
239                log.debug(displayName(desc) + " IGNORED");
240            };
241
242            @Override
243            public void testFinished(Description desc) throws Exception {
244                log.debug(displayName(desc) + " FINISHED");
245            };
246        };
247
248        protected final Statement base;
249
250        protected int serial;
251
252        protected Description description;
253
254        protected RepeatStatement(Repeat someParams, RunNotifier aNotifier, Statement aStatement,
255                Description aDescription) {
256            params = someParams;
257            notifier = aNotifier;
258            base = aStatement;
259            description = aDescription;
260        }
261
262        protected String displayName(Description desc) {
263            String displayName = desc.getClassName().substring(desc.getClassName().lastIndexOf(".") + 1);
264            if (desc.isTest()) {
265                displayName += "." + desc.getMethodName();
266            }
267            return displayName;
268        }
269
270        protected void onEnter(int aSerial) {
271            MDC.put("fRepeat", serial = aSerial);
272        }
273
274        protected void onLeave() {
275            MDC.remove("fRepeat");
276        }
277
278        @Override
279        public void evaluate() throws Throwable {
280            Error error = error();
281            notifier.addListener(listener);
282            try {
283                log.debug(displayName(description) + " STARTED");
284                for (int retry = 1; retry <= retryCount(); retry++) {
285                    gotFailure = false;
286                    onEnter(retry);
287                    try {
288                        log.debug(displayName(description) + " retry " + retry);
289                        base.evaluate();
290                    } catch (AssumptionViolatedException cause) {
291                        Throwable t = new Throwable("On retry " + retry).initCause(cause);
292                        error.addSuppressed(t);
293                        notifier.fireTestAssumptionFailed(new Failure(description, t));
294                    } catch (Throwable cause) {
295                        // Repeat annotation is on method (else the Throwable is not raised up to here)
296                        Throwable t = new Throwable("On retry " + retry).initCause(cause);
297                        error.addSuppressed(t);
298                        if (returnOnFailure()) {
299                            notifier.fireTestFailure(new Failure(description, t));
300                        } else {
301                            gotFailure = true;
302                            log.debug(displayName(description) + " FAILURE SWALLOW");
303                            log.trace(t, t);
304                        }
305                    } finally {
306                        onLeave();
307                    }
308                    if (gotFailure && returnOnFailure()) {
309                        log.debug(displayName(description) + " returnOnFailure");
310                        return;
311                    }
312                    if (!gotFailure && returnOnSuccess()) {
313                        log.debug(displayName(description) + " returnOnSuccess");
314                        return;
315                    }
316                }
317            } finally {
318                log.debug(displayName(description) + " FINISHED");
319                notifier.removeListener(listener);
320            }
321            log.trace("throw " + error);
322            throw error;
323        }
324
325        protected abstract Error error();
326
327        protected abstract int retryCount();
328
329        protected abstract boolean returnOnSuccess();
330
331        protected abstract boolean returnOnFailure();
332    }
333
334    protected class RepeatOnFailure extends RepeatStatement {
335        protected String issue;
336
337        protected RepeatOnFailure(Repeat someParams, RunNotifier aNotifier, Statement aStatement,
338                Description description) {
339            super(someParams, aNotifier, aStatement, description);
340        }
341
342        @Override
343        protected Error error() {
344            return new AssertionError(String.format("No success after %d tries. Either the bug is not random "
345                    + "or you should increase the 'onFailure' value.\n" + "Issue: %s", params.onFailure(), issue));
346        }
347
348        @Override
349        protected int retryCount() {
350            return params.onFailure();
351        }
352
353        @Override
354        protected boolean returnOnFailure() {
355            return false;
356        }
357
358        @Override
359        protected boolean returnOnSuccess() {
360            return true;
361        }
362    }
363
364    protected class RepeatOnSuccess extends RepeatStatement {
365        protected RepeatOnSuccess(Repeat someParams, RunNotifier aNotifier, Statement aStatement,
366                Description description) {
367            super(someParams, aNotifier, aStatement, description);
368        }
369
370        @Override
371        protected Error error() {
372            return new AssertionError(String.format("No failure after %d tries. Either the bug is fixed "
373                    + "or you should increase the 'onSuccess' value.\n" + "Issue: %s", params.onSuccess(),
374                    params.issue()));
375        }
376
377        @Override
378        protected boolean returnOnFailure() {
379            return true;
380        }
381
382        @Override
383        protected boolean returnOnSuccess() {
384            return false;
385        }
386
387        @Override
388        protected int retryCount() {
389            return params.onSuccess();
390        }
391    }
392
393    protected class Bypass extends RepeatStatement {
394        public Bypass(Repeat someParams, RunNotifier aNotifier, Statement aStatement, Description description) {
395            super(someParams, aNotifier, aStatement, description);
396        }
397
398        @Override
399        public void evaluate() throws Throwable {
400            notifier.fireTestIgnored(description);
401        }
402
403        @Override
404        protected Error error() {
405            throw new UnsupportedOperationException();
406        }
407
408        @Override
409        protected int retryCount() {
410            throw new UnsupportedOperationException();
411        }
412
413        @Override
414        protected boolean returnOnSuccess() {
415            throw new UnsupportedOperationException();
416        }
417
418        @Override
419        protected boolean returnOnFailure() {
420            return false;
421        }
422    }
423
424    protected RepeatStatement onRepeat(Repeat someParams, RunNotifier aNotifier, Statement aStatement,
425            Description description) {
426        if (someParams.bypass()) {
427            return new Bypass(someParams, aNotifier, aStatement, description);
428        }
429        switch (fetchMode()) {
430        case BYPASS:
431            return new Bypass(someParams, aNotifier, aStatement, description);
432        case STRICT:
433            return new RepeatOnSuccess(someParams, aNotifier, aStatement, description);
434        case RELAX:
435            return new RepeatOnFailure(someParams, aNotifier, aStatement, description);
436        }
437        throw new IllegalArgumentException("no such mode");
438    }
439
440}