001/*
002 * (C) Copyright 2014-2015 Nuxeo SA (http://nuxeo.com/) and others.
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 *     http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 *
016 * Contributors:
017 *     slacoin, jcarsique
018 *
019 */
020package org.nuxeo.runtime.test.runner;
021
022import java.lang.annotation.ElementType;
023import java.lang.annotation.Inherited;
024import java.lang.annotation.Retention;
025import java.lang.annotation.RetentionPolicy;
026import java.lang.annotation.Target;
027
028import javax.inject.Inject;
029
030import org.apache.commons.logging.Log;
031import org.apache.commons.logging.LogFactory;
032import org.apache.log4j.MDC;
033import org.junit.ClassRule;
034import org.junit.Ignore;
035import org.junit.Rule;
036import org.junit.internal.AssumptionViolatedException;
037import org.junit.rules.MethodRule;
038import org.junit.rules.TestRule;
039import org.junit.runner.Description;
040import org.junit.runner.notification.Failure;
041import org.junit.runner.notification.RunListener;
042import org.junit.runner.notification.RunNotifier;
043import org.junit.runners.model.FrameworkMethod;
044import org.junit.runners.model.Statement;
045
046/**
047 * Define execution rules for an annotated random bug.
048 * <p>
049 * Principle is to increase consistency on tests which have a random behavior. Such test is a headache because:
050 * <ul>
051 * <li>some developers may ask to ignore a random test since it's not reliable and produces useless noise most of the
052 * time,</li>
053 * <li>however, the test may still be useful in continuous integration for checking the non-random part of code it
054 * covers,</li>
055 * <li>and, after all, there's a random bug which should be fixed!</li>
056 * </ul>
057 * </p>
058 * <p>
059 * Compared to the @{@link Ignore} JUnit annotation, the advantage is to provide different behaviors for different use
060 * cases. The wanted behavior depending on whereas:
061 * <ul>
062 * <li>we are working on something else and don't want being bothered by an unreliable test,</li>
063 * <li>we are working on the covered code and want to be warned in case of regression,</li>
064 * <li>we are working on the random bug and want to reproduce it.</li>
065 * </ul>
066 * </p>
067 * That means that a random bug cannot be ignored. But must attempt to reproduce or hide its random aspect, depending on
068 * its execution context. For instance: <blockquote>
069 *
070 * <pre>
071 * <code>
072 * import org.nuxeo.runtime.test.runner.FeaturesRunner;
073 * import org.nuxeo.runtime.test.RandomBugRule;
074 *
075 * {@literal @}RunWith(FeaturesRunner.class)
076 * public class TestSample {
077 *     public static final String NXP99999 = "Some comment or description";
078 *
079 *     {@literal @}Test
080 *     {@literal @}RandomBug.Repeat(issue = NXP99999, onFailure=5, onSuccess=50)
081 *     public void testWhichFailsSometimes() throws Exception {
082 *         assertTrue(java.lang.Math.random() > 0.2);
083 *     }
084 * }</code>
085 * </pre>
086 *
087 * </blockquote>
088 * <p>
089 * In the above example, the test fails sometimes. With the {@link RandomBug.Repeat} annotation, it will be repeated in
090 * case of failure up to 5 times until success. This is the default {@link Mode#RELAX} mode. In order to reproduce the
091 * bug, use the {@link Mode#STRICT} mode. It will be repeated in case of success up to 50 times until failure. In
092 * {@link Mode#BYPASS} mode, the test is ignored.
093 * </p>
094 * <p>
095 * You may also repeat a whole suite in the same way by annotating the class itself. You may want also want to skip some
096 * tests, then you can annotate them and set {@link Repeat#bypass()} to true.
097 * </p>
098 *
099 * @see Mode
100 * @since 5.9.5
101 */
102public class RandomBug {
103    private static final Log log = LogFactory.getLog(RandomBug.class);
104
105    protected static final RandomBug self = new RandomBug();
106
107    /**
108     * Repeat condition based on
109     *
110     * @see Mode
111     */
112    @Retention(RetentionPolicy.RUNTIME)
113    @Target({ ElementType.METHOD, ElementType.TYPE })
114    @Inherited
115    public @interface Repeat {
116        /**
117         * Reference in issue management system. Recommendation is to use a constant which name is the issue reference
118         * and value is a description or comment.
119         */
120        String issue();
121
122        /**
123         * Times to repeat until failure in case of success
124         */
125        int onSuccess() default 30;
126
127        /**
128         * Times to repeat until success in case of failure
129         */
130        int onFailure() default 10;
131
132        /**
133         * Bypass a method/suite ....
134         */
135        boolean bypass() default false;
136    }
137
138    public static class Feature extends SimpleFeature {
139        @ClassRule
140        public static TestRule onClass() {
141            return self.onTest();
142        }
143
144        @Rule
145        public MethodRule onMethod() {
146            return self.onMethod();
147        }
148    }
149
150    public class RepeatRule implements TestRule, MethodRule {
151        @Inject
152        protected RunNotifier notifier;
153
154        public RepeatStatement statement;
155
156        @Override
157        public Statement apply(Statement base, Description description) {
158            final Repeat actual = description.getAnnotation(Repeat.class);
159            if (actual == null) {
160                return base;
161            }
162            return statement = onRepeat(actual, notifier, base, description);
163        }
164
165        @Override
166        public Statement apply(Statement base, FrameworkMethod method, Object fixtureTarget) {
167            final Repeat actual = method.getAnnotation(Repeat.class);
168            if (actual == null) {
169                return base;
170            }
171            Class<?> fixtureType = fixtureTarget.getClass();
172            Description description = Description.createTestDescription(fixtureType, method.getName(),
173                    method.getAnnotations());
174            return statement = onRepeat(actual, notifier, base, description);
175        }
176    }
177
178    protected RepeatRule onTest() {
179        return new RepeatRule();
180    }
181
182    protected RepeatRule onMethod() {
183        return new RepeatRule();
184    }
185
186    public static final String MODE_PROPERTY = "nuxeo.tests.random.mode";
187
188    /**
189     * <ul>
190     * <li>BYPASS: the test is ignored. Like with @{@link Ignore} JUnit annotation.</li>
191     * <li>STRICT: the test must fail. On success, the test is repeated until failure or the limit number of tries
192     * {@link Repeat#onSuccess()} is reached. If it does not fail during the tries, then the whole test class is marked
193     * as failed.</li>
194     * <li>RELAX: the test must succeed. On failure, the test is repeated until success or the limit number of tries
195     * {@link Repeat#onFailure()} is reached.</li>
196     * </ul>
197     * Could be set by the environment using the <em>nuxeo.tests.random.mode</em>T system property.
198     */
199    public static enum Mode {
200        BYPASS, STRICT, RELAX
201    };
202
203    /**
204     * The default mode if {@link #MODE_PROPERTY} is not set.
205     */
206    public final Mode DEFAULT = Mode.RELAX;
207
208    protected Mode fetchMode() {
209        String mode = System.getProperty(MODE_PROPERTY, DEFAULT.name());
210        return Mode.valueOf(mode.toUpperCase());
211    }
212
213    protected abstract class RepeatStatement extends Statement {
214        protected final Repeat params;
215
216        protected final RunNotifier notifier;
217
218        protected boolean gotFailure;
219
220        protected final RunListener listener = new RunListener() {
221            @Override
222            public void testStarted(Description desc) throws Exception {
223                log.debug(displayName(desc) + " STARTED");
224            };
225
226            @Override
227            public void testFailure(Failure failure) throws Exception {
228                gotFailure = true;
229                log.debug(displayName(failure.getDescription()) + " FAILURE");
230                log.trace(failure, failure.getException());
231            }
232
233            @Override
234            public void testAssumptionFailure(Failure failure) {
235                log.debug(displayName(failure.getDescription()) + " ASSUMPTION FAILURE");
236                log.trace(failure, failure.getException());
237            }
238
239            @Override
240            public void testIgnored(Description desc) throws Exception {
241                log.debug(displayName(desc) + " IGNORED");
242            };
243
244            @Override
245            public void testFinished(Description desc) throws Exception {
246                log.debug(displayName(desc) + " FINISHED");
247            };
248        };
249
250        protected final Statement base;
251
252        protected int serial;
253
254        protected Description description;
255
256        protected RepeatStatement(Repeat someParams, RunNotifier aNotifier, Statement aStatement,
257                Description aDescription) {
258            params = someParams;
259            notifier = aNotifier;
260            base = aStatement;
261            description = aDescription;
262        }
263
264        protected String displayName(Description desc) {
265            String displayName = desc.getClassName().substring(desc.getClassName().lastIndexOf(".") + 1);
266            if (desc.isTest()) {
267                displayName += "." + desc.getMethodName();
268            }
269            return displayName;
270        }
271
272        protected void onEnter(int aSerial) {
273            MDC.put("fRepeat", serial = aSerial);
274        }
275
276        protected void onLeave() {
277            MDC.remove("fRepeat");
278        }
279
280        @Override
281        public void evaluate() throws Throwable {
282            Error error = error();
283            notifier.addListener(listener);
284            try {
285                log.debug(displayName(description) + " STARTED");
286                for (int retry = 1; retry <= retryCount(); retry++) {
287                    gotFailure = false;
288                    onEnter(retry);
289                    try {
290                        log.debug(displayName(description) + " retry " + retry);
291                        base.evaluate();
292                    } catch (AssumptionViolatedException cause) {
293                        Throwable t = new Throwable("On retry " + retry).initCause(cause);
294                        error.addSuppressed(t);
295                        notifier.fireTestAssumptionFailed(new Failure(description, t));
296                    } catch (Throwable cause) {
297                        // Repeat annotation is on method (else the Throwable is not raised up to here)
298                        Throwable t = new Throwable("On retry " + retry).initCause(cause);
299                        error.addSuppressed(t);
300                        if (returnOnFailure()) {
301                            notifier.fireTestFailure(new Failure(description, t));
302                        } else {
303                            gotFailure = true;
304                            log.debug(displayName(description) + " FAILURE SWALLOW");
305                            log.trace(t, t);
306                        }
307                    } finally {
308                        onLeave();
309                    }
310                    if (gotFailure && returnOnFailure()) {
311                        log.debug(displayName(description) + " returnOnFailure");
312                        return;
313                    }
314                    if (!gotFailure && returnOnSuccess()) {
315                        log.debug(displayName(description) + " returnOnSuccess");
316                        return;
317                    }
318                }
319            } finally {
320                log.debug(displayName(description) + " FINISHED");
321                notifier.removeListener(listener);
322            }
323            log.trace("throw " + error);
324            throw error;
325        }
326
327        protected abstract Error error();
328
329        protected abstract int retryCount();
330
331        protected abstract boolean returnOnSuccess();
332
333        protected abstract boolean returnOnFailure();
334    }
335
336    protected class RepeatOnFailure extends RepeatStatement {
337        protected String issue;
338
339        protected RepeatOnFailure(Repeat someParams, RunNotifier aNotifier, Statement aStatement,
340                Description description) {
341            super(someParams, aNotifier, aStatement, description);
342        }
343
344        @Override
345        protected Error error() {
346            return new AssertionError(String.format("No success after %d tries. Either the bug is not random "
347                    + "or you should increase the 'onFailure' value.\n" + "Issue: %s", params.onFailure(), issue));
348        }
349
350        @Override
351        protected int retryCount() {
352            return params.onFailure();
353        }
354
355        @Override
356        protected boolean returnOnFailure() {
357            return false;
358        }
359
360        @Override
361        protected boolean returnOnSuccess() {
362            return true;
363        }
364    }
365
366    protected class RepeatOnSuccess extends RepeatStatement {
367        protected RepeatOnSuccess(Repeat someParams, RunNotifier aNotifier, Statement aStatement,
368                Description description) {
369            super(someParams, aNotifier, aStatement, description);
370        }
371
372        @Override
373        protected Error error() {
374            return new AssertionError(String.format("No failure after %d tries. Either the bug is fixed "
375                    + "or you should increase the 'onSuccess' value.\n" + "Issue: %s", params.onSuccess(),
376                    params.issue()));
377        }
378
379        @Override
380        protected boolean returnOnFailure() {
381            return true;
382        }
383
384        @Override
385        protected boolean returnOnSuccess() {
386            return false;
387        }
388
389        @Override
390        protected int retryCount() {
391            return params.onSuccess();
392        }
393    }
394
395    protected class Bypass extends RepeatStatement {
396        public Bypass(Repeat someParams, RunNotifier aNotifier, Statement aStatement, Description description) {
397            super(someParams, aNotifier, aStatement, description);
398        }
399
400        @Override
401        public void evaluate() throws Throwable {
402            notifier.fireTestIgnored(description);
403        }
404
405        @Override
406        protected Error error() {
407            throw new UnsupportedOperationException();
408        }
409
410        @Override
411        protected int retryCount() {
412            throw new UnsupportedOperationException();
413        }
414
415        @Override
416        protected boolean returnOnSuccess() {
417            throw new UnsupportedOperationException();
418        }
419
420        @Override
421        protected boolean returnOnFailure() {
422            return false;
423        }
424    }
425
426    protected RepeatStatement onRepeat(Repeat someParams, RunNotifier aNotifier, Statement aStatement,
427            Description description) {
428        if (someParams.bypass()) {
429            return new Bypass(someParams, aNotifier, aStatement, description);
430        }
431        switch (fetchMode()) {
432        case BYPASS:
433            return new Bypass(someParams, aNotifier, aStatement, description);
434        case STRICT:
435            return new RepeatOnSuccess(someParams, aNotifier, aStatement, description);
436        case RELAX:
437            return new RepeatOnFailure(someParams, aNotifier, aStatement, description);
438        }
439        throw new IllegalArgumentException("no such mode");
440    }
441
442}