001/*
002 * (C) Copyright 2014-2018 Nuxeo (http://nuxeo.com/) and others.
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 *     http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 *
016 * Contributors:
017 *     slacoin, jcarsique
018 *
019 */
020package org.nuxeo.runtime.test.runner;
021
022import java.lang.annotation.ElementType;
023import java.lang.annotation.Inherited;
024import java.lang.annotation.Retention;
025import java.lang.annotation.RetentionPolicy;
026import java.lang.annotation.Target;
027
028import javax.inject.Inject;
029
030import org.apache.commons.logging.Log;
031import org.apache.commons.logging.LogFactory;
032import org.apache.logging.log4j.ThreadContext;
033import org.junit.ClassRule;
034import org.junit.Ignore;
035import org.junit.Rule;
036import org.junit.internal.AssumptionViolatedException;
037import org.junit.rules.MethodRule;
038import org.junit.rules.TestRule;
039import org.junit.runner.Description;
040import org.junit.runner.notification.Failure;
041import org.junit.runner.notification.RunListener;
042import org.junit.runner.notification.RunNotifier;
043import org.junit.runners.model.FrameworkMethod;
044import org.junit.runners.model.Statement;
045
046/**
047 * Define execution rules for an annotated random bug.
048 * <p>
049 * Principle is to increase consistency on tests which have a random behavior. Such test is a headache because:
050 * <ul>
051 * <li>some developers may ask to ignore a random test since it's not reliable and produces useless noise most of the
052 * time,</li>
053 * <li>however, the test may still be useful in continuous integration for checking the non-random part of code it
054 * covers,</li>
055 * <li>and, after all, there's a random bug which should be fixed!</li>
056 * </ul>
057 * <p>
058 * Compared to the @{@link Ignore} JUnit annotation, the advantage is to provide different behaviors for different use
059 * cases. The wanted behavior depending on whereas:
060 * <ul>
061 * <li>we are working on something else and don't want being bothered by an unreliable test,</li>
062 * <li>we are working on the covered code and want to be warned in case of regression,</li>
063 * <li>we are working on the random bug and want to reproduce it.</li>
064 * </ul>
065 * <p>
066 * That means that a random bug cannot be ignored. But must attempt to reproduce or hide its random aspect, depending on
067 * its execution context. For instance: <blockquote>
068 *
069 * <pre>
070 * <code>
071 * import org.nuxeo.runtime.test.runner.FeaturesRunner;
072 * import org.nuxeo.runtime.test.RandomBugRule;
073 *
074 * {@literal @}RunWith(FeaturesRunner.class)
075 * public class TestSample {
076 *     public static final String NXP99999 = "Some comment or description";
077 *
078 *     {@literal @}Test
079 *     {@literal @}RandomBug.Repeat(issue = NXP99999, onFailure=5, onSuccess=50)
080 *     public void testWhichFailsSometimes() throws Exception {
081 *         assertTrue(java.lang.Math.random() &gt; 0.2);
082 *     }
083 * }</code>
084 * </pre>
085 *
086 * </blockquote>
087 * <p>
088 * In the above example, the test fails sometimes. With the {@link RandomBug.Repeat} annotation, it will be repeated in
089 * case of failure up to 5 times until success. This is the default {@link Mode#RELAX} mode. In order to reproduce the
090 * bug, use the {@link Mode#STRICT} mode. It will be repeated in case of success up to 50 times until failure. In
091 * {@link Mode#BYPASS} mode, the test is ignored.
092 * </p>
093 * <p>
094 * You may also repeat a whole suite in the same way by annotating the class itself. You may want also want to skip some
095 * tests, then you can annotate them and set {@link Repeat#bypass()} to true.
096 * </p>
097 *
098 * @see Mode
099 * @since 5.9.5
100 */
101public class RandomBug {
102    private static final Log log = LogFactory.getLog(RandomBug.class);
103
104    protected static final RandomBug self = new RandomBug();
105
106    /**
107     * Repeat condition based on
108     *
109     * @see Mode
110     */
111    @Retention(RetentionPolicy.RUNTIME)
112    @Target({ ElementType.METHOD, ElementType.TYPE })
113    @Inherited
114    public @interface Repeat {
115        /**
116         * Reference in issue management system. Recommendation is to use a constant which name is the issue reference
117         * and value is a description or comment.
118         */
119        String issue();
120
121        /**
122         * Times to repeat until failure in case of success
123         */
124        int onSuccess() default 30;
125
126        /**
127         * Times to repeat until success in case of failure
128         */
129        int onFailure() default 10;
130
131        /**
132         * Bypass a method/suite ....
133         */
134        boolean bypass() default false;
135    }
136
137    public static class Feature implements RunnerFeature {
138        @ClassRule
139        public static TestRule onClass() {
140            return self.onTest();
141        }
142
143        @Rule
144        public MethodRule onMethod() {
145            return self.onMethod();
146        }
147    }
148
149    public class RepeatRule implements TestRule, MethodRule {
150        @Inject
151        protected RunNotifier notifier;
152
153        @Inject
154        FeaturesRunner runner;
155
156        public RepeatStatement statement;
157
158        @Override
159        public Statement apply(Statement base, Description description) {
160            Repeat actual = runner.getConfig(Repeat.class);
161            if (actual.issue() == null) {
162                return base;
163            }
164            return statement = onRepeat(actual, notifier, base, description);
165        }
166
167        @Override
168        public Statement apply(Statement base, FrameworkMethod method, Object fixtureTarget) {
169            Repeat actual = method.getAnnotation(Repeat.class);
170            if (actual == null) {
171                return base;
172            }
173            Class<?> fixtureType = fixtureTarget.getClass();
174            Description description = Description.createTestDescription(fixtureType, method.getName(),
175                    method.getAnnotations());
176            return statement = onRepeat(actual, notifier, base, description);
177        }
178    }
179
180    protected RepeatRule onTest() {
181        return new RepeatRule();
182    }
183
184    protected RepeatRule onMethod() {
185        return new RepeatRule();
186    }
187
188    public static final String MODE_PROPERTY = "nuxeo.tests.random.mode";
189
190    /**
191     * <ul>
192     * <li>BYPASS: the test is ignored. Like with @{@link Ignore} JUnit annotation.</li>
193     * <li>STRICT: the test must fail. On success, the test is repeated until failure or the limit number of tries
194     * {@link Repeat#onSuccess()} is reached. If it does not fail during the tries, then the whole test class is marked
195     * as failed.</li>
196     * <li>RELAX: the test must succeed. On failure, the test is repeated until success or the limit number of tries
197     * {@link Repeat#onFailure()} is reached.</li>
198     * </ul>
199     * Could be set by the environment using the <em>nuxeo.tests.random.mode</em>T system property.
200     */
201    public enum Mode {
202        BYPASS, STRICT, RELAX
203    }
204
205    /**
206     * The default mode if {@link #MODE_PROPERTY} is not set.
207     */
208    public final Mode DEFAULT = Mode.RELAX;
209
210    protected Mode fetchMode() {
211        String mode = System.getProperty(MODE_PROPERTY, DEFAULT.name());
212        return Mode.valueOf(mode.toUpperCase());
213    }
214
215    protected abstract class RepeatStatement extends Statement {
216        protected final Repeat params;
217
218        protected final RunNotifier notifier;
219
220        protected boolean gotFailure;
221
222        protected final RunListener listener = new RunListener() {
223            @Override
224            public void testStarted(Description desc) {
225                log.debug(displayName(desc) + " STARTED");
226            }
227
228            @Override
229            public void testFailure(Failure failure) {
230                gotFailure = true;
231                log.debug(displayName(failure.getDescription()) + " FAILURE");
232                log.trace(failure, failure.getException());
233            }
234
235            @Override
236            public void testAssumptionFailure(Failure failure) {
237                log.debug(displayName(failure.getDescription()) + " ASSUMPTION FAILURE");
238                log.trace(failure, failure.getException());
239            }
240
241            @Override
242            public void testIgnored(Description desc) {
243                log.debug(displayName(desc) + " IGNORED");
244            }
245
246            @Override
247            public void testFinished(Description desc) {
248                log.debug(displayName(desc) + " FINISHED");
249            }
250        };
251
252        protected final Statement base;
253
254        protected int serial;
255
256        protected Description description;
257
258        protected RepeatStatement(Repeat someParams, RunNotifier aNotifier, Statement aStatement,
259                Description aDescription) {
260            params = someParams;
261            notifier = aNotifier;
262            base = aStatement;
263            description = aDescription;
264        }
265
266        protected String displayName(Description desc) {
267            String displayName = desc.getClassName().substring(desc.getClassName().lastIndexOf(".") + 1);
268            if (desc.isTest()) {
269                displayName += "." + desc.getMethodName();
270            }
271            return displayName;
272        }
273
274        protected void onEnter(int serial) {
275            this.serial = serial;
276            ThreadContext.put("fRepeat", Integer.toString(serial));
277        }
278
279        protected void onLeave() {
280            ThreadContext.remove("fRepeat");
281        }
282
283        @Override
284        public void evaluate() {
285            Error error = error();
286            notifier.addListener(listener);
287            try {
288                log.debug(displayName(description) + " STARTED");
289                for (int retry = 1; retry <= retryCount(); retry++) {
290                    gotFailure = false;
291                    onEnter(retry);
292                    try {
293                        log.debug(displayName(description) + " retry " + retry);
294                        base.evaluate();
295                    } catch (AssumptionViolatedException cause) {
296                        Throwable t = new Throwable("On retry " + retry, cause);
297                        error.addSuppressed(t);
298                        notifier.fireTestAssumptionFailed(new Failure(description, t));
299                    } catch (Throwable cause) {
300                        // Repeat annotation is on method (else the Throwable is not raised up to here)
301                        Throwable t = new Throwable("On retry " + retry, cause);
302                        error.addSuppressed(t);
303                        if (returnOnFailure()) {
304                            notifier.fireTestFailure(new Failure(description, t));
305                        } else {
306                            gotFailure = true;
307                            log.debug(displayName(description) + " FAILURE SWALLOW");
308                            log.trace(t, t);
309                        }
310                    } finally {
311                        onLeave();
312                    }
313                    if (gotFailure && returnOnFailure()) {
314                        log.debug(displayName(description) + " returnOnFailure");
315                        return;
316                    }
317                    if (!gotFailure && returnOnSuccess()) {
318                        log.debug(displayName(description) + " returnOnSuccess");
319                        return;
320                    }
321                }
322            } finally {
323                log.debug(displayName(description) + " FINISHED");
324                notifier.removeListener(listener);
325            }
326            log.trace("throw " + error);
327            throw error;
328        }
329
330        protected abstract Error error();
331
332        protected abstract int retryCount();
333
334        protected abstract boolean returnOnSuccess();
335
336        protected abstract boolean returnOnFailure();
337    }
338
339    protected class RepeatOnFailure extends RepeatStatement {
340        protected String issue;
341
342        protected RepeatOnFailure(Repeat someParams, RunNotifier aNotifier, Statement aStatement,
343                Description description) {
344            super(someParams, aNotifier, aStatement, description);
345        }
346
347        @Override
348        protected Error error() {
349            return new AssertionError(String.format(
350                    "No success after %d tries. Either the bug is not random "
351                            + "or you should increase the 'onFailure' value.\n" + "Issue: %s",
352                    params.onFailure(), issue));
353        }
354
355        @Override
356        protected int retryCount() {
357            return params.onFailure();
358        }
359
360        @Override
361        protected boolean returnOnFailure() {
362            return false;
363        }
364
365        @Override
366        protected boolean returnOnSuccess() {
367            return true;
368        }
369    }
370
371    protected class RepeatOnSuccess extends RepeatStatement {
372        protected RepeatOnSuccess(Repeat someParams, RunNotifier aNotifier, Statement aStatement,
373                Description description) {
374            super(someParams, aNotifier, aStatement, description);
375        }
376
377        @Override
378        protected Error error() {
379            return new AssertionError(String.format(
380                    "No failure after %d tries. Either the bug is fixed "
381                            + "or you should increase the 'onSuccess' value.\n" + "Issue: %s",
382                    params.onSuccess(), params.issue()));
383        }
384
385        @Override
386        protected boolean returnOnFailure() {
387            return true;
388        }
389
390        @Override
391        protected boolean returnOnSuccess() {
392            return false;
393        }
394
395        @Override
396        protected int retryCount() {
397            return params.onSuccess();
398        }
399    }
400
401    protected class Bypass extends RepeatStatement {
402        public Bypass(Repeat someParams, RunNotifier aNotifier, Statement aStatement, Description description) {
403            super(someParams, aNotifier, aStatement, description);
404        }
405
406        @Override
407        public void evaluate() {
408            notifier.fireTestIgnored(description);
409        }
410
411        @Override
412        protected Error error() {
413            throw new UnsupportedOperationException();
414        }
415
416        @Override
417        protected int retryCount() {
418            throw new UnsupportedOperationException();
419        }
420
421        @Override
422        protected boolean returnOnSuccess() {
423            throw new UnsupportedOperationException();
424        }
425
426        @Override
427        protected boolean returnOnFailure() {
428            return false;
429        }
430    }
431
432    protected RepeatStatement onRepeat(Repeat someParams, RunNotifier aNotifier, Statement aStatement,
433            Description description) {
434        if (someParams.bypass()) {
435            return new Bypass(someParams, aNotifier, aStatement, description);
436        }
437        switch (fetchMode()) {
438        case BYPASS:
439            return new Bypass(someParams, aNotifier, aStatement, description);
440        case STRICT:
441            return new RepeatOnSuccess(someParams, aNotifier, aStatement, description);
442        case RELAX:
443            return new RepeatOnFailure(someParams, aNotifier, aStatement, description);
444        }
445        throw new IllegalArgumentException("no such mode");
446    }
447
448}