| /** |
| * @license |
| * Copyright 2013 Google Inc. All rights reserved. |
| * Licensed under the Apache License, Version 2.0 (the "License"); |
| * you may not use this file except in compliance with the License. |
| * You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| package com.google.security.wycheproof; |
| |
| import java.lang.annotation.ElementType; |
| import java.lang.annotation.Retention; |
| import java.lang.annotation.RetentionPolicy; |
| import java.lang.annotation.Target; |
| import java.util.Arrays; |
| import org.junit.runner.Description; |
| import org.junit.runner.manipulation.Filter; |
| import org.junit.runner.manipulation.NoTestsRemainException; |
| import org.junit.runners.Suite; |
| import org.junit.runners.model.InitializationError; |
| import org.junit.runners.model.RunnerBuilder; |
| |
| /** |
| * <p>A custom JUnit4 runner that, with annotations, allows choosing tests to run on a specific |
| * provider. To use it, annotate a runner class with {@code RunWith(WycheproofRunner.class)}, and |
| * {@code SuiteClasses({AesGcmTest.class, ...})}. When you run this class, it will run all the tests |
| * in all the suite classes. |
| * |
| * <p>To exclude certain tests, a runner class should be annotated with {@code @Provider} which |
| * indicates the target provider. Test exclusion is defined as follows: |
| * <ul>@Fast test runners skip @SlowTest test functions. |
| * <ul>@Presubmit test runners skip @NoPresubmitTest test functions. |
| * <ul>All test runners skip @ExcludedTest test functions. |
| * |
| * @author thaidn@google.com (Thai Duong) |
| */ |
| public class WycheproofRunner extends Suite { |
| |
| /** List of supported providers. */ |
| public enum ProviderType { |
| BOUNCY_CASTLE, |
| CONSCRYPT, |
| OPENJDK, |
| SPONGY_CASTLE, |
| } |
| |
| // Annotations for test runners. |
| |
| /** |
| * Annotation to specify the target provider of a test runner. |
| * |
| * <p>Usage: @Provider(ProviderType.BOUNCY_CASTLE) |
| */ |
| @Retention(RetentionPolicy.RUNTIME) |
| @Target({ElementType.TYPE}) |
| public @interface Provider { |
| ProviderType value(); |
| } |
| |
| /** |
| * Annotation to specify presubmit test runners that exclude {@code @NoPresubmitTets} tests. |
| * |
| * <p>Usage: @Presubmit(ProviderType.BOUNCY_CASTLE) |
| */ |
| @Retention(RetentionPolicy.RUNTIME) |
| @Target({ElementType.TYPE}) |
| public @interface Presubmit {} |
| |
| /** |
| * Annotation to specify fast test runners that exclude {@code @SlowTest} tests. |
| * |
| * <p>Usage: @Fast |
| */ |
| @Retention(RetentionPolicy.RUNTIME) |
| @Target({ElementType.TYPE}) |
| public @interface Fast {} |
| |
| // Annotations for test functions |
| |
| /** |
| * Tests that take too much time to run, should be excluded from TAP and wildcard target patterns |
| * like:..., :*, or :all. |
| * |
| * <p>Usage: @SlowTest(providers = {ProviderType.BOUNCY_CASTLE, ...}) |
| */ |
| @Retention(RetentionPolicy.RUNTIME) |
| @Target({ElementType.METHOD}) |
| public @interface SlowTest { |
| ProviderType[] providers(); |
| } |
| |
| /** |
| * Tests that should be excluded from presubmit checks on specific providers. |
| * |
| * <p>Usage: @NoPresubmitTest( |
| * providers = {ProviderType.BOUNCY_CASTLE, ...}, |
| * bugs = {"b/123456789"} |
| * ) |
| */ |
| @Retention(RetentionPolicy.RUNTIME) |
| @Target({ElementType.METHOD, ElementType.FIELD}) |
| public @interface NoPresubmitTest { |
| /** List of providers that this test method should not run as presubmit check. */ |
| ProviderType[] providers(); |
| |
| /** List of blocking bugs (and comments). */ |
| String[] bugs(); |
| } |
| |
| /** |
| * Annotation to specify test functions that should be excluded on specific providers. |
| * |
| * <p>Usage: @ExcludedTest(providers = {ProviderType.BOUNCY_CASTLE, ProviderType.OPENJDK}) |
| */ |
| @Retention(RetentionPolicy.RUNTIME) |
| @Target({ElementType.METHOD}) |
| public @interface ExcludedTest { |
| ProviderType[] providers(); |
| String comment(); |
| } |
| |
| /** |
| * Custom filter to exclude certain test functions. |
| * |
| */ |
| public static class ExcludeTestFilter extends Filter { |
| |
| Class<?> runnerClass; |
| Provider targetProvider; |
| Fast fast; |
| Presubmit presubmit; |
| |
| public ExcludeTestFilter(Class<?> runnerClass) { |
| this.runnerClass = runnerClass; |
| this.targetProvider = runnerClass.getAnnotation(Provider.class); |
| this.fast = runnerClass.getAnnotation(Fast.class); |
| this.presubmit = runnerClass.getAnnotation(Presubmit.class); |
| } |
| |
| @Override |
| public String describe() { |
| return "exclude certain tests on specific providers"; |
| } |
| |
| @Override |
| public boolean shouldRun(Description description) { |
| return isOkayToRunTest(description); |
| } |
| |
| private boolean isOkayToRunTest(Description description) { |
| if (targetProvider == null) { |
| // Run all test functions if the test runner is not annotated with {@code @Provider}. |
| return true; |
| } |
| // Skip @ExcludedTest tests |
| ExcludedTest excludedTest = description.getAnnotation(ExcludedTest.class); |
| if (excludedTest != null |
| && Arrays.asList(excludedTest.providers()).contains(targetProvider.value())) { |
| return false; |
| } |
| |
| // If the runner class is annotated with @Presubmit, skip non-presubmit tests |
| if (presubmit != null) { |
| NoPresubmitTest ignoreOn = description.getAnnotation(NoPresubmitTest.class); |
| if (ignoreOn != null |
| && Arrays.asList(ignoreOn.providers()).contains(targetProvider.value())) { |
| return false; |
| } |
| } |
| |
| // If the runner class is annotated with @Fast, skip slow tests |
| if (fast != null) { |
| SlowTest ignoreOn = description.getAnnotation(SlowTest.class); |
| if (ignoreOn != null |
| && Arrays.asList(ignoreOn.providers()).contains(targetProvider.value())) { |
| return false; |
| } |
| } |
| |
| // run everything else |
| return true; |
| } |
| } |
| |
| /** Required constructor: called by JUnit reflectively. */ |
| public WycheproofRunner(Class<?> runnerClass, RunnerBuilder builder) throws InitializationError { |
| super(runnerClass, builder); |
| addFilter(new ExcludeTestFilter(runnerClass)); |
| } |
| |
| private void addFilter(Filter filter) { |
| try { |
| filter(filter); |
| } catch (NoTestsRemainException ex) { |
| System.out.println("No tests remain exception: " + ex); |
| } |
| } |
| } |