tests/src/com/android/tradefed/testtype/GoogleBenchmarkTestTest.java

/*
 * Copyright (C) 2016 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.android.tradefed.testtype;

import com.android.tradefed.device.CollectingOutputReceiver;
import com.android.tradefed.device.DeviceNotAvailableException;
import com.android.tradefed.device.ITestDevice;
import com.android.tradefed.device.MockFileUtil;
import com.android.tradefed.invoker.TestInformation;
import com.android.tradefed.metrics.proto.MetricMeasurement.Metric;
import com.android.tradefed.result.ITestInvocationListener;

import junit.framework.TestCase;

import org.easymock.EasyMock;

import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.TimeUnit;

/**
 * Unit tests for {@link GoogleBenchmarkTest}.
 */
public class GoogleBenchmarkTestTest extends TestCase {

    private ITestInvocationListener mMockInvocationListener = null;
    private CollectingOutputReceiver mMockReceiver = null;
    private ITestDevice mMockITestDevice = null;
    private GoogleBenchmarkTest mGoogleBenchmarkTest;
    private TestInformation mTestInfo;

    /**
     * Helper to initialize the various EasyMocks we'll need.
     */
    @Override
    protected void setUp() throws Exception {
        super.setUp();
        mMockInvocationListener = EasyMock.createMock(ITestInvocationListener.class);
        mMockReceiver = new CollectingOutputReceiver();
        mMockITestDevice = EasyMock.createMock(ITestDevice.class);
        EasyMock.expect(mMockITestDevice.getSerialNumber()).andStubReturn("serial");
        mGoogleBenchmarkTest = new GoogleBenchmarkTest() {
            @Override
            CollectingOutputReceiver createOutputCollector() {
                return mMockReceiver;
            }
            @Override
            GoogleBenchmarkResultParser createResultParser(String runName,
                    ITestInvocationListener listener) {
                return new GoogleBenchmarkResultParser(runName, listener) {
                    @Override
                    public Map<String, String> parse(CollectingOutputReceiver output) {
                        return Collections.emptyMap();
                    }
                };
            }
        };
        mGoogleBenchmarkTest.setDevice(mMockITestDevice);
        mTestInfo = TestInformation.newBuilder().build();
    }

    /**
     * Helper that replays all mocks.
     */
    private void replayMocks() {
      EasyMock.replay(mMockInvocationListener, mMockITestDevice);
    }

    /**
     * Helper that verifies all mocks.
     */
    private void verifyMocks() {
      EasyMock.verify(mMockInvocationListener, mMockITestDevice);
    }

    /**
     * Test the run method for a couple tests
     */
    public void testRun() throws DeviceNotAvailableException {
        final String nativeTestPath = GoogleBenchmarkTest.DEFAULT_TEST_PATH;
        final String test1 = "test1";
        final String test2 = "test2";
        MockFileUtil.setMockDirContents(mMockITestDevice, nativeTestPath, test1, test2);
        EasyMock.expect(mMockITestDevice.doesFileExist(nativeTestPath)).andReturn(true);
        EasyMock.expect(mMockITestDevice.isDirectory(nativeTestPath)).andReturn(true);
        EasyMock.expect(mMockITestDevice.isDirectory(nativeTestPath + "/test1")).andReturn(false);
        EasyMock.expect(mMockITestDevice.isDirectory(nativeTestPath + "/test2")).andReturn(false);
        String[] files = new String[] {"test1", "test2"};
        EasyMock.expect(mMockITestDevice.getChildren(nativeTestPath)).andReturn(files);
        EasyMock.expect(mMockITestDevice.executeShellCommand(EasyMock.contains("chmod")))
                .andReturn("").times(2);
        mMockITestDevice.executeShellCommand(EasyMock.contains(test1), EasyMock.same(mMockReceiver),
                EasyMock.anyLong(), (TimeUnit)EasyMock.anyObject(), EasyMock.anyInt());
        mMockITestDevice.executeShellCommand(EasyMock.contains(test2), EasyMock.same(mMockReceiver),
                EasyMock.anyLong(), (TimeUnit)EasyMock.anyObject(), EasyMock.anyInt());

        EasyMock.expect(mMockITestDevice.isExecutable(nativeTestPath + "/test1")).andReturn(true);
        EasyMock.expect(
                        mMockITestDevice.executeShellCommand(
                                String.format(
                                        "%s/test1 --benchmark_list_tests=true", nativeTestPath)))
                .andReturn("method1\nmethod2\nmethod3");
        EasyMock.expect(mMockITestDevice.isExecutable(nativeTestPath + "/test2")).andReturn(true);

        EasyMock.expect(
                        mMockITestDevice.executeShellCommand(
                                String.format(
                                        "%s/test2 --benchmark_list_tests=true", nativeTestPath)))
                .andReturn("method1\nmethod2\n");
        mMockInvocationListener.testRunStarted(test1, 3);
        mMockInvocationListener.testRunStarted(test2, 2);
        mMockInvocationListener.testRunEnded(
                EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>anyObject());
        EasyMock.expectLastCall().times(2);
        replayMocks();

        mGoogleBenchmarkTest.run(mTestInfo, mMockInvocationListener);
        verifyMocks();
    }

    /**
     * Test the run method when no device is set.
     */
    public void testRun_noDevice() throws DeviceNotAvailableException {
        mGoogleBenchmarkTest.setDevice(null);
        try {
            mGoogleBenchmarkTest.run(mTestInfo, mMockInvocationListener);
        } catch (IllegalArgumentException e) {
            assertEquals("Device has not been set", e.getMessage());
            return;
        }
        fail();
    }

    /**
     * Test the run method for a couple tests
     */
    public void testRun_noBenchmarkDir() throws DeviceNotAvailableException {
        EasyMock.expect(mMockITestDevice.doesFileExist(GoogleBenchmarkTest.DEFAULT_TEST_PATH))
                .andReturn(false);
        replayMocks();
        try {
            mGoogleBenchmarkTest.run(mTestInfo, mMockInvocationListener);
            fail("Should have thrown an exception.");
        } catch (RuntimeException e) {
            // expected
        }
        verifyMocks();
    }

    /**
     * Test the run method for a couple tests with a module name
     */
    public void testRun_withModuleName() throws DeviceNotAvailableException {
        final String moduleName = "module";
        final String nativeTestPath =
                String.format("%s/%s", GoogleBenchmarkTest.DEFAULT_TEST_PATH, moduleName);
        mGoogleBenchmarkTest.setModuleName(moduleName);
        final String test1 = "test1";
        final String test2 = "test2";
        MockFileUtil.setMockDirContents(mMockITestDevice, nativeTestPath, test1, test2);
        EasyMock.expect(mMockITestDevice.doesFileExist(nativeTestPath)).andReturn(true);
        EasyMock.expect(mMockITestDevice.isDirectory(nativeTestPath)).andReturn(true);
        EasyMock.expect(mMockITestDevice.isDirectory(nativeTestPath + "/test1")).andReturn(false);
        EasyMock.expect(mMockITestDevice.isDirectory(nativeTestPath + "/test2")).andReturn(false);
        String[] files = new String[] {"test1", "test2"};
        EasyMock.expect(mMockITestDevice.getChildren(nativeTestPath)).andReturn(files);
        EasyMock.expect(mMockITestDevice.executeShellCommand(EasyMock.contains("chmod")))
                .andReturn("").times(2);
        mMockITestDevice.executeShellCommand(EasyMock.contains(test1), EasyMock.same(mMockReceiver),
                EasyMock.anyLong(), (TimeUnit)EasyMock.anyObject(), EasyMock.anyInt());
        mMockITestDevice.executeShellCommand(EasyMock.contains(test2), EasyMock.same(mMockReceiver),
                EasyMock.anyLong(), (TimeUnit)EasyMock.anyObject(), EasyMock.anyInt());
        EasyMock.expect(mMockITestDevice.isExecutable(nativeTestPath + "/test1")).andReturn(true);
        EasyMock.expect(
                        mMockITestDevice.executeShellCommand(
                                String.format(
                                        "%s/test1 --benchmark_list_tests=true", nativeTestPath)))
                .andReturn("\nmethod1\nmethod2\nmethod3\n\n");
        EasyMock.expect(mMockITestDevice.isExecutable(nativeTestPath + "/test2")).andReturn(true);
        EasyMock.expect(
                        mMockITestDevice.executeShellCommand(
                                String.format(
                                        "%s/test2 --benchmark_list_tests=true", nativeTestPath)))
                .andReturn("method1\nmethod2\n");
        mMockInvocationListener.testRunStarted(test1, 3);
        mMockInvocationListener.testRunStarted(test2, 2);
        mMockInvocationListener.testRunEnded(
                EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>anyObject());
        EasyMock.expectLastCall().times(2);
        replayMocks();

        mGoogleBenchmarkTest.run(mTestInfo, mMockInvocationListener);
        verifyMocks();
    }

    /**
     * Test the run method for a couple tests with a module name
     */
    public void testRun_withRunReportName() throws DeviceNotAvailableException {
        final String nativeTestPath = GoogleBenchmarkTest.DEFAULT_TEST_PATH;
        final String test1 = "test1";
        final String reportName = "reportName";
        mGoogleBenchmarkTest.setReportRunName(reportName);
        MockFileUtil.setMockDirContents(mMockITestDevice, nativeTestPath, test1);
        EasyMock.expect(mMockITestDevice.doesFileExist(nativeTestPath)).andReturn(true);
        EasyMock.expect(mMockITestDevice.isDirectory(nativeTestPath)).andReturn(true);
        EasyMock.expect(mMockITestDevice.isDirectory(nativeTestPath + "/test1")).andReturn(false);
        String[] files = new String[] {"test1"};
        EasyMock.expect(mMockITestDevice.getChildren(nativeTestPath)).andReturn(files);
        EasyMock.expect(mMockITestDevice.executeShellCommand(EasyMock.contains("chmod")))
                .andReturn("");
        mMockITestDevice.executeShellCommand(EasyMock.contains(test1), EasyMock.same(mMockReceiver),
                EasyMock.anyLong(), (TimeUnit)EasyMock.anyObject(), EasyMock.anyInt());
        EasyMock.expect(mMockITestDevice.isExecutable(nativeTestPath + "/test1")).andReturn(true);
        EasyMock.expect(
                        mMockITestDevice.executeShellCommand(
                                String.format(
                                        "%s/test1 --benchmark_list_tests=true", nativeTestPath)))
                .andReturn("method1\nmethod2\nmethod3");
        // Expect reportName instead of test name
        mMockInvocationListener.testRunStarted(reportName, 3);
        mMockInvocationListener.testRunEnded(
                EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>anyObject());
        EasyMock.expectLastCall();
        replayMocks();

        mGoogleBenchmarkTest.run(mTestInfo, mMockInvocationListener);
        verifyMocks();
    }

    /**
     * Test the run method when exec shell throw exeception.
     */
    public void testRun_exceptionDuringExecShell() throws DeviceNotAvailableException {
        final String nativeTestPath = GoogleBenchmarkTest.DEFAULT_TEST_PATH;
        final String test1 = "test1";
        MockFileUtil.setMockDirContents(mMockITestDevice, nativeTestPath, test1);
        EasyMock.expect(mMockITestDevice.doesFileExist(nativeTestPath)).andReturn(true);
        EasyMock.expect(mMockITestDevice.isDirectory(nativeTestPath)).andReturn(true);
        EasyMock.expect(mMockITestDevice.isDirectory(nativeTestPath + "/test1")).andReturn(false);
        String[] files = new String[] {"test1"};
        EasyMock.expect(mMockITestDevice.getChildren(nativeTestPath)).andReturn(files);
        EasyMock.expect(mMockITestDevice.executeShellCommand(EasyMock.contains("chmod")))
                .andReturn("");
        mMockITestDevice.executeShellCommand(EasyMock.contains(test1), EasyMock.same(mMockReceiver),
                EasyMock.anyLong(), (TimeUnit)EasyMock.anyObject(), EasyMock.anyInt());
        EasyMock.expectLastCall().andThrow(new DeviceNotAvailableException("dnae", "serial"));
        EasyMock.expect(mMockITestDevice.isExecutable(nativeTestPath + "/test1")).andReturn(true);
        EasyMock.expect(
                        mMockITestDevice.executeShellCommand(
                                String.format(
                                        "%s/test1 --benchmark_list_tests=true", nativeTestPath)))
                .andReturn("method1\nmethod2\nmethod3");
        mMockInvocationListener.testRunStarted(test1, 3);
        mMockInvocationListener.testRunFailed((String) EasyMock.anyObject());
        // Even with exception testrunEnded is expected.
        mMockInvocationListener.testRunEnded(
                EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>anyObject());
        EasyMock.expectLastCall();
        replayMocks();

        try {
            mGoogleBenchmarkTest.run(mTestInfo, mMockInvocationListener);
            fail();
        } catch (DeviceNotAvailableException e) {
            // expected
        }
        verifyMocks();
    }

    /**
     * File exclusion regex filter should skip matched filepaths.
     */
    public void testFileExclusionRegexFilter_skipMatched() {
        // Skip files ending in .txt
        mGoogleBenchmarkTest.addFileExclusionFilterRegex(".*\\.txt");
        assertFalse(mGoogleBenchmarkTest.shouldSkipFile("/some/path/file/binary"));
        assertFalse(mGoogleBenchmarkTest.shouldSkipFile("/some/path/file/random.dat"));
        assertTrue(mGoogleBenchmarkTest.shouldSkipFile("/some/path/file/test.txt"));
        // Always skip files ending in .config
        assertTrue(mGoogleBenchmarkTest.shouldSkipFile("/some/path/file/random.config"));
    }

    /**
     * File exclusion regex filter for multi filters.
     */
    public void testFileExclusionRegexFilter_skipMultiMatched() {
        // Skip files ending in .txt
        mGoogleBenchmarkTest.addFileExclusionFilterRegex(".*\\.txt");
        // Also skip files ending in .dat
        mGoogleBenchmarkTest.addFileExclusionFilterRegex(".*\\.dat");
        assertFalse(mGoogleBenchmarkTest.shouldSkipFile("/some/path/file/binary"));
        assertTrue(mGoogleBenchmarkTest.shouldSkipFile("/some/path/file/random.dat"));
        assertTrue(mGoogleBenchmarkTest.shouldSkipFile("/some/path/file/test.txt"));
        // Always skip files ending in .config
        assertTrue(mGoogleBenchmarkTest.shouldSkipFile("/some/path/file/random.config"));
    }

    /** File exclusion regex filter should always skip .config file. */
    public void testFileExclusionRegexFilter_skipDefaultMatched() {
        // Always skip files ending in .config
        assertTrue(mGoogleBenchmarkTest.shouldSkipFile("/some/path/file/random.config"));
        // Other file should not be skipped
        assertFalse(mGoogleBenchmarkTest.shouldSkipFile("/some/path/file/random.configs"));
        assertFalse(mGoogleBenchmarkTest.shouldSkipFile("/some/path/file/binary"));
        assertFalse(mGoogleBenchmarkTest.shouldSkipFile("/some/path/file/random.dat"));
        assertFalse(mGoogleBenchmarkTest.shouldSkipFile("/some/path/file/test.txt"));

    }
}