tests: avoid putting build products into source directory (#2353)

* tests: keep source dir clean

* ci: make first build inplace

* ci: drop dev setting (wasn't doing anything)

* tests: warn if source directory is dirty
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 895cfe7..72de210 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -239,7 +239,6 @@
   endif()
 endforeach()
 
-set(testdir ${CMAKE_CURRENT_SOURCE_DIR})
 foreach(target ${test_targets})
   set(test_files ${PYBIND11_TEST_FILES})
   if(NOT "${target}" STREQUAL "pybind11_tests")
@@ -250,6 +249,18 @@
   pybind11_add_module(${target} THIN_LTO ${target}.cpp ${test_files} ${PYBIND11_HEADERS})
   pybind11_enable_warnings(${target})
 
+  if(NOT CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_CURRENT_BINARY_DIR)
+    get_property(
+      suffix
+      TARGET ${target}
+      PROPERTY SUFFIX)
+    set(source_output "${CMAKE_CURRENT_SOURCE_DIR}/${target}${suffix}")
+    if(suffix AND EXISTS "${source_output}")
+      message(WARNING "Output file also in source directory; "
+                      "please remove to avoid confusion: ${source_output}")
+    endif()
+  endif()
+
   if(MSVC)
     target_compile_options(${target} PRIVATE /utf-8)
   endif()
@@ -266,10 +277,12 @@
 
   # Always write the output file directly into the 'tests' directory (even on MSVC)
   if(NOT CMAKE_LIBRARY_OUTPUT_DIRECTORY)
-    set_target_properties(${target} PROPERTIES LIBRARY_OUTPUT_DIRECTORY "${testdir}")
+    set_target_properties(${target} PROPERTIES LIBRARY_OUTPUT_DIRECTORY
+                                               "${CMAKE_CURRENT_BINARY_DIR}")
     foreach(config ${CMAKE_CONFIGURATION_TYPES})
       string(TOUPPER ${config} config)
-      set_target_properties(${target} PROPERTIES LIBRARY_OUTPUT_DIRECTORY_${config} "${testdir}")
+      set_target_properties(${target} PROPERTIES LIBRARY_OUTPUT_DIRECTORY_${config}
+                                                 "${CMAKE_CURRENT_BINARY_DIR}")
     endforeach()
   endif()
 endforeach()
@@ -293,12 +306,26 @@
       CACHE INTERNAL "")
 endif()
 
+if(NOT CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_CURRENT_BINARY_DIR)
+  # This is not used later in the build, so it's okay to regenerate each time.
+  configure_file("${CMAKE_CURRENT_SOURCE_DIR}/pytest.ini" "${CMAKE_CURRENT_BINARY_DIR}/pytest.ini"
+                 COPYONLY)
+  file(APPEND "${CMAKE_CURRENT_BINARY_DIR}/pytest.ini"
+       "\ntestpaths = \"${CMAKE_CURRENT_SOURCE_DIR}\"")
+
+endif()
+
+# cmake 3.12 added list(transform <list> prepend
+# but we can't use it yet
+string(REPLACE "test_" "${CMAKE_CURRENT_BINARY_DIR}/test_" PYBIND11_BINARY_TEST_FILES
+               "${PYBIND11_PYTEST_FILES}")
+
 # A single command to compile and run the tests
 add_custom_target(
   pytest
-  COMMAND ${PYTHON_EXECUTABLE} -m pytest ${PYBIND11_PYTEST_FILES}
+  COMMAND ${PYTHON_EXECUTABLE} -m pytest ${PYBIND11_BINARY_PYTEST_FILES}
   DEPENDS ${test_targets}
-  WORKING_DIRECTORY ${testdir}
+  WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
   USES_TERMINAL)
 
 if(PYBIND11_TEST_OVERRIDE)
diff --git a/tests/conftest.py b/tests/conftest.py
index 8b6e47d..a2350d0 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -13,6 +13,8 @@
 
 import pytest
 
+import env
+
 # Early diagnostic for failed imports
 import pybind11_tests  # noqa: F401
 
@@ -20,6 +22,11 @@
 _long_marker = re.compile(r'([0-9])L')
 _hexadecimal = re.compile(r'0x[0-9a-fA-F]+')
 
+# Avoid collecting Python3 only files
+collect_ignore = []
+if env.PY2:
+    collect_ignore.append("test_async.py")
+
 
 def _strip_and_dedent(s):
     """For triple-quote strings"""
diff --git a/tests/test_embed/CMakeLists.txt b/tests/test_embed/CMakeLists.txt
index 1495c77..2e298fa 100644
--- a/tests/test_embed/CMakeLists.txt
+++ b/tests/test_embed/CMakeLists.txt
@@ -21,18 +21,22 @@
 
 target_link_libraries(test_embed PRIVATE pybind11::embed Catch2::Catch2 Threads::Threads)
 
+if(NOT CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_CURRENT_BINARY_DIR)
+  file(COPY test_interpreter.py DESTINATION "${CMAKE_CURRENT_BINARY_DIR}")
+endif()
+
 add_custom_target(
   cpptest
   COMMAND "$<TARGET_FILE:test_embed>"
-  WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}")
+  WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}")
 
 pybind11_add_module(external_module THIN_LTO external_module.cpp)
 set_target_properties(external_module PROPERTIES LIBRARY_OUTPUT_DIRECTORY
-                                                 "${CMAKE_CURRENT_SOURCE_DIR}")
+                                                 "${CMAKE_CURRENT_BINARY_DIR}")
 foreach(config ${CMAKE_CONFIGURATION_TYPES})
   string(TOUPPER ${config} config)
   set_target_properties(external_module PROPERTIES LIBRARY_OUTPUT_DIRECTORY_${config}
-                                                   "${CMAKE_CURRENT_SOURCE_DIR}")
+                                                   "${CMAKE_CURRENT_BINARY_DIR}")
 endforeach()
 add_dependencies(cpptest external_module)