Don't try to load backends if the test doesn't need it
diff --git a/tests/conftest.py b/tests/conftest.py
index 6599a64..bdd17fb 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -12,10 +12,10 @@
 
 
 def pytest_generate_tests(metafunc):
-    names = metafunc.config.getoption("--backend")
-    selected_backends = select_backends(names, _available_backends())
-
     if "backend" in metafunc.fixturenames:
+        names = metafunc.config.getoption("--backend")
+        selected_backends = select_backends(names, _available_backends())
+
         filtered_backends = []
         required = metafunc.function.requires_backend_interface
         required_interfaces = [