build: Get rid of SRC_DIRS

Tested-by: Emil Velikov <emil.l.velikov@gmail.com>
Reviewed-and-Tested-by: Andreas Boll <andreas.boll.dev@gmail.com>
Reviewed-by: Jordan Justen <jordan.l.justen@intel.com>
diff --git a/src/Makefile.am b/src/Makefile.am
index d096f29..b6449b9 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -25,4 +25,27 @@
 SUBDIRS += glsl mesa
 endif
 
-SUBDIRS += $(SRC_DIRS)
+if HAVE_DRI_GLX
+SUBDIRS += glx
+endif
+
+if HAVE_GBM
+SUBDIRS += gbm
+endif
+
+if HAVE_EGL
+SUBDIRS += egl
+endif
+
+if HAVE_GALLIUM
+SUBDIRS +=			\
+	gallium			\
+	gallium/winsys		\
+	gallium/targets
+
+if HAVE_GALLIUM_TESTS
+SUBDIRS +=			\
+	gallium/tests/trivial	\
+	gallium/tests/unit
+endif
+endif