8014819: set max size for jtreg testvms

Reviewed-by: alanb, darcy
diff --git a/jdk/test/Makefile b/jdk/test/Makefile
index 51529b7..de6b4a7 100644
--- a/jdk/test/Makefile
+++ b/jdk/test/Makefile
@@ -657,9 +657,12 @@
 # Multiple by 4 the timeout numbers
 JTREG_TIMEOUT_OPTION =  -timeoutFactor:4
 JTREG_BASIC_OPTIONS += $(JTREG_TIMEOUT_OPTION)
-# Boost the max memory for jtreg to avoid gc thrashing
+# Set the max memory for jtreg control vm
 JTREG_MEMORY_OPTION = -J-Xmx512m
 JTREG_BASIC_OPTIONS += $(JTREG_MEMORY_OPTION)
+# Set the max memory for jtreg target test vms
+JTREG_TESTVM_MEMORY_OPTION = -vmoption:-Xmx512m
+JTREG_TEST_OPTIONS += $(JTREG_TESTVM_MEMORY_OPTION)
 
 # Make sure jtreg exists
 $(JTREG): $(JT_HOME)