Fix shutting down a never-started ServerImpl
diff --git a/core/src/main/java/io/grpc/ServerImpl.java b/core/src/main/java/io/grpc/ServerImpl.java
index 9b2125b..0063187 100644
--- a/core/src/main/java/io/grpc/ServerImpl.java
+++ b/core/src/main/java/io/grpc/ServerImpl.java
@@ -132,13 +132,21 @@
    * Initiates an orderly shutdown in which preexisting calls continue but new calls are rejected.
    */
   public ServerImpl shutdown() {
+    boolean shutdownTransportServer;
     synchronized (lock) {
       if (shutdown) {
         return this;
       }
       shutdown = true;
+      shutdownTransportServer = started;
+      if (!shutdownTransportServer) {
+        transportServerTerminated = true;
+        checkForTermination();
+      }
     }
-    transportServer.shutdown();
+    if (shutdownTransportServer) {
+      transportServer.shutdown();
+    }
     SharedResourceHolder.release(TIMER_SERVICE, timeoutService);
     return this;
   }
diff --git a/core/src/test/java/io/grpc/ServerImplTest.java b/core/src/test/java/io/grpc/ServerImplTest.java
index a0f381e..7b86d7a 100644
--- a/core/src/test/java/io/grpc/ServerImplTest.java
+++ b/core/src/test/java/io/grpc/ServerImplTest.java
@@ -116,6 +116,20 @@
   }
 
   @Test
+  public void stopImmediate() {
+    transportServer = new SimpleServer() {
+      @Override
+      public void shutdown() {
+        throw new AssertionError("Should not be called, because wasn't started");
+      }
+    };
+    ServerImpl server = new ServerImpl(executor, registry, transportServer);
+    server.shutdown();
+    assertTrue(server.isShutdown());
+    assertTrue(server.isTerminated());
+  }
+
+  @Test
   public void startStopImmediateWithChildTransport() throws IOException {
     ServerImpl server = new ServerImpl(executor, registry, transportServer);
     server.start();