docs: update docs (#916)

* fix: re-run script

* test: fix noxfile
diff --git a/docs/dyn/dataflow_v1b3.projects.templates.html b/docs/dyn/dataflow_v1b3.projects.templates.html
index 8055b70..da13aae 100644
--- a/docs/dyn/dataflow_v1b3.projects.templates.html
+++ b/docs/dyn/dataflow_v1b3.projects.templates.html
@@ -94,17 +94,35 @@
     The object takes the form of:
 
 { # A request to create a Cloud Dataflow job from a template.
+    "jobName": "A String", # Required. The job name to use for the created job.
+    "gcsPath": "A String", # Required. A Cloud Storage path to the template from which to
+        # create the job.
+        # Must be a valid Cloud Storage URL, beginning with `gs://`.
     "environment": { # The environment values to set at runtime. # The runtime environment for the job.
+      "machineType": "A String", # The machine type to use for the job. Defaults to the value from the
+          # template if not specified.
+      "subnetwork": "A String", # Subnetwork to which VMs will be assigned, if desired.  Expected to be of
+          # the form "regions/REGION/subnetworks/SUBNETWORK".
+      "ipConfiguration": "A String", # Configuration for VM IPs.
+      "kmsKeyName": "A String", # Optional. Name for the Cloud KMS key for the job.
+          # Key format is:
+          # projects/<project>/locations/<location>/keyRings/<keyring>/cryptoKeys/<key>
+      "tempLocation": "A String", # The Cloud Storage path to use for temporary files.
+          # Must be a valid Cloud Storage URL, beginning with `gs://`.
+      "bypassTempDirValidation": True or False, # Whether to bypass the safety checks for the job's temporary directory.
+          # Use with caution.
+      "network": "A String", # Network to which VMs will be assigned.  If empty or unspecified,
+          # the service will use the network "default".
       "workerRegion": "A String", # The Compute Engine region
           # (https://cloud.google.com/compute/docs/regions-zones/regions-zones) in
           # which worker processing should occur, e.g. "us-west1". Mutually exclusive
           # with worker_zone. If neither worker_region nor worker_zone is specified,
           # default to the control plane's region.
-      "numWorkers": 42, # The initial number of Google Compute Engine instnaces for the job.
       "zone": "A String", # The Compute Engine [availability
           # zone](https://cloud.google.com/compute/docs/regions-zones/regions-zones)
           # for launching worker instances to run your pipeline.
           # In the future, worker_zone will take precedence.
+      "numWorkers": 42, # The initial number of Google Compute Engine instnaces for the job.
       "workerZone": "A String", # The Compute Engine zone
           # (https://cloud.google.com/compute/docs/regions-zones/regions-zones) in
           # which worker processing should occur, e.g. "us-west1-a". Mutually exclusive
@@ -123,20 +141,6 @@
       "maxWorkers": 42, # The maximum number of Google Compute Engine instances to be made
           # available to your pipeline during execution, from 1 to 1000.
       "serviceAccountEmail": "A String", # The email address of the service account to run the job as.
-      "machineType": "A String", # The machine type to use for the job. Defaults to the value from the
-          # template if not specified.
-      "subnetwork": "A String", # Subnetwork to which VMs will be assigned, if desired.  Expected to be of
-          # the form "regions/REGION/subnetworks/SUBNETWORK".
-      "ipConfiguration": "A String", # Configuration for VM IPs.
-      "kmsKeyName": "A String", # Optional. Name for the Cloud KMS key for the job.
-          # Key format is:
-          # projects/<project>/locations/<location>/keyRings/<keyring>/cryptoKeys/<key>
-      "bypassTempDirValidation": True or False, # Whether to bypass the safety checks for the job's temporary directory.
-          # Use with caution.
-      "tempLocation": "A String", # The Cloud Storage path to use for temporary files.
-          # Must be a valid Cloud Storage URL, beginning with `gs://`.
-      "network": "A String", # Network to which VMs will be assigned.  If empty or unspecified,
-          # the service will use the network "default".
     },
     "location": "A String", # The [regional endpoint]
         # (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) to
@@ -144,10 +148,6 @@
     "parameters": { # The runtime parameters to pass to the job.
       "a_key": "A String",
     },
-    "jobName": "A String", # Required. The job name to use for the created job.
-    "gcsPath": "A String", # Required. A Cloud Storage path to the template from which to
-        # create the job.
-        # Must be a valid Cloud Storage URL, beginning with `gs://`.
   }
 
   x__xgafv: string, V1 error format.
@@ -176,219 +176,6 @@
       "a_key": "A String",
     },
     "environment": { # Describes the environment in which a Dataflow Job runs. # The environment for the job.
-      "sdkPipelineOptions": { # The Cloud Dataflow SDK pipeline options specified by the user. These
-          # options are passed through the service and are used to recreate the
-          # SDK pipeline options on the worker in a language agnostic and platform
-          # independent way.
-        "a_key": "", # Properties of the object.
-      },
-      "flexResourceSchedulingGoal": "A String", # Which Flexible Resource Scheduling mode to run in.
-      "workerPools": [ # The worker pools. At least one "harness" worker pool must be
-          # specified in order for the job to have workers.
-        { # Describes one particular pool of Cloud Dataflow workers to be
-            # instantiated by the Cloud Dataflow service in order to perform the
-            # computations required by a job.  Note that a workflow job may use
-            # multiple pools, in order to match the various computational
-            # requirements of the various stages of the job.
-          "defaultPackageSet": "A String", # The default package set to install.  This allows the service to
-              # select a default set of packages which are useful to worker
-              # harnesses written in a particular language.
-          "network": "A String", # Network to which VMs will be assigned.  If empty or unspecified,
-              # the service will use the network "default".
-          "zone": "A String", # Zone to run the worker pools in.  If empty or unspecified, the service
-              # will attempt to choose a reasonable default.
-          "numWorkers": 42, # Number of Google Compute Engine workers in this pool needed to
-              # execute the job.  If zero or unspecified, the service will
-              # attempt to choose a reasonable default.
-          "numThreadsPerWorker": 42, # The number of threads per worker harness. If empty or unspecified, the
-              # service will choose a number of threads (according to the number of cores
-              # on the selected machine type for batch, or 1 by convention for streaming).
-          "diskSourceImage": "A String", # Fully qualified source image for disks.
-          "packages": [ # Packages to be installed on workers.
-            { # The packages that must be installed in order for a worker to run the
-                # steps of the Cloud Dataflow job that will be assigned to its worker
-                # pool.
-                #
-                # This is the mechanism by which the Cloud Dataflow SDK causes code to
-                # be loaded onto the workers. For example, the Cloud Dataflow Java SDK
-                # might use this to install jars containing the user's code and all of the
-                # various dependencies (libraries, data files, etc.) required in order
-                # for that code to run.
-              "location": "A String", # The resource to read the package from. The supported resource type is:
-                  #
-                  # Google Cloud Storage:
-                  #
-                  #   storage.googleapis.com/{bucket}
-                  #   bucket.storage.googleapis.com/
-              "name": "A String", # The name of the package.
-            },
-          ],
-          "teardownPolicy": "A String", # Sets the policy for determining when to turndown worker pool.
-              # Allowed values are: `TEARDOWN_ALWAYS`, `TEARDOWN_ON_SUCCESS`, and
-              # `TEARDOWN_NEVER`.
-              # `TEARDOWN_ALWAYS` means workers are always torn down regardless of whether
-              # the job succeeds. `TEARDOWN_ON_SUCCESS` means workers are torn down
-              # if the job succeeds. `TEARDOWN_NEVER` means the workers are never torn
-              # down.
-              #
-              # If the workers are not torn down by the service, they will
-              # continue to run and use Google Compute Engine VM resources in the
-              # user's project until they are explicitly terminated by the user.
-              # Because of this, Google recommends using the `TEARDOWN_ALWAYS`
-              # policy except for small, manually supervised test jobs.
-              #
-              # If unknown or unspecified, the service will attempt to choose a reasonable
-              # default.
-          "onHostMaintenance": "A String", # The action to take on host maintenance, as defined by the Google
-              # Compute Engine API.
-          "poolArgs": { # Extra arguments for this worker pool.
-            "a_key": "", # Properties of the object. Contains field @type with type URL.
-          },
-          "diskSizeGb": 42, # Size of root disk for VMs, in GB.  If zero or unspecified, the service will
-              # attempt to choose a reasonable default.
-          "workerHarnessContainerImage": "A String", # Required. Docker container image that executes the Cloud Dataflow worker
-              # harness, residing in Google Container Registry.
-              #
-              # Deprecated for the Fn API path. Use sdk_harness_container_images instead.
-          "diskType": "A String", # Type of root disk for VMs.  If empty or unspecified, the service will
-              # attempt to choose a reasonable default.
-          "machineType": "A String", # Machine type (e.g. "n1-standard-1").  If empty or unspecified, the
-              # service will attempt to choose a reasonable default.
-          "kind": "A String", # The kind of the worker pool; currently only `harness` and `shuffle`
-              # are supported.
-          "dataDisks": [ # Data disks that are used by a VM in this workflow.
-            { # Describes the data disk used by a workflow job.
-              "sizeGb": 42, # Size of disk in GB.  If zero or unspecified, the service will
-                  # attempt to choose a reasonable default.
-              "diskType": "A String", # Disk storage type, as defined by Google Compute Engine.  This
-                  # must be a disk type appropriate to the project and zone in which
-                  # the workers will run.  If unknown or unspecified, the service
-                  # will attempt to choose a reasonable default.
-                  #
-                  # For example, the standard persistent disk type is a resource name
-                  # typically ending in "pd-standard".  If SSD persistent disks are
-                  # available, the resource name typically ends with "pd-ssd".  The
-                  # actual valid values are defined the Google Compute Engine API,
-                  # not by the Cloud Dataflow API; consult the Google Compute Engine
-                  # documentation for more information about determining the set of
-                  # available disk types for a particular project and zone.
-                  #
-                  # Google Compute Engine Disk types are local to a particular
-                  # project in a particular zone, and so the resource name will
-                  # typically look something like this:
-                  #
-                  # compute.googleapis.com/projects/project-id/zones/zone/diskTypes/pd-standard
-              "mountPoint": "A String", # Directory in a VM where disk is mounted.
-            },
-          ],
-          "sdkHarnessContainerImages": [ # Set of SDK harness containers needed to execute this pipeline. This will
-              # only be set in the Fn API path. For non-cross-language pipelines this
-              # should have only one entry. Cross-language pipelines will have two or more
-              # entries.
-            { # Defines a SDK harness container for executing Dataflow pipelines.
-              "containerImage": "A String", # A docker container image that resides in Google Container Registry.
-              "useSingleCorePerContainer": True or False, # If true, recommends the Dataflow service to use only one core per SDK
-                  # container instance with this image. If false (or unset) recommends using
-                  # more than one core per SDK container instance with this image for
-                  # efficiency. Note that Dataflow service may choose to override this property
-                  # if needed.
-            },
-          ],
-          "subnetwork": "A String", # Subnetwork to which VMs will be assigned, if desired.  Expected to be of
-              # the form "regions/REGION/subnetworks/SUBNETWORK".
-          "ipConfiguration": "A String", # Configuration for VM IPs.
-          "taskrunnerSettings": { # Taskrunner configuration settings. # Settings passed through to Google Compute Engine workers when
-              # using the standard Dataflow task runner.  Users should ignore
-              # this field.
-            "alsologtostderr": True or False, # Whether to also send taskrunner log info to stderr.
-            "taskGroup": "A String", # The UNIX group ID on the worker VM to use for tasks launched by
-                # taskrunner; e.g. "wheel".
-            "harnessCommand": "A String", # The command to launch the worker harness.
-            "logDir": "A String", # The directory on the VM to store logs.
-            "oauthScopes": [ # The OAuth2 scopes to be requested by the taskrunner in order to
-                # access the Cloud Dataflow API.
-              "A String",
-            ],
-            "dataflowApiVersion": "A String", # The API version of endpoint, e.g. "v1b3"
-            "logUploadLocation": "A String", # Indicates where to put logs.  If this is not specified, the logs
-                # will not be uploaded.
-                #
-                # The supported resource type is:
-                #
-                # Google Cloud Storage:
-                #   storage.googleapis.com/{bucket}/{object}
-                #   bucket.storage.googleapis.com/{object}
-            "streamingWorkerMainClass": "A String", # The streaming worker main class name.
-            "workflowFileName": "A String", # The file to store the workflow in.
-            "baseTaskDir": "A String", # The location on the worker for task-specific subdirectories.
-            "tempStoragePrefix": "A String", # The prefix of the resources the taskrunner should use for
-                # temporary storage.
-                #
-                # The supported resource type is:
-                #
-                # Google Cloud Storage:
-                #   storage.googleapis.com/{bucket}/{object}
-                #   bucket.storage.googleapis.com/{object}
-            "commandlinesFileName": "A String", # The file to store preprocessing commands in.
-            "languageHint": "A String", # The suggested backend language.
-            "baseUrl": "A String", # The base URL for the taskrunner to use when accessing Google Cloud APIs.
-                #
-                # When workers access Google Cloud APIs, they logically do so via
-                # relative URLs.  If this field is specified, it supplies the base
-                # URL to use for resolving these relative URLs.  The normative
-                # algorithm used is defined by RFC 1808, "Relative Uniform Resource
-                # Locators".
-                #
-                # If not specified, the default value is "http://www.googleapis.com/"
-            "logToSerialconsole": True or False, # Whether to send taskrunner log info to Google Compute Engine VM serial
-                # console.
-            "continueOnException": True or False, # Whether to continue taskrunner if an exception is hit.
-            "parallelWorkerSettings": { # Provides data to pass through to the worker harness. # The settings to pass to the parallel worker harness.
-              "baseUrl": "A String", # The base URL for accessing Google Cloud APIs.
-                  #
-                  # When workers access Google Cloud APIs, they logically do so via
-                  # relative URLs.  If this field is specified, it supplies the base
-                  # URL to use for resolving these relative URLs.  The normative
-                  # algorithm used is defined by RFC 1808, "Relative Uniform Resource
-                  # Locators".
-                  #
-                  # If not specified, the default value is "http://www.googleapis.com/"
-              "reportingEnabled": True or False, # Whether to send work progress updates to the service.
-              "servicePath": "A String", # The Cloud Dataflow service path relative to the root URL, for example,
-                  # "dataflow/v1b3/projects".
-              "shuffleServicePath": "A String", # The Shuffle service path relative to the root URL, for example,
-                  # "shuffle/v1beta1".
-              "workerId": "A String", # The ID of the worker running this pipeline.
-              "tempStoragePrefix": "A String", # The prefix of the resources the system should use for temporary
-                  # storage.
-                  #
-                  # The supported resource type is:
-                  #
-                  # Google Cloud Storage:
-                  #
-                  #   storage.googleapis.com/{bucket}/{object}
-                  #   bucket.storage.googleapis.com/{object}
-            },
-            "vmId": "A String", # The ID string of the VM.
-            "taskUser": "A String", # The UNIX user ID on the worker VM to use for tasks launched by
-                # taskrunner; e.g. "root".
-          },
-          "autoscalingSettings": { # Settings for WorkerPool autoscaling. # Settings for autoscaling of this WorkerPool.
-            "maxNumWorkers": 42, # The maximum number of workers to cap scaling at.
-            "algorithm": "A String", # The algorithm to use for autoscaling.
-          },
-          "metadata": { # Metadata to set on the Google Compute Engine VMs.
-            "a_key": "A String",
-          },
-        },
-      ],
-      "dataset": "A String", # The dataset for the current project where various workflow
-          # related tables are stored.
-          #
-          # The supported resource type is:
-          #
-          # Google BigQuery:
-          #   bigquery.googleapis.com/{dataset}
       "internalExperiments": { # Experimental settings.
         "a_key": "", # Properties of the object. Contains field @type with type URL.
       },
@@ -435,50 +222,249 @@
         "a_key": "", # Properties of the object.
       },
       "serviceAccountEmail": "A String", # Identity to run virtual machines as. Defaults to the default account.
+      "sdkPipelineOptions": { # The Cloud Dataflow SDK pipeline options specified by the user. These
+          # options are passed through the service and are used to recreate the
+          # SDK pipeline options on the worker in a language agnostic and platform
+          # independent way.
+        "a_key": "", # Properties of the object.
+      },
+      "flexResourceSchedulingGoal": "A String", # Which Flexible Resource Scheduling mode to run in.
+      "workerPools": [ # The worker pools. At least one "harness" worker pool must be
+          # specified in order for the job to have workers.
+        { # Describes one particular pool of Cloud Dataflow workers to be
+            # instantiated by the Cloud Dataflow service in order to perform the
+            # computations required by a job.  Note that a workflow job may use
+            # multiple pools, in order to match the various computational
+            # requirements of the various stages of the job.
+          "numThreadsPerWorker": 42, # The number of threads per worker harness. If empty or unspecified, the
+              # service will choose a number of threads (according to the number of cores
+              # on the selected machine type for batch, or 1 by convention for streaming).
+          "numWorkers": 42, # Number of Google Compute Engine workers in this pool needed to
+              # execute the job.  If zero or unspecified, the service will
+              # attempt to choose a reasonable default.
+          "zone": "A String", # Zone to run the worker pools in.  If empty or unspecified, the service
+              # will attempt to choose a reasonable default.
+          "diskSourceImage": "A String", # Fully qualified source image for disks.
+          "packages": [ # Packages to be installed on workers.
+            { # The packages that must be installed in order for a worker to run the
+                # steps of the Cloud Dataflow job that will be assigned to its worker
+                # pool.
+                #
+                # This is the mechanism by which the Cloud Dataflow SDK causes code to
+                # be loaded onto the workers. For example, the Cloud Dataflow Java SDK
+                # might use this to install jars containing the user's code and all of the
+                # various dependencies (libraries, data files, etc.) required in order
+                # for that code to run.
+              "name": "A String", # The name of the package.
+              "location": "A String", # The resource to read the package from. The supported resource type is:
+                  #
+                  # Google Cloud Storage:
+                  #
+                  #   storage.googleapis.com/{bucket}
+                  #   bucket.storage.googleapis.com/
+            },
+          ],
+          "teardownPolicy": "A String", # Sets the policy for determining when to turndown worker pool.
+              # Allowed values are: `TEARDOWN_ALWAYS`, `TEARDOWN_ON_SUCCESS`, and
+              # `TEARDOWN_NEVER`.
+              # `TEARDOWN_ALWAYS` means workers are always torn down regardless of whether
+              # the job succeeds. `TEARDOWN_ON_SUCCESS` means workers are torn down
+              # if the job succeeds. `TEARDOWN_NEVER` means the workers are never torn
+              # down.
+              #
+              # If the workers are not torn down by the service, they will
+              # continue to run and use Google Compute Engine VM resources in the
+              # user's project until they are explicitly terminated by the user.
+              # Because of this, Google recommends using the `TEARDOWN_ALWAYS`
+              # policy except for small, manually supervised test jobs.
+              #
+              # If unknown or unspecified, the service will attempt to choose a reasonable
+              # default.
+          "onHostMaintenance": "A String", # The action to take on host maintenance, as defined by the Google
+              # Compute Engine API.
+          "poolArgs": { # Extra arguments for this worker pool.
+            "a_key": "", # Properties of the object. Contains field @type with type URL.
+          },
+          "diskSizeGb": 42, # Size of root disk for VMs, in GB.  If zero or unspecified, the service will
+              # attempt to choose a reasonable default.
+          "workerHarnessContainerImage": "A String", # Required. Docker container image that executes the Cloud Dataflow worker
+              # harness, residing in Google Container Registry.
+              #
+              # Deprecated for the Fn API path. Use sdk_harness_container_images instead.
+          "diskType": "A String", # Type of root disk for VMs.  If empty or unspecified, the service will
+              # attempt to choose a reasonable default.
+          "machineType": "A String", # Machine type (e.g. "n1-standard-1").  If empty or unspecified, the
+              # service will attempt to choose a reasonable default.
+          "kind": "A String", # The kind of the worker pool; currently only `harness` and `shuffle`
+              # are supported.
+          "sdkHarnessContainerImages": [ # Set of SDK harness containers needed to execute this pipeline. This will
+              # only be set in the Fn API path. For non-cross-language pipelines this
+              # should have only one entry. Cross-language pipelines will have two or more
+              # entries.
+            { # Defines a SDK harness container for executing Dataflow pipelines.
+              "containerImage": "A String", # A docker container image that resides in Google Container Registry.
+              "useSingleCorePerContainer": True or False, # If true, recommends the Dataflow service to use only one core per SDK
+                  # container instance with this image. If false (or unset) recommends using
+                  # more than one core per SDK container instance with this image for
+                  # efficiency. Note that Dataflow service may choose to override this property
+                  # if needed.
+            },
+          ],
+          "dataDisks": [ # Data disks that are used by a VM in this workflow.
+            { # Describes the data disk used by a workflow job.
+              "diskType": "A String", # Disk storage type, as defined by Google Compute Engine.  This
+                  # must be a disk type appropriate to the project and zone in which
+                  # the workers will run.  If unknown or unspecified, the service
+                  # will attempt to choose a reasonable default.
+                  #
+                  # For example, the standard persistent disk type is a resource name
+                  # typically ending in "pd-standard".  If SSD persistent disks are
+                  # available, the resource name typically ends with "pd-ssd".  The
+                  # actual valid values are defined the Google Compute Engine API,
+                  # not by the Cloud Dataflow API; consult the Google Compute Engine
+                  # documentation for more information about determining the set of
+                  # available disk types for a particular project and zone.
+                  #
+                  # Google Compute Engine Disk types are local to a particular
+                  # project in a particular zone, and so the resource name will
+                  # typically look something like this:
+                  #
+                  # compute.googleapis.com/projects/project-id/zones/zone/diskTypes/pd-standard
+              "sizeGb": 42, # Size of disk in GB.  If zero or unspecified, the service will
+                  # attempt to choose a reasonable default.
+              "mountPoint": "A String", # Directory in a VM where disk is mounted.
+            },
+          ],
+          "subnetwork": "A String", # Subnetwork to which VMs will be assigned, if desired.  Expected to be of
+              # the form "regions/REGION/subnetworks/SUBNETWORK".
+          "ipConfiguration": "A String", # Configuration for VM IPs.
+          "taskrunnerSettings": { # Taskrunner configuration settings. # Settings passed through to Google Compute Engine workers when
+              # using the standard Dataflow task runner.  Users should ignore
+              # this field.
+            "alsologtostderr": True or False, # Whether to also send taskrunner log info to stderr.
+            "taskGroup": "A String", # The UNIX group ID on the worker VM to use for tasks launched by
+                # taskrunner; e.g. "wheel".
+            "harnessCommand": "A String", # The command to launch the worker harness.
+            "logDir": "A String", # The directory on the VM to store logs.
+            "oauthScopes": [ # The OAuth2 scopes to be requested by the taskrunner in order to
+                # access the Cloud Dataflow API.
+              "A String",
+            ],
+            "dataflowApiVersion": "A String", # The API version of endpoint, e.g. "v1b3"
+            "logUploadLocation": "A String", # Indicates where to put logs.  If this is not specified, the logs
+                # will not be uploaded.
+                #
+                # The supported resource type is:
+                #
+                # Google Cloud Storage:
+                #   storage.googleapis.com/{bucket}/{object}
+                #   bucket.storage.googleapis.com/{object}
+            "streamingWorkerMainClass": "A String", # The streaming worker main class name.
+            "workflowFileName": "A String", # The file to store the workflow in.
+            "languageHint": "A String", # The suggested backend language.
+            "commandlinesFileName": "A String", # The file to store preprocessing commands in.
+            "baseTaskDir": "A String", # The location on the worker for task-specific subdirectories.
+            "tempStoragePrefix": "A String", # The prefix of the resources the taskrunner should use for
+                # temporary storage.
+                #
+                # The supported resource type is:
+                #
+                # Google Cloud Storage:
+                #   storage.googleapis.com/{bucket}/{object}
+                #   bucket.storage.googleapis.com/{object}
+            "baseUrl": "A String", # The base URL for the taskrunner to use when accessing Google Cloud APIs.
+                #
+                # When workers access Google Cloud APIs, they logically do so via
+                # relative URLs.  If this field is specified, it supplies the base
+                # URL to use for resolving these relative URLs.  The normative
+                # algorithm used is defined by RFC 1808, "Relative Uniform Resource
+                # Locators".
+                #
+                # If not specified, the default value is "http://www.googleapis.com/"
+            "logToSerialconsole": True or False, # Whether to send taskrunner log info to Google Compute Engine VM serial
+                # console.
+            "continueOnException": True or False, # Whether to continue taskrunner if an exception is hit.
+            "parallelWorkerSettings": { # Provides data to pass through to the worker harness. # The settings to pass to the parallel worker harness.
+              "tempStoragePrefix": "A String", # The prefix of the resources the system should use for temporary
+                  # storage.
+                  #
+                  # The supported resource type is:
+                  #
+                  # Google Cloud Storage:
+                  #
+                  #   storage.googleapis.com/{bucket}/{object}
+                  #   bucket.storage.googleapis.com/{object}
+              "reportingEnabled": True or False, # Whether to send work progress updates to the service.
+              "baseUrl": "A String", # The base URL for accessing Google Cloud APIs.
+                  #
+                  # When workers access Google Cloud APIs, they logically do so via
+                  # relative URLs.  If this field is specified, it supplies the base
+                  # URL to use for resolving these relative URLs.  The normative
+                  # algorithm used is defined by RFC 1808, "Relative Uniform Resource
+                  # Locators".
+                  #
+                  # If not specified, the default value is "http://www.googleapis.com/"
+              "servicePath": "A String", # The Cloud Dataflow service path relative to the root URL, for example,
+                  # "dataflow/v1b3/projects".
+              "shuffleServicePath": "A String", # The Shuffle service path relative to the root URL, for example,
+                  # "shuffle/v1beta1".
+              "workerId": "A String", # The ID of the worker running this pipeline.
+            },
+            "taskUser": "A String", # The UNIX user ID on the worker VM to use for tasks launched by
+                # taskrunner; e.g. "root".
+            "vmId": "A String", # The ID string of the VM.
+          },
+          "autoscalingSettings": { # Settings for WorkerPool autoscaling. # Settings for autoscaling of this WorkerPool.
+            "algorithm": "A String", # The algorithm to use for autoscaling.
+            "maxNumWorkers": 42, # The maximum number of workers to cap scaling at.
+          },
+          "metadata": { # Metadata to set on the Google Compute Engine VMs.
+            "a_key": "A String",
+          },
+          "defaultPackageSet": "A String", # The default package set to install.  This allows the service to
+              # select a default set of packages which are useful to worker
+              # harnesses written in a particular language.
+          "network": "A String", # Network to which VMs will be assigned.  If empty or unspecified,
+              # the service will use the network "default".
+        },
+      ],
+      "dataset": "A String", # The dataset for the current project where various workflow
+          # related tables are stored.
+          #
+          # The supported resource type is:
+          #
+          # Google BigQuery:
+          #   bigquery.googleapis.com/{dataset}
     },
     "stageStates": [ # This field may be mutated by the Cloud Dataflow service;
         # callers cannot mutate it.
       { # A message describing the state of a particular execution stage.
-        "executionStageName": "A String", # The name of the execution stage.
         "currentStateTime": "A String", # The time at which the stage transitioned to this state.
         "executionStageState": "A String", # Executions stage states allow the same set of values as JobState.
+        "executionStageName": "A String", # The name of the execution stage.
       },
     ],
     "jobMetadata": { # Metadata available primarily for filtering jobs. Will be included in the # This field is populated by the Dataflow service to support filtering jobs
         # by the metadata values provided here. Populated for ListJobs and all GetJob
         # views SUMMARY and higher.
         # ListJob response and Job SUMMARY view.
-      "bigTableDetails": [ # Identification of a BigTable source used in the Dataflow job.
-        { # Metadata for a BigTable connector used by the job.
-          "tableId": "A String", # TableId accessed in the connection.
-          "projectId": "A String", # ProjectId accessed in the connection.
-          "instanceId": "A String", # InstanceId accessed in the connection.
-        },
-      ],
-      "spannerDetails": [ # Identification of a Spanner source used in the Dataflow job.
-        { # Metadata for a Spanner connector used by the job.
-          "databaseId": "A String", # DatabaseId accessed in the connection.
-          "instanceId": "A String", # InstanceId accessed in the connection.
-          "projectId": "A String", # ProjectId accessed in the connection.
-        },
-      ],
       "datastoreDetails": [ # Identification of a Datastore source used in the Dataflow job.
         { # Metadata for a Datastore connector used by the job.
-          "projectId": "A String", # ProjectId accessed in the connection.
           "namespace": "A String", # Namespace used in the connection.
+          "projectId": "A String", # ProjectId accessed in the connection.
         },
       ],
       "sdkVersion": { # The version of the SDK used to run the job. # The SDK version used to run the job.
-        "versionDisplayName": "A String", # A readable string describing the version of the SDK.
-        "sdkSupportStatus": "A String", # The support status for this SDK version.
         "version": "A String", # The version of the SDK used to run the job.
+        "sdkSupportStatus": "A String", # The support status for this SDK version.
+        "versionDisplayName": "A String", # A readable string describing the version of the SDK.
       },
       "bigqueryDetails": [ # Identification of a BigQuery source used in the Dataflow job.
         { # Metadata for a BigQuery connector used by the job.
           "table": "A String", # Table accessed in the connection.
           "dataset": "A String", # Dataset accessed in the connection.
-          "projectId": "A String", # Project accessed in the connection.
           "query": "A String", # Query used to access data in the connection.
+          "projectId": "A String", # Project accessed in the connection.
         },
       ],
       "fileDetails": [ # Identification of a File source used in the Dataflow job.
@@ -488,15 +474,29 @@
       ],
       "pubsubDetails": [ # Identification of a PubSub source used in the Dataflow job.
         { # Metadata for a PubSub connector used by the job.
-          "subscription": "A String", # Subscription used in the connection.
           "topic": "A String", # Topic accessed in the connection.
+          "subscription": "A String", # Subscription used in the connection.
+        },
+      ],
+      "bigTableDetails": [ # Identification of a BigTable source used in the Dataflow job.
+        { # Metadata for a BigTable connector used by the job.
+          "projectId": "A String", # ProjectId accessed in the connection.
+          "instanceId": "A String", # InstanceId accessed in the connection.
+          "tableId": "A String", # TableId accessed in the connection.
+        },
+      ],
+      "spannerDetails": [ # Identification of a Spanner source used in the Dataflow job.
+        { # Metadata for a Spanner connector used by the job.
+          "instanceId": "A String", # InstanceId accessed in the connection.
+          "projectId": "A String", # ProjectId accessed in the connection.
+          "databaseId": "A String", # DatabaseId accessed in the connection.
         },
       ],
     },
+    "type": "A String", # The type of Cloud Dataflow job.
+    "projectId": "A String", # The ID of the Cloud Platform project that the job belongs to.
     "createdFromSnapshotId": "A String", # If this is specified, the job's initial state is populated from the given
         # snapshot.
-    "projectId": "A String", # The ID of the Cloud Platform project that the job belongs to.
-    "type": "A String", # The type of Cloud Dataflow job.
     "pipelineDescription": { # A descriptive representation of submitted pipeline as well as the executed # Preliminary field: The format of this data may change at any time.
         # A description of the user pipeline and stages through which it is executed.
         # Created by Cloud Dataflow service.  Only retrieved with
@@ -507,6 +507,25 @@
         { # Description of the composing transforms, names/ids, and input/outputs of a
             # stage of execution.  Some composing transforms and sources may have been
             # generated by the Dataflow service during execution planning.
+          "outputSource": [ # Output sources for this stage.
+            { # Description of an input or output of an execution stage.
+              "sizeBytes": "A String", # Size of the source, if measurable.
+              "name": "A String", # Dataflow service generated name for this source.
+              "userName": "A String", # Human-readable name for this source; may be user or system generated.
+              "originalTransformOrCollection": "A String", # User name for the original user transform or collection with which this
+                  # source is most closely associated.
+            },
+          ],
+          "name": "A String", # Dataflow service generated name for this stage.
+          "inputSource": [ # Input sources for this stage.
+            { # Description of an input or output of an execution stage.
+              "sizeBytes": "A String", # Size of the source, if measurable.
+              "name": "A String", # Dataflow service generated name for this source.
+              "userName": "A String", # Human-readable name for this source; may be user or system generated.
+              "originalTransformOrCollection": "A String", # User name for the original user transform or collection with which this
+                  # source is most closely associated.
+            },
+          ],
           "id": "A String", # Dataflow service generated id for this stage.
           "componentTransform": [ # Transforms that comprise this execution stage.
             { # Description of a transform executed as part of an execution stage.
@@ -526,25 +545,6 @@
             },
           ],
           "kind": "A String", # Type of tranform this stage is executing.
-          "outputSource": [ # Output sources for this stage.
-            { # Description of an input or output of an execution stage.
-              "originalTransformOrCollection": "A String", # User name for the original user transform or collection with which this
-                  # source is most closely associated.
-              "name": "A String", # Dataflow service generated name for this source.
-              "sizeBytes": "A String", # Size of the source, if measurable.
-              "userName": "A String", # Human-readable name for this source; may be user or system generated.
-            },
-          ],
-          "name": "A String", # Dataflow service generated name for this stage.
-          "inputSource": [ # Input sources for this stage.
-            { # Description of an input or output of an execution stage.
-              "originalTransformOrCollection": "A String", # User name for the original user transform or collection with which this
-                  # source is most closely associated.
-              "name": "A String", # Dataflow service generated name for this source.
-              "sizeBytes": "A String", # Size of the source, if measurable.
-              "userName": "A String", # Human-readable name for this source; may be user or system generated.
-            },
-          ],
         },
       ],
       "originalPipelineTransform": [ # Description of each transform in the pipeline and collections between them.
@@ -557,12 +557,8 @@
           "id": "A String", # SDK generated id of this transform instance.
           "displayData": [ # Transform-specific display data.
             { # Data provided with a pipeline or transform to provide descriptive info.
-              "timestampValue": "A String", # Contains value if the data is of timestamp type.
-              "boolValue": True or False, # Contains value if the data is of a boolean type.
-              "javaClassValue": "A String", # Contains value if the data is of java class type.
-              "strValue": "A String", # Contains value if the data is of string type.
-              "int64Value": "A String", # Contains value if the data is of int64 type.
               "durationValue": "A String", # Contains value if the data is of duration type.
+              "int64Value": "A String", # Contains value if the data is of int64 type.
               "namespace": "A String", # The namespace for the key. This is usually a class name or programming
                   # language namespace (i.e. python module) which defines the display data.
                   # This allows a dax monitoring system to specially handle the data
@@ -579,6 +575,10 @@
                   # will be displayed as a tooltip.
               "url": "A String", # An optional full URL.
               "label": "A String", # An optional label to display in a dax UI for the element.
+              "timestampValue": "A String", # Contains value if the data is of timestamp type.
+              "boolValue": True or False, # Contains value if the data is of a boolean type.
+              "javaClassValue": "A String", # Contains value if the data is of java class type.
+              "strValue": "A String", # Contains value if the data is of string type.
             },
           ],
           "outputCollectionName": [ # User  names for all collection outputs to this transform.
@@ -588,12 +588,8 @@
       ],
       "displayData": [ # Pipeline level display data.
         { # Data provided with a pipeline or transform to provide descriptive info.
-          "timestampValue": "A String", # Contains value if the data is of timestamp type.
-          "boolValue": True or False, # Contains value if the data is of a boolean type.
-          "javaClassValue": "A String", # Contains value if the data is of java class type.
-          "strValue": "A String", # Contains value if the data is of string type.
-          "int64Value": "A String", # Contains value if the data is of int64 type.
           "durationValue": "A String", # Contains value if the data is of duration type.
+          "int64Value": "A String", # Contains value if the data is of int64 type.
           "namespace": "A String", # The namespace for the key. This is usually a class name or programming
               # language namespace (i.e. python module) which defines the display data.
               # This allows a dax monitoring system to specially handle the data
@@ -610,6 +606,10 @@
               # will be displayed as a tooltip.
           "url": "A String", # An optional full URL.
           "label": "A String", # An optional label to display in a dax UI for the element.
+          "timestampValue": "A String", # Contains value if the data is of timestamp type.
+          "boolValue": True or False, # Contains value if the data is of a boolean type.
+          "javaClassValue": "A String", # Contains value if the data is of java class type.
+          "strValue": "A String", # Contains value if the data is of string type.
         },
       ],
     },
@@ -761,29 +761,8 @@
   An object of the form:
 
     { # The response to a GetTemplate request.
-    "status": { # The `Status` type defines a logical error model that is suitable for # The status of the get template request. Any problems with the
-        # request will be indicated in the error_details.
-        # different programming environments, including REST APIs and RPC APIs. It is
-        # used by [gRPC](https://github.com/grpc). Each `Status` message contains
-        # three pieces of data: error code, error message, and error details.
-        #
-        # You can find out more about this error model and how to work with it in the
-        # [API Design Guide](https://cloud.google.com/apis/design/errors).
-      "message": "A String", # A developer-facing error message, which should be in English. Any
-          # user-facing error message should be localized and sent in the
-          # google.rpc.Status.details field, or localized by the client.
-      "details": [ # A list of messages that carry the error details.  There is a common set of
-          # message types for APIs to use.
-        {
-          "a_key": "", # Properties of the object. Contains field @type with type URL.
-        },
-      ],
-      "code": 42, # The status code, which should be an enum value of google.rpc.Code.
-    },
-    "templateType": "A String", # Template Type.
     "metadata": { # Metadata describing a template. # The template metadata describing the template name, available
         # parameters, etc.
-      "name": "A String", # Required. The name of the template.
       "parameters": [ # The parameters for the template.
         { # Metadata for a specific parameter.
           "label": "A String", # Required. The label to display for the parameter.
@@ -797,6 +776,7 @@
           "isOptional": True or False, # Optional. Whether the parameter is optional. Defaults to false.
         },
       ],
+      "name": "A String", # Required. The name of the template.
       "description": "A String", # Optional. A description of the template.
     },
     "runtimeMetadata": { # RuntimeMetadata describing a runtime environment. # Describes the runtime metadata with SDKInfo and available parameters.
@@ -818,6 +798,26 @@
         },
       ],
     },
+    "templateType": "A String", # Template Type.
+    "status": { # The `Status` type defines a logical error model that is suitable for # The status of the get template request. Any problems with the
+        # request will be indicated in the error_details.
+        # different programming environments, including REST APIs and RPC APIs. It is
+        # used by [gRPC](https://github.com/grpc). Each `Status` message contains
+        # three pieces of data: error code, error message, and error details.
+        #
+        # You can find out more about this error model and how to work with it in the
+        # [API Design Guide](https://cloud.google.com/apis/design/errors).
+      "code": 42, # The status code, which should be an enum value of google.rpc.Code.
+      "message": "A String", # A developer-facing error message, which should be in English. Any
+          # user-facing error message should be localized and sent in the
+          # google.rpc.Status.details field, or localized by the client.
+      "details": [ # A list of messages that carry the error details.  There is a common set of
+          # message types for APIs to use.
+        {
+          "a_key": "", # Properties of the object. Contains field @type with type URL.
+        },
+      ],
+    },
   }</pre>
 </div>
 
@@ -831,21 +831,39 @@
     The object takes the form of:
 
 { # Parameters to provide to the template being launched.
+    &quot;parameters&quot;: { # The runtime parameters to pass to the job.
+      &quot;a_key&quot;: &quot;A String&quot;,
+    },
+    &quot;jobName&quot;: &quot;A String&quot;, # Required. The job name to use for the created job.
     &quot;transformNameMapping&quot;: { # Only applicable when updating a pipeline. Map of transform name prefixes of
         # the job to be replaced to the corresponding name prefixes of the new job.
       &quot;a_key&quot;: &quot;A String&quot;,
     },
     &quot;environment&quot;: { # The environment values to set at runtime. # The runtime environment for the job.
+      &quot;machineType&quot;: &quot;A String&quot;, # The machine type to use for the job. Defaults to the value from the
+          # template if not specified.
+      &quot;subnetwork&quot;: &quot;A String&quot;, # Subnetwork to which VMs will be assigned, if desired.  Expected to be of
+          # the form &quot;regions/REGION/subnetworks/SUBNETWORK&quot;.
+      &quot;ipConfiguration&quot;: &quot;A String&quot;, # Configuration for VM IPs.
+      &quot;kmsKeyName&quot;: &quot;A String&quot;, # Optional. Name for the Cloud KMS key for the job.
+          # Key format is:
+          # projects/&lt;project&gt;/locations/&lt;location&gt;/keyRings/&lt;keyring&gt;/cryptoKeys/&lt;key&gt;
+      &quot;tempLocation&quot;: &quot;A String&quot;, # The Cloud Storage path to use for temporary files.
+          # Must be a valid Cloud Storage URL, beginning with `gs://`.
+      &quot;bypassTempDirValidation&quot;: True or False, # Whether to bypass the safety checks for the job&#x27;s temporary directory.
+          # Use with caution.
+      &quot;network&quot;: &quot;A String&quot;, # Network to which VMs will be assigned.  If empty or unspecified,
+          # the service will use the network &quot;default&quot;.
       &quot;workerRegion&quot;: &quot;A String&quot;, # The Compute Engine region
           # (https://cloud.google.com/compute/docs/regions-zones/regions-zones) in
           # which worker processing should occur, e.g. &quot;us-west1&quot;. Mutually exclusive
           # with worker_zone. If neither worker_region nor worker_zone is specified,
           # default to the control plane&#x27;s region.
-      &quot;numWorkers&quot;: 42, # The initial number of Google Compute Engine instnaces for the job.
       &quot;zone&quot;: &quot;A String&quot;, # The Compute Engine [availability
           # zone](https://cloud.google.com/compute/docs/regions-zones/regions-zones)
           # for launching worker instances to run your pipeline.
           # In the future, worker_zone will take precedence.
+      &quot;numWorkers&quot;: 42, # The initial number of Google Compute Engine instnaces for the job.
       &quot;workerZone&quot;: &quot;A String&quot;, # The Compute Engine zone
           # (https://cloud.google.com/compute/docs/regions-zones/regions-zones) in
           # which worker processing should occur, e.g. &quot;us-west1-a&quot;. Mutually exclusive
@@ -864,27 +882,9 @@
       &quot;maxWorkers&quot;: 42, # The maximum number of Google Compute Engine instances to be made
           # available to your pipeline during execution, from 1 to 1000.
       &quot;serviceAccountEmail&quot;: &quot;A String&quot;, # The email address of the service account to run the job as.
-      &quot;machineType&quot;: &quot;A String&quot;, # The machine type to use for the job. Defaults to the value from the
-          # template if not specified.
-      &quot;subnetwork&quot;: &quot;A String&quot;, # Subnetwork to which VMs will be assigned, if desired.  Expected to be of
-          # the form &quot;regions/REGION/subnetworks/SUBNETWORK&quot;.
-      &quot;ipConfiguration&quot;: &quot;A String&quot;, # Configuration for VM IPs.
-      &quot;kmsKeyName&quot;: &quot;A String&quot;, # Optional. Name for the Cloud KMS key for the job.
-          # Key format is:
-          # projects/&lt;project&gt;/locations/&lt;location&gt;/keyRings/&lt;keyring&gt;/cryptoKeys/&lt;key&gt;
-      &quot;bypassTempDirValidation&quot;: True or False, # Whether to bypass the safety checks for the job&#x27;s temporary directory.
-          # Use with caution.
-      &quot;tempLocation&quot;: &quot;A String&quot;, # The Cloud Storage path to use for temporary files.
-          # Must be a valid Cloud Storage URL, beginning with `gs://`.
-      &quot;network&quot;: &quot;A String&quot;, # Network to which VMs will be assigned.  If empty or unspecified,
-          # the service will use the network &quot;default&quot;.
     },
     &quot;update&quot;: True or False, # If set, replace the existing pipeline with the name specified by jobName
         # with this pipeline, preserving state.
-    &quot;parameters&quot;: { # The runtime parameters to pass to the job.
-      &quot;a_key&quot;: &quot;A String&quot;,
-    },
-    &quot;jobName&quot;: &quot;A String&quot;, # Required. The job name to use for the created job.
   }
 
   validateOnly: boolean, If true, the request is validated but not actually executed.
@@ -927,219 +927,6 @@
         &quot;a_key&quot;: &quot;A String&quot;,
       },
       &quot;environment&quot;: { # Describes the environment in which a Dataflow Job runs. # The environment for the job.
-        &quot;sdkPipelineOptions&quot;: { # The Cloud Dataflow SDK pipeline options specified by the user. These
-            # options are passed through the service and are used to recreate the
-            # SDK pipeline options on the worker in a language agnostic and platform
-            # independent way.
-          &quot;a_key&quot;: &quot;&quot;, # Properties of the object.
-        },
-        &quot;flexResourceSchedulingGoal&quot;: &quot;A String&quot;, # Which Flexible Resource Scheduling mode to run in.
-        &quot;workerPools&quot;: [ # The worker pools. At least one &quot;harness&quot; worker pool must be
-            # specified in order for the job to have workers.
-          { # Describes one particular pool of Cloud Dataflow workers to be
-              # instantiated by the Cloud Dataflow service in order to perform the
-              # computations required by a job.  Note that a workflow job may use
-              # multiple pools, in order to match the various computational
-              # requirements of the various stages of the job.
-            &quot;defaultPackageSet&quot;: &quot;A String&quot;, # The default package set to install.  This allows the service to
-                # select a default set of packages which are useful to worker
-                # harnesses written in a particular language.
-            &quot;network&quot;: &quot;A String&quot;, # Network to which VMs will be assigned.  If empty or unspecified,
-                # the service will use the network &quot;default&quot;.
-            &quot;zone&quot;: &quot;A String&quot;, # Zone to run the worker pools in.  If empty or unspecified, the service
-                # will attempt to choose a reasonable default.
-            &quot;numWorkers&quot;: 42, # Number of Google Compute Engine workers in this pool needed to
-                # execute the job.  If zero or unspecified, the service will
-                # attempt to choose a reasonable default.
-            &quot;numThreadsPerWorker&quot;: 42, # The number of threads per worker harness. If empty or unspecified, the
-                # service will choose a number of threads (according to the number of cores
-                # on the selected machine type for batch, or 1 by convention for streaming).
-            &quot;diskSourceImage&quot;: &quot;A String&quot;, # Fully qualified source image for disks.
-            &quot;packages&quot;: [ # Packages to be installed on workers.
-              { # The packages that must be installed in order for a worker to run the
-                  # steps of the Cloud Dataflow job that will be assigned to its worker
-                  # pool.
-                  #
-                  # This is the mechanism by which the Cloud Dataflow SDK causes code to
-                  # be loaded onto the workers. For example, the Cloud Dataflow Java SDK
-                  # might use this to install jars containing the user&#x27;s code and all of the
-                  # various dependencies (libraries, data files, etc.) required in order
-                  # for that code to run.
-                &quot;location&quot;: &quot;A String&quot;, # The resource to read the package from. The supported resource type is:
-                    #
-                    # Google Cloud Storage:
-                    #
-                    #   storage.googleapis.com/{bucket}
-                    #   bucket.storage.googleapis.com/
-                &quot;name&quot;: &quot;A String&quot;, # The name of the package.
-              },
-            ],
-            &quot;teardownPolicy&quot;: &quot;A String&quot;, # Sets the policy for determining when to turndown worker pool.
-                # Allowed values are: `TEARDOWN_ALWAYS`, `TEARDOWN_ON_SUCCESS`, and
-                # `TEARDOWN_NEVER`.
-                # `TEARDOWN_ALWAYS` means workers are always torn down regardless of whether
-                # the job succeeds. `TEARDOWN_ON_SUCCESS` means workers are torn down
-                # if the job succeeds. `TEARDOWN_NEVER` means the workers are never torn
-                # down.
-                #
-                # If the workers are not torn down by the service, they will
-                # continue to run and use Google Compute Engine VM resources in the
-                # user&#x27;s project until they are explicitly terminated by the user.
-                # Because of this, Google recommends using the `TEARDOWN_ALWAYS`
-                # policy except for small, manually supervised test jobs.
-                #
-                # If unknown or unspecified, the service will attempt to choose a reasonable
-                # default.
-            &quot;onHostMaintenance&quot;: &quot;A String&quot;, # The action to take on host maintenance, as defined by the Google
-                # Compute Engine API.
-            &quot;poolArgs&quot;: { # Extra arguments for this worker pool.
-              &quot;a_key&quot;: &quot;&quot;, # Properties of the object. Contains field @type with type URL.
-            },
-            &quot;diskSizeGb&quot;: 42, # Size of root disk for VMs, in GB.  If zero or unspecified, the service will
-                # attempt to choose a reasonable default.
-            &quot;workerHarnessContainerImage&quot;: &quot;A String&quot;, # Required. Docker container image that executes the Cloud Dataflow worker
-                # harness, residing in Google Container Registry.
-                #
-                # Deprecated for the Fn API path. Use sdk_harness_container_images instead.
-            &quot;diskType&quot;: &quot;A String&quot;, # Type of root disk for VMs.  If empty or unspecified, the service will
-                # attempt to choose a reasonable default.
-            &quot;machineType&quot;: &quot;A String&quot;, # Machine type (e.g. &quot;n1-standard-1&quot;).  If empty or unspecified, the
-                # service will attempt to choose a reasonable default.
-            &quot;kind&quot;: &quot;A String&quot;, # The kind of the worker pool; currently only `harness` and `shuffle`
-                # are supported.
-            &quot;dataDisks&quot;: [ # Data disks that are used by a VM in this workflow.
-              { # Describes the data disk used by a workflow job.
-                &quot;sizeGb&quot;: 42, # Size of disk in GB.  If zero or unspecified, the service will
-                    # attempt to choose a reasonable default.
-                &quot;diskType&quot;: &quot;A String&quot;, # Disk storage type, as defined by Google Compute Engine.  This
-                    # must be a disk type appropriate to the project and zone in which
-                    # the workers will run.  If unknown or unspecified, the service
-                    # will attempt to choose a reasonable default.
-                    #
-                    # For example, the standard persistent disk type is a resource name
-                    # typically ending in &quot;pd-standard&quot;.  If SSD persistent disks are
-                    # available, the resource name typically ends with &quot;pd-ssd&quot;.  The
-                    # actual valid values are defined the Google Compute Engine API,
-                    # not by the Cloud Dataflow API; consult the Google Compute Engine
-                    # documentation for more information about determining the set of
-                    # available disk types for a particular project and zone.
-                    #
-                    # Google Compute Engine Disk types are local to a particular
-                    # project in a particular zone, and so the resource name will
-                    # typically look something like this:
-                    #
-                    # compute.googleapis.com/projects/project-id/zones/zone/diskTypes/pd-standard
-                &quot;mountPoint&quot;: &quot;A String&quot;, # Directory in a VM where disk is mounted.
-              },
-            ],
-            &quot;sdkHarnessContainerImages&quot;: [ # Set of SDK harness containers needed to execute this pipeline. This will
-                # only be set in the Fn API path. For non-cross-language pipelines this
-                # should have only one entry. Cross-language pipelines will have two or more
-                # entries.
-              { # Defines a SDK harness container for executing Dataflow pipelines.
-                &quot;containerImage&quot;: &quot;A String&quot;, # A docker container image that resides in Google Container Registry.
-                &quot;useSingleCorePerContainer&quot;: True or False, # If true, recommends the Dataflow service to use only one core per SDK
-                    # container instance with this image. If false (or unset) recommends using
-                    # more than one core per SDK container instance with this image for
-                    # efficiency. Note that Dataflow service may choose to override this property
-                    # if needed.
-              },
-            ],
-            &quot;subnetwork&quot;: &quot;A String&quot;, # Subnetwork to which VMs will be assigned, if desired.  Expected to be of
-                # the form &quot;regions/REGION/subnetworks/SUBNETWORK&quot;.
-            &quot;ipConfiguration&quot;: &quot;A String&quot;, # Configuration for VM IPs.
-            &quot;taskrunnerSettings&quot;: { # Taskrunner configuration settings. # Settings passed through to Google Compute Engine workers when
-                # using the standard Dataflow task runner.  Users should ignore
-                # this field.
-              &quot;alsologtostderr&quot;: True or False, # Whether to also send taskrunner log info to stderr.
-              &quot;taskGroup&quot;: &quot;A String&quot;, # The UNIX group ID on the worker VM to use for tasks launched by
-                  # taskrunner; e.g. &quot;wheel&quot;.
-              &quot;harnessCommand&quot;: &quot;A String&quot;, # The command to launch the worker harness.
-              &quot;logDir&quot;: &quot;A String&quot;, # The directory on the VM to store logs.
-              &quot;oauthScopes&quot;: [ # The OAuth2 scopes to be requested by the taskrunner in order to
-                  # access the Cloud Dataflow API.
-                &quot;A String&quot;,
-              ],
-              &quot;dataflowApiVersion&quot;: &quot;A String&quot;, # The API version of endpoint, e.g. &quot;v1b3&quot;
-              &quot;logUploadLocation&quot;: &quot;A String&quot;, # Indicates where to put logs.  If this is not specified, the logs
-                  # will not be uploaded.
-                  #
-                  # The supported resource type is:
-                  #
-                  # Google Cloud Storage:
-                  #   storage.googleapis.com/{bucket}/{object}
-                  #   bucket.storage.googleapis.com/{object}
-              &quot;streamingWorkerMainClass&quot;: &quot;A String&quot;, # The streaming worker main class name.
-              &quot;workflowFileName&quot;: &quot;A String&quot;, # The file to store the workflow in.
-              &quot;baseTaskDir&quot;: &quot;A String&quot;, # The location on the worker for task-specific subdirectories.
-              &quot;tempStoragePrefix&quot;: &quot;A String&quot;, # The prefix of the resources the taskrunner should use for
-                  # temporary storage.
-                  #
-                  # The supported resource type is:
-                  #
-                  # Google Cloud Storage:
-                  #   storage.googleapis.com/{bucket}/{object}
-                  #   bucket.storage.googleapis.com/{object}
-              &quot;commandlinesFileName&quot;: &quot;A String&quot;, # The file to store preprocessing commands in.
-              &quot;languageHint&quot;: &quot;A String&quot;, # The suggested backend language.
-              &quot;baseUrl&quot;: &quot;A String&quot;, # The base URL for the taskrunner to use when accessing Google Cloud APIs.
-                  #
-                  # When workers access Google Cloud APIs, they logically do so via
-                  # relative URLs.  If this field is specified, it supplies the base
-                  # URL to use for resolving these relative URLs.  The normative
-                  # algorithm used is defined by RFC 1808, &quot;Relative Uniform Resource
-                  # Locators&quot;.
-                  #
-                  # If not specified, the default value is &quot;http://www.googleapis.com/&quot;
-              &quot;logToSerialconsole&quot;: True or False, # Whether to send taskrunner log info to Google Compute Engine VM serial
-                  # console.
-              &quot;continueOnException&quot;: True or False, # Whether to continue taskrunner if an exception is hit.
-              &quot;parallelWorkerSettings&quot;: { # Provides data to pass through to the worker harness. # The settings to pass to the parallel worker harness.
-                &quot;baseUrl&quot;: &quot;A String&quot;, # The base URL for accessing Google Cloud APIs.
-                    #
-                    # When workers access Google Cloud APIs, they logically do so via
-                    # relative URLs.  If this field is specified, it supplies the base
-                    # URL to use for resolving these relative URLs.  The normative
-                    # algorithm used is defined by RFC 1808, &quot;Relative Uniform Resource
-                    # Locators&quot;.
-                    #
-                    # If not specified, the default value is &quot;http://www.googleapis.com/&quot;
-                &quot;reportingEnabled&quot;: True or False, # Whether to send work progress updates to the service.
-                &quot;servicePath&quot;: &quot;A String&quot;, # The Cloud Dataflow service path relative to the root URL, for example,
-                    # &quot;dataflow/v1b3/projects&quot;.
-                &quot;shuffleServicePath&quot;: &quot;A String&quot;, # The Shuffle service path relative to the root URL, for example,
-                    # &quot;shuffle/v1beta1&quot;.
-                &quot;workerId&quot;: &quot;A String&quot;, # The ID of the worker running this pipeline.
-                &quot;tempStoragePrefix&quot;: &quot;A String&quot;, # The prefix of the resources the system should use for temporary
-                    # storage.
-                    #
-                    # The supported resource type is:
-                    #
-                    # Google Cloud Storage:
-                    #
-                    #   storage.googleapis.com/{bucket}/{object}
-                    #   bucket.storage.googleapis.com/{object}
-              },
-              &quot;vmId&quot;: &quot;A String&quot;, # The ID string of the VM.
-              &quot;taskUser&quot;: &quot;A String&quot;, # The UNIX user ID on the worker VM to use for tasks launched by
-                  # taskrunner; e.g. &quot;root&quot;.
-            },
-            &quot;autoscalingSettings&quot;: { # Settings for WorkerPool autoscaling. # Settings for autoscaling of this WorkerPool.
-              &quot;maxNumWorkers&quot;: 42, # The maximum number of workers to cap scaling at.
-              &quot;algorithm&quot;: &quot;A String&quot;, # The algorithm to use for autoscaling.
-            },
-            &quot;metadata&quot;: { # Metadata to set on the Google Compute Engine VMs.
-              &quot;a_key&quot;: &quot;A String&quot;,
-            },
-          },
-        ],
-        &quot;dataset&quot;: &quot;A String&quot;, # The dataset for the current project where various workflow
-            # related tables are stored.
-            #
-            # The supported resource type is:
-            #
-            # Google BigQuery:
-            #   bigquery.googleapis.com/{dataset}
         &quot;internalExperiments&quot;: { # Experimental settings.
           &quot;a_key&quot;: &quot;&quot;, # Properties of the object. Contains field @type with type URL.
         },
@@ -1186,50 +973,249 @@
           &quot;a_key&quot;: &quot;&quot;, # Properties of the object.
         },
         &quot;serviceAccountEmail&quot;: &quot;A String&quot;, # Identity to run virtual machines as. Defaults to the default account.
+        &quot;sdkPipelineOptions&quot;: { # The Cloud Dataflow SDK pipeline options specified by the user. These
+            # options are passed through the service and are used to recreate the
+            # SDK pipeline options on the worker in a language agnostic and platform
+            # independent way.
+          &quot;a_key&quot;: &quot;&quot;, # Properties of the object.
+        },
+        &quot;flexResourceSchedulingGoal&quot;: &quot;A String&quot;, # Which Flexible Resource Scheduling mode to run in.
+        &quot;workerPools&quot;: [ # The worker pools. At least one &quot;harness&quot; worker pool must be
+            # specified in order for the job to have workers.
+          { # Describes one particular pool of Cloud Dataflow workers to be
+              # instantiated by the Cloud Dataflow service in order to perform the
+              # computations required by a job.  Note that a workflow job may use
+              # multiple pools, in order to match the various computational
+              # requirements of the various stages of the job.
+            &quot;numThreadsPerWorker&quot;: 42, # The number of threads per worker harness. If empty or unspecified, the
+                # service will choose a number of threads (according to the number of cores
+                # on the selected machine type for batch, or 1 by convention for streaming).
+            &quot;numWorkers&quot;: 42, # Number of Google Compute Engine workers in this pool needed to
+                # execute the job.  If zero or unspecified, the service will
+                # attempt to choose a reasonable default.
+            &quot;zone&quot;: &quot;A String&quot;, # Zone to run the worker pools in.  If empty or unspecified, the service
+                # will attempt to choose a reasonable default.
+            &quot;diskSourceImage&quot;: &quot;A String&quot;, # Fully qualified source image for disks.
+            &quot;packages&quot;: [ # Packages to be installed on workers.
+              { # The packages that must be installed in order for a worker to run the
+                  # steps of the Cloud Dataflow job that will be assigned to its worker
+                  # pool.
+                  #
+                  # This is the mechanism by which the Cloud Dataflow SDK causes code to
+                  # be loaded onto the workers. For example, the Cloud Dataflow Java SDK
+                  # might use this to install jars containing the user&#x27;s code and all of the
+                  # various dependencies (libraries, data files, etc.) required in order
+                  # for that code to run.
+                &quot;name&quot;: &quot;A String&quot;, # The name of the package.
+                &quot;location&quot;: &quot;A String&quot;, # The resource to read the package from. The supported resource type is:
+                    #
+                    # Google Cloud Storage:
+                    #
+                    #   storage.googleapis.com/{bucket}
+                    #   bucket.storage.googleapis.com/
+              },
+            ],
+            &quot;teardownPolicy&quot;: &quot;A String&quot;, # Sets the policy for determining when to turndown worker pool.
+                # Allowed values are: `TEARDOWN_ALWAYS`, `TEARDOWN_ON_SUCCESS`, and
+                # `TEARDOWN_NEVER`.
+                # `TEARDOWN_ALWAYS` means workers are always torn down regardless of whether
+                # the job succeeds. `TEARDOWN_ON_SUCCESS` means workers are torn down
+                # if the job succeeds. `TEARDOWN_NEVER` means the workers are never torn
+                # down.
+                #
+                # If the workers are not torn down by the service, they will
+                # continue to run and use Google Compute Engine VM resources in the
+                # user&#x27;s project until they are explicitly terminated by the user.
+                # Because of this, Google recommends using the `TEARDOWN_ALWAYS`
+                # policy except for small, manually supervised test jobs.
+                #
+                # If unknown or unspecified, the service will attempt to choose a reasonable
+                # default.
+            &quot;onHostMaintenance&quot;: &quot;A String&quot;, # The action to take on host maintenance, as defined by the Google
+                # Compute Engine API.
+            &quot;poolArgs&quot;: { # Extra arguments for this worker pool.
+              &quot;a_key&quot;: &quot;&quot;, # Properties of the object. Contains field @type with type URL.
+            },
+            &quot;diskSizeGb&quot;: 42, # Size of root disk for VMs, in GB.  If zero or unspecified, the service will
+                # attempt to choose a reasonable default.
+            &quot;workerHarnessContainerImage&quot;: &quot;A String&quot;, # Required. Docker container image that executes the Cloud Dataflow worker
+                # harness, residing in Google Container Registry.
+                #
+                # Deprecated for the Fn API path. Use sdk_harness_container_images instead.
+            &quot;diskType&quot;: &quot;A String&quot;, # Type of root disk for VMs.  If empty or unspecified, the service will
+                # attempt to choose a reasonable default.
+            &quot;machineType&quot;: &quot;A String&quot;, # Machine type (e.g. &quot;n1-standard-1&quot;).  If empty or unspecified, the
+                # service will attempt to choose a reasonable default.
+            &quot;kind&quot;: &quot;A String&quot;, # The kind of the worker pool; currently only `harness` and `shuffle`
+                # are supported.
+            &quot;sdkHarnessContainerImages&quot;: [ # Set of SDK harness containers needed to execute this pipeline. This will
+                # only be set in the Fn API path. For non-cross-language pipelines this
+                # should have only one entry. Cross-language pipelines will have two or more
+                # entries.
+              { # Defines a SDK harness container for executing Dataflow pipelines.
+                &quot;containerImage&quot;: &quot;A String&quot;, # A docker container image that resides in Google Container Registry.
+                &quot;useSingleCorePerContainer&quot;: True or False, # If true, recommends the Dataflow service to use only one core per SDK
+                    # container instance with this image. If false (or unset) recommends using
+                    # more than one core per SDK container instance with this image for
+                    # efficiency. Note that Dataflow service may choose to override this property
+                    # if needed.
+              },
+            ],
+            &quot;dataDisks&quot;: [ # Data disks that are used by a VM in this workflow.
+              { # Describes the data disk used by a workflow job.
+                &quot;diskType&quot;: &quot;A String&quot;, # Disk storage type, as defined by Google Compute Engine.  This
+                    # must be a disk type appropriate to the project and zone in which
+                    # the workers will run.  If unknown or unspecified, the service
+                    # will attempt to choose a reasonable default.
+                    #
+                    # For example, the standard persistent disk type is a resource name
+                    # typically ending in &quot;pd-standard&quot;.  If SSD persistent disks are
+                    # available, the resource name typically ends with &quot;pd-ssd&quot;.  The
+                    # actual valid values are defined the Google Compute Engine API,
+                    # not by the Cloud Dataflow API; consult the Google Compute Engine
+                    # documentation for more information about determining the set of
+                    # available disk types for a particular project and zone.
+                    #
+                    # Google Compute Engine Disk types are local to a particular
+                    # project in a particular zone, and so the resource name will
+                    # typically look something like this:
+                    #
+                    # compute.googleapis.com/projects/project-id/zones/zone/diskTypes/pd-standard
+                &quot;sizeGb&quot;: 42, # Size of disk in GB.  If zero or unspecified, the service will
+                    # attempt to choose a reasonable default.
+                &quot;mountPoint&quot;: &quot;A String&quot;, # Directory in a VM where disk is mounted.
+              },
+            ],
+            &quot;subnetwork&quot;: &quot;A String&quot;, # Subnetwork to which VMs will be assigned, if desired.  Expected to be of
+                # the form &quot;regions/REGION/subnetworks/SUBNETWORK&quot;.
+            &quot;ipConfiguration&quot;: &quot;A String&quot;, # Configuration for VM IPs.
+            &quot;taskrunnerSettings&quot;: { # Taskrunner configuration settings. # Settings passed through to Google Compute Engine workers when
+                # using the standard Dataflow task runner.  Users should ignore
+                # this field.
+              &quot;alsologtostderr&quot;: True or False, # Whether to also send taskrunner log info to stderr.
+              &quot;taskGroup&quot;: &quot;A String&quot;, # The UNIX group ID on the worker VM to use for tasks launched by
+                  # taskrunner; e.g. &quot;wheel&quot;.
+              &quot;harnessCommand&quot;: &quot;A String&quot;, # The command to launch the worker harness.
+              &quot;logDir&quot;: &quot;A String&quot;, # The directory on the VM to store logs.
+              &quot;oauthScopes&quot;: [ # The OAuth2 scopes to be requested by the taskrunner in order to
+                  # access the Cloud Dataflow API.
+                &quot;A String&quot;,
+              ],
+              &quot;dataflowApiVersion&quot;: &quot;A String&quot;, # The API version of endpoint, e.g. &quot;v1b3&quot;
+              &quot;logUploadLocation&quot;: &quot;A String&quot;, # Indicates where to put logs.  If this is not specified, the logs
+                  # will not be uploaded.
+                  #
+                  # The supported resource type is:
+                  #
+                  # Google Cloud Storage:
+                  #   storage.googleapis.com/{bucket}/{object}
+                  #   bucket.storage.googleapis.com/{object}
+              &quot;streamingWorkerMainClass&quot;: &quot;A String&quot;, # The streaming worker main class name.
+              &quot;workflowFileName&quot;: &quot;A String&quot;, # The file to store the workflow in.
+              &quot;languageHint&quot;: &quot;A String&quot;, # The suggested backend language.
+              &quot;commandlinesFileName&quot;: &quot;A String&quot;, # The file to store preprocessing commands in.
+              &quot;baseTaskDir&quot;: &quot;A String&quot;, # The location on the worker for task-specific subdirectories.
+              &quot;tempStoragePrefix&quot;: &quot;A String&quot;, # The prefix of the resources the taskrunner should use for
+                  # temporary storage.
+                  #
+                  # The supported resource type is:
+                  #
+                  # Google Cloud Storage:
+                  #   storage.googleapis.com/{bucket}/{object}
+                  #   bucket.storage.googleapis.com/{object}
+              &quot;baseUrl&quot;: &quot;A String&quot;, # The base URL for the taskrunner to use when accessing Google Cloud APIs.
+                  #
+                  # When workers access Google Cloud APIs, they logically do so via
+                  # relative URLs.  If this field is specified, it supplies the base
+                  # URL to use for resolving these relative URLs.  The normative
+                  # algorithm used is defined by RFC 1808, &quot;Relative Uniform Resource
+                  # Locators&quot;.
+                  #
+                  # If not specified, the default value is &quot;http://www.googleapis.com/&quot;
+              &quot;logToSerialconsole&quot;: True or False, # Whether to send taskrunner log info to Google Compute Engine VM serial
+                  # console.
+              &quot;continueOnException&quot;: True or False, # Whether to continue taskrunner if an exception is hit.
+              &quot;parallelWorkerSettings&quot;: { # Provides data to pass through to the worker harness. # The settings to pass to the parallel worker harness.
+                &quot;tempStoragePrefix&quot;: &quot;A String&quot;, # The prefix of the resources the system should use for temporary
+                    # storage.
+                    #
+                    # The supported resource type is:
+                    #
+                    # Google Cloud Storage:
+                    #
+                    #   storage.googleapis.com/{bucket}/{object}
+                    #   bucket.storage.googleapis.com/{object}
+                &quot;reportingEnabled&quot;: True or False, # Whether to send work progress updates to the service.
+                &quot;baseUrl&quot;: &quot;A String&quot;, # The base URL for accessing Google Cloud APIs.
+                    #
+                    # When workers access Google Cloud APIs, they logically do so via
+                    # relative URLs.  If this field is specified, it supplies the base
+                    # URL to use for resolving these relative URLs.  The normative
+                    # algorithm used is defined by RFC 1808, &quot;Relative Uniform Resource
+                    # Locators&quot;.
+                    #
+                    # If not specified, the default value is &quot;http://www.googleapis.com/&quot;
+                &quot;servicePath&quot;: &quot;A String&quot;, # The Cloud Dataflow service path relative to the root URL, for example,
+                    # &quot;dataflow/v1b3/projects&quot;.
+                &quot;shuffleServicePath&quot;: &quot;A String&quot;, # The Shuffle service path relative to the root URL, for example,
+                    # &quot;shuffle/v1beta1&quot;.
+                &quot;workerId&quot;: &quot;A String&quot;, # The ID of the worker running this pipeline.
+              },
+              &quot;taskUser&quot;: &quot;A String&quot;, # The UNIX user ID on the worker VM to use for tasks launched by
+                  # taskrunner; e.g. &quot;root&quot;.
+              &quot;vmId&quot;: &quot;A String&quot;, # The ID string of the VM.
+            },
+            &quot;autoscalingSettings&quot;: { # Settings for WorkerPool autoscaling. # Settings for autoscaling of this WorkerPool.
+              &quot;algorithm&quot;: &quot;A String&quot;, # The algorithm to use for autoscaling.
+              &quot;maxNumWorkers&quot;: 42, # The maximum number of workers to cap scaling at.
+            },
+            &quot;metadata&quot;: { # Metadata to set on the Google Compute Engine VMs.
+              &quot;a_key&quot;: &quot;A String&quot;,
+            },
+            &quot;defaultPackageSet&quot;: &quot;A String&quot;, # The default package set to install.  This allows the service to
+                # select a default set of packages which are useful to worker
+                # harnesses written in a particular language.
+            &quot;network&quot;: &quot;A String&quot;, # Network to which VMs will be assigned.  If empty or unspecified,
+                # the service will use the network &quot;default&quot;.
+          },
+        ],
+        &quot;dataset&quot;: &quot;A String&quot;, # The dataset for the current project where various workflow
+            # related tables are stored.
+            #
+            # The supported resource type is:
+            #
+            # Google BigQuery:
+            #   bigquery.googleapis.com/{dataset}
       },
       &quot;stageStates&quot;: [ # This field may be mutated by the Cloud Dataflow service;
           # callers cannot mutate it.
         { # A message describing the state of a particular execution stage.
-          &quot;executionStageName&quot;: &quot;A String&quot;, # The name of the execution stage.
           &quot;currentStateTime&quot;: &quot;A String&quot;, # The time at which the stage transitioned to this state.
           &quot;executionStageState&quot;: &quot;A String&quot;, # Executions stage states allow the same set of values as JobState.
+          &quot;executionStageName&quot;: &quot;A String&quot;, # The name of the execution stage.
         },
       ],
       &quot;jobMetadata&quot;: { # Metadata available primarily for filtering jobs. Will be included in the # This field is populated by the Dataflow service to support filtering jobs
           # by the metadata values provided here. Populated for ListJobs and all GetJob
           # views SUMMARY and higher.
           # ListJob response and Job SUMMARY view.
-        &quot;bigTableDetails&quot;: [ # Identification of a BigTable source used in the Dataflow job.
-          { # Metadata for a BigTable connector used by the job.
-            &quot;tableId&quot;: &quot;A String&quot;, # TableId accessed in the connection.
-            &quot;projectId&quot;: &quot;A String&quot;, # ProjectId accessed in the connection.
-            &quot;instanceId&quot;: &quot;A String&quot;, # InstanceId accessed in the connection.
-          },
-        ],
-        &quot;spannerDetails&quot;: [ # Identification of a Spanner source used in the Dataflow job.
-          { # Metadata for a Spanner connector used by the job.
-            &quot;databaseId&quot;: &quot;A String&quot;, # DatabaseId accessed in the connection.
-            &quot;instanceId&quot;: &quot;A String&quot;, # InstanceId accessed in the connection.
-            &quot;projectId&quot;: &quot;A String&quot;, # ProjectId accessed in the connection.
-          },
-        ],
         &quot;datastoreDetails&quot;: [ # Identification of a Datastore source used in the Dataflow job.
           { # Metadata for a Datastore connector used by the job.
-            &quot;projectId&quot;: &quot;A String&quot;, # ProjectId accessed in the connection.
             &quot;namespace&quot;: &quot;A String&quot;, # Namespace used in the connection.
+            &quot;projectId&quot;: &quot;A String&quot;, # ProjectId accessed in the connection.
           },
         ],
         &quot;sdkVersion&quot;: { # The version of the SDK used to run the job. # The SDK version used to run the job.
-          &quot;versionDisplayName&quot;: &quot;A String&quot;, # A readable string describing the version of the SDK.
-          &quot;sdkSupportStatus&quot;: &quot;A String&quot;, # The support status for this SDK version.
           &quot;version&quot;: &quot;A String&quot;, # The version of the SDK used to run the job.
+          &quot;sdkSupportStatus&quot;: &quot;A String&quot;, # The support status for this SDK version.
+          &quot;versionDisplayName&quot;: &quot;A String&quot;, # A readable string describing the version of the SDK.
         },
         &quot;bigqueryDetails&quot;: [ # Identification of a BigQuery source used in the Dataflow job.
           { # Metadata for a BigQuery connector used by the job.
             &quot;table&quot;: &quot;A String&quot;, # Table accessed in the connection.
             &quot;dataset&quot;: &quot;A String&quot;, # Dataset accessed in the connection.
-            &quot;projectId&quot;: &quot;A String&quot;, # Project accessed in the connection.
             &quot;query&quot;: &quot;A String&quot;, # Query used to access data in the connection.
+            &quot;projectId&quot;: &quot;A String&quot;, # Project accessed in the connection.
           },
         ],
         &quot;fileDetails&quot;: [ # Identification of a File source used in the Dataflow job.
@@ -1239,15 +1225,29 @@
         ],
         &quot;pubsubDetails&quot;: [ # Identification of a PubSub source used in the Dataflow job.
           { # Metadata for a PubSub connector used by the job.
-            &quot;subscription&quot;: &quot;A String&quot;, # Subscription used in the connection.
             &quot;topic&quot;: &quot;A String&quot;, # Topic accessed in the connection.
+            &quot;subscription&quot;: &quot;A String&quot;, # Subscription used in the connection.
+          },
+        ],
+        &quot;bigTableDetails&quot;: [ # Identification of a BigTable source used in the Dataflow job.
+          { # Metadata for a BigTable connector used by the job.
+            &quot;projectId&quot;: &quot;A String&quot;, # ProjectId accessed in the connection.
+            &quot;instanceId&quot;: &quot;A String&quot;, # InstanceId accessed in the connection.
+            &quot;tableId&quot;: &quot;A String&quot;, # TableId accessed in the connection.
+          },
+        ],
+        &quot;spannerDetails&quot;: [ # Identification of a Spanner source used in the Dataflow job.
+          { # Metadata for a Spanner connector used by the job.
+            &quot;instanceId&quot;: &quot;A String&quot;, # InstanceId accessed in the connection.
+            &quot;projectId&quot;: &quot;A String&quot;, # ProjectId accessed in the connection.
+            &quot;databaseId&quot;: &quot;A String&quot;, # DatabaseId accessed in the connection.
           },
         ],
       },
+      &quot;type&quot;: &quot;A String&quot;, # The type of Cloud Dataflow job.
+      &quot;projectId&quot;: &quot;A String&quot;, # The ID of the Cloud Platform project that the job belongs to.
       &quot;createdFromSnapshotId&quot;: &quot;A String&quot;, # If this is specified, the job&#x27;s initial state is populated from the given
           # snapshot.
-      &quot;projectId&quot;: &quot;A String&quot;, # The ID of the Cloud Platform project that the job belongs to.
-      &quot;type&quot;: &quot;A String&quot;, # The type of Cloud Dataflow job.
       &quot;pipelineDescription&quot;: { # A descriptive representation of submitted pipeline as well as the executed # Preliminary field: The format of this data may change at any time.
           # A description of the user pipeline and stages through which it is executed.
           # Created by Cloud Dataflow service.  Only retrieved with
@@ -1258,6 +1258,25 @@
           { # Description of the composing transforms, names/ids, and input/outputs of a
               # stage of execution.  Some composing transforms and sources may have been
               # generated by the Dataflow service during execution planning.
+            &quot;outputSource&quot;: [ # Output sources for this stage.
+              { # Description of an input or output of an execution stage.
+                &quot;sizeBytes&quot;: &quot;A String&quot;, # Size of the source, if measurable.
+                &quot;name&quot;: &quot;A String&quot;, # Dataflow service generated name for this source.
+                &quot;userName&quot;: &quot;A String&quot;, # Human-readable name for this source; may be user or system generated.
+                &quot;originalTransformOrCollection&quot;: &quot;A String&quot;, # User name for the original user transform or collection with which this
+                    # source is most closely associated.
+              },
+            ],
+            &quot;name&quot;: &quot;A String&quot;, # Dataflow service generated name for this stage.
+            &quot;inputSource&quot;: [ # Input sources for this stage.
+              { # Description of an input or output of an execution stage.
+                &quot;sizeBytes&quot;: &quot;A String&quot;, # Size of the source, if measurable.
+                &quot;name&quot;: &quot;A String&quot;, # Dataflow service generated name for this source.
+                &quot;userName&quot;: &quot;A String&quot;, # Human-readable name for this source; may be user or system generated.
+                &quot;originalTransformOrCollection&quot;: &quot;A String&quot;, # User name for the original user transform or collection with which this
+                    # source is most closely associated.
+              },
+            ],
             &quot;id&quot;: &quot;A String&quot;, # Dataflow service generated id for this stage.
             &quot;componentTransform&quot;: [ # Transforms that comprise this execution stage.
               { # Description of a transform executed as part of an execution stage.
@@ -1277,25 +1296,6 @@
               },
             ],
             &quot;kind&quot;: &quot;A String&quot;, # Type of tranform this stage is executing.
-            &quot;outputSource&quot;: [ # Output sources for this stage.
-              { # Description of an input or output of an execution stage.
-                &quot;originalTransformOrCollection&quot;: &quot;A String&quot;, # User name for the original user transform or collection with which this
-                    # source is most closely associated.
-                &quot;name&quot;: &quot;A String&quot;, # Dataflow service generated name for this source.
-                &quot;sizeBytes&quot;: &quot;A String&quot;, # Size of the source, if measurable.
-                &quot;userName&quot;: &quot;A String&quot;, # Human-readable name for this source; may be user or system generated.
-              },
-            ],
-            &quot;name&quot;: &quot;A String&quot;, # Dataflow service generated name for this stage.
-            &quot;inputSource&quot;: [ # Input sources for this stage.
-              { # Description of an input or output of an execution stage.
-                &quot;originalTransformOrCollection&quot;: &quot;A String&quot;, # User name for the original user transform or collection with which this
-                    # source is most closely associated.
-                &quot;name&quot;: &quot;A String&quot;, # Dataflow service generated name for this source.
-                &quot;sizeBytes&quot;: &quot;A String&quot;, # Size of the source, if measurable.
-                &quot;userName&quot;: &quot;A String&quot;, # Human-readable name for this source; may be user or system generated.
-              },
-            ],
           },
         ],
         &quot;originalPipelineTransform&quot;: [ # Description of each transform in the pipeline and collections between them.
@@ -1308,12 +1308,8 @@
             &quot;id&quot;: &quot;A String&quot;, # SDK generated id of this transform instance.
             &quot;displayData&quot;: [ # Transform-specific display data.
               { # Data provided with a pipeline or transform to provide descriptive info.
-                &quot;timestampValue&quot;: &quot;A String&quot;, # Contains value if the data is of timestamp type.
-                &quot;boolValue&quot;: True or False, # Contains value if the data is of a boolean type.
-                &quot;javaClassValue&quot;: &quot;A String&quot;, # Contains value if the data is of java class type.
-                &quot;strValue&quot;: &quot;A String&quot;, # Contains value if the data is of string type.
-                &quot;int64Value&quot;: &quot;A String&quot;, # Contains value if the data is of int64 type.
                 &quot;durationValue&quot;: &quot;A String&quot;, # Contains value if the data is of duration type.
+                &quot;int64Value&quot;: &quot;A String&quot;, # Contains value if the data is of int64 type.
                 &quot;namespace&quot;: &quot;A String&quot;, # The namespace for the key. This is usually a class name or programming
                     # language namespace (i.e. python module) which defines the display data.
                     # This allows a dax monitoring system to specially handle the data
@@ -1330,6 +1326,10 @@
                     # will be displayed as a tooltip.
                 &quot;url&quot;: &quot;A String&quot;, # An optional full URL.
                 &quot;label&quot;: &quot;A String&quot;, # An optional label to display in a dax UI for the element.
+                &quot;timestampValue&quot;: &quot;A String&quot;, # Contains value if the data is of timestamp type.
+                &quot;boolValue&quot;: True or False, # Contains value if the data is of a boolean type.
+                &quot;javaClassValue&quot;: &quot;A String&quot;, # Contains value if the data is of java class type.
+                &quot;strValue&quot;: &quot;A String&quot;, # Contains value if the data is of string type.
               },
             ],
             &quot;outputCollectionName&quot;: [ # User  names for all collection outputs to this transform.
@@ -1339,12 +1339,8 @@
         ],
         &quot;displayData&quot;: [ # Pipeline level display data.
           { # Data provided with a pipeline or transform to provide descriptive info.
-            &quot;timestampValue&quot;: &quot;A String&quot;, # Contains value if the data is of timestamp type.
-            &quot;boolValue&quot;: True or False, # Contains value if the data is of a boolean type.
-            &quot;javaClassValue&quot;: &quot;A String&quot;, # Contains value if the data is of java class type.
-            &quot;strValue&quot;: &quot;A String&quot;, # Contains value if the data is of string type.
-            &quot;int64Value&quot;: &quot;A String&quot;, # Contains value if the data is of int64 type.
             &quot;durationValue&quot;: &quot;A String&quot;, # Contains value if the data is of duration type.
+            &quot;int64Value&quot;: &quot;A String&quot;, # Contains value if the data is of int64 type.
             &quot;namespace&quot;: &quot;A String&quot;, # The namespace for the key. This is usually a class name or programming
                 # language namespace (i.e. python module) which defines the display data.
                 # This allows a dax monitoring system to specially handle the data
@@ -1361,6 +1357,10 @@
                 # will be displayed as a tooltip.
             &quot;url&quot;: &quot;A String&quot;, # An optional full URL.
             &quot;label&quot;: &quot;A String&quot;, # An optional label to display in a dax UI for the element.
+            &quot;timestampValue&quot;: &quot;A String&quot;, # Contains value if the data is of timestamp type.
+            &quot;boolValue&quot;: True or False, # Contains value if the data is of a boolean type.
+            &quot;javaClassValue&quot;: &quot;A String&quot;, # Contains value if the data is of java class type.
+            &quot;strValue&quot;: &quot;A String&quot;, # Contains value if the data is of string type.
           },
         ],
       },