Blame - docs/dyn/genomics_v1alpha2.pipelines.html - platform/external/python/google-api-python-client

2016-10-21 13:59:49 -0700

[diff] [blame]

239

},

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

240

},

241

],

242

"inputParameters": [ # Input parameters of the pipeline.

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

243

{ # Parameters facilitate setting and delivering data into the

244

# pipeline's execution environment. They are defined at create time,

245

# with optional defaults, and can be overridden at run time.

246

#

247

# If `localCopy` is unset, then the parameter specifies a string that

248

# is passed as-is into the pipeline, as the value of the environment

249

# variable with the given name. A default value can be optionally

250

# specified at create time. The default can be overridden at run time

251

# using the inputs map. If no default is given, a value must be

252

# supplied at runtime.

253

#

254

# If `localCopy` is defined, then the parameter specifies a data

255

# source or sink, both in Google Cloud Storage and on the Docker container

256

# where the pipeline computation is run. The service account associated with

257

# the Pipeline (by

258

# default the project's Compute Engine service account) must have access to the

259

# Google Cloud Storage paths.

260

#

261

# At run time, the Google Cloud Storage paths can be overridden if a default

262

# was provided at create time, or must be set otherwise. The pipeline runner

263

# should add a key/value pair to either the inputs or outputs map. The

264

# indicated data copies will be carried out before/after pipeline execution,

265

# just as if the corresponding arguments were provided to `gsutil cp`.

266

#

267

# For example: Given the following `PipelineParameter`, specified

268

# in the `inputParameters` list:

269

#

270

# ```

271

# {name: "input_file", localCopy: {path: "file.txt", disk: "pd1"}}

272

# ```

273

#

274

# where `disk` is defined in the `PipelineResources` object as:

275

#

276

# ```

277

# {name: "pd1", mountPoint: "/mnt/disk/"}

278

# ```

279

#

280

# We create a disk named `pd1`, mount it on the host VM, and map

281

# `/mnt/pd1` to `/mnt/disk` in the docker container. At

282

# runtime, an entry for `input_file` would be required in the inputs

# map, such as:

#

# ```

# inputs["input_file"] = "gs://my-bucket/bar.txt"

287

# ```

288

#

289

# This would generate the following gsutil call:

290

#

291

# ```

292

# gsutil cp gs://my-bucket/bar.txt /mnt/pd1/file.txt

293

# ```

294

#

295

# The file `/mnt/pd1/file.txt` maps to `/mnt/disk/file.txt` in the

296

# Docker container. Acceptable paths are:

#

# <table>

# <thead>

# <tr><th>Google Cloud storage path</th><th>Local path</th></tr>

301

# </thead>

302

# <tbody>

303

# <tr><td>file</td><td>file</td></tr>

304

# <tr><td>glob</td><td>directory</td></tr>

# </tbody>

# </table>

#

# For outputs, the direction of the copy is reversed:

309

#

310

# ```

311

# gsutil cp /mnt/disk/file.txt gs://my-bucket/bar.txt

312

# ```

313

#

314

# Acceptable paths are:

#

# <table>

# <thead>

# <tr><th>Local path</th><th>Google Cloud Storage path</th></tr>

319

# </thead>

320

# <tbody>

321

# <tr><td>file</td><td>file</td></tr>

322

# <tr>

323

# <td>file</td>

324

# <td>directory - directory must already exist</td>

# </tr>

# <tr>

# <td>glob</td>

# <td>directory - directory will be created if it doesn't exist</td></tr>

# </tbody>

# </table>

#

# One restriction due to docker limitations, is that for outputs that are found

333

# on the boot disk, the local path cannot be a glob and must be a file.

334

"defaultValue": "A String", # The default value for this parameter. Can be overridden at runtime.

335

# If `localCopy` is present, then this must be a Google Cloud Storage path

336

# beginning with `gs://`.

Sai Cheemalapati

2017-06-06 18:46:08 -0400

[diff] [blame^]

337

"description": "A String", # Human-readable description.

338

"name": "A String", # Required. Name of the parameter - the pipeline runner uses this string

339

# as the key to the input and output maps in RunPipeline.

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

340

"localCopy": { # LocalCopy defines how a remote file should be copied to and from the VM. # If present, this parameter is marked for copying to and from the VM.

341

# `LocalCopy` indicates where on the VM the file should be. The value

342

# given to this parameter (either at runtime or using `defaultValue`)

343

# must be the remote path where the file should be.

344

"path": "A String", # Required. The path within the user's docker container where

345

# this input should be localized to and from, relative to the specified

346

# disk's mount point. For example: file.txt,

347

"disk": "A String", # Required. The name of the disk where this parameter is

348

# located. Can be the name of one of the disks specified in the

349

# Resources field, or "boot", which represents the Docker

350

# instance's boot disk and has a mount point of `/`.

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

351

},

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

352

},

353

],

354

"docker": { # The Docker execuctor specification. # Specifies the docker run information.

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

355

"cmd": "A String", # Required. The command or newline delimited script to run. The command

356

# string will be executed within a bash shell.

357

#

358

# If the command exits with a non-zero exit code, output parameter

359

# de-localization will be skipped and the pipeline operation's

360

# `error` field will be populated.

361

#

362

# Maximum command string length is 16384.

363

"imageName": "A String", # Required. Image name from either Docker Hub or Google Container Registry.

364

# Users that run pipelines must have READ access to the image.

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

365

},

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

366

"resources": { # The system resources for the pipeline run. # Required. Specifies resource requirements for the pipeline run.

# Required fields:

#

# *

# minimumCpuCores

#

# *

# minimumRamGb

"noAddress": True or False, # Whether to assign an external IP to the instance. This is an experimental

375

# feature that may go away. Defaults to false.

376

# Corresponds to `--no_address` flag for [gcloud compute instances create]

377

# (https://cloud.google.com/sdk/gcloud/reference/compute/instances/create).

378

# In order to use this, must be true for both create time and run time.

379

# Cannot be true at run time if false at create time. If you need to ssh into

380

# a private IP VM for debugging, you can ssh to a public VM and then ssh into

381

# the private VM's Internal IP. If noAddress is set, this pipeline run may

382

# only load docker images from Google Container Registry and not Docker Hub.

383

# Before using this, you must

384

# [configure access to Google services from internal IPs](https://cloud.google.com/compute/docs/configure-private-google-access#configuring_access_to_google_services_from_internal_ips).

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

385

"minimumRamGb": 3.14, # The minimum amount of RAM to use. Defaults to 3.75 (GB)

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

386

"preemptible": True or False, # Whether to use preemptible VMs. Defaults to `false`. In order to use this,

387

# must be true for both create time and run time. Cannot be true at run time

388

# if false at create time.

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

389

"disks": [ # Disks to attach.

390

{ # A Google Compute Engine disk resource specification.

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

391

"autoDelete": True or False, # Deprecated. Disks created by the Pipelines API will be deleted at the end

392

# of the pipeline run, regardless of what this field is set to.

Sai Cheemalapati

2017-06-06 18:46:08 -0400

[diff] [blame^]

393

"sizeGb": 42, # The size of the disk. Defaults to 500 (GB).

394

# This field is not applicable for local SSD.

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

395

"source": "A String", # The full or partial URL of the persistent disk to attach. See

396

# https://cloud.google.com/compute/docs/reference/latest/instances#resource

397

# and

398

# https://cloud.google.com/compute/docs/disks/persistent-disks#snapshots

399

# for more details.

Sai Cheemalapati

2017-06-06 18:46:08 -0400

[diff] [blame^]

400

"readOnly": True or False, # Specifies how a sourced-base persistent disk will be mounted. See

401

# https://cloud.google.com/compute/docs/disks/persistent-disks#use_multi_instances

402

# for more details.

403

# Can only be set at create time.

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

404

"mountPoint": "A String", # Required at create time and cannot be overridden at run time.

405

# Specifies the path in the docker container where files on

406

# this disk should be located. For example, if `mountPoint`

407

# is `/mnt/disk`, and the parameter has `localPath`

408

# `inputs/file.txt`, the docker container can access the data at

409

# `/mnt/disk/inputs/file.txt`.

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

410

"type": "A String", # Required. The type of the disk to create.

Sai Cheemalapati

2017-06-06 18:46:08 -0400

[diff] [blame^]

411

"name": "A String", # Required. The name of the disk that can be used in the pipeline

412

# parameters. Must be 1 - 63 characters.

413

# The name "boot" is reserved for system use.

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

414

},

415

],

416

"bootDiskSizeGb": 42, # The size of the boot disk. Defaults to 10 (GB).

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

417

"zones": [ # List of Google Compute Engine availability zones to which resource

418

# creation will restricted. If empty, any zone may be chosen.

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

419

"A String",

420

],

421

"minimumCpuCores": 42, # The minimum number of cores to use. Defaults to 1.

422

},

423

"description": "A String", # User-specified description.

424

}

425

426

x__xgafv: string, V1 error format.

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

427

Allowed values

428

1 - v1 error format

429

2 - v2 error format

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

430

431

Returns:

432

An object of the form:

433

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

434

{ # The pipeline object. Represents a transformation from a set of input

435

# parameters to a set of output parameters. The transformation is defined

436

# as a docker image and command to run within that image. Each pipeline

437

# is run on a Google Compute Engine VM. A pipeline can be created with the

438

# `create` method and then later run with the `run` method, or a pipeline can

439

# be defined and run all at once with the `run` method.

440

"pipelineId": "A String", # Unique pipeline id that is generated by the service when CreatePipeline

441

# is called. Cannot be specified in the Pipeline used in the

442

# CreatePipelineRequest, and will be populated in the response to

443

# CreatePipeline and all subsequent Get and List calls. Indicates that the

444

# service has registered this pipeline.

445

"name": "A String", # Required. A user specified pipeline name that does not have to be unique.

446

# This name can be used for filtering Pipelines in ListPipelines.

447

"projectId": "A String", # Required. The project in which to create the pipeline. The caller must have

448

# WRITE access.

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

449

"outputParameters": [ # Output parameters of the pipeline.

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

450

{ # Parameters facilitate setting and delivering data into the

451

# pipeline's execution environment. They are defined at create time,

452

# with optional defaults, and can be overridden at run time.

453

#

454

# If `localCopy` is unset, then the parameter specifies a string that

455

# is passed as-is into the pipeline, as the value of the environment

456

# variable with the given name. A default value can be optionally

457

# specified at create time. The default can be overridden at run time

458

# using the inputs map. If no default is given, a value must be

459

# supplied at runtime.

460

#

461

# If `localCopy` is defined, then the parameter specifies a data

462

# source or sink, both in Google Cloud Storage and on the Docker container

463

# where the pipeline computation is run. The service account associated with

464

# the Pipeline (by

465

# default the project's Compute Engine service account) must have access to the

466

# Google Cloud Storage paths.

467

#

468

# At run time, the Google Cloud Storage paths can be overridden if a default

469

# was provided at create time, or must be set otherwise. The pipeline runner

470

# should add a key/value pair to either the inputs or outputs map. The

471

# indicated data copies will be carried out before/after pipeline execution,

472

# just as if the corresponding arguments were provided to `gsutil cp`.

473

#

474

# For example: Given the following `PipelineParameter`, specified

475

# in the `inputParameters` list:

476

#

477

# ```

478

# {name: "input_file", localCopy: {path: "file.txt", disk: "pd1"}}

479

# ```

480

#

481

# where `disk` is defined in the `PipelineResources` object as:

482

#

483

# ```

484

# {name: "pd1", mountPoint: "/mnt/disk/"}

485

# ```

486

#

487

# We create a disk named `pd1`, mount it on the host VM, and map

488

# `/mnt/pd1` to `/mnt/disk` in the docker container. At

489

# runtime, an entry for `input_file` would be required in the inputs

# map, such as:

#

# ```

# inputs["input_file"] = "gs://my-bucket/bar.txt"

494

# ```

495

#

496

# This would generate the following gsutil call:

497

#

498

# ```

499

# gsutil cp gs://my-bucket/bar.txt /mnt/pd1/file.txt

500

# ```

501

#

502

# The file `/mnt/pd1/file.txt` maps to `/mnt/disk/file.txt` in the

503

# Docker container. Acceptable paths are:

#

# <table>

# <thead>

# <tr><th>Google Cloud storage path</th><th>Local path</th></tr>

508

# </thead>

509

# <tbody>

510

# <tr><td>file</td><td>file</td></tr>

511

# <tr><td>glob</td><td>directory</td></tr>

# </tbody>

# </table>

#

# For outputs, the direction of the copy is reversed:

516

#

517

# ```

518

# gsutil cp /mnt/disk/file.txt gs://my-bucket/bar.txt

519

# ```

520

#

521

# Acceptable paths are:

#

# <table>

# <thead>

# <tr><th>Local path</th><th>Google Cloud Storage path</th></tr>

526

# </thead>

527

# <tbody>

528

# <tr><td>file</td><td>file</td></tr>

529

# <tr>

530

# <td>file</td>

531

# <td>directory - directory must already exist</td>

# </tr>

# <tr>

# <td>glob</td>

# <td>directory - directory will be created if it doesn't exist</td></tr>

# </tbody>

# </table>

#

# One restriction due to docker limitations, is that for outputs that are found

540

# on the boot disk, the local path cannot be a glob and must be a file.

541

"defaultValue": "A String", # The default value for this parameter. Can be overridden at runtime.

542

# If `localCopy` is present, then this must be a Google Cloud Storage path

543

# beginning with `gs://`.

Sai Cheemalapati

2017-06-06 18:46:08 -0400

[diff] [blame^]

544

"description": "A String", # Human-readable description.

545

"name": "A String", # Required. Name of the parameter - the pipeline runner uses this string

546

# as the key to the input and output maps in RunPipeline.

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

547

"localCopy": { # LocalCopy defines how a remote file should be copied to and from the VM. # If present, this parameter is marked for copying to and from the VM.

548

# `LocalCopy` indicates where on the VM the file should be. The value

549

# given to this parameter (either at runtime or using `defaultValue`)

550

# must be the remote path where the file should be.

551

"path": "A String", # Required. The path within the user's docker container where

552

# this input should be localized to and from, relative to the specified

553

# disk's mount point. For example: file.txt,

554

"disk": "A String", # Required. The name of the disk where this parameter is

555

# located. Can be the name of one of the disks specified in the

556

# Resources field, or "boot", which represents the Docker

557

# instance's boot disk and has a mount point of `/`.

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

558

},

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

559

},

560

],

561

"inputParameters": [ # Input parameters of the pipeline.

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

562

{ # Parameters facilitate setting and delivering data into the

563

# pipeline's execution environment. They are defined at create time,

564

# with optional defaults, and can be overridden at run time.

565

#

566

# If `localCopy` is unset, then the parameter specifies a string that

567

# is passed as-is into the pipeline, as the value of the environment

568

# variable with the given name. A default value can be optionally

569

# specified at create time. The default can be overridden at run time

570

# using the inputs map. If no default is given, a value must be

571

# supplied at runtime.

572

#

573

# If `localCopy` is defined, then the parameter specifies a data

574

# source or sink, both in Google Cloud Storage and on the Docker container

575

# where the pipeline computation is run. The service account associated with

576

# the Pipeline (by

577

# default the project's Compute Engine service account) must have access to the

578

# Google Cloud Storage paths.

579

#

580

# At run time, the Google Cloud Storage paths can be overridden if a default

581

# was provided at create time, or must be set otherwise. The pipeline runner

582

# should add a key/value pair to either the inputs or outputs map. The

583

# indicated data copies will be carried out before/after pipeline execution,

584

# just as if the corresponding arguments were provided to `gsutil cp`.

585

#

586

# For example: Given the following `PipelineParameter`, specified

587

# in the `inputParameters` list:

588

#

589

# ```

590

# {name: "input_file", localCopy: {path: "file.txt", disk: "pd1"}}

591

# ```

592

#

593

# where `disk` is defined in the `PipelineResources` object as:

594

#

595

# ```

596

# {name: "pd1", mountPoint: "/mnt/disk/"}

597

# ```

598

#

599

# We create a disk named `pd1`, mount it on the host VM, and map

600

# `/mnt/pd1` to `/mnt/disk` in the docker container. At

601

# runtime, an entry for `input_file` would be required in the inputs

# map, such as:

#

# ```

# inputs["input_file"] = "gs://my-bucket/bar.txt"

606

# ```

607

#

608

# This would generate the following gsutil call:

609

#

610

# ```

611

# gsutil cp gs://my-bucket/bar.txt /mnt/pd1/file.txt

612

# ```

613

#

614

# The file `/mnt/pd1/file.txt` maps to `/mnt/disk/file.txt` in the

615

# Docker container. Acceptable paths are:

#

# <table>

# <thead>

# <tr><th>Google Cloud storage path</th><th>Local path</th></tr>

620

# </thead>

621

# <tbody>

622

# <tr><td>file</td><td>file</td></tr>

623

# <tr><td>glob</td><td>directory</td></tr>

# </tbody>

# </table>

#

# For outputs, the direction of the copy is reversed:

628

#

629

# ```

630

# gsutil cp /mnt/disk/file.txt gs://my-bucket/bar.txt

631

# ```

632

#

633

# Acceptable paths are:

#

# <table>

# <thead>

# <tr><th>Local path</th><th>Google Cloud Storage path</th></tr>

638

# </thead>

639

# <tbody>

640

# <tr><td>file</td><td>file</td></tr>

641

# <tr>

642

# <td>file</td>

643

# <td>directory - directory must already exist</td>

# </tr>

# <tr>

# <td>glob</td>

# <td>directory - directory will be created if it doesn't exist</td></tr>

# </tbody>

# </table>

#

# One restriction due to docker limitations, is that for outputs that are found

652

# on the boot disk, the local path cannot be a glob and must be a file.

653

"defaultValue": "A String", # The default value for this parameter. Can be overridden at runtime.

654

# If `localCopy` is present, then this must be a Google Cloud Storage path

655

# beginning with `gs://`.

Sai Cheemalapati

2017-06-06 18:46:08 -0400

[diff] [blame^]

656

"description": "A String", # Human-readable description.

657

"name": "A String", # Required. Name of the parameter - the pipeline runner uses this string

658

# as the key to the input and output maps in RunPipeline.

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

659

"localCopy": { # LocalCopy defines how a remote file should be copied to and from the VM. # If present, this parameter is marked for copying to and from the VM.

660

# `LocalCopy` indicates where on the VM the file should be. The value

661

# given to this parameter (either at runtime or using `defaultValue`)

662

# must be the remote path where the file should be.

663

"path": "A String", # Required. The path within the user's docker container where

664

# this input should be localized to and from, relative to the specified

665

# disk's mount point. For example: file.txt,

666

"disk": "A String", # Required. The name of the disk where this parameter is

667

# located. Can be the name of one of the disks specified in the

668

# Resources field, or "boot", which represents the Docker

669

# instance's boot disk and has a mount point of `/`.

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

670

},

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

671

},

672

],

673

"docker": { # The Docker execuctor specification. # Specifies the docker run information.

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

674

"cmd": "A String", # Required. The command or newline delimited script to run. The command

675

# string will be executed within a bash shell.

676

#

677

# If the command exits with a non-zero exit code, output parameter

678

# de-localization will be skipped and the pipeline operation's

679

# `error` field will be populated.

680

#

681

# Maximum command string length is 16384.

682

"imageName": "A String", # Required. Image name from either Docker Hub or Google Container Registry.

683

# Users that run pipelines must have READ access to the image.

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

684

},

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

685

"resources": { # The system resources for the pipeline run. # Required. Specifies resource requirements for the pipeline run.

# Required fields:

#

# *

# minimumCpuCores

#

# *

# minimumRamGb

"noAddress": True or False, # Whether to assign an external IP to the instance. This is an experimental

694

# feature that may go away. Defaults to false.

695

# Corresponds to `--no_address` flag for [gcloud compute instances create]

696

# (https://cloud.google.com/sdk/gcloud/reference/compute/instances/create).

697

# In order to use this, must be true for both create time and run time.

698

# Cannot be true at run time if false at create time. If you need to ssh into

699

# a private IP VM for debugging, you can ssh to a public VM and then ssh into

700

# the private VM's Internal IP. If noAddress is set, this pipeline run may

701

# only load docker images from Google Container Registry and not Docker Hub.

702

# Before using this, you must

703

# [configure access to Google services from internal IPs](https://cloud.google.com/compute/docs/configure-private-google-access#configuring_access_to_google_services_from_internal_ips).

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

704

"minimumRamGb": 3.14, # The minimum amount of RAM to use. Defaults to 3.75 (GB)

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

705

"preemptible": True or False, # Whether to use preemptible VMs. Defaults to `false`. In order to use this,

706

# must be true for both create time and run time. Cannot be true at run time

707

# if false at create time.

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

708

"disks": [ # Disks to attach.

709

{ # A Google Compute Engine disk resource specification.

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

710

"autoDelete": True or False, # Deprecated. Disks created by the Pipelines API will be deleted at the end

711

# of the pipeline run, regardless of what this field is set to.

Sai Cheemalapati

2017-06-06 18:46:08 -0400

[diff] [blame^]

712

"sizeGb": 42, # The size of the disk. Defaults to 500 (GB).

713

# This field is not applicable for local SSD.

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

714

"source": "A String", # The full or partial URL of the persistent disk to attach. See

715

# https://cloud.google.com/compute/docs/reference/latest/instances#resource

716

# and

717

# https://cloud.google.com/compute/docs/disks/persistent-disks#snapshots

718

# for more details.

Sai Cheemalapati

2017-06-06 18:46:08 -0400

[diff] [blame^]

719

"readOnly": True or False, # Specifies how a sourced-base persistent disk will be mounted. See

720

# https://cloud.google.com/compute/docs/disks/persistent-disks#use_multi_instances

721

# for more details.

722

# Can only be set at create time.

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

723

"mountPoint": "A String", # Required at create time and cannot be overridden at run time.

724

# Specifies the path in the docker container where files on

725

# this disk should be located. For example, if `mountPoint`

726

# is `/mnt/disk`, and the parameter has `localPath`

727

# `inputs/file.txt`, the docker container can access the data at

728

# `/mnt/disk/inputs/file.txt`.

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

729

"type": "A String", # Required. The type of the disk to create.

Sai Cheemalapati

2017-06-06 18:46:08 -0400

[diff] [blame^]

730

"name": "A String", # Required. The name of the disk that can be used in the pipeline

731

# parameters. Must be 1 - 63 characters.

732

# The name "boot" is reserved for system use.

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

733

},

734

],

735

"bootDiskSizeGb": 42, # The size of the boot disk. Defaults to 10 (GB).

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

736

"zones": [ # List of Google Compute Engine availability zones to which resource

737

# creation will restricted. If empty, any zone may be chosen.

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

738

"A String",

739

],

740

"minimumCpuCores": 42, # The minimum number of cores to use. Defaults to 1.

741

},

742

"description": "A String", # User-specified description.

}</pre>

</div>

Thomas Coffee

2017-03-27 10:39:26 -0700

[diff] [blame]

747

<code class="details" id="delete">delete(pipelineId, x__xgafv=None)</code>

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

748

<pre>Deletes a pipeline based on ID.

749

750

Caller must have WRITE permission to the project.

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

751

752

Args:

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

753

pipelineId: string, Caller must have WRITE access to the project in which this pipeline

754

is defined. (required)

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

755

x__xgafv: string, V1 error format.

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

756

Allowed values

757

1 - v1 error format

758

2 - v2 error format

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

759

760

Returns:

761

An object of the form:

762

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

763

{ # A generic empty message that you can re-use to avoid defining duplicated

764

# empty messages in your APIs. A typical example is to use it as the request

765

# or the response type of an API method. For instance:

766

#

767

# service Foo {

768

# rpc Bar(google.protobuf.Empty) returns (google.protobuf.Empty);

769

# }

770

#

771

# The JSON representation for `Empty` is empty JSON object `{}`.

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

}</pre>

</div>

Thomas Coffee

2017-03-27 10:39:26 -0700

[diff] [blame]

776

<code class="details" id="get">get(pipelineId, x__xgafv=None)</code>

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

777

<pre>Retrieves a pipeline based on ID.

778

779

Caller must have READ permission to the project.

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

780

781

Args:

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

782

pipelineId: string, Caller must have READ access to the project in which this pipeline

783

is defined. (required)

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

784

x__xgafv: string, V1 error format.

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

785

Allowed values

786

1 - v1 error format

787

2 - v2 error format

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

788

789

Returns:

790

An object of the form:

791

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

792

{ # The pipeline object. Represents a transformation from a set of input

793

# parameters to a set of output parameters. The transformation is defined

794

# as a docker image and command to run within that image. Each pipeline

795

# is run on a Google Compute Engine VM. A pipeline can be created with the

796

# `create` method and then later run with the `run` method, or a pipeline can

797

# be defined and run all at once with the `run` method.

798

"pipelineId": "A String", # Unique pipeline id that is generated by the service when CreatePipeline

799

# is called. Cannot be specified in the Pipeline used in the

800

# CreatePipelineRequest, and will be populated in the response to

801

# CreatePipeline and all subsequent Get and List calls. Indicates that the

802

# service has registered this pipeline.

803

"name": "A String", # Required. A user specified pipeline name that does not have to be unique.

804

# This name can be used for filtering Pipelines in ListPipelines.

805

"projectId": "A String", # Required. The project in which to create the pipeline. The caller must have

806

# WRITE access.

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

807

"outputParameters": [ # Output parameters of the pipeline.

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

808

{ # Parameters facilitate setting and delivering data into the

809

# pipeline's execution environment. They are defined at create time,

810

# with optional defaults, and can be overridden at run time.

811

#

812

# If `localCopy` is unset, then the parameter specifies a string that

813

# is passed as-is into the pipeline, as the value of the environment

814

# variable with the given name. A default value can be optionally

815

# specified at create time. The default can be overridden at run time

816

# using the inputs map. If no default is given, a value must be

817

# supplied at runtime.

818

#

819

# If `localCopy` is defined, then the parameter specifies a data

820

# source or sink, both in Google Cloud Storage and on the Docker container

821

# where the pipeline computation is run. The service account associated with

822

# the Pipeline (by

823

# default the project's Compute Engine service account) must have access to the

824

# Google Cloud Storage paths.

825

#

826

# At run time, the Google Cloud Storage paths can be overridden if a default

827

# was provided at create time, or must be set otherwise. The pipeline runner

828

# should add a key/value pair to either the inputs or outputs map. The

829

# indicated data copies will be carried out before/after pipeline execution,

830

# just as if the corresponding arguments were provided to `gsutil cp`.

831

#

832

# For example: Given the following `PipelineParameter`, specified

833

# in the `inputParameters` list:

834

#

835

# ```

836

# {name: "input_file", localCopy: {path: "file.txt", disk: "pd1"}}

837

# ```

838

#

839

# where `disk` is defined in the `PipelineResources` object as:

840

#

841

# ```

842

# {name: "pd1", mountPoint: "/mnt/disk/"}

843

# ```

844

#

845

# We create a disk named `pd1`, mount it on the host VM, and map

846

# `/mnt/pd1` to `/mnt/disk` in the docker container. At

847

# runtime, an entry for `input_file` would be required in the inputs

# map, such as:

#

# ```

# inputs["input_file"] = "gs://my-bucket/bar.txt"

852

# ```

853

#

854

# This would generate the following gsutil call:

855

#

856

# ```

857

# gsutil cp gs://my-bucket/bar.txt /mnt/pd1/file.txt

858

# ```

859

#

860

# The file `/mnt/pd1/file.txt` maps to `/mnt/disk/file.txt` in the

861

# Docker container. Acceptable paths are:

#

# <table>

# <thead>

# <tr><th>Google Cloud storage path</th><th>Local path</th></tr>

866

# </thead>

867

# <tbody>

868

# <tr><td>file</td><td>file</td></tr>

869

# <tr><td>glob</td><td>directory</td></tr>

# </tbody>

# </table>

#

# For outputs, the direction of the copy is reversed:

874

#

875

# ```

876

# gsutil cp /mnt/disk/file.txt gs://my-bucket/bar.txt

877

# ```

878

#

879

# Acceptable paths are:

#

# <table>

# <thead>

# <tr><th>Local path</th><th>Google Cloud Storage path</th></tr>

884

# </thead>

885

# <tbody>

886

# <tr><td>file</td><td>file</td></tr>

887

# <tr>

888

# <td>file</td>

889

# <td>directory - directory must already exist</td>

# </tr>

# <tr>

# <td>glob</td>

# <td>directory - directory will be created if it doesn't exist</td></tr>

# </tbody>

# </table>

#

# One restriction due to docker limitations, is that for outputs that are found

898

# on the boot disk, the local path cannot be a glob and must be a file.

899

"defaultValue": "A String", # The default value for this parameter. Can be overridden at runtime.

900

# If `localCopy` is present, then this must be a Google Cloud Storage path

901

# beginning with `gs://`.

Sai Cheemalapati

2017-06-06 18:46:08 -0400

[diff] [blame^]

902

"description": "A String", # Human-readable description.

903

"name": "A String", # Required. Name of the parameter - the pipeline runner uses this string

904

# as the key to the input and output maps in RunPipeline.

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

905

"localCopy": { # LocalCopy defines how a remote file should be copied to and from the VM. # If present, this parameter is marked for copying to and from the VM.

906

# `LocalCopy` indicates where on the VM the file should be. The value

907

# given to this parameter (either at runtime or using `defaultValue`)

908

# must be the remote path where the file should be.

909

"path": "A String", # Required. The path within the user's docker container where

910

# this input should be localized to and from, relative to the specified

911

# disk's mount point. For example: file.txt,

912

"disk": "A String", # Required. The name of the disk where this parameter is

913

# located. Can be the name of one of the disks specified in the

914

# Resources field, or "boot", which represents the Docker

915

# instance's boot disk and has a mount point of `/`.

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

916

},

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

917

},

918

],

919

"inputParameters": [ # Input parameters of the pipeline.

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

920

{ # Parameters facilitate setting and delivering data into the

921

# pipeline's execution environment. They are defined at create time,

922

# with optional defaults, and can be overridden at run time.

923

#

924

# If `localCopy` is unset, then the parameter specifies a string that

925

# is passed as-is into the pipeline, as the value of the environment

926

# variable with the given name. A default value can be optionally

927

# specified at create time. The default can be overridden at run time

928

# using the inputs map. If no default is given, a value must be

929

# supplied at runtime.

930

#

931

# If `localCopy` is defined, then the parameter specifies a data

932

# source or sink, both in Google Cloud Storage and on the Docker container

933

# where the pipeline computation is run. The service account associated with

934

# the Pipeline (by

935

# default the project's Compute Engine service account) must have access to the

936

# Google Cloud Storage paths.

937

#

938

# At run time, the Google Cloud Storage paths can be overridden if a default

939

# was provided at create time, or must be set otherwise. The pipeline runner

940

# should add a key/value pair to either the inputs or outputs map. The

941

# indicated data copies will be carried out before/after pipeline execution,

942

# just as if the corresponding arguments were provided to `gsutil cp`.

943

#

944

# For example: Given the following `PipelineParameter`, specified

945

# in the `inputParameters` list:

946

#

947

# ```

948

# {name: "input_file", localCopy: {path: "file.txt", disk: "pd1"}}

949

# ```

950

#

951

# where `disk` is defined in the `PipelineResources` object as:

952

#

953

# ```

954

# {name: "pd1", mountPoint: "/mnt/disk/"}

955

# ```

956

#

957

# We create a disk named `pd1`, mount it on the host VM, and map

958

# `/mnt/pd1` to `/mnt/disk` in the docker container. At

959

# runtime, an entry for `input_file` would be required in the inputs

# map, such as:

#

# ```

# inputs["input_file"] = "gs://my-bucket/bar.txt"

964

# ```

965

#

966

# This would generate the following gsutil call:

967

#

968

# ```

969

# gsutil cp gs://my-bucket/bar.txt /mnt/pd1/file.txt

970

# ```

971

#

972

# The file `/mnt/pd1/file.txt` maps to `/mnt/disk/file.txt` in the

973

# Docker container. Acceptable paths are:

#

# <table>

# <thead>

# <tr><th>Google Cloud storage path</th><th>Local path</th></tr>

978

# </thead>

979

# <tbody>

980

# <tr><td>file</td><td>file</td></tr>

981

# <tr><td>glob</td><td>directory</td></tr>

# </tbody>

# </table>

#

# For outputs, the direction of the copy is reversed:

986

#

987

# ```

988

# gsutil cp /mnt/disk/file.txt gs://my-bucket/bar.txt

989

# ```

990

#

991

# Acceptable paths are:

#

# <table>

# <thead>

# <tr><th>Local path</th><th>Google Cloud Storage path</th></tr>

996

# </thead>

997

# <tbody>

998

# <tr><td>file</td><td>file</td></tr>

999

# <tr>

1000

# <td>file</td>

1001

# <td>directory - directory must already exist</td>

# </tr>

# <tr>

# <td>glob</td>

# <td>directory - directory will be created if it doesn't exist</td></tr>

# </tbody>

# </table>

#

# One restriction due to docker limitations, is that for outputs that are found

1010

# on the boot disk, the local path cannot be a glob and must be a file.

1011

"defaultValue": "A String", # The default value for this parameter. Can be overridden at runtime.

1012

# If `localCopy` is present, then this must be a Google Cloud Storage path

1013

# beginning with `gs://`.

Sai Cheemalapati

2017-06-06 18:46:08 -0400

[diff] [blame^]

1014

"description": "A String", # Human-readable description.

1015

"name": "A String", # Required. Name of the parameter - the pipeline runner uses this string

1016

# as the key to the input and output maps in RunPipeline.

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

1017

"localCopy": { # LocalCopy defines how a remote file should be copied to and from the VM. # If present, this parameter is marked for copying to and from the VM.

1018

# `LocalCopy` indicates where on the VM the file should be. The value

1019

# given to this parameter (either at runtime or using `defaultValue`)

1020

# must be the remote path where the file should be.

1021

"path": "A String", # Required. The path within the user's docker container where

1022

# this input should be localized to and from, relative to the specified

1023

# disk's mount point. For example: file.txt,

1024

"disk": "A String", # Required. The name of the disk where this parameter is

1025

# located. Can be the name of one of the disks specified in the

1026

# Resources field, or "boot", which represents the Docker

1027

# instance's boot disk and has a mount point of `/`.

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

1028

},

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

1029

},

1030

],

1031

"docker": { # The Docker execuctor specification. # Specifies the docker run information.

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

1032

"cmd": "A String", # Required. The command or newline delimited script to run. The command

1033

# string will be executed within a bash shell.

1034

#

1035

# If the command exits with a non-zero exit code, output parameter

1036

# de-localization will be skipped and the pipeline operation's

1037

# `error` field will be populated.

1038

#

1039

# Maximum command string length is 16384.

1040

"imageName": "A String", # Required. Image name from either Docker Hub or Google Container Registry.

1041

# Users that run pipelines must have READ access to the image.

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

1042

},

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

1043

"resources": { # The system resources for the pipeline run. # Required. Specifies resource requirements for the pipeline run.

# Required fields:

#

# *

# minimumCpuCores

#

# *

# minimumRamGb

"noAddress": True or False, # Whether to assign an external IP to the instance. This is an experimental

1052

# feature that may go away. Defaults to false.

1053

# Corresponds to `--no_address` flag for [gcloud compute instances create]

1054

# (https://cloud.google.com/sdk/gcloud/reference/compute/instances/create).

1055

# In order to use this, must be true for both create time and run time.

1056

# Cannot be true at run time if false at create time. If you need to ssh into

1057

# a private IP VM for debugging, you can ssh to a public VM and then ssh into

1058

# the private VM's Internal IP. If noAddress is set, this pipeline run may

1059

# only load docker images from Google Container Registry and not Docker Hub.

1060

# Before using this, you must

1061

# [configure access to Google services from internal IPs](https://cloud.google.com/compute/docs/configure-private-google-access#configuring_access_to_google_services_from_internal_ips).

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

1062

"minimumRamGb": 3.14, # The minimum amount of RAM to use. Defaults to 3.75 (GB)

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

1063

"preemptible": True or False, # Whether to use preemptible VMs. Defaults to `false`. In order to use this,

1064

# must be true for both create time and run time. Cannot be true at run time

1065

# if false at create time.

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

1066

"disks": [ # Disks to attach.

1067

{ # A Google Compute Engine disk resource specification.

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

1068

"autoDelete": True or False, # Deprecated. Disks created by the Pipelines API will be deleted at the end

1069

# of the pipeline run, regardless of what this field is set to.

Sai Cheemalapati

2017-06-06 18:46:08 -0400

[diff] [blame^]

1070

"sizeGb": 42, # The size of the disk. Defaults to 500 (GB).

1071

# This field is not applicable for local SSD.

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

1072

"source": "A String", # The full or partial URL of the persistent disk to attach. See

1073

# https://cloud.google.com/compute/docs/reference/latest/instances#resource

1074

# and

1075

# https://cloud.google.com/compute/docs/disks/persistent-disks#snapshots

1076

# for more details.

Sai Cheemalapati

2017-06-06 18:46:08 -0400

[diff] [blame^]

1077

"readOnly": True or False, # Specifies how a sourced-base persistent disk will be mounted. See

1078

# https://cloud.google.com/compute/docs/disks/persistent-disks#use_multi_instances

1079

# for more details.

1080

# Can only be set at create time.

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

1081

"mountPoint": "A String", # Required at create time and cannot be overridden at run time.

1082

# Specifies the path in the docker container where files on

1083

# this disk should be located. For example, if `mountPoint`

1084

# is `/mnt/disk`, and the parameter has `localPath`

1085

# `inputs/file.txt`, the docker container can access the data at

1086

# `/mnt/disk/inputs/file.txt`.

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

1087

"type": "A String", # Required. The type of the disk to create.

Sai Cheemalapati

2017-06-06 18:46:08 -0400

[diff] [blame^]

1088

"name": "A String", # Required. The name of the disk that can be used in the pipeline

1089

# parameters. Must be 1 - 63 characters.

1090

# The name "boot" is reserved for system use.

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

1091

},

1092

],

1093

"bootDiskSizeGb": 42, # The size of the boot disk. Defaults to 10 (GB).

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

1094

"zones": [ # List of Google Compute Engine availability zones to which resource

1095

# creation will restricted. If empty, any zone may be chosen.

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

1096

"A String",

1097

],

1098

"minimumCpuCores": 42, # The minimum number of cores to use. Defaults to 1.

1099

},

1100

"description": "A String", # User-specified description.

}</pre>

</div>

<code class="details" id="getControllerConfig">getControllerConfig(validationToken=None, x__xgafv=None, operationId=None)</code>

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

1106

<pre>Gets controller configuration information. Should only be called

1107

by VMs created by the Pipelines Service and not by end users.

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

1108

1109

Args:

1110

validationToken: string, A parameter

1111

x__xgafv: string, V1 error format.

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

1112

Allowed values

1113

1 - v1 error format

1114

2 - v2 error format

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

1115

operationId: string, The operation to retrieve controller configuration for.

1116

1117

Returns:

1118

An object of the form:

1119

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

1120

{ # Stores the information that the controller will fetch from the

1121

# server in order to run. Should only be used by VMs created by the

1122

# Pipelines Service and not by end users.

Thomas Coffee

2f24537

2017-03-27 10:39:26 -0700

[diff] [blame]

1123

"cmd": "A String",

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

1124

"machineType": "A String",

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

1125

"vars": {

1126

"a_key": "A String",

1127

},

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

1128

"gcsLogPath": "A String",

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

"gcsSinks": {

"a_key": {

"values": [

"A String",

],

},

},

"image": "A String",

Thomas Coffee

2f24537

2017-03-27 10:39:26 -0700

[diff] [blame]

1137

"disks": {

1138

"a_key": "A String",

1139

},

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

"gcsSources": {

"a_key": {

"values": [

"A String",

],

},

},

}</pre>

</div>

<code class="details" id="list">list(pageSize=None, projectId=None, namePrefix=None, pageToken=None, x__xgafv=None)</code>

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

1152

<pre>Lists pipelines.

1153

1154

Caller must have READ permission to the project.

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

1155

1156

Args:

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

1157

pageSize: integer, Number of pipelines to return at once. Defaults to 256, and max

1158

is 2048.

1159

projectId: string, Required. The name of the project to search for pipelines. Caller

1160

must have READ access to this project.

1161

namePrefix: string, Pipelines with names that match this prefix should be

1162

returned. If unspecified, all pipelines in the project, up to

1163

`pageSize`, will be returned.

1164

pageToken: string, Token to use to indicate where to start getting results.

1165

If unspecified, returns the first page of results.

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

1166

x__xgafv: string, V1 error format.

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

1167

Allowed values

1168

1 - v1 error format

1169

2 - v2 error format

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

1170

1171

Returns:

1172

An object of the form:

1173

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

1174

{ # The response of ListPipelines. Contains at most `pageSize`

1175

# pipelines. If it contains `pageSize` pipelines, and more pipelines

1176

# exist, then `nextPageToken` will be populated and should be

1177

# used as the `pageToken` argument to a subsequent ListPipelines

1178

# request.

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

1179

"nextPageToken": "A String", # The token to use to get the next page of results.

1180

"pipelines": [ # The matched pipelines.

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

1181

{ # The pipeline object. Represents a transformation from a set of input

1182

# parameters to a set of output parameters. The transformation is defined

1183

# as a docker image and command to run within that image. Each pipeline

1184

# is run on a Google Compute Engine VM. A pipeline can be created with the

1185

# `create` method and then later run with the `run` method, or a pipeline can

1186

# be defined and run all at once with the `run` method.

1187

"pipelineId": "A String", # Unique pipeline id that is generated by the service when CreatePipeline

1188

# is called. Cannot be specified in the Pipeline used in the

1189

# CreatePipelineRequest, and will be populated in the response to

1190

# CreatePipeline and all subsequent Get and List calls. Indicates that the

1191

# service has registered this pipeline.

1192

"name": "A String", # Required. A user specified pipeline name that does not have to be unique.

1193

# This name can be used for filtering Pipelines in ListPipelines.

1194

"projectId": "A String", # Required. The project in which to create the pipeline. The caller must have

1195

# WRITE access.

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

1196

"outputParameters": [ # Output parameters of the pipeline.

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

1197

{ # Parameters facilitate setting and delivering data into the

1198

# pipeline's execution environment. They are defined at create time,

1199

# with optional defaults, and can be overridden at run time.

1200

#

1201

# If `localCopy` is unset, then the parameter specifies a string that

1202

# is passed as-is into the pipeline, as the value of the environment

1203

# variable with the given name. A default value can be optionally

1204

# specified at create time. The default can be overridden at run time

1205

# using the inputs map. If no default is given, a value must be

1206

# supplied at runtime.

1207

#

1208

# If `localCopy` is defined, then the parameter specifies a data

1209

# source or sink, both in Google Cloud Storage and on the Docker container

1210

# where the pipeline computation is run. The service account associated with

1211

# the Pipeline (by

1212

# default the project's Compute Engine service account) must have access to the

1213

# Google Cloud Storage paths.

1214

#

1215

# At run time, the Google Cloud Storage paths can be overridden if a default

1216

# was provided at create time, or must be set otherwise. The pipeline runner

1217

# should add a key/value pair to either the inputs or outputs map. The

1218

# indicated data copies will be carried out before/after pipeline execution,

1219

# just as if the corresponding arguments were provided to `gsutil cp`.

1220

#

1221

# For example: Given the following `PipelineParameter`, specified

1222

# in the `inputParameters` list:

1223

#

1224

# ```

1225

# {name: "input_file", localCopy: {path: "file.txt", disk: "pd1"}}

1226

# ```

1227

#

1228

# where `disk` is defined in the `PipelineResources` object as:

1229

#

1230

# ```

1231

# {name: "pd1", mountPoint: "/mnt/disk/"}

1232

# ```

1233

#

1234

# We create a disk named `pd1`, mount it on the host VM, and map

1235

# `/mnt/pd1` to `/mnt/disk` in the docker container. At

1236

# runtime, an entry for `input_file` would be required in the inputs

# map, such as:

#

# ```

# inputs["input_file"] = "gs://my-bucket/bar.txt"

1241

# ```

1242

#

1243

# This would generate the following gsutil call:

1244

#

1245

# ```

1246

# gsutil cp gs://my-bucket/bar.txt /mnt/pd1/file.txt

1247

# ```

1248

#

1249

# The file `/mnt/pd1/file.txt` maps to `/mnt/disk/file.txt` in the

1250

# Docker container. Acceptable paths are:

#

# <table>

# <thead>

# <tr><th>Google Cloud storage path</th><th>Local path</th></tr>

1255

# </thead>

1256

# <tbody>

1257

# <tr><td>file</td><td>file</td></tr>

1258

# <tr><td>glob</td><td>directory</td></tr>

# </tbody>

# </table>

#

# For outputs, the direction of the copy is reversed:

1263

#

1264

# ```

1265

# gsutil cp /mnt/disk/file.txt gs://my-bucket/bar.txt

1266

# ```

1267

#

1268

# Acceptable paths are:

#

# <table>

# <thead>

# <tr><th>Local path</th><th>Google Cloud Storage path</th></tr>

1273

# </thead>

1274

# <tbody>

1275

# <tr><td>file</td><td>file</td></tr>

1276

# <tr>

1277

# <td>file</td>

1278

# <td>directory - directory must already exist</td>

# </tr>

# <tr>

# <td>glob</td>

# <td>directory - directory will be created if it doesn't exist</td></tr>

# </tbody>

# </table>

#

# One restriction due to docker limitations, is that for outputs that are found

1287

# on the boot disk, the local path cannot be a glob and must be a file.

1288

"defaultValue": "A String", # The default value for this parameter. Can be overridden at runtime.

1289

# If `localCopy` is present, then this must be a Google Cloud Storage path

1290

# beginning with `gs://`.

Sai Cheemalapati

2017-06-06 18:46:08 -0400

[diff] [blame^]

1291

"description": "A String", # Human-readable description.

1292

"name": "A String", # Required. Name of the parameter - the pipeline runner uses this string

1293

# as the key to the input and output maps in RunPipeline.

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

1294

"localCopy": { # LocalCopy defines how a remote file should be copied to and from the VM. # If present, this parameter is marked for copying to and from the VM.

1295

# `LocalCopy` indicates where on the VM the file should be. The value

1296

# given to this parameter (either at runtime or using `defaultValue`)

1297

# must be the remote path where the file should be.

1298

"path": "A String", # Required. The path within the user's docker container where

1299

# this input should be localized to and from, relative to the specified

1300

# disk's mount point. For example: file.txt,

1301

"disk": "A String", # Required. The name of the disk where this parameter is

1302

# located. Can be the name of one of the disks specified in the

1303

# Resources field, or "boot", which represents the Docker

1304

# instance's boot disk and has a mount point of `/`.

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

1305

},

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

1306

},

1307

],

1308

"inputParameters": [ # Input parameters of the pipeline.

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

1309

{ # Parameters facilitate setting and delivering data into the

1310

# pipeline's execution environment. They are defined at create time,

1311

# with optional defaults, and can be overridden at run time.

1312

#

1313

# If `localCopy` is unset, then the parameter specifies a string that

1314

# is passed as-is into the pipeline, as the value of the environment

1315

# variable with the given name. A default value can be optionally

1316

# specified at create time. The default can be overridden at run time

1317

# using the inputs map. If no default is given, a value must be

1318

# supplied at runtime.

1319

#

1320

# If `localCopy` is defined, then the parameter specifies a data

1321

# source or sink, both in Google Cloud Storage and on the Docker container

1322

# where the pipeline computation is run. The service account associated with

1323

# the Pipeline (by

1324

# default the project's Compute Engine service account) must have access to the

1325

# Google Cloud Storage paths.

1326

#

1327

# At run time, the Google Cloud Storage paths can be overridden if a default

1328

# was provided at create time, or must be set otherwise. The pipeline runner

1329

# should add a key/value pair to either the inputs or outputs map. The

1330

# indicated data copies will be carried out before/after pipeline execution,

1331

# just as if the corresponding arguments were provided to `gsutil cp`.

1332

#

1333

# For example: Given the following `PipelineParameter`, specified

1334

# in the `inputParameters` list:

1335

#

1336

# ```

1337

# {name: "input_file", localCopy: {path: "file.txt", disk: "pd1"}}

1338

# ```

1339

#

1340

# where `disk` is defined in the `PipelineResources` object as:

1341

#

1342

# ```

1343

# {name: "pd1", mountPoint: "/mnt/disk/"}

1344

# ```

1345

#

1346

# We create a disk named `pd1`, mount it on the host VM, and map

1347

# `/mnt/pd1` to `/mnt/disk` in the docker container. At

1348

# runtime, an entry for `input_file` would be required in the inputs

# map, such as:

#

# ```

# inputs["input_file"] = "gs://my-bucket/bar.txt"

1353

# ```

1354

#

1355

# This would generate the following gsutil call:

1356

#

1357

# ```

1358

# gsutil cp gs://my-bucket/bar.txt /mnt/pd1/file.txt

1359

# ```

1360

#

1361

# The file `/mnt/pd1/file.txt` maps to `/mnt/disk/file.txt` in the

1362

# Docker container. Acceptable paths are:

#

# <table>

# <thead>

# <tr><th>Google Cloud storage path</th><th>Local path</th></tr>

1367

# </thead>

1368

# <tbody>

1369

# <tr><td>file</td><td>file</td></tr>

1370

# <tr><td>glob</td><td>directory</td></tr>

# </tbody>

# </table>

#

# For outputs, the direction of the copy is reversed:

1375

#

1376

# ```

1377

# gsutil cp /mnt/disk/file.txt gs://my-bucket/bar.txt

1378

# ```

1379

#

1380

# Acceptable paths are:

#

# <table>

# <thead>

# <tr><th>Local path</th><th>Google Cloud Storage path</th></tr>

1385

# </thead>

1386

# <tbody>

1387

# <tr><td>file</td><td>file</td></tr>

1388

# <tr>

1389

# <td>file</td>

1390

# <td>directory - directory must already exist</td>

# </tr>

# <tr>

# <td>glob</td>

# <td>directory - directory will be created if it doesn't exist</td></tr>

# </tbody>

# </table>

#

# One restriction due to docker limitations, is that for outputs that are found

1399

# on the boot disk, the local path cannot be a glob and must be a file.

1400

"defaultValue": "A String", # The default value for this parameter. Can be overridden at runtime.

1401

# If `localCopy` is present, then this must be a Google Cloud Storage path

1402

# beginning with `gs://`.

Sai Cheemalapati

2017-06-06 18:46:08 -0400

[diff] [blame^]

1403

"description": "A String", # Human-readable description.

1404

"name": "A String", # Required. Name of the parameter - the pipeline runner uses this string

1405

# as the key to the input and output maps in RunPipeline.

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

1406

"localCopy": { # LocalCopy defines how a remote file should be copied to and from the VM. # If present, this parameter is marked for copying to and from the VM.

1407

# `LocalCopy` indicates where on the VM the file should be. The value

1408

# given to this parameter (either at runtime or using `defaultValue`)

1409

# must be the remote path where the file should be.

1410

"path": "A String", # Required. The path within the user's docker container where

1411

# this input should be localized to and from, relative to the specified

1412

# disk's mount point. For example: file.txt,

1413

"disk": "A String", # Required. The name of the disk where this parameter is

1414

# located. Can be the name of one of the disks specified in the

1415

# Resources field, or "boot", which represents the Docker

1416

# instance's boot disk and has a mount point of `/`.

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

1417

},

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

1418

},

1419

],

1420

"docker": { # The Docker execuctor specification. # Specifies the docker run information.

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

1421

"cmd": "A String", # Required. The command or newline delimited script to run. The command

1422

# string will be executed within a bash shell.

1423

#

1424

# If the command exits with a non-zero exit code, output parameter

1425

# de-localization will be skipped and the pipeline operation's

1426

# `error` field will be populated.

1427

#

1428

# Maximum command string length is 16384.

1429

"imageName": "A String", # Required. Image name from either Docker Hub or Google Container Registry.

1430

# Users that run pipelines must have READ access to the image.

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

1431

},

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

1432

"resources": { # The system resources for the pipeline run. # Required. Specifies resource requirements for the pipeline run.

# Required fields:

#

# *

# minimumCpuCores

#

# *

# minimumRamGb

"noAddress": True or False, # Whether to assign an external IP to the instance. This is an experimental

1441

# feature that may go away. Defaults to false.

1442

# Corresponds to `--no_address` flag for [gcloud compute instances create]

1443

# (https://cloud.google.com/sdk/gcloud/reference/compute/instances/create).

1444

# In order to use this, must be true for both create time and run time.

1445

# Cannot be true at run time if false at create time. If you need to ssh into

1446

# a private IP VM for debugging, you can ssh to a public VM and then ssh into

1447

# the private VM's Internal IP. If noAddress is set, this pipeline run may

1448

# only load docker images from Google Container Registry and not Docker Hub.

1449

# Before using this, you must

1450

# [configure access to Google services from internal IPs](https://cloud.google.com/compute/docs/configure-private-google-access#configuring_access_to_google_services_from_internal_ips).

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

1451

"minimumRamGb": 3.14, # The minimum amount of RAM to use. Defaults to 3.75 (GB)

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

1452

"preemptible": True or False, # Whether to use preemptible VMs. Defaults to `false`. In order to use this,

1453

# must be true for both create time and run time. Cannot be true at run time

1454

# if false at create time.

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

1455

"disks": [ # Disks to attach.

1456

{ # A Google Compute Engine disk resource specification.

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

1457

"autoDelete": True or False, # Deprecated. Disks created by the Pipelines API will be deleted at the end

1458

# of the pipeline run, regardless of what this field is set to.

Sai Cheemalapati

2017-06-06 18:46:08 -0400

[diff] [blame^]

1459

"sizeGb": 42, # The size of the disk. Defaults to 500 (GB).

1460

# This field is not applicable for local SSD.

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

1461

"source": "A String", # The full or partial URL of the persistent disk to attach. See

1462

# https://cloud.google.com/compute/docs/reference/latest/instances#resource

1463

# and

1464

# https://cloud.google.com/compute/docs/disks/persistent-disks#snapshots

1465

# for more details.

Sai Cheemalapati

2017-06-06 18:46:08 -0400

[diff] [blame^]

1466

"readOnly": True or False, # Specifies how a sourced-base persistent disk will be mounted. See

1467

# https://cloud.google.com/compute/docs/disks/persistent-disks#use_multi_instances

1468

# for more details.

1469

# Can only be set at create time.

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

1470

"mountPoint": "A String", # Required at create time and cannot be overridden at run time.

1471

# Specifies the path in the docker container where files on

1472

# this disk should be located. For example, if `mountPoint`

1473

# is `/mnt/disk`, and the parameter has `localPath`

1474

# `inputs/file.txt`, the docker container can access the data at

1475

# `/mnt/disk/inputs/file.txt`.

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

1476

"type": "A String", # Required. The type of the disk to create.

Sai Cheemalapati

2017-06-06 18:46:08 -0400

[diff] [blame^]

1477

"name": "A String", # Required. The name of the disk that can be used in the pipeline

1478

# parameters. Must be 1 - 63 characters.

1479

# The name "boot" is reserved for system use.

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

1480

},

1481

],

1482

"bootDiskSizeGb": 42, # The size of the boot disk. Defaults to 10 (GB).

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

1483

"zones": [ # List of Google Compute Engine availability zones to which resource

1484

# creation will restricted. If empty, any zone may be chosen.

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

1485

"A String",

1486

],

1487

"minimumCpuCores": 42, # The minimum number of cores to use. Defaults to 1.

1488

},

1489

"description": "A String", # User-specified description.

},

],

}</pre>

</div>

<code class="details" id="list_next">list_next(previous_request, previous_response)</code>

1497

<pre>Retrieves the next page of results.

1498

1499

Args:

1500

previous_request: The request for the previous page. (required)

1501

previous_response: The response from the request for the previous page. (required)

1502

1503

Returns:

1504

A request object that you can call 'execute()' on to request the next

1505

page. Returns None if there are no more items in the collection.

</pre>

</div>

<code class="details" id="run">run(body, x__xgafv=None)</code>

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

1511

<pre>Runs a pipeline. If `pipelineId` is specified in the request, then

1512

run a saved pipeline. If `ephemeralPipeline` is specified, then run

1513

that pipeline once without saving a copy.

1514

1515

The caller must have READ permission to the project where the pipeline

1516

is stored and WRITE permission to the project where the pipeline will be

1517

run, as VMs will be created and storage will be used.

1518

1519

If a pipeline operation is still running after 6 days, it will be canceled.

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

1520

1521

Args:

1522

body: object, The request body. (required)

1523

The object takes the form of:

1524

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

1525

{ # The request to run a pipeline. If `pipelineId` is specified, it

1526

# refers to a saved pipeline created with CreatePipeline and set as

1527

# the `pipelineId` of the returned Pipeline object. If

1528

# `ephemeralPipeline` is specified, that pipeline is run once

1529

# with the given args and not saved. It is an error to specify both

1530

# `pipelineId` and `ephemeralPipeline`. `pipelineArgs`

1531

# must be specified.

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

1532

"pipelineArgs": { # The pipeline run arguments. # The arguments to use when running this pipeline.

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

1533

"inputs": { # Pipeline input arguments; keys are defined in the pipeline documentation.

1534

# All input parameters that do not have default values must be specified.

1535

# If parameters with defaults are specified here, the defaults will be

1536

# overridden.

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

1537

"a_key": "A String",

1538

},

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

1539

"logging": { # The logging options for the pipeline run. # Required. Logging options. Used by the service to communicate results

1540

# to the user.

1541

"gcsPath": "A String", # The location in Google Cloud Storage to which the pipeline logs

1542

# will be copied. Can be specified as a fully qualified directory

1543

# path, in which case logs will be output with a unique identifier

1544

# as the filename in that directory, or as a fully specified path,

1545

# which must end in `.log`, in which case that path will be

1546

# used, and the user must ensure that logs are not

1547

# overwritten. Stdout and stderr logs from the run are also

1548

# generated and output as `-stdout.log` and `-stderr.log`.

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

1549

},

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

1550

"outputs": { # Pipeline output arguments; keys are defined in the pipeline

1551

# documentation. All output parameters of without default values

1552

# must be specified. If parameters with defaults are specified

1553

# here, the defaults will be overridden.

Jon Wayne Parrott

692617a

2017-01-06 09:58:29 -0800

[diff] [blame]

1554

"a_key": "A String",

1555

},

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

1556

"labels": { # Labels to apply to this pipeline run. Labels will also be applied to

1557

# compute resources (VM, disks) created by this pipeline run. When listing

1558

# operations, operations can filtered by labels.

1559

# Label keys may not be empty; label values may be empty. Non-empty labels

1560

# must be 1-63 characters long, and comply with [RFC1035]

1561

# (https://www.ietf.org/rfc/rfc1035.txt).

1562

# Specifically, the name must be 1-63 characters long and match the regular

1563

# expression `[a-z]([-a-z0-9]*[a-z0-9])?` which means the first

1564

# character must be a lowercase letter, and all following characters must be

1565

# a dash, lowercase letter, or digit, except the last character, which cannot

# be a dash.

"a_key": "A String",

},

"serviceAccount": { # A Google Cloud Service Account. # The Google Cloud Service Account that will be used to access data and

1570

# services. By default, the compute service account associated with

1571

# `projectId` is used.

1572

"scopes": [ # List of scopes to be enabled for this service account on the VM.

1573

# The following scopes are automatically included:

1574

#

1575

# * https://www.googleapis.com/auth/compute

1576

# * https://www.googleapis.com/auth/devstorage.full_control

1577

# * https://www.googleapis.com/auth/genomics

1578

# * https://www.googleapis.com/auth/logging.write

1579

# * https://www.googleapis.com/auth/monitoring.write

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

1580

"A String",

1581

],

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

1582

"email": "A String", # Email address of the service account. Defaults to `default`,

1583

# which uses the compute service account associated with the project.

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

1584

},

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

1585

"clientId": "A String", # This field is deprecated. Use `labels` instead. Client-specified pipeline

1586

# operation identifier.

1587

"projectId": "A String", # Required. The project in which to run the pipeline. The caller must have

1588

# WRITER access to all Google Cloud services and resources (e.g. Google

1589

# Compute Engine) will be used.

1590

"keepVmAliveOnFailureDuration": "A String", # How long to keep the VM up after a failure (for example docker command

1591

# failed, copying input or output files failed, etc). While the VM is up, one

1592

# can ssh into the VM to debug. Default is 0; maximum allowed value is 1 day.

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

1593

"resources": { # The system resources for the pipeline run. # Specifies resource requirements/overrides for the pipeline run.

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

1594

"noAddress": True or False, # Whether to assign an external IP to the instance. This is an experimental

1595

# feature that may go away. Defaults to false.

1596

# Corresponds to `--no_address` flag for [gcloud compute instances create]

1597

# (https://cloud.google.com/sdk/gcloud/reference/compute/instances/create).

1598

# In order to use this, must be true for both create time and run time.

1599

# Cannot be true at run time if false at create time. If you need to ssh into

1600

# a private IP VM for debugging, you can ssh to a public VM and then ssh into

1601

# the private VM's Internal IP. If noAddress is set, this pipeline run may

1602

# only load docker images from Google Container Registry and not Docker Hub.

1603

# Before using this, you must

1604

# [configure access to Google services from internal IPs](https://cloud.google.com/compute/docs/configure-private-google-access#configuring_access_to_google_services_from_internal_ips).

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

1605

"minimumRamGb": 3.14, # The minimum amount of RAM to use. Defaults to 3.75 (GB)

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

1606

"preemptible": True or False, # Whether to use preemptible VMs. Defaults to `false`. In order to use this,

1607

# must be true for both create time and run time. Cannot be true at run time

1608

# if false at create time.

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

1609

"disks": [ # Disks to attach.

1610

{ # A Google Compute Engine disk resource specification.

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

1611

"autoDelete": True or False, # Deprecated. Disks created by the Pipelines API will be deleted at the end

1612

# of the pipeline run, regardless of what this field is set to.

Sai Cheemalapati

2017-06-06 18:46:08 -0400

[diff] [blame^]

1613

"sizeGb": 42, # The size of the disk. Defaults to 500 (GB).

1614

# This field is not applicable for local SSD.

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

1615

"source": "A String", # The full or partial URL of the persistent disk to attach. See

1616

# https://cloud.google.com/compute/docs/reference/latest/instances#resource

1617

# and

1618

# https://cloud.google.com/compute/docs/disks/persistent-disks#snapshots

1619

# for more details.

Sai Cheemalapati

2017-06-06 18:46:08 -0400

[diff] [blame^]

1620

"readOnly": True or False, # Specifies how a sourced-base persistent disk will be mounted. See

1621

# https://cloud.google.com/compute/docs/disks/persistent-disks#use_multi_instances

1622

# for more details.

1623

# Can only be set at create time.

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

1624

"mountPoint": "A String", # Required at create time and cannot be overridden at run time.

1625

# Specifies the path in the docker container where files on

1626

# this disk should be located. For example, if `mountPoint`

1627

# is `/mnt/disk`, and the parameter has `localPath`

1628

# `inputs/file.txt`, the docker container can access the data at

1629

# `/mnt/disk/inputs/file.txt`.

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

1630

"type": "A String", # Required. The type of the disk to create.

Sai Cheemalapati

2017-06-06 18:46:08 -0400

[diff] [blame^]

1631

"name": "A String", # Required. The name of the disk that can be used in the pipeline

1632

# parameters. Must be 1 - 63 characters.

1633

# The name "boot" is reserved for system use.

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

1634

},

1635

],

1636

"bootDiskSizeGb": 42, # The size of the boot disk. Defaults to 10 (GB).

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

1637

"zones": [ # List of Google Compute Engine availability zones to which resource

1638

# creation will restricted. If empty, any zone may be chosen.

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

1639

"A String",

1640

],

1641

"minimumCpuCores": 42, # The minimum number of cores to use. Defaults to 1.

1642

},

1643

},

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

1644

"ephemeralPipeline": { # The pipeline object. Represents a transformation from a set of input # A new pipeline object to run once and then delete.

1645

# parameters to a set of output parameters. The transformation is defined

1646

# as a docker image and command to run within that image. Each pipeline

1647

# is run on a Google Compute Engine VM. A pipeline can be created with the

1648

# `create` method and then later run with the `run` method, or a pipeline can

1649

# be defined and run all at once with the `run` method.

1650

"pipelineId": "A String", # Unique pipeline id that is generated by the service when CreatePipeline

1651

# is called. Cannot be specified in the Pipeline used in the

1652

# CreatePipelineRequest, and will be populated in the response to

1653

# CreatePipeline and all subsequent Get and List calls. Indicates that the

1654

# service has registered this pipeline.

1655

"name": "A String", # Required. A user specified pipeline name that does not have to be unique.

1656

# This name can be used for filtering Pipelines in ListPipelines.

1657

"projectId": "A String", # Required. The project in which to create the pipeline. The caller must have

1658

# WRITE access.

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

1659

"outputParameters": [ # Output parameters of the pipeline.

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

1660

{ # Parameters facilitate setting and delivering data into the

1661

# pipeline's execution environment. They are defined at create time,

1662

# with optional defaults, and can be overridden at run time.

1663

#

1664

# If `localCopy` is unset, then the parameter specifies a string that

1665

# is passed as-is into the pipeline, as the value of the environment

1666

# variable with the given name. A default value can be optionally

1667

# specified at create time. The default can be overridden at run time

1668

# using the inputs map. If no default is given, a value must be

1669

# supplied at runtime.

1670

#

1671

# If `localCopy` is defined, then the parameter specifies a data

1672

# source or sink, both in Google Cloud Storage and on the Docker container

1673

# where the pipeline computation is run. The service account associated with

1674

# the Pipeline (by

1675

# default the project's Compute Engine service account) must have access to the

1676

# Google Cloud Storage paths.

1677

#

1678

# At run time, the Google Cloud Storage paths can be overridden if a default

1679

# was provided at create time, or must be set otherwise. The pipeline runner

1680

# should add a key/value pair to either the inputs or outputs map. The

1681

# indicated data copies will be carried out before/after pipeline execution,

1682

# just as if the corresponding arguments were provided to `gsutil cp`.

1683

#

1684

# For example: Given the following `PipelineParameter`, specified

1685

# in the `inputParameters` list:

1686

#

1687

# ```

1688

# {name: "input_file", localCopy: {path: "file.txt", disk: "pd1"}}

1689

# ```

1690

#

1691

# where `disk` is defined in the `PipelineResources` object as:

1692

#

1693

# ```

1694

# {name: "pd1", mountPoint: "/mnt/disk/"}

1695

# ```

1696

#

1697

# We create a disk named `pd1`, mount it on the host VM, and map

1698

# `/mnt/pd1` to `/mnt/disk` in the docker container. At

1699

# runtime, an entry for `input_file` would be required in the inputs

# map, such as:

#

# ```

# inputs["input_file"] = "gs://my-bucket/bar.txt"

1704

# ```

1705

#

1706

# This would generate the following gsutil call:

1707

#

1708

# ```

1709

# gsutil cp gs://my-bucket/bar.txt /mnt/pd1/file.txt

1710

# ```

1711

#

1712

# The file `/mnt/pd1/file.txt` maps to `/mnt/disk/file.txt` in the

1713

# Docker container. Acceptable paths are:

#

# <table>

# <thead>

# <tr><th>Google Cloud storage path</th><th>Local path</th></tr>

1718

# </thead>

1719

# <tbody>

1720

# <tr><td>file</td><td>file</td></tr>

1721

# <tr><td>glob</td><td>directory</td></tr>

# </tbody>

# </table>

#

# For outputs, the direction of the copy is reversed:

1726

#

1727

# ```

1728

# gsutil cp /mnt/disk/file.txt gs://my-bucket/bar.txt

1729

# ```

1730

#

1731

# Acceptable paths are:

#

# <table>

# <thead>

# <tr><th>Local path</th><th>Google Cloud Storage path</th></tr>

1736

# </thead>

1737

# <tbody>

1738

# <tr><td>file</td><td>file</td></tr>

1739

# <tr>

1740

# <td>file</td>

1741

# <td>directory - directory must already exist</td>

# </tr>

# <tr>

# <td>glob</td>

# <td>directory - directory will be created if it doesn't exist</td></tr>

# </tbody>

# </table>

#

# One restriction due to docker limitations, is that for outputs that are found

1750

# on the boot disk, the local path cannot be a glob and must be a file.

1751

"defaultValue": "A String", # The default value for this parameter. Can be overridden at runtime.

1752

# If `localCopy` is present, then this must be a Google Cloud Storage path

1753

# beginning with `gs://`.

Sai Cheemalapati

2017-06-06 18:46:08 -0400

[diff] [blame^]

1754

"description": "A String", # Human-readable description.

1755

"name": "A String", # Required. Name of the parameter - the pipeline runner uses this string

1756

# as the key to the input and output maps in RunPipeline.

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

1757

"localCopy": { # LocalCopy defines how a remote file should be copied to and from the VM. # If present, this parameter is marked for copying to and from the VM.

1758

# `LocalCopy` indicates where on the VM the file should be. The value

1759

# given to this parameter (either at runtime or using `defaultValue`)

1760

# must be the remote path where the file should be.

1761

"path": "A String", # Required. The path within the user's docker container where

1762

# this input should be localized to and from, relative to the specified

1763

# disk's mount point. For example: file.txt,

1764

"disk": "A String", # Required. The name of the disk where this parameter is

1765

# located. Can be the name of one of the disks specified in the

1766

# Resources field, or "boot", which represents the Docker

1767

# instance's boot disk and has a mount point of `/`.

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

1768

},

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

1769

},

1770

],

1771

"inputParameters": [ # Input parameters of the pipeline.

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

1772

{ # Parameters facilitate setting and delivering data into the

1773

# pipeline's execution environment. They are defined at create time,

1774

# with optional defaults, and can be overridden at run time.

1775

#

1776

# If `localCopy` is unset, then the parameter specifies a string that

1777

# is passed as-is into the pipeline, as the value of the environment

1778

# variable with the given name. A default value can be optionally

1779

# specified at create time. The default can be overridden at run time

1780

# using the inputs map. If no default is given, a value must be

1781

# supplied at runtime.

1782

#

1783

# If `localCopy` is defined, then the parameter specifies a data

1784

# source or sink, both in Google Cloud Storage and on the Docker container

1785

# where the pipeline computation is run. The service account associated with

1786

# the Pipeline (by

1787

# default the project's Compute Engine service account) must have access to the

1788

# Google Cloud Storage paths.

1789

#

1790

# At run time, the Google Cloud Storage paths can be overridden if a default

1791

# was provided at create time, or must be set otherwise. The pipeline runner

1792

# should add a key/value pair to either the inputs or outputs map. The

1793

# indicated data copies will be carried out before/after pipeline execution,

1794

# just as if the corresponding arguments were provided to `gsutil cp`.

1795

#

1796

# For example: Given the following `PipelineParameter`, specified

1797

# in the `inputParameters` list:

1798

#

1799

# ```

1800

# {name: "input_file", localCopy: {path: "file.txt", disk: "pd1"}}

1801

# ```

1802

#

1803

# where `disk` is defined in the `PipelineResources` object as:

1804

#

1805

# ```

1806

# {name: "pd1", mountPoint: "/mnt/disk/"}

1807

# ```

1808

#

1809

# We create a disk named `pd1`, mount it on the host VM, and map

1810

# `/mnt/pd1` to `/mnt/disk` in the docker container. At

1811

# runtime, an entry for `input_file` would be required in the inputs

# map, such as:

#

# ```

# inputs["input_file"] = "gs://my-bucket/bar.txt"

1816

# ```

1817

#

1818

# This would generate the following gsutil call:

1819

#

1820

# ```

1821

# gsutil cp gs://my-bucket/bar.txt /mnt/pd1/file.txt

1822

# ```

1823

#

1824

# The file `/mnt/pd1/file.txt` maps to `/mnt/disk/file.txt` in the

1825

# Docker container. Acceptable paths are:

#

# <table>

# <thead>

# <tr><th>Google Cloud storage path</th><th>Local path</th></tr>

1830

# </thead>

1831

# <tbody>

1832

# <tr><td>file</td><td>file</td></tr>

1833

# <tr><td>glob</td><td>directory</td></tr>

# </tbody>

# </table>

#

# For outputs, the direction of the copy is reversed:

1838

#

1839

# ```

1840

# gsutil cp /mnt/disk/file.txt gs://my-bucket/bar.txt

1841

# ```

1842

#

1843

# Acceptable paths are:

#

# <table>

# <thead>

# <tr><th>Local path</th><th>Google Cloud Storage path</th></tr>

1848

# </thead>

1849

# <tbody>

1850

# <tr><td>file</td><td>file</td></tr>

1851

# <tr>

1852

# <td>file</td>

1853

# <td>directory - directory must already exist</td>

# </tr>

# <tr>

# <td>glob</td>

# <td>directory - directory will be created if it doesn't exist</td></tr>

# </tbody>

# </table>

#

# One restriction due to docker limitations, is that for outputs that are found

1862

# on the boot disk, the local path cannot be a glob and must be a file.

1863

"defaultValue": "A String", # The default value for this parameter. Can be overridden at runtime.

1864

# If `localCopy` is present, then this must be a Google Cloud Storage path

1865

# beginning with `gs://`.

Sai Cheemalapati

2017-06-06 18:46:08 -0400

[diff] [blame^]

1866

"description": "A String", # Human-readable description.

1867

"name": "A String", # Required. Name of the parameter - the pipeline runner uses this string

1868

# as the key to the input and output maps in RunPipeline.

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

1869

"localCopy": { # LocalCopy defines how a remote file should be copied to and from the VM. # If present, this parameter is marked for copying to and from the VM.

1870

# `LocalCopy` indicates where on the VM the file should be. The value

1871

# given to this parameter (either at runtime or using `defaultValue`)

1872

# must be the remote path where the file should be.

1873

"path": "A String", # Required. The path within the user's docker container where

1874

# this input should be localized to and from, relative to the specified

1875

# disk's mount point. For example: file.txt,

1876

"disk": "A String", # Required. The name of the disk where this parameter is

1877

# located. Can be the name of one of the disks specified in the

1878

# Resources field, or "boot", which represents the Docker

1879

# instance's boot disk and has a mount point of `/`.

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

1880

},

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

1881

},

1882

],

1883

"docker": { # The Docker execuctor specification. # Specifies the docker run information.

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

1884

"cmd": "A String", # Required. The command or newline delimited script to run. The command

1885

# string will be executed within a bash shell.

1886

#

1887

# If the command exits with a non-zero exit code, output parameter

1888

# de-localization will be skipped and the pipeline operation's

1889

# `error` field will be populated.

1890

#

1891

# Maximum command string length is 16384.

1892

"imageName": "A String", # Required. Image name from either Docker Hub or Google Container Registry.

1893

# Users that run pipelines must have READ access to the image.

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

1894

},

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

1895

"resources": { # The system resources for the pipeline run. # Required. Specifies resource requirements for the pipeline run.

# Required fields:

#

# *

# minimumCpuCores

#

# *

# minimumRamGb

"noAddress": True or False, # Whether to assign an external IP to the instance. This is an experimental

1904

# feature that may go away. Defaults to false.

1905

# Corresponds to `--no_address` flag for [gcloud compute instances create]

1906

# (https://cloud.google.com/sdk/gcloud/reference/compute/instances/create).

1907

# In order to use this, must be true for both create time and run time.

1908

# Cannot be true at run time if false at create time. If you need to ssh into

1909

# a private IP VM for debugging, you can ssh to a public VM and then ssh into

1910

# the private VM's Internal IP. If noAddress is set, this pipeline run may

1911

# only load docker images from Google Container Registry and not Docker Hub.

1912

# Before using this, you must

1913

# [configure access to Google services from internal IPs](https://cloud.google.com/compute/docs/configure-private-google-access#configuring_access_to_google_services_from_internal_ips).

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

1914

"minimumRamGb": 3.14, # The minimum amount of RAM to use. Defaults to 3.75 (GB)

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

1915

"preemptible": True or False, # Whether to use preemptible VMs. Defaults to `false`. In order to use this,

1916

# must be true for both create time and run time. Cannot be true at run time

1917

# if false at create time.

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

1918

"disks": [ # Disks to attach.

1919

{ # A Google Compute Engine disk resource specification.

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

1920

"autoDelete": True or False, # Deprecated. Disks created by the Pipelines API will be deleted at the end

1921

# of the pipeline run, regardless of what this field is set to.

Sai Cheemalapati

2017-06-06 18:46:08 -0400

[diff] [blame^]

1922

"sizeGb": 42, # The size of the disk. Defaults to 500 (GB).

1923

# This field is not applicable for local SSD.

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

1924

"source": "A String", # The full or partial URL of the persistent disk to attach. See

1925

# https://cloud.google.com/compute/docs/reference/latest/instances#resource

1926

# and

1927

# https://cloud.google.com/compute/docs/disks/persistent-disks#snapshots

1928

# for more details.

Sai Cheemalapati

2017-06-06 18:46:08 -0400

[diff] [blame^]

1929

"readOnly": True or False, # Specifies how a sourced-base persistent disk will be mounted. See

1930

# https://cloud.google.com/compute/docs/disks/persistent-disks#use_multi_instances

1931

# for more details.

1932

# Can only be set at create time.

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

1933

"mountPoint": "A String", # Required at create time and cannot be overridden at run time.

1934

# Specifies the path in the docker container where files on

1935

# this disk should be located. For example, if `mountPoint`

1936

# is `/mnt/disk`, and the parameter has `localPath`

1937

# `inputs/file.txt`, the docker container can access the data at

1938

# `/mnt/disk/inputs/file.txt`.

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

1939

"type": "A String", # Required. The type of the disk to create.

Sai Cheemalapati

2017-06-06 18:46:08 -0400

[diff] [blame^]

1940

"name": "A String", # Required. The name of the disk that can be used in the pipeline

1941

# parameters. Must be 1 - 63 characters.

1942

# The name "boot" is reserved for system use.

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

1943

},

1944

],

1945

"bootDiskSizeGb": 42, # The size of the boot disk. Defaults to 10 (GB).

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

1946

"zones": [ # List of Google Compute Engine availability zones to which resource

1947

# creation will restricted. If empty, any zone may be chosen.

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

1948

"A String",

1949

],

1950

"minimumCpuCores": 42, # The minimum number of cores to use. Defaults to 1.

1951

},

1952

"description": "A String", # User-specified description.

1953

},

1954

"pipelineId": "A String", # The already created pipeline to run.

1955

}

1956

1957

x__xgafv: string, V1 error format.

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

1958

Allowed values

1959

1 - v1 error format

1960

2 - v2 error format

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

1961

1962

Returns:

1963

An object of the form:

1964

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

1965

{ # This resource represents a long-running operation that is the result of a

1966

# network API call.

1967

"metadata": { # An OperationMetadata object. This will always be returned with the Operation.

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

1968

"a_key": "", # Properties of the object. Contains field @type with type URL.

1969

},

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

1970

"error": { # The `Status` type defines a logical error model that is suitable for different # The error result of the operation in case of failure or cancellation.

1971

# programming environments, including REST APIs and RPC APIs. It is used by

1972

# [gRPC](https://github.com/grpc). The error model is designed to be:

1973

#

1974

# - Simple to use and understand for most users

1975

# - Flexible enough to meet unexpected needs

#

# # Overview

#

# The `Status` message contains three pieces of data: error code, error message,

1980

# and error details. The error code should be an enum value of

1981

# google.rpc.Code, but it may accept additional error codes if needed. The

1982

# error message should be a developer-facing English message that helps

1983

# developers *understand* and *resolve* the error. If a localized user-facing

1984

# error message is needed, put the localized message in the error details or

1985

# localize it in the client. The optional error details may contain arbitrary

1986

# information about the error. There is a predefined set of error detail types

Sai Cheemalapati

2017-06-06 18:46:08 -0400

[diff] [blame^]

1987

# in the package `google.rpc` that can be used for common error conditions.

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

#

# # Language mapping

#

# The `Status` message is the logical representation of the error model, but it

1992

# is not necessarily the actual wire format. When the `Status` message is

1993

# exposed in different client libraries and different wire protocols, it can be

1994

# mapped differently. For example, it will likely be mapped to some exceptions

1995

# in Java, but more likely mapped to some error codes in C.

#

# # Other uses

#

# The error model and the `Status` message can be used in a variety of

2000

# environments, either with or without APIs, to provide a

2001

# consistent developer experience across different environments.

2002

#

2003

# Example uses of this error model include:

2004

#

2005

# - Partial errors. If a service needs to return partial errors to the client,

2006

# it may embed the `Status` in the normal response to indicate the partial

2007

# errors.

2008

#

2009

# - Workflow errors. A typical workflow has multiple steps. Each step may

Sai Cheemalapati

2017-06-06 18:46:08 -0400

[diff] [blame^]

2010

# have a `Status` message for error reporting.

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

2011

#

2012

# - Batch operations. If a client uses batch request and batch response, the

2013

# `Status` message should be used directly inside batch response, one for

2014

# each error sub-response.

2015

#

2016

# - Asynchronous operations. If an API call embeds asynchronous operation

2017

# results in its response, the status of those operations should be

2018

# represented directly using the `Status` message.

2019

#

2020

# - Logging. If some API errors are stored in logs, the message `Status` could

2021

# be used directly after any stripping needed for security/privacy reasons.

2022

"message": "A String", # A developer-facing error message, which should be in English. Any

2023

# user-facing error message should be localized and sent in the

2024

# google.rpc.Status.details field, or localized by the client.

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

2025

"code": 42, # The status code, which should be an enum value of google.rpc.Code.

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

2026

"details": [ # A list of messages that carry the error details. There will be a

2027

# common set of message types for APIs to use.

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

2028

{

2029

"a_key": "", # Properties of the object. Contains field @type with type URL.

2030

},

2031

],

2032

},

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

2033

"done": True or False, # If the value is `false`, it means the operation is still in progress.

2034

# If true, the operation is completed, and either `error` or `response` is

2035

# available.

2036

"response": { # If importing ReadGroupSets, an ImportReadGroupSetsResponse is returned. If importing Variants, an ImportVariantsResponse is returned. For pipelines and exports, an empty response is returned.

Sai Cheemalapati

2016-10-21 13:59:49 -0700

[diff] [blame]

2037

"a_key": "", # Properties of the object. Contains field @type with type URL.

2038

},

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame]

2039

"name": "A String", # The server-assigned name, which is only unique within the same service that originally returns it. For example: `operations/CJHU7Oi_ChDrveSpBRjfuL-qzoWAgEw`

Sai Cheemalapati