Blame - docs/dyn/genomics_v1alpha2.pipelines.html - platform/external/python/google-api-python-client

"localCopy": { # LocalCopy defines how a remote file should be copied to and from the VM. # If present, this parameter is marked for copying to and from the VM.

226

# `LocalCopy` indicates where on the VM the file should be. The value

227

# given to this parameter (either at runtime or using `defaultValue`)

228

# must be the remote path where the file should be.

229

"path": "A String", # Required. The path within the user's docker container where

230

# this input should be localized to and from, relative to the specified

231

# disk's mount point. For example: file.txt,

232

"disk": "A String", # Required. The name of the disk where this parameter is

233

# located. Can be the name of one of the disks specified in the

234

# Resources field, or "boot", which represents the Docker

235

# instance's boot disk and has a mount point of `/`.

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

236

},

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

237

"name": "A String", # Required. Name of the parameter - the pipeline runner uses this string

238

# as the key to the input and output maps in RunPipeline.

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

239

"description": "A String", # Human-readable description.

240

},

241

],

242

"inputParameters": [ # Input parameters of the pipeline.

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

243

{ # Parameters facilitate setting and delivering data into the

244

# pipeline's execution environment. They are defined at create time,

245

# with optional defaults, and can be overridden at run time.

246

#

247

# If `localCopy` is unset, then the parameter specifies a string that

248

# is passed as-is into the pipeline, as the value of the environment

249

# variable with the given name. A default value can be optionally

250

# specified at create time. The default can be overridden at run time

251

# using the inputs map. If no default is given, a value must be

252

# supplied at runtime.

253

#

254

# If `localCopy` is defined, then the parameter specifies a data

255

# source or sink, both in Google Cloud Storage and on the Docker container

256

# where the pipeline computation is run. The service account associated with

257

# the Pipeline (by

258

# default the project's Compute Engine service account) must have access to the

259

# Google Cloud Storage paths.

260

#

261

# At run time, the Google Cloud Storage paths can be overridden if a default

262

# was provided at create time, or must be set otherwise. The pipeline runner

263

# should add a key/value pair to either the inputs or outputs map. The

264

# indicated data copies will be carried out before/after pipeline execution,

265

# just as if the corresponding arguments were provided to `gsutil cp`.

266

#

267

# For example: Given the following `PipelineParameter`, specified

268

# in the `inputParameters` list:

269

#

270

# ```

271

# {name: "input_file", localCopy: {path: "file.txt", disk: "pd1"}}

272

# ```

273

#

274

# where `disk` is defined in the `PipelineResources` object as:

275

#

276

# ```

277

# {name: "pd1", mountPoint: "/mnt/disk/"}

278

# ```

279

#

280

# We create a disk named `pd1`, mount it on the host VM, and map

281

# `/mnt/pd1` to `/mnt/disk` in the docker container. At

282

# runtime, an entry for `input_file` would be required in the inputs

# map, such as:

#

# ```

# inputs["input_file"] = "gs://my-bucket/bar.txt"

287

# ```

288

#

289

# This would generate the following gsutil call:

290

#

291

# ```

292

# gsutil cp gs://my-bucket/bar.txt /mnt/pd1/file.txt

293

# ```

294

#

295

# The file `/mnt/pd1/file.txt` maps to `/mnt/disk/file.txt` in the

296

# Docker container. Acceptable paths are:

#

# <table>

# <thead>

# <tr><th>Google Cloud storage path</th><th>Local path</th></tr>

301

# </thead>

302

# <tbody>

303

# <tr><td>file</td><td>file</td></tr>

304

# <tr><td>glob</td><td>directory</td></tr>

# </tbody>

# </table>

#

# For outputs, the direction of the copy is reversed:

309

#

310

# ```

311

# gsutil cp /mnt/disk/file.txt gs://my-bucket/bar.txt

312

# ```

313

#

314

# Acceptable paths are:

#

# <table>

# <thead>

# <tr><th>Local path</th><th>Google Cloud Storage path</th></tr>

319

# </thead>

320

# <tbody>

321

# <tr><td>file</td><td>file</td></tr>

322

# <tr>

323

# <td>file</td>

324

# <td>directory - directory must already exist</td>

# </tr>

# <tr>

# <td>glob</td>

# <td>directory - directory will be created if it doesn't exist</td></tr>

# </tbody>

# </table>

#

# One restriction due to docker limitations, is that for outputs that are found

333

# on the boot disk, the local path cannot be a glob and must be a file.

334

"defaultValue": "A String", # The default value for this parameter. Can be overridden at runtime.

335

# If `localCopy` is present, then this must be a Google Cloud Storage path

336

# beginning with `gs://`.

337

"localCopy": { # LocalCopy defines how a remote file should be copied to and from the VM. # If present, this parameter is marked for copying to and from the VM.

338

# `LocalCopy` indicates where on the VM the file should be. The value

339

# given to this parameter (either at runtime or using `defaultValue`)

340

# must be the remote path where the file should be.

341

"path": "A String", # Required. The path within the user's docker container where

342

# this input should be localized to and from, relative to the specified

343

# disk's mount point. For example: file.txt,

344

"disk": "A String", # Required. The name of the disk where this parameter is

345

# located. Can be the name of one of the disks specified in the

346

# Resources field, or "boot", which represents the Docker

347

# instance's boot disk and has a mount point of `/`.

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

348

},

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

349

"name": "A String", # Required. Name of the parameter - the pipeline runner uses this string

350

# as the key to the input and output maps in RunPipeline.

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

351

"description": "A String", # Human-readable description.

352

},

353

],

354

"docker": { # The Docker execuctor specification. # Specifies the docker run information.

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

355

"cmd": "A String", # Required. The command or newline delimited script to run. The command

356

# string will be executed within a bash shell.

357

#

358

# If the command exits with a non-zero exit code, output parameter

359

# de-localization will be skipped and the pipeline operation's

360

# `error` field will be populated.

361

#

362

# Maximum command string length is 16384.

363

"imageName": "A String", # Required. Image name from either Docker Hub or Google Container Registry.

364

# Users that run pipelines must have READ access to the image.

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

365

},

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

366

"resources": { # The system resources for the pipeline run. # Required. Specifies resource requirements for the pipeline run.

# Required fields:

#

# *

# minimumCpuCores

#

# *

# minimumRamGb

"noAddress": True or False, # Whether to assign an external IP to the instance. This is an experimental

375

# feature that may go away. Defaults to false.

376

# Corresponds to `--no_address` flag for [gcloud compute instances create]

377

# (https://cloud.google.com/sdk/gcloud/reference/compute/instances/create).

378

# In order to use this, must be true for both create time and run time.

379

# Cannot be true at run time if false at create time. If you need to ssh into

380

# a private IP VM for debugging, you can ssh to a public VM and then ssh into

381

# the private VM's Internal IP. If noAddress is set, this pipeline run may

382

# only load docker images from Google Container Registry and not Docker Hub.

383

# Before using this, you must

384

# [configure access to Google services from internal IPs](https://cloud.google.com/compute/docs/configure-private-google-access#configuring_access_to_google_services_from_internal_ips).

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

385

"minimumRamGb": 3.14, # The minimum amount of RAM to use. Defaults to 3.75 (GB)

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

386

"preemptible": True or False, # Whether to use preemptible VMs. Defaults to `false`. In order to use this,

387

# must be true for both create time and run time. Cannot be true at run time

388

# if false at create time.

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

389

"disks": [ # Disks to attach.

390

{ # A Google Compute Engine disk resource specification.

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

391

"name": "A String", # Required. The name of the disk that can be used in the pipeline

392

# parameters. Must be 1 - 63 characters.

393

# The name "boot" is reserved for system use.

394

"sizeGb": 42, # The size of the disk. Defaults to 500 (GB).

395

# This field is not applicable for local SSD.

396

"autoDelete": True or False, # Deprecated. Disks created by the Pipelines API will be deleted at the end

397

# of the pipeline run, regardless of what this field is set to.

398

"source": "A String", # The full or partial URL of the persistent disk to attach. See

399

# https://cloud.google.com/compute/docs/reference/latest/instances#resource

400

# and

401

# https://cloud.google.com/compute/docs/disks/persistent-disks#snapshots

402

# for more details.

403

"mountPoint": "A String", # Required at create time and cannot be overridden at run time.

404

# Specifies the path in the docker container where files on

405

# this disk should be located. For example, if `mountPoint`

406

# is `/mnt/disk`, and the parameter has `localPath`

407

# `inputs/file.txt`, the docker container can access the data at

408

# `/mnt/disk/inputs/file.txt`.

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

409

"type": "A String", # Required. The type of the disk to create.

410

},

411

],

412

"bootDiskSizeGb": 42, # The size of the boot disk. Defaults to 10 (GB).

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

413

"zones": [ # List of Google Compute Engine availability zones to which resource

414

# creation will restricted. If empty, any zone may be chosen.

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

415

"A String",

416

],

417

"minimumCpuCores": 42, # The minimum number of cores to use. Defaults to 1.

418

},

419

"description": "A String", # User-specified description.

420

}

421

422

x__xgafv: string, V1 error format.

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

423

Allowed values

424

1 - v1 error format

425

2 - v2 error format

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

426

427

Returns:

428

An object of the form:

429

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

430

{ # The pipeline object. Represents a transformation from a set of input

431

# parameters to a set of output parameters. The transformation is defined

432

# as a docker image and command to run within that image. Each pipeline

433

# is run on a Google Compute Engine VM. A pipeline can be created with the

434

# `create` method and then later run with the `run` method, or a pipeline can

435

# be defined and run all at once with the `run` method.

436

"pipelineId": "A String", # Unique pipeline id that is generated by the service when CreatePipeline

437

# is called. Cannot be specified in the Pipeline used in the

438

# CreatePipelineRequest, and will be populated in the response to

439

# CreatePipeline and all subsequent Get and List calls. Indicates that the

440

# service has registered this pipeline.

441

"name": "A String", # Required. A user specified pipeline name that does not have to be unique.

442

# This name can be used for filtering Pipelines in ListPipelines.

443

"projectId": "A String", # Required. The project in which to create the pipeline. The caller must have

444

# WRITE access.

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

445

"outputParameters": [ # Output parameters of the pipeline.

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

446

{ # Parameters facilitate setting and delivering data into the

447

# pipeline's execution environment. They are defined at create time,

448

# with optional defaults, and can be overridden at run time.

449

#

450

# If `localCopy` is unset, then the parameter specifies a string that

451

# is passed as-is into the pipeline, as the value of the environment

452

# variable with the given name. A default value can be optionally

453

# specified at create time. The default can be overridden at run time

454

# using the inputs map. If no default is given, a value must be

455

# supplied at runtime.

456

#

457

# If `localCopy` is defined, then the parameter specifies a data

458

# source or sink, both in Google Cloud Storage and on the Docker container

459

# where the pipeline computation is run. The service account associated with

460

# the Pipeline (by

461

# default the project's Compute Engine service account) must have access to the

462

# Google Cloud Storage paths.

463

#

464

# At run time, the Google Cloud Storage paths can be overridden if a default

465

# was provided at create time, or must be set otherwise. The pipeline runner

466

# should add a key/value pair to either the inputs or outputs map. The

467

# indicated data copies will be carried out before/after pipeline execution,

468

# just as if the corresponding arguments were provided to `gsutil cp`.

469

#

470

# For example: Given the following `PipelineParameter`, specified

471

# in the `inputParameters` list:

472

#

473

# ```

474

# {name: "input_file", localCopy: {path: "file.txt", disk: "pd1"}}

475

# ```

476

#

477

# where `disk` is defined in the `PipelineResources` object as:

478

#

479

# ```

480

# {name: "pd1", mountPoint: "/mnt/disk/"}

481

# ```

482

#

483

# We create a disk named `pd1`, mount it on the host VM, and map

484

# `/mnt/pd1` to `/mnt/disk` in the docker container. At

485

# runtime, an entry for `input_file` would be required in the inputs

# map, such as:

#

# ```

# inputs["input_file"] = "gs://my-bucket/bar.txt"

490

# ```

491

#

492

# This would generate the following gsutil call:

493

#

494

# ```

495

# gsutil cp gs://my-bucket/bar.txt /mnt/pd1/file.txt

496

# ```

497

#

498

# The file `/mnt/pd1/file.txt` maps to `/mnt/disk/file.txt` in the

499

# Docker container. Acceptable paths are:

#

# <table>

# <thead>

# <tr><th>Google Cloud storage path</th><th>Local path</th></tr>

504

# </thead>

505

# <tbody>

506

# <tr><td>file</td><td>file</td></tr>

507

# <tr><td>glob</td><td>directory</td></tr>

# </tbody>

# </table>

#

# For outputs, the direction of the copy is reversed:

512

#

513

# ```

514

# gsutil cp /mnt/disk/file.txt gs://my-bucket/bar.txt

515

# ```

516

#

517

# Acceptable paths are:

#

# <table>

# <thead>

# <tr><th>Local path</th><th>Google Cloud Storage path</th></tr>

522

# </thead>

523

# <tbody>

524

# <tr><td>file</td><td>file</td></tr>

525

# <tr>

526

# <td>file</td>

527

# <td>directory - directory must already exist</td>

# </tr>

# <tr>

# <td>glob</td>

# <td>directory - directory will be created if it doesn't exist</td></tr>

# </tbody>

# </table>

#

# One restriction due to docker limitations, is that for outputs that are found

536

# on the boot disk, the local path cannot be a glob and must be a file.

537

"defaultValue": "A String", # The default value for this parameter. Can be overridden at runtime.

538

# If `localCopy` is present, then this must be a Google Cloud Storage path

539

# beginning with `gs://`.

540

"localCopy": { # LocalCopy defines how a remote file should be copied to and from the VM. # If present, this parameter is marked for copying to and from the VM.

541

# `LocalCopy` indicates where on the VM the file should be. The value

542

# given to this parameter (either at runtime or using `defaultValue`)

543

# must be the remote path where the file should be.

544

"path": "A String", # Required. The path within the user's docker container where

545

# this input should be localized to and from, relative to the specified

546

# disk's mount point. For example: file.txt,

547

"disk": "A String", # Required. The name of the disk where this parameter is

548

# located. Can be the name of one of the disks specified in the

549

# Resources field, or "boot", which represents the Docker

550

# instance's boot disk and has a mount point of `/`.

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

551

},

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

552

"name": "A String", # Required. Name of the parameter - the pipeline runner uses this string

553

# as the key to the input and output maps in RunPipeline.

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

554

"description": "A String", # Human-readable description.

555

},

556

],

557

"inputParameters": [ # Input parameters of the pipeline.

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

558

{ # Parameters facilitate setting and delivering data into the

559

# pipeline's execution environment. They are defined at create time,

560

# with optional defaults, and can be overridden at run time.

561

#

562

# If `localCopy` is unset, then the parameter specifies a string that

563

# is passed as-is into the pipeline, as the value of the environment

564

# variable with the given name. A default value can be optionally

565

# specified at create time. The default can be overridden at run time

566

# using the inputs map. If no default is given, a value must be

567

# supplied at runtime.

568

#

569

# If `localCopy` is defined, then the parameter specifies a data

570

# source or sink, both in Google Cloud Storage and on the Docker container

571

# where the pipeline computation is run. The service account associated with

572

# the Pipeline (by

573

# default the project's Compute Engine service account) must have access to the

574

# Google Cloud Storage paths.

575

#

576

# At run time, the Google Cloud Storage paths can be overridden if a default

577

# was provided at create time, or must be set otherwise. The pipeline runner

578

# should add a key/value pair to either the inputs or outputs map. The

579

# indicated data copies will be carried out before/after pipeline execution,

580

# just as if the corresponding arguments were provided to `gsutil cp`.

581

#

582

# For example: Given the following `PipelineParameter`, specified

583

# in the `inputParameters` list:

584

#

585

# ```

586

# {name: "input_file", localCopy: {path: "file.txt", disk: "pd1"}}

587

# ```

588

#

589

# where `disk` is defined in the `PipelineResources` object as:

590

#

591

# ```

592

# {name: "pd1", mountPoint: "/mnt/disk/"}

593

# ```

594

#

595

# We create a disk named `pd1`, mount it on the host VM, and map

596

# `/mnt/pd1` to `/mnt/disk` in the docker container. At

597

# runtime, an entry for `input_file` would be required in the inputs

# map, such as:

#

# ```

# inputs["input_file"] = "gs://my-bucket/bar.txt"

602

# ```

603

#

604

# This would generate the following gsutil call:

605

#

606

# ```

607

# gsutil cp gs://my-bucket/bar.txt /mnt/pd1/file.txt

608

# ```

609

#

610

# The file `/mnt/pd1/file.txt` maps to `/mnt/disk/file.txt` in the

611

# Docker container. Acceptable paths are:

#

# <table>

# <thead>

# <tr><th>Google Cloud storage path</th><th>Local path</th></tr>

616

# </thead>

617

# <tbody>

618

# <tr><td>file</td><td>file</td></tr>

619

# <tr><td>glob</td><td>directory</td></tr>

# </tbody>

# </table>

#

# For outputs, the direction of the copy is reversed:

624

#

625

# ```

626

# gsutil cp /mnt/disk/file.txt gs://my-bucket/bar.txt

627

# ```

628

#

629

# Acceptable paths are:

#

# <table>

# <thead>

# <tr><th>Local path</th><th>Google Cloud Storage path</th></tr>

634

# </thead>

635

# <tbody>

636

# <tr><td>file</td><td>file</td></tr>

637

# <tr>

638

# <td>file</td>

639

# <td>directory - directory must already exist</td>

# </tr>

# <tr>

# <td>glob</td>

# <td>directory - directory will be created if it doesn't exist</td></tr>

# </tbody>

# </table>

#

# One restriction due to docker limitations, is that for outputs that are found

648

# on the boot disk, the local path cannot be a glob and must be a file.

649

"defaultValue": "A String", # The default value for this parameter. Can be overridden at runtime.

650

# If `localCopy` is present, then this must be a Google Cloud Storage path

651

# beginning with `gs://`.

652

"localCopy": { # LocalCopy defines how a remote file should be copied to and from the VM. # If present, this parameter is marked for copying to and from the VM.

653

# `LocalCopy` indicates where on the VM the file should be. The value

654

# given to this parameter (either at runtime or using `defaultValue`)

655

# must be the remote path where the file should be.

656

"path": "A String", # Required. The path within the user's docker container where

657

# this input should be localized to and from, relative to the specified

658

# disk's mount point. For example: file.txt,

659

"disk": "A String", # Required. The name of the disk where this parameter is

660

# located. Can be the name of one of the disks specified in the

661

# Resources field, or "boot", which represents the Docker

662

# instance's boot disk and has a mount point of `/`.

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

663

},

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

664

"name": "A String", # Required. Name of the parameter - the pipeline runner uses this string

665

# as the key to the input and output maps in RunPipeline.

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

666

"description": "A String", # Human-readable description.

667

},

668

],

669

"docker": { # The Docker execuctor specification. # Specifies the docker run information.

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

670

"cmd": "A String", # Required. The command or newline delimited script to run. The command

671

# string will be executed within a bash shell.

672

#

673

# If the command exits with a non-zero exit code, output parameter

674

# de-localization will be skipped and the pipeline operation's

675

# `error` field will be populated.

676

#

677

# Maximum command string length is 16384.

678

"imageName": "A String", # Required. Image name from either Docker Hub or Google Container Registry.

679

# Users that run pipelines must have READ access to the image.

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

680

},

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

681

"resources": { # The system resources for the pipeline run. # Required. Specifies resource requirements for the pipeline run.

# Required fields:

#

# *

# minimumCpuCores

#

# *

# minimumRamGb

"noAddress": True or False, # Whether to assign an external IP to the instance. This is an experimental

690

# feature that may go away. Defaults to false.

691

# Corresponds to `--no_address` flag for [gcloud compute instances create]

692

# (https://cloud.google.com/sdk/gcloud/reference/compute/instances/create).

693

# In order to use this, must be true for both create time and run time.

694

# Cannot be true at run time if false at create time. If you need to ssh into

695

# a private IP VM for debugging, you can ssh to a public VM and then ssh into

696

# the private VM's Internal IP. If noAddress is set, this pipeline run may

697

# only load docker images from Google Container Registry and not Docker Hub.

698

# Before using this, you must

699

# [configure access to Google services from internal IPs](https://cloud.google.com/compute/docs/configure-private-google-access#configuring_access_to_google_services_from_internal_ips).

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

700

"minimumRamGb": 3.14, # The minimum amount of RAM to use. Defaults to 3.75 (GB)

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

701

"preemptible": True or False, # Whether to use preemptible VMs. Defaults to `false`. In order to use this,

702

# must be true for both create time and run time. Cannot be true at run time

703

# if false at create time.

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

704

"disks": [ # Disks to attach.

705

{ # A Google Compute Engine disk resource specification.

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

706

"name": "A String", # Required. The name of the disk that can be used in the pipeline

707

# parameters. Must be 1 - 63 characters.

708

# The name "boot" is reserved for system use.

709

"sizeGb": 42, # The size of the disk. Defaults to 500 (GB).

710

# This field is not applicable for local SSD.

711

"autoDelete": True or False, # Deprecated. Disks created by the Pipelines API will be deleted at the end

712

# of the pipeline run, regardless of what this field is set to.

713

"source": "A String", # The full or partial URL of the persistent disk to attach. See

714

# https://cloud.google.com/compute/docs/reference/latest/instances#resource

715

# and

716

# https://cloud.google.com/compute/docs/disks/persistent-disks#snapshots

717

# for more details.

718

"mountPoint": "A String", # Required at create time and cannot be overridden at run time.

719

# Specifies the path in the docker container where files on

720

# this disk should be located. For example, if `mountPoint`

721

# is `/mnt/disk`, and the parameter has `localPath`

722

# `inputs/file.txt`, the docker container can access the data at

723

# `/mnt/disk/inputs/file.txt`.

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

724

"type": "A String", # Required. The type of the disk to create.

725

},

726

],

727

"bootDiskSizeGb": 42, # The size of the boot disk. Defaults to 10 (GB).

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

728

"zones": [ # List of Google Compute Engine availability zones to which resource

729

# creation will restricted. If empty, any zone may be chosen.

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

730

"A String",

731

],

732

"minimumCpuCores": 42, # The minimum number of cores to use. Defaults to 1.

733

},

734

"description": "A String", # User-specified description.

}</pre>

</div>

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame^]

739

<code class="details" id="delete">delete(pipelineId=None, x__xgafv=None)</code>

740

<pre>Deletes a pipeline based on ID.

741

742

Caller must have WRITE permission to the project.

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

743

744

Args:

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

745

pipelineId: string, Caller must have WRITE access to the project in which this pipeline

746

is defined. (required)

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

747

x__xgafv: string, V1 error format.

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

748

Allowed values

749

1 - v1 error format

750

2 - v2 error format

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

751

752

Returns:

753

An object of the form:

754

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

755

{ # A generic empty message that you can re-use to avoid defining duplicated

756

# empty messages in your APIs. A typical example is to use it as the request

757

# or the response type of an API method. For instance:

758

#

759

# service Foo {

760

# rpc Bar(google.protobuf.Empty) returns (google.protobuf.Empty);

761

# }

762

#

763

# The JSON representation for `Empty` is empty JSON object `{}`.

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

}</pre>

</div>

Sai Cheemalapati

2017-03-13 12:12:03 -0400

[diff] [blame^]

768

<code class="details" id="get">get(pipelineId=None, x__xgafv=None)</code>

769

<pre>Retrieves a pipeline based on ID.

770

771

Caller must have READ permission to the project.

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

772

773

Args:

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

774

pipelineId: string, Caller must have READ access to the project in which this pipeline

775

is defined. (required)

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

776

x__xgafv: string, V1 error format.

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

777

Allowed values

778

1 - v1 error format

779

2 - v2 error format

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

780

781

Returns:

782

An object of the form:

783

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

784

{ # The pipeline object. Represents a transformation from a set of input

785

# parameters to a set of output parameters. The transformation is defined

786

# as a docker image and command to run within that image. Each pipeline

787

# is run on a Google Compute Engine VM. A pipeline can be created with the

788

# `create` method and then later run with the `run` method, or a pipeline can

789

# be defined and run all at once with the `run` method.

790

"pipelineId": "A String", # Unique pipeline id that is generated by the service when CreatePipeline

791

# is called. Cannot be specified in the Pipeline used in the

792

# CreatePipelineRequest, and will be populated in the response to

793

# CreatePipeline and all subsequent Get and List calls. Indicates that the

794

# service has registered this pipeline.

795

"name": "A String", # Required. A user specified pipeline name that does not have to be unique.

796

# This name can be used for filtering Pipelines in ListPipelines.

797

"projectId": "A String", # Required. The project in which to create the pipeline. The caller must have

798

# WRITE access.

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

799

"outputParameters": [ # Output parameters of the pipeline.

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

800

{ # Parameters facilitate setting and delivering data into the

801

# pipeline's execution environment. They are defined at create time,

802

# with optional defaults, and can be overridden at run time.

803

#

804

# If `localCopy` is unset, then the parameter specifies a string that

805

# is passed as-is into the pipeline, as the value of the environment

806

# variable with the given name. A default value can be optionally

807

# specified at create time. The default can be overridden at run time

808

# using the inputs map. If no default is given, a value must be

809

# supplied at runtime.

810

#

811

# If `localCopy` is defined, then the parameter specifies a data

812

# source or sink, both in Google Cloud Storage and on the Docker container

813

# where the pipeline computation is run. The service account associated with

814

# the Pipeline (by

815

# default the project's Compute Engine service account) must have access to the

816

# Google Cloud Storage paths.

817

#

818

# At run time, the Google Cloud Storage paths can be overridden if a default

819

# was provided at create time, or must be set otherwise. The pipeline runner

820

# should add a key/value pair to either the inputs or outputs map. The

821

# indicated data copies will be carried out before/after pipeline execution,

822

# just as if the corresponding arguments were provided to `gsutil cp`.

823

#

824

# For example: Given the following `PipelineParameter`, specified

825

# in the `inputParameters` list:

826

#

827

# ```

828

# {name: "input_file", localCopy: {path: "file.txt", disk: "pd1"}}

829

# ```

830

#

831

# where `disk` is defined in the `PipelineResources` object as:

832

#

833

# ```

834

# {name: "pd1", mountPoint: "/mnt/disk/"}

835

# ```

836

#

837

# We create a disk named `pd1`, mount it on the host VM, and map

838

# `/mnt/pd1` to `/mnt/disk` in the docker container. At

839

# runtime, an entry for `input_file` would be required in the inputs

# map, such as:

#

# ```

# inputs["input_file"] = "gs://my-bucket/bar.txt"

844

# ```

845

#

846

# This would generate the following gsutil call:

847

#

848

# ```

849

# gsutil cp gs://my-bucket/bar.txt /mnt/pd1/file.txt

850

# ```

851

#

852

# The file `/mnt/pd1/file.txt` maps to `/mnt/disk/file.txt` in the

853

# Docker container. Acceptable paths are:

#

# <table>

# <thead>

# <tr><th>Google Cloud storage path</th><th>Local path</th></tr>

858

# </thead>

859

# <tbody>

860

# <tr><td>file</td><td>file</td></tr>

861

# <tr><td>glob</td><td>directory</td></tr>

# </tbody>

# </table>

#

# For outputs, the direction of the copy is reversed:

866

#

867

# ```

868

# gsutil cp /mnt/disk/file.txt gs://my-bucket/bar.txt

869

# ```

870

#

871

# Acceptable paths are:

#

# <table>

# <thead>

# <tr><th>Local path</th><th>Google Cloud Storage path</th></tr>

876

# </thead>

877

# <tbody>

878

# <tr><td>file</td><td>file</td></tr>

879

# <tr>

880

# <td>file</td>

881

# <td>directory - directory must already exist</td>

# </tr>

# <tr>

# <td>glob</td>

# <td>directory - directory will be created if it doesn't exist</td></tr>

# </tbody>

# </table>

#

# One restriction due to docker limitations, is that for outputs that are found

890

# on the boot disk, the local path cannot be a glob and must be a file.

891

"defaultValue": "A String", # The default value for this parameter. Can be overridden at runtime.

892

# If `localCopy` is present, then this must be a Google Cloud Storage path

893

# beginning with `gs://`.

894

"localCopy": { # LocalCopy defines how a remote file should be copied to and from the VM. # If present, this parameter is marked for copying to and from the VM.

895

# `LocalCopy` indicates where on the VM the file should be. The value

896

# given to this parameter (either at runtime or using `defaultValue`)

897

# must be the remote path where the file should be.

898

"path": "A String", # Required. The path within the user's docker container where

899

# this input should be localized to and from, relative to the specified

900

# disk's mount point. For example: file.txt,

901

"disk": "A String", # Required. The name of the disk where this parameter is

902

# located. Can be the name of one of the disks specified in the

903

# Resources field, or "boot", which represents the Docker

904

# instance's boot disk and has a mount point of `/`.

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

905

},

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

906

"name": "A String", # Required. Name of the parameter - the pipeline runner uses this string

907

# as the key to the input and output maps in RunPipeline.

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

908

"description": "A String", # Human-readable description.

909

},

910

],

911

"inputParameters": [ # Input parameters of the pipeline.

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

912

{ # Parameters facilitate setting and delivering data into the

913

# pipeline's execution environment. They are defined at create time,

914

# with optional defaults, and can be overridden at run time.

915

#

916

# If `localCopy` is unset, then the parameter specifies a string that

917

# is passed as-is into the pipeline, as the value of the environment

918

# variable with the given name. A default value can be optionally

919

# specified at create time. The default can be overridden at run time

920

# using the inputs map. If no default is given, a value must be

921

# supplied at runtime.

922

#

923

# If `localCopy` is defined, then the parameter specifies a data

924

# source or sink, both in Google Cloud Storage and on the Docker container

925

# where the pipeline computation is run. The service account associated with

926

# the Pipeline (by

927

# default the project's Compute Engine service account) must have access to the

928

# Google Cloud Storage paths.

929

#

930

# At run time, the Google Cloud Storage paths can be overridden if a default

931

# was provided at create time, or must be set otherwise. The pipeline runner

932

# should add a key/value pair to either the inputs or outputs map. The

933

# indicated data copies will be carried out before/after pipeline execution,

934

# just as if the corresponding arguments were provided to `gsutil cp`.

935

#

936

# For example: Given the following `PipelineParameter`, specified

937

# in the `inputParameters` list:

938

#

939

# ```

940

# {name: "input_file", localCopy: {path: "file.txt", disk: "pd1"}}

941

# ```

942

#

943

# where `disk` is defined in the `PipelineResources` object as:

944

#

945

# ```

946

# {name: "pd1", mountPoint: "/mnt/disk/"}

947

# ```

948

#

949

# We create a disk named `pd1`, mount it on the host VM, and map

950

# `/mnt/pd1` to `/mnt/disk` in the docker container. At

951

# runtime, an entry for `input_file` would be required in the inputs

# map, such as:

#

# ```

# inputs["input_file"] = "gs://my-bucket/bar.txt"

956

# ```

957

#

958

# This would generate the following gsutil call:

959

#

960

# ```

961

# gsutil cp gs://my-bucket/bar.txt /mnt/pd1/file.txt

962

# ```

963

#

964

# The file `/mnt/pd1/file.txt` maps to `/mnt/disk/file.txt` in the

965

# Docker container. Acceptable paths are:

#

# <table>

# <thead>

# <tr><th>Google Cloud storage path</th><th>Local path</th></tr>

970

# </thead>

971

# <tbody>

972

# <tr><td>file</td><td>file</td></tr>

973

# <tr><td>glob</td><td>directory</td></tr>

# </tbody>

# </table>

#

# For outputs, the direction of the copy is reversed:

978

#

979

# ```

980

# gsutil cp /mnt/disk/file.txt gs://my-bucket/bar.txt

981

# ```

982

#

983

# Acceptable paths are:

#

# <table>

# <thead>

# <tr><th>Local path</th><th>Google Cloud Storage path</th></tr>

988

# </thead>

989

# <tbody>

990

# <tr><td>file</td><td>file</td></tr>

991

# <tr>

992

# <td>file</td>

993

# <td>directory - directory must already exist</td>

# </tr>

# <tr>

# <td>glob</td>

# <td>directory - directory will be created if it doesn't exist</td></tr>

# </tbody>

# </table>

#

# One restriction due to docker limitations, is that for outputs that are found

1002

# on the boot disk, the local path cannot be a glob and must be a file.

1003

"defaultValue": "A String", # The default value for this parameter. Can be overridden at runtime.

1004

# If `localCopy` is present, then this must be a Google Cloud Storage path

1005

# beginning with `gs://`.

1006

"localCopy": { # LocalCopy defines how a remote file should be copied to and from the VM. # If present, this parameter is marked for copying to and from the VM.

1007

# `LocalCopy` indicates where on the VM the file should be. The value

1008

# given to this parameter (either at runtime or using `defaultValue`)

1009

# must be the remote path where the file should be.

1010

"path": "A String", # Required. The path within the user's docker container where

1011

# this input should be localized to and from, relative to the specified

1012

# disk's mount point. For example: file.txt,

1013

"disk": "A String", # Required. The name of the disk where this parameter is

1014

# located. Can be the name of one of the disks specified in the

1015

# Resources field, or "boot", which represents the Docker

1016

# instance's boot disk and has a mount point of `/`.

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

1017

},

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

1018

"name": "A String", # Required. Name of the parameter - the pipeline runner uses this string

1019

# as the key to the input and output maps in RunPipeline.

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

1020

"description": "A String", # Human-readable description.

1021

},

1022

],

1023

"docker": { # The Docker execuctor specification. # Specifies the docker run information.

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

1024

"cmd": "A String", # Required. The command or newline delimited script to run. The command

1025

# string will be executed within a bash shell.

1026

#

1027

# If the command exits with a non-zero exit code, output parameter

1028

# de-localization will be skipped and the pipeline operation's

1029

# `error` field will be populated.

1030

#

1031

# Maximum command string length is 16384.

1032

"imageName": "A String", # Required. Image name from either Docker Hub or Google Container Registry.

1033

# Users that run pipelines must have READ access to the image.

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

1034

},

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

1035

"resources": { # The system resources for the pipeline run. # Required. Specifies resource requirements for the pipeline run.

# Required fields:

#

# *

# minimumCpuCores

#

# *

# minimumRamGb

"noAddress": True or False, # Whether to assign an external IP to the instance. This is an experimental

1044

# feature that may go away. Defaults to false.

1045

# Corresponds to `--no_address` flag for [gcloud compute instances create]

1046

# (https://cloud.google.com/sdk/gcloud/reference/compute/instances/create).

1047

# In order to use this, must be true for both create time and run time.

1048

# Cannot be true at run time if false at create time. If you need to ssh into

1049

# a private IP VM for debugging, you can ssh to a public VM and then ssh into

1050

# the private VM's Internal IP. If noAddress is set, this pipeline run may

1051

# only load docker images from Google Container Registry and not Docker Hub.

1052

# Before using this, you must

1053

# [configure access to Google services from internal IPs](https://cloud.google.com/compute/docs/configure-private-google-access#configuring_access_to_google_services_from_internal_ips).

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

1054

"minimumRamGb": 3.14, # The minimum amount of RAM to use. Defaults to 3.75 (GB)

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

1055

"preemptible": True or False, # Whether to use preemptible VMs. Defaults to `false`. In order to use this,

1056

# must be true for both create time and run time. Cannot be true at run time

1057

# if false at create time.

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

1058

"disks": [ # Disks to attach.

1059

{ # A Google Compute Engine disk resource specification.

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

1060

"name": "A String", # Required. The name of the disk that can be used in the pipeline

1061

# parameters. Must be 1 - 63 characters.

1062

# The name "boot" is reserved for system use.

1063

"sizeGb": 42, # The size of the disk. Defaults to 500 (GB).

1064

# This field is not applicable for local SSD.

1065

"autoDelete": True or False, # Deprecated. Disks created by the Pipelines API will be deleted at the end

1066

# of the pipeline run, regardless of what this field is set to.

1067

"source": "A String", # The full or partial URL of the persistent disk to attach. See

1068

# https://cloud.google.com/compute/docs/reference/latest/instances#resource

1069

# and

1070

# https://cloud.google.com/compute/docs/disks/persistent-disks#snapshots

1071

# for more details.

1072

"mountPoint": "A String", # Required at create time and cannot be overridden at run time.

1073

# Specifies the path in the docker container where files on

1074

# this disk should be located. For example, if `mountPoint`

1075

# is `/mnt/disk`, and the parameter has `localPath`

1076

# `inputs/file.txt`, the docker container can access the data at

1077

# `/mnt/disk/inputs/file.txt`.

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

1078

"type": "A String", # Required. The type of the disk to create.

1079

},

1080

],

1081

"bootDiskSizeGb": 42, # The size of the boot disk. Defaults to 10 (GB).

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

1082

"zones": [ # List of Google Compute Engine availability zones to which resource

1083

# creation will restricted. If empty, any zone may be chosen.

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

1084

"A String",

1085

],

1086

"minimumCpuCores": 42, # The minimum number of cores to use. Defaults to 1.

1087

},

1088

"description": "A String", # User-specified description.

}</pre>

</div>

<code class="details" id="getControllerConfig">getControllerConfig(validationToken=None, x__xgafv=None, operationId=None)</code>

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

1094

<pre>Gets controller configuration information. Should only be called

1095

by VMs created by the Pipelines Service and not by end users.

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

1096

1097

Args:

1098

validationToken: string, A parameter

1099

x__xgafv: string, V1 error format.

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

1100

Allowed values

1101

1 - v1 error format

1102

2 - v2 error format

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

1103

operationId: string, The operation to retrieve controller configuration for.

1104

1105

Returns:

1106

An object of the form:

1107

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

1108

{ # Stores the information that the controller will fetch from the

1109

# server in order to run. Should only be used by VMs created by the

1110

# Pipelines Service and not by end users.

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

1111

"cmd": "A String",

1112

"machineType": "A String",

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

1113

"vars": {

1114

"a_key": "A String",

1115

},

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

1116

"gcsLogPath": "A String",

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

"gcsSinks": {

"a_key": {

"values": [

"A String",

],

},

},

"image": "A String",

"disks": {

"a_key": "A String",

},

"gcsSources": {

"a_key": {

"values": [

"A String",

],

},

},

}</pre>

</div>

<code class="details" id="list">list(pageSize=None, projectId=None, namePrefix=None, pageToken=None, x__xgafv=None)</code>

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

1140

<pre>Lists pipelines.

1141

1142

Caller must have READ permission to the project.

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

1143

1144

Args:

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

1145

pageSize: integer, Number of pipelines to return at once. Defaults to 256, and max

1146

is 2048.

1147

projectId: string, Required. The name of the project to search for pipelines. Caller

1148

must have READ access to this project.

1149

namePrefix: string, Pipelines with names that match this prefix should be

1150

returned. If unspecified, all pipelines in the project, up to

1151

`pageSize`, will be returned.

1152

pageToken: string, Token to use to indicate where to start getting results.

1153

If unspecified, returns the first page of results.

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

1154

x__xgafv: string, V1 error format.

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

1155

Allowed values

1156

1 - v1 error format

1157

2 - v2 error format

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

1158

1159

Returns:

1160

An object of the form:

1161

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

1162

{ # The response of ListPipelines. Contains at most `pageSize`

1163

# pipelines. If it contains `pageSize` pipelines, and more pipelines

1164

# exist, then `nextPageToken` will be populated and should be

1165

# used as the `pageToken` argument to a subsequent ListPipelines

1166

# request.

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

1167

"nextPageToken": "A String", # The token to use to get the next page of results.

1168

"pipelines": [ # The matched pipelines.

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

1169

{ # The pipeline object. Represents a transformation from a set of input

1170

# parameters to a set of output parameters. The transformation is defined

1171

# as a docker image and command to run within that image. Each pipeline

1172

# is run on a Google Compute Engine VM. A pipeline can be created with the

1173

# `create` method and then later run with the `run` method, or a pipeline can

1174

# be defined and run all at once with the `run` method.

1175

"pipelineId": "A String", # Unique pipeline id that is generated by the service when CreatePipeline

1176

# is called. Cannot be specified in the Pipeline used in the

1177

# CreatePipelineRequest, and will be populated in the response to

1178

# CreatePipeline and all subsequent Get and List calls. Indicates that the

1179

# service has registered this pipeline.

1180

"name": "A String", # Required. A user specified pipeline name that does not have to be unique.

1181

# This name can be used for filtering Pipelines in ListPipelines.

1182

"projectId": "A String", # Required. The project in which to create the pipeline. The caller must have

1183

# WRITE access.

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

1184

"outputParameters": [ # Output parameters of the pipeline.

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

1185

{ # Parameters facilitate setting and delivering data into the

1186

# pipeline's execution environment. They are defined at create time,

1187

# with optional defaults, and can be overridden at run time.

1188

#

1189

# If `localCopy` is unset, then the parameter specifies a string that

1190

# is passed as-is into the pipeline, as the value of the environment

1191

# variable with the given name. A default value can be optionally

1192

# specified at create time. The default can be overridden at run time

1193

# using the inputs map. If no default is given, a value must be

1194

# supplied at runtime.

1195

#

1196

# If `localCopy` is defined, then the parameter specifies a data

1197

# source or sink, both in Google Cloud Storage and on the Docker container

1198

# where the pipeline computation is run. The service account associated with

1199

# the Pipeline (by

1200

# default the project's Compute Engine service account) must have access to the

1201

# Google Cloud Storage paths.

1202

#

1203

# At run time, the Google Cloud Storage paths can be overridden if a default

1204

# was provided at create time, or must be set otherwise. The pipeline runner

1205

# should add a key/value pair to either the inputs or outputs map. The

1206

# indicated data copies will be carried out before/after pipeline execution,

1207

# just as if the corresponding arguments were provided to `gsutil cp`.

1208

#

1209

# For example: Given the following `PipelineParameter`, specified

1210

# in the `inputParameters` list:

1211

#

1212

# ```

1213

# {name: "input_file", localCopy: {path: "file.txt", disk: "pd1"}}

1214

# ```

1215

#

1216

# where `disk` is defined in the `PipelineResources` object as:

1217

#

1218

# ```

1219

# {name: "pd1", mountPoint: "/mnt/disk/"}

1220

# ```

1221

#

1222

# We create a disk named `pd1`, mount it on the host VM, and map

1223

# `/mnt/pd1` to `/mnt/disk` in the docker container. At

1224

# runtime, an entry for `input_file` would be required in the inputs

# map, such as:

#

# ```

# inputs["input_file"] = "gs://my-bucket/bar.txt"

1229

# ```

1230

#

1231

# This would generate the following gsutil call:

1232

#

1233

# ```

1234

# gsutil cp gs://my-bucket/bar.txt /mnt/pd1/file.txt

1235

# ```

1236

#

1237

# The file `/mnt/pd1/file.txt` maps to `/mnt/disk/file.txt` in the

1238

# Docker container. Acceptable paths are:

#

# <table>

# <thead>

# <tr><th>Google Cloud storage path</th><th>Local path</th></tr>

1243

# </thead>

1244

# <tbody>

1245

# <tr><td>file</td><td>file</td></tr>

1246

# <tr><td>glob</td><td>directory</td></tr>

# </tbody>

# </table>

#

# For outputs, the direction of the copy is reversed:

1251

#

1252

# ```

1253

# gsutil cp /mnt/disk/file.txt gs://my-bucket/bar.txt

1254

# ```

1255

#

1256

# Acceptable paths are:

#

# <table>

# <thead>

# <tr><th>Local path</th><th>Google Cloud Storage path</th></tr>

1261

# </thead>

1262

# <tbody>

1263

# <tr><td>file</td><td>file</td></tr>

1264

# <tr>

1265

# <td>file</td>

1266

# <td>directory - directory must already exist</td>

# </tr>

# <tr>

# <td>glob</td>

# <td>directory - directory will be created if it doesn't exist</td></tr>

# </tbody>

# </table>

#

# One restriction due to docker limitations, is that for outputs that are found

1275

# on the boot disk, the local path cannot be a glob and must be a file.

1276

"defaultValue": "A String", # The default value for this parameter. Can be overridden at runtime.

1277

# If `localCopy` is present, then this must be a Google Cloud Storage path

1278

# beginning with `gs://`.

1279

"localCopy": { # LocalCopy defines how a remote file should be copied to and from the VM. # If present, this parameter is marked for copying to and from the VM.

1280

# `LocalCopy` indicates where on the VM the file should be. The value

1281

# given to this parameter (either at runtime or using `defaultValue`)

1282

# must be the remote path where the file should be.

1283

"path": "A String", # Required. The path within the user's docker container where

1284

# this input should be localized to and from, relative to the specified

1285

# disk's mount point. For example: file.txt,

1286

"disk": "A String", # Required. The name of the disk where this parameter is

1287

# located. Can be the name of one of the disks specified in the

1288

# Resources field, or "boot", which represents the Docker

1289

# instance's boot disk and has a mount point of `/`.

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

1290

},

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

1291

"name": "A String", # Required. Name of the parameter - the pipeline runner uses this string

1292

# as the key to the input and output maps in RunPipeline.

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

1293

"description": "A String", # Human-readable description.

1294

},

1295

],

1296

"inputParameters": [ # Input parameters of the pipeline.

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

1297

{ # Parameters facilitate setting and delivering data into the

1298

# pipeline's execution environment. They are defined at create time,

1299

# with optional defaults, and can be overridden at run time.

1300

#

1301

# If `localCopy` is unset, then the parameter specifies a string that

1302

# is passed as-is into the pipeline, as the value of the environment

1303

# variable with the given name. A default value can be optionally

1304

# specified at create time. The default can be overridden at run time

1305

# using the inputs map. If no default is given, a value must be

1306

# supplied at runtime.

1307

#

1308

# If `localCopy` is defined, then the parameter specifies a data

1309

# source or sink, both in Google Cloud Storage and on the Docker container

1310

# where the pipeline computation is run. The service account associated with

1311

# the Pipeline (by

1312

# default the project's Compute Engine service account) must have access to the

1313

# Google Cloud Storage paths.

1314

#

1315

# At run time, the Google Cloud Storage paths can be overridden if a default

1316

# was provided at create time, or must be set otherwise. The pipeline runner

1317

# should add a key/value pair to either the inputs or outputs map. The

1318

# indicated data copies will be carried out before/after pipeline execution,

1319

# just as if the corresponding arguments were provided to `gsutil cp`.

1320

#

1321

# For example: Given the following `PipelineParameter`, specified

1322

# in the `inputParameters` list:

1323

#

1324

# ```

1325

# {name: "input_file", localCopy: {path: "file.txt", disk: "pd1"}}

1326

# ```

1327

#

1328

# where `disk` is defined in the `PipelineResources` object as:

1329

#

1330

# ```

1331

# {name: "pd1", mountPoint: "/mnt/disk/"}

1332

# ```

1333

#

1334

# We create a disk named `pd1`, mount it on the host VM, and map

1335

# `/mnt/pd1` to `/mnt/disk` in the docker container. At

1336

# runtime, an entry for `input_file` would be required in the inputs

# map, such as:

#

# ```

# inputs["input_file"] = "gs://my-bucket/bar.txt"

1341

# ```

1342

#

1343

# This would generate the following gsutil call:

1344

#

1345

# ```

1346

# gsutil cp gs://my-bucket/bar.txt /mnt/pd1/file.txt

1347

# ```

1348

#

1349

# The file `/mnt/pd1/file.txt` maps to `/mnt/disk/file.txt` in the

1350

# Docker container. Acceptable paths are:

#

# <table>

# <thead>

# <tr><th>Google Cloud storage path</th><th>Local path</th></tr>

1355

# </thead>

1356

# <tbody>

1357

# <tr><td>file</td><td>file</td></tr>

1358

# <tr><td>glob</td><td>directory</td></tr>

# </tbody>

# </table>

#

# For outputs, the direction of the copy is reversed:

1363

#

1364

# ```

1365

# gsutil cp /mnt/disk/file.txt gs://my-bucket/bar.txt

1366

# ```

1367

#

1368

# Acceptable paths are:

#

# <table>

# <thead>

# <tr><th>Local path</th><th>Google Cloud Storage path</th></tr>

1373

# </thead>

1374

# <tbody>

1375

# <tr><td>file</td><td>file</td></tr>

1376

# <tr>

1377

# <td>file</td>

1378

# <td>directory - directory must already exist</td>

# </tr>

# <tr>

# <td>glob</td>

# <td>directory - directory will be created if it doesn't exist</td></tr>

# </tbody>

# </table>

#

# One restriction due to docker limitations, is that for outputs that are found

1387

# on the boot disk, the local path cannot be a glob and must be a file.

1388

"defaultValue": "A String", # The default value for this parameter. Can be overridden at runtime.

1389

# If `localCopy` is present, then this must be a Google Cloud Storage path

1390

# beginning with `gs://`.

1391

"localCopy": { # LocalCopy defines how a remote file should be copied to and from the VM. # If present, this parameter is marked for copying to and from the VM.

1392

# `LocalCopy` indicates where on the VM the file should be. The value

1393

# given to this parameter (either at runtime or using `defaultValue`)

1394

# must be the remote path where the file should be.

1395

"path": "A String", # Required. The path within the user's docker container where

1396

# this input should be localized to and from, relative to the specified

1397

# disk's mount point. For example: file.txt,

1398

"disk": "A String", # Required. The name of the disk where this parameter is

1399

# located. Can be the name of one of the disks specified in the

1400

# Resources field, or "boot", which represents the Docker

1401

# instance's boot disk and has a mount point of `/`.

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

1402

},

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

1403

"name": "A String", # Required. Name of the parameter - the pipeline runner uses this string

1404

# as the key to the input and output maps in RunPipeline.

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

1405

"description": "A String", # Human-readable description.

1406

},

1407

],

1408

"docker": { # The Docker execuctor specification. # Specifies the docker run information.

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

1409

"cmd": "A String", # Required. The command or newline delimited script to run. The command

1410

# string will be executed within a bash shell.

1411

#

1412

# If the command exits with a non-zero exit code, output parameter

1413

# de-localization will be skipped and the pipeline operation's

1414

# `error` field will be populated.

1415

#

1416

# Maximum command string length is 16384.

1417

"imageName": "A String", # Required. Image name from either Docker Hub or Google Container Registry.

1418

# Users that run pipelines must have READ access to the image.

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

1419

},

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

1420

"resources": { # The system resources for the pipeline run. # Required. Specifies resource requirements for the pipeline run.

# Required fields:

#

# *

# minimumCpuCores

#

# *

# minimumRamGb

"noAddress": True or False, # Whether to assign an external IP to the instance. This is an experimental

1429

# feature that may go away. Defaults to false.

1430

# Corresponds to `--no_address` flag for [gcloud compute instances create]

1431

# (https://cloud.google.com/sdk/gcloud/reference/compute/instances/create).

1432

# In order to use this, must be true for both create time and run time.

1433

# Cannot be true at run time if false at create time. If you need to ssh into

1434

# a private IP VM for debugging, you can ssh to a public VM and then ssh into

1435

# the private VM's Internal IP. If noAddress is set, this pipeline run may

1436

# only load docker images from Google Container Registry and not Docker Hub.

1437

# Before using this, you must

1438

# [configure access to Google services from internal IPs](https://cloud.google.com/compute/docs/configure-private-google-access#configuring_access_to_google_services_from_internal_ips).

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

1439

"minimumRamGb": 3.14, # The minimum amount of RAM to use. Defaults to 3.75 (GB)

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

1440

"preemptible": True or False, # Whether to use preemptible VMs. Defaults to `false`. In order to use this,

1441

# must be true for both create time and run time. Cannot be true at run time

1442

# if false at create time.

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

1443

"disks": [ # Disks to attach.

1444

{ # A Google Compute Engine disk resource specification.

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

1445

"name": "A String", # Required. The name of the disk that can be used in the pipeline

1446

# parameters. Must be 1 - 63 characters.

1447

# The name "boot" is reserved for system use.

1448

"sizeGb": 42, # The size of the disk. Defaults to 500 (GB).

1449

# This field is not applicable for local SSD.

1450

"autoDelete": True or False, # Deprecated. Disks created by the Pipelines API will be deleted at the end

1451

# of the pipeline run, regardless of what this field is set to.

1452

"source": "A String", # The full or partial URL of the persistent disk to attach. See

1453

# https://cloud.google.com/compute/docs/reference/latest/instances#resource

1454

# and

1455

# https://cloud.google.com/compute/docs/disks/persistent-disks#snapshots

1456

# for more details.

1457

"mountPoint": "A String", # Required at create time and cannot be overridden at run time.

1458

# Specifies the path in the docker container where files on

1459

# this disk should be located. For example, if `mountPoint`

1460

# is `/mnt/disk`, and the parameter has `localPath`

1461

# `inputs/file.txt`, the docker container can access the data at

1462

# `/mnt/disk/inputs/file.txt`.

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

1463

"type": "A String", # Required. The type of the disk to create.

1464

},

1465

],

1466

"bootDiskSizeGb": 42, # The size of the boot disk. Defaults to 10 (GB).

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

1467

"zones": [ # List of Google Compute Engine availability zones to which resource

1468

# creation will restricted. If empty, any zone may be chosen.

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

1469

"A String",

1470

],

1471

"minimumCpuCores": 42, # The minimum number of cores to use. Defaults to 1.

1472

},

1473

"description": "A String", # User-specified description.

},

],

}</pre>

</div>

<code class="details" id="list_next">list_next(previous_request, previous_response)</code>

1481

<pre>Retrieves the next page of results.

1482

1483

Args:

1484

previous_request: The request for the previous page. (required)

1485

previous_response: The response from the request for the previous page. (required)

1486

1487

Returns:

1488

A request object that you can call 'execute()' on to request the next

1489

page. Returns None if there are no more items in the collection.

</pre>

</div>

<code class="details" id="run">run(body, x__xgafv=None)</code>

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

1495

<pre>Runs a pipeline. If `pipelineId` is specified in the request, then

1496

run a saved pipeline. If `ephemeralPipeline` is specified, then run

1497

that pipeline once without saving a copy.

1498

1499

The caller must have READ permission to the project where the pipeline

1500

is stored and WRITE permission to the project where the pipeline will be

1501

run, as VMs will be created and storage will be used.

1502

1503

If a pipeline operation is still running after 6 days, it will be canceled.

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

1504

1505

Args:

1506

body: object, The request body. (required)

1507

The object takes the form of:

1508

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

1509

{ # The request to run a pipeline. If `pipelineId` is specified, it

1510

# refers to a saved pipeline created with CreatePipeline and set as

1511

# the `pipelineId` of the returned Pipeline object. If

1512

# `ephemeralPipeline` is specified, that pipeline is run once

1513

# with the given args and not saved. It is an error to specify both

1514

# `pipelineId` and `ephemeralPipeline`. `pipelineArgs`

1515

# must be specified.

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

1516

"pipelineArgs": { # The pipeline run arguments. # The arguments to use when running this pipeline.

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

1517

"inputs": { # Pipeline input arguments; keys are defined in the pipeline documentation.

1518

# All input parameters that do not have default values must be specified.

1519

# If parameters with defaults are specified here, the defaults will be

1520

# overridden.

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

1521

"a_key": "A String",

1522

},

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

1523

"logging": { # The logging options for the pipeline run. # Required. Logging options. Used by the service to communicate results

1524

# to the user.

1525

"gcsPath": "A String", # The location in Google Cloud Storage to which the pipeline logs

1526

# will be copied. Can be specified as a fully qualified directory

1527

# path, in which case logs will be output with a unique identifier

1528

# as the filename in that directory, or as a fully specified path,

1529

# which must end in `.log`, in which case that path will be

1530

# used, and the user must ensure that logs are not

1531

# overwritten. Stdout and stderr logs from the run are also

1532

# generated and output as `-stdout.log` and `-stderr.log`.

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

1533

},

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

1534

"outputs": { # Pipeline output arguments; keys are defined in the pipeline

1535

# documentation. All output parameters of without default values

1536

# must be specified. If parameters with defaults are specified

1537

# here, the defaults will be overridden.

Jon Wayne Parrott

692617a

2017-01-06 09:58:29 -0800

[diff] [blame]

1538

"a_key": "A String",

1539

},

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

1540

"labels": { # Labels to apply to this pipeline run. Labels will also be applied to

1541

# compute resources (VM, disks) created by this pipeline run. When listing

1542

# operations, operations can filtered by labels.

1543

# Label keys may not be empty; label values may be empty. Non-empty labels

1544

# must be 1-63 characters long, and comply with [RFC1035]

1545

# (https://www.ietf.org/rfc/rfc1035.txt).

1546

# Specifically, the name must be 1-63 characters long and match the regular

1547

# expression `[a-z]([-a-z0-9]*[a-z0-9])?` which means the first

1548

# character must be a lowercase letter, and all following characters must be

1549

# a dash, lowercase letter, or digit, except the last character, which cannot

# be a dash.

"a_key": "A String",

},

"serviceAccount": { # A Google Cloud Service Account. # The Google Cloud Service Account that will be used to access data and

1554

# services. By default, the compute service account associated with

1555

# `projectId` is used.

1556

"scopes": [ # List of scopes to be enabled for this service account on the VM.

1557

# The following scopes are automatically included:

1558

#

1559

# * https://www.googleapis.com/auth/compute

1560

# * https://www.googleapis.com/auth/devstorage.full_control

1561

# * https://www.googleapis.com/auth/genomics

1562

# * https://www.googleapis.com/auth/logging.write

1563

# * https://www.googleapis.com/auth/monitoring.write

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

1564

"A String",

1565

],

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

1566

"email": "A String", # Email address of the service account. Defaults to `default`,

1567

# which uses the compute service account associated with the project.

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

1568

},

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

1569

"clientId": "A String", # This field is deprecated. Use `labels` instead. Client-specified pipeline

1570

# operation identifier.

1571

"projectId": "A String", # Required. The project in which to run the pipeline. The caller must have

1572

# WRITER access to all Google Cloud services and resources (e.g. Google

1573

# Compute Engine) will be used.

1574

"keepVmAliveOnFailureDuration": "A String", # How long to keep the VM up after a failure (for example docker command

1575

# failed, copying input or output files failed, etc). While the VM is up, one

1576

# can ssh into the VM to debug. Default is 0; maximum allowed value is 1 day.

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

1577

"resources": { # The system resources for the pipeline run. # Specifies resource requirements/overrides for the pipeline run.

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

1578

"noAddress": True or False, # Whether to assign an external IP to the instance. This is an experimental

1579

# feature that may go away. Defaults to false.

1580

# Corresponds to `--no_address` flag for [gcloud compute instances create]

1581

# (https://cloud.google.com/sdk/gcloud/reference/compute/instances/create).

1582

# In order to use this, must be true for both create time and run time.

1583

# Cannot be true at run time if false at create time. If you need to ssh into

1584

# a private IP VM for debugging, you can ssh to a public VM and then ssh into

1585

# the private VM's Internal IP. If noAddress is set, this pipeline run may

1586

# only load docker images from Google Container Registry and not Docker Hub.

1587

# Before using this, you must

1588

# [configure access to Google services from internal IPs](https://cloud.google.com/compute/docs/configure-private-google-access#configuring_access_to_google_services_from_internal_ips).

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

1589

"minimumRamGb": 3.14, # The minimum amount of RAM to use. Defaults to 3.75 (GB)

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

1590

"preemptible": True or False, # Whether to use preemptible VMs. Defaults to `false`. In order to use this,

1591

# must be true for both create time and run time. Cannot be true at run time

1592

# if false at create time.

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

1593

"disks": [ # Disks to attach.

1594

{ # A Google Compute Engine disk resource specification.

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

1595

"name": "A String", # Required. The name of the disk that can be used in the pipeline

1596

# parameters. Must be 1 - 63 characters.

1597

# The name "boot" is reserved for system use.

1598

"sizeGb": 42, # The size of the disk. Defaults to 500 (GB).

1599

# This field is not applicable for local SSD.

1600

"autoDelete": True or False, # Deprecated. Disks created by the Pipelines API will be deleted at the end

1601

# of the pipeline run, regardless of what this field is set to.

1602

"source": "A String", # The full or partial URL of the persistent disk to attach. See

1603

# https://cloud.google.com/compute/docs/reference/latest/instances#resource

1604

# and

1605

# https://cloud.google.com/compute/docs/disks/persistent-disks#snapshots

1606

# for more details.

1607

"mountPoint": "A String", # Required at create time and cannot be overridden at run time.

1608

# Specifies the path in the docker container where files on

1609

# this disk should be located. For example, if `mountPoint`

1610

# is `/mnt/disk`, and the parameter has `localPath`

1611

# `inputs/file.txt`, the docker container can access the data at

1612

# `/mnt/disk/inputs/file.txt`.

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

1613

"type": "A String", # Required. The type of the disk to create.

1614

},

1615

],

1616

"bootDiskSizeGb": 42, # The size of the boot disk. Defaults to 10 (GB).

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

1617

"zones": [ # List of Google Compute Engine availability zones to which resource

1618

# creation will restricted. If empty, any zone may be chosen.

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

1619

"A String",

1620

],

1621

"minimumCpuCores": 42, # The minimum number of cores to use. Defaults to 1.

1622

},

1623

},

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

1624

"ephemeralPipeline": { # The pipeline object. Represents a transformation from a set of input # A new pipeline object to run once and then delete.

1625

# parameters to a set of output parameters. The transformation is defined

1626

# as a docker image and command to run within that image. Each pipeline

1627

# is run on a Google Compute Engine VM. A pipeline can be created with the

1628

# `create` method and then later run with the `run` method, or a pipeline can

1629

# be defined and run all at once with the `run` method.

1630

"pipelineId": "A String", # Unique pipeline id that is generated by the service when CreatePipeline

1631

# is called. Cannot be specified in the Pipeline used in the

1632

# CreatePipelineRequest, and will be populated in the response to

1633

# CreatePipeline and all subsequent Get and List calls. Indicates that the

1634

# service has registered this pipeline.

1635

"name": "A String", # Required. A user specified pipeline name that does not have to be unique.

1636

# This name can be used for filtering Pipelines in ListPipelines.

1637

"projectId": "A String", # Required. The project in which to create the pipeline. The caller must have

1638

# WRITE access.

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

1639

"outputParameters": [ # Output parameters of the pipeline.

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

1640

{ # Parameters facilitate setting and delivering data into the

1641

# pipeline's execution environment. They are defined at create time,

1642

# with optional defaults, and can be overridden at run time.

1643

#

1644

# If `localCopy` is unset, then the parameter specifies a string that

1645

# is passed as-is into the pipeline, as the value of the environment

1646

# variable with the given name. A default value can be optionally

1647

# specified at create time. The default can be overridden at run time

1648

# using the inputs map. If no default is given, a value must be

1649

# supplied at runtime.

1650

#

1651

# If `localCopy` is defined, then the parameter specifies a data

1652

# source or sink, both in Google Cloud Storage and on the Docker container

1653

# where the pipeline computation is run. The service account associated with

1654

# the Pipeline (by

1655

# default the project's Compute Engine service account) must have access to the

1656

# Google Cloud Storage paths.

1657

#

1658

# At run time, the Google Cloud Storage paths can be overridden if a default

1659

# was provided at create time, or must be set otherwise. The pipeline runner

1660

# should add a key/value pair to either the inputs or outputs map. The

1661

# indicated data copies will be carried out before/after pipeline execution,

1662

# just as if the corresponding arguments were provided to `gsutil cp`.

1663

#

1664

# For example: Given the following `PipelineParameter`, specified

1665

# in the `inputParameters` list:

1666

#

1667

# ```

1668

# {name: "input_file", localCopy: {path: "file.txt", disk: "pd1"}}

1669

# ```

1670

#

1671

# where `disk` is defined in the `PipelineResources` object as:

1672

#

1673

# ```

1674

# {name: "pd1", mountPoint: "/mnt/disk/"}

1675

# ```

1676

#

1677

# We create a disk named `pd1`, mount it on the host VM, and map

1678

# `/mnt/pd1` to `/mnt/disk` in the docker container. At

1679

# runtime, an entry for `input_file` would be required in the inputs

# map, such as:

#

# ```

# inputs["input_file"] = "gs://my-bucket/bar.txt"

1684

# ```

1685

#

1686

# This would generate the following gsutil call:

1687

#

1688

# ```

1689

# gsutil cp gs://my-bucket/bar.txt /mnt/pd1/file.txt

1690

# ```

1691

#

1692

# The file `/mnt/pd1/file.txt` maps to `/mnt/disk/file.txt` in the

1693

# Docker container. Acceptable paths are:

#

# <table>

# <thead>

# <tr><th>Google Cloud storage path</th><th>Local path</th></tr>

1698

# </thead>

1699

# <tbody>

1700

# <tr><td>file</td><td>file</td></tr>

1701

# <tr><td>glob</td><td>directory</td></tr>

# </tbody>

# </table>

#

# For outputs, the direction of the copy is reversed:

1706

#

1707

# ```

1708

# gsutil cp /mnt/disk/file.txt gs://my-bucket/bar.txt

1709

# ```

1710

#

1711

# Acceptable paths are:

#

# <table>

# <thead>

# <tr><th>Local path</th><th>Google Cloud Storage path</th></tr>

1716

# </thead>

1717

# <tbody>

1718

# <tr><td>file</td><td>file</td></tr>

1719

# <tr>

1720

# <td>file</td>

1721

# <td>directory - directory must already exist</td>

# </tr>

# <tr>

# <td>glob</td>

# <td>directory - directory will be created if it doesn't exist</td></tr>

# </tbody>

# </table>

#

# One restriction due to docker limitations, is that for outputs that are found

1730

# on the boot disk, the local path cannot be a glob and must be a file.

1731

"defaultValue": "A String", # The default value for this parameter. Can be overridden at runtime.

1732

# If `localCopy` is present, then this must be a Google Cloud Storage path

1733

# beginning with `gs://`.

1734

"localCopy": { # LocalCopy defines how a remote file should be copied to and from the VM. # If present, this parameter is marked for copying to and from the VM.

1735

# `LocalCopy` indicates where on the VM the file should be. The value

1736

# given to this parameter (either at runtime or using `defaultValue`)

1737

# must be the remote path where the file should be.

1738

"path": "A String", # Required. The path within the user's docker container where

1739

# this input should be localized to and from, relative to the specified

1740

# disk's mount point. For example: file.txt,

1741

"disk": "A String", # Required. The name of the disk where this parameter is

1742

# located. Can be the name of one of the disks specified in the

1743

# Resources field, or "boot", which represents the Docker

1744

# instance's boot disk and has a mount point of `/`.

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

1745

},

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

1746

"name": "A String", # Required. Name of the parameter - the pipeline runner uses this string

1747

# as the key to the input and output maps in RunPipeline.

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

1748

"description": "A String", # Human-readable description.

1749

},

1750

],

1751

"inputParameters": [ # Input parameters of the pipeline.

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

1752

{ # Parameters facilitate setting and delivering data into the

1753

# pipeline's execution environment. They are defined at create time,

1754

# with optional defaults, and can be overridden at run time.

1755

#

1756

# If `localCopy` is unset, then the parameter specifies a string that

1757

# is passed as-is into the pipeline, as the value of the environment

1758

# variable with the given name. A default value can be optionally

1759

# specified at create time. The default can be overridden at run time

1760

# using the inputs map. If no default is given, a value must be

1761

# supplied at runtime.

1762

#

1763

# If `localCopy` is defined, then the parameter specifies a data

1764

# source or sink, both in Google Cloud Storage and on the Docker container

1765

# where the pipeline computation is run. The service account associated with

1766

# the Pipeline (by

1767

# default the project's Compute Engine service account) must have access to the

1768

# Google Cloud Storage paths.

1769

#

1770

# At run time, the Google Cloud Storage paths can be overridden if a default

1771

# was provided at create time, or must be set otherwise. The pipeline runner

1772

# should add a key/value pair to either the inputs or outputs map. The

1773

# indicated data copies will be carried out before/after pipeline execution,

1774

# just as if the corresponding arguments were provided to `gsutil cp`.

1775

#

1776

# For example: Given the following `PipelineParameter`, specified

1777

# in the `inputParameters` list:

1778

#

1779

# ```

1780

# {name: "input_file", localCopy: {path: "file.txt", disk: "pd1"}}

1781

# ```

1782

#

1783

# where `disk` is defined in the `PipelineResources` object as:

1784

#

1785

# ```

1786

# {name: "pd1", mountPoint: "/mnt/disk/"}

1787

# ```

1788

#

1789

# We create a disk named `pd1`, mount it on the host VM, and map

1790

# `/mnt/pd1` to `/mnt/disk` in the docker container. At

1791

# runtime, an entry for `input_file` would be required in the inputs

# map, such as:

#

# ```

# inputs["input_file"] = "gs://my-bucket/bar.txt"

1796

# ```

1797

#

1798

# This would generate the following gsutil call:

1799

#

1800

# ```

1801

# gsutil cp gs://my-bucket/bar.txt /mnt/pd1/file.txt

1802

# ```

1803

#

1804

# The file `/mnt/pd1/file.txt` maps to `/mnt/disk/file.txt` in the

1805

# Docker container. Acceptable paths are:

#

# <table>

# <thead>

# <tr><th>Google Cloud storage path</th><th>Local path</th></tr>

1810

# </thead>

1811

# <tbody>

1812

# <tr><td>file</td><td>file</td></tr>

1813

# <tr><td>glob</td><td>directory</td></tr>

# </tbody>

# </table>

#

# For outputs, the direction of the copy is reversed:

1818

#

1819

# ```

1820

# gsutil cp /mnt/disk/file.txt gs://my-bucket/bar.txt

1821

# ```

1822

#

1823

# Acceptable paths are:

#

# <table>

# <thead>

# <tr><th>Local path</th><th>Google Cloud Storage path</th></tr>

1828

# </thead>

1829

# <tbody>

1830

# <tr><td>file</td><td>file</td></tr>

1831

# <tr>

1832

# <td>file</td>

1833

# <td>directory - directory must already exist</td>

# </tr>

# <tr>

# <td>glob</td>

# <td>directory - directory will be created if it doesn't exist</td></tr>

# </tbody>

# </table>

#

# One restriction due to docker limitations, is that for outputs that are found

1842

# on the boot disk, the local path cannot be a glob and must be a file.

1843

"defaultValue": "A String", # The default value for this parameter. Can be overridden at runtime.

1844

# If `localCopy` is present, then this must be a Google Cloud Storage path

1845

# beginning with `gs://`.

1846

"localCopy": { # LocalCopy defines how a remote file should be copied to and from the VM. # If present, this parameter is marked for copying to and from the VM.

1847

# `LocalCopy` indicates where on the VM the file should be. The value

1848

# given to this parameter (either at runtime or using `defaultValue`)

1849

# must be the remote path where the file should be.

1850

"path": "A String", # Required. The path within the user's docker container where

1851

# this input should be localized to and from, relative to the specified

1852

# disk's mount point. For example: file.txt,

1853

"disk": "A String", # Required. The name of the disk where this parameter is

1854

# located. Can be the name of one of the disks specified in the

1855

# Resources field, or "boot", which represents the Docker

1856

# instance's boot disk and has a mount point of `/`.

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

1857

},

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

1858

"name": "A String", # Required. Name of the parameter - the pipeline runner uses this string

1859

# as the key to the input and output maps in RunPipeline.

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

1860

"description": "A String", # Human-readable description.

1861

},

1862

],

1863

"docker": { # The Docker execuctor specification. # Specifies the docker run information.

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

1864

"cmd": "A String", # Required. The command or newline delimited script to run. The command

1865

# string will be executed within a bash shell.

1866

#

1867

# If the command exits with a non-zero exit code, output parameter

1868

# de-localization will be skipped and the pipeline operation's

1869

# `error` field will be populated.

1870

#

1871

# Maximum command string length is 16384.

1872

"imageName": "A String", # Required. Image name from either Docker Hub or Google Container Registry.

1873

# Users that run pipelines must have READ access to the image.

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

1874

},

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

1875

"resources": { # The system resources for the pipeline run. # Required. Specifies resource requirements for the pipeline run.

# Required fields:

#

# *

# minimumCpuCores

#

# *

# minimumRamGb

"noAddress": True or False, # Whether to assign an external IP to the instance. This is an experimental

1884

# feature that may go away. Defaults to false.

1885

# Corresponds to `--no_address` flag for [gcloud compute instances create]

1886

# (https://cloud.google.com/sdk/gcloud/reference/compute/instances/create).

1887

# In order to use this, must be true for both create time and run time.

1888

# Cannot be true at run time if false at create time. If you need to ssh into

1889

# a private IP VM for debugging, you can ssh to a public VM and then ssh into

1890

# the private VM's Internal IP. If noAddress is set, this pipeline run may

1891

# only load docker images from Google Container Registry and not Docker Hub.

1892

# Before using this, you must

1893

# [configure access to Google services from internal IPs](https://cloud.google.com/compute/docs/configure-private-google-access#configuring_access_to_google_services_from_internal_ips).

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

1894

"minimumRamGb": 3.14, # The minimum amount of RAM to use. Defaults to 3.75 (GB)

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

1895

"preemptible": True or False, # Whether to use preemptible VMs. Defaults to `false`. In order to use this,

1896

# must be true for both create time and run time. Cannot be true at run time

1897

# if false at create time.

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

1898

"disks": [ # Disks to attach.

1899

{ # A Google Compute Engine disk resource specification.

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

1900

"name": "A String", # Required. The name of the disk that can be used in the pipeline

1901

# parameters. Must be 1 - 63 characters.

1902

# The name "boot" is reserved for system use.

1903

"sizeGb": 42, # The size of the disk. Defaults to 500 (GB).

1904

# This field is not applicable for local SSD.

1905

"autoDelete": True or False, # Deprecated. Disks created by the Pipelines API will be deleted at the end

1906

# of the pipeline run, regardless of what this field is set to.

1907

"source": "A String", # The full or partial URL of the persistent disk to attach. See

1908

# https://cloud.google.com/compute/docs/reference/latest/instances#resource

1909

# and

1910

# https://cloud.google.com/compute/docs/disks/persistent-disks#snapshots

1911

# for more details.

1912

"mountPoint": "A String", # Required at create time and cannot be overridden at run time.

1913

# Specifies the path in the docker container where files on

1914

# this disk should be located. For example, if `mountPoint`

1915

# is `/mnt/disk`, and the parameter has `localPath`

1916

# `inputs/file.txt`, the docker container can access the data at

1917

# `/mnt/disk/inputs/file.txt`.

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

1918

"type": "A String", # Required. The type of the disk to create.

1919

},

1920

],

1921

"bootDiskSizeGb": 42, # The size of the boot disk. Defaults to 10 (GB).

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

1922

"zones": [ # List of Google Compute Engine availability zones to which resource

1923

# creation will restricted. If empty, any zone may be chosen.

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

1924

"A String",

1925

],

1926

"minimumCpuCores": 42, # The minimum number of cores to use. Defaults to 1.

1927

},

1928

"description": "A String", # User-specified description.

1929

},

1930

"pipelineId": "A String", # The already created pipeline to run.

1931

}

1932

1933

x__xgafv: string, V1 error format.

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

1934

Allowed values

1935

1 - v1 error format

1936

2 - v2 error format

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

1937

1938

Returns:

1939

An object of the form:

1940

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

1941

{ # This resource represents a long-running operation that is the result of a

1942

# network API call.

1943

"metadata": { # An OperationMetadata object. This will always be returned with the Operation.

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

1944

"a_key": "", # Properties of the object. Contains field @type with type URL.

1945

},

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

1946

"error": { # The `Status` type defines a logical error model that is suitable for different # The error result of the operation in case of failure or cancellation.

1947

# programming environments, including REST APIs and RPC APIs. It is used by

1948

# [gRPC](https://github.com/grpc). The error model is designed to be:

1949

#

1950

# - Simple to use and understand for most users

1951

# - Flexible enough to meet unexpected needs

#

# # Overview

#

# The `Status` message contains three pieces of data: error code, error message,

1956

# and error details. The error code should be an enum value of

1957

# google.rpc.Code, but it may accept additional error codes if needed. The

1958

# error message should be a developer-facing English message that helps

1959

# developers *understand* and *resolve* the error. If a localized user-facing

1960

# error message is needed, put the localized message in the error details or

1961

# localize it in the client. The optional error details may contain arbitrary

1962

# information about the error. There is a predefined set of error detail types

1963

# in the package `google.rpc` which can be used for common error conditions.

#

# # Language mapping

#

# The `Status` message is the logical representation of the error model, but it

1968

# is not necessarily the actual wire format. When the `Status` message is

1969

# exposed in different client libraries and different wire protocols, it can be

1970

# mapped differently. For example, it will likely be mapped to some exceptions

1971

# in Java, but more likely mapped to some error codes in C.

#

# # Other uses

#

# The error model and the `Status` message can be used in a variety of

1976

# environments, either with or without APIs, to provide a

1977

# consistent developer experience across different environments.

1978

#

1979

# Example uses of this error model include:

1980

#

1981

# - Partial errors. If a service needs to return partial errors to the client,

1982

# it may embed the `Status` in the normal response to indicate the partial

1983

# errors.

1984

#

1985

# - Workflow errors. A typical workflow has multiple steps. Each step may

1986

# have a `Status` message for error reporting purpose.

1987

#

1988

# - Batch operations. If a client uses batch request and batch response, the

1989

# `Status` message should be used directly inside batch response, one for

1990

# each error sub-response.

1991

#

1992

# - Asynchronous operations. If an API call embeds asynchronous operation

1993

# results in its response, the status of those operations should be

1994

# represented directly using the `Status` message.

1995

#

1996

# - Logging. If some API errors are stored in logs, the message `Status` could

1997

# be used directly after any stripping needed for security/privacy reasons.

1998

"message": "A String", # A developer-facing error message, which should be in English. Any

1999

# user-facing error message should be localized and sent in the

2000

# google.rpc.Status.details field, or localized by the client.

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

2001

"code": 42, # The status code, which should be an enum value of google.rpc.Code.

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

2002

"details": [ # A list of messages that carry the error details. There will be a

2003

# common set of message types for APIs to use.

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

2004

{

2005

"a_key": "", # Properties of the object. Contains field @type with type URL.

2006

},

2007

],

2008

},

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

2009

"done": True or False, # If the value is `false`, it means the operation is still in progress.

2010

# If true, the operation is completed, and either `error` or `response` is

2011

# available.

2012

"response": { # If importing ReadGroupSets, an ImportReadGroupSetsResponse is returned. If importing Variants, an ImportVariantsResponse is returned. For pipelines and exports, an empty response is returned.

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

2013

"a_key": "", # Properties of the object. Contains field @type with type URL.

2014

},

Sai Cheemalapati

c30d2b5

2017-03-13 12:12:03 -0400

[diff] [blame^]

2015

"name": "A String", # The server-assigned name, which is only unique within the same service that originally returns it. For example: `operations/CJHU7Oi_ChDrveSpBRjfuL-qzoWAgEw`

Sai Cheemalapati

df61397

2016-10-21 13:59:49 -0700

[diff] [blame]

}</pre>

</div>

<code class="details" id="setOperationStatus">setOperationStatus(body, x__xgafv=None)</code>