workflows

Paperspace Gradient CLI reference guide for workflows.

create

Create workflow

Examples

CLI
curl
Python

gradient workflows create \
  --name "NewWorkflow" \
  --projectId "prjpkflqz"

curl -X POST 'https://api.paperspace.io/workflows' \
-H 'x-api-key: d44808a2785d6a...' \
-H 'Content-Type: application/json' \
--data-raw '{
    "name": "New-Workflow",
    "projectId": "prjpkflqz"
}'

from gradient import WorkflowsClient

api_key='d44808a2785d6a...'

workflow_client = WorkflowsClient(api_key)

print(workflow_client.create(
    name='New-Workflow',
    project_id='prjpkflqz'
))

Options

Name	Type	Attributes	Description
`--name`	string	required	Workflow name
`--projectId`	string	required	Project ID
`--apiKey`	string	optional	API key to use this time only
`--optionsFile`	string	optional	Path to YAML with predefined options
`--createOptionsFile`	string	optional	Generate template options file

Response

CLI
curl
Python

+--------------+--------------------------------------+
| Name         | ID                                   |
+--------------+--------------------------------------+
| New-Workflow | 075615ac-a036-408c-a95e-36e5b18c73aa |
+--------------+--------------------------------------+

{
  "id": "075615ac-a036-408c-a95e-36e5b18c73aa",
  "name": "New-Workflow",
  "dtCreated": "2022-03-22T08:15:17.147Z",
  "dtModified": "2022-03-22T08:15:17.147Z"
}

{
  "id": "075615ac-a036-408c-a95e-36e5b18c73aa",
  "name": "New-Workflow",
  "dtCreated": "2022-03-22T08:15:17.147Z",
  "dtModified": "2022-03-22T08:15:17.147Z"
}

get

Get workflow info

Examples

CLI
curl
Python

gradient workflows get \
  --id "075615ac-a036-408c-a95e-36e5b18c73aa"

curl -X GET 'https://api.paperspace.io/workflows/075615ac-a036-408c-a95e-36e5b18c73aa' \
-H 'x-api-key: d44808a2785d6a...'

from gradient import WorkflowsClient

api_key='d44808a2785d6a...'

workflow_client = WorkflowsClient(api_key)

print(workflow_client.get(
    workflow_id='075615ac-a036-408c-a95e-36e5b18c73aa'
))

Options

Name	Type	Attributes	Description
`--id`	string	required	Workflow ID
`--show-runs`		optional	Fetch runs
`--run`	string	optional	Specify workload run
`--apiKey`	string	optional	API key to use this time only
`--optionsFile`	string	optional	Path to YAML with predefined options
`--createOptionsFile`	string	optional	Generate template options file

Response

CLI
curl
Python

{
  "id": "075615ac-a036-408c-a95e-36e5b18c73aa",
  "name": "New-Workflow",
  "dtCreated": "2022-03-22T08:10:29.677Z",
  "dtModified": "2022-03-22T08:10:29.677Z"
}

{
  "id": "075615ac-a036-408c-a95e-36e5b18c73aa",
  "name": "New-Workflow",
  "dtCreated": "2022-03-22T08:10:29.677Z",
  "dtModified": "2022-03-22T08:10:29.677Z"
}

{
  "id": "075615ac-a036-408c-a95e-36e5b18c73aa",
  "name": "New-Workflow",
  "dtCreated": "2022-03-22T08:10:29.677Z",
  "dtModified": "2022-03-22T08:10:29.677Z"
}

list

List workflows

Examples

CLI
curl
Python

gradient workflows list \
  --projectId "prjpkflqz"

curl -g -X GET 'https://api.paperspace.io/workflows?filter[where][projectId]=prjpkflqz' \
-H 'x-api-key: d44808a2785d6a...'

from gradient import WorkflowsClient

api_key='d44808a2785d6a...'

workflow_client = WorkflowsClient(api_key)

print(workflow_client.list(
    project_id='prjpkflqz'
))

Options

Name	Type	Attributes	Description
`--projectId`	string	required	Project ID
`--apiKey`	string	optional	API key to use this time only
`--optionsFile`	string	optional	Path to YAML with predefined options
`--createOptionsFile`	string	optional	Generate template options file

Response

CLI
curl
Python

+---------------+--------------------------------------+
| Name          | ID                                   |
+---------------+--------------------------------------+
| New-Workflow  | 075615ac-a036-408c-a95e-36e5b18c73aa |
+---------------+--------------------------------------+

[
  {
    "id": "075615ac-a036-408c-a95e-36e5b18c73aa",
    "name": "New-Workflow",
    "dtCreated": "2022-03-22T08:10:29.677Z",
    "dtModified": "2022-03-22T08:10:29.677Z"
  }
]

[
  Workflow(
    (id = "075615ac-a036-408c-a95e-36e5b18c73aa"),
    (team_id = None),
    (project_id = None),
    (name = "New-Workflow"),
    (workflow_spec_id = None),
    (dt_deleted = None),
    (dt_created = datetime.datetime(
      2022,
      3,
      22,
      8,
      10,
      29,
      677000,
      (tzinfo = tzutc())
    )),
    (dt_modified = datetime.datetime(
      2022,
      3,
      22,
      8,
      10,
      29,
      677000,
      (tzinfo = tzutc())
    ))
  ),
];

logs

List logs for specific workflow

Examples

CLI
curl
Python

gradient workflows logs \
  --id "075615ac-a036-408c-a95e-36e5b18c73aa" \
  --run 1

# First query to get logId values which will be used for jobId in subsequent steps

curl -X GET 'https://api.paperspace.io/workflows/075615ac-a036-408c-a95e-36e5b18c73aa/runs/1' \
-H 'x-api-key: d44808a2785d6a...'

# Subsequent queries to get logs for each job

curl -X GET 'https://logs.paperspace.io/jobs/logs?jobId=wfrj67d8451ea1bf49749925755f5e61a46e' \
-H 'x-api-key: d44808a2785d6a...'

# Another query

curl -X GET 'https://logs.paperspace.io/jobs/logs?jobId=wfrjaedc275373ec4d239f3ea1af029dfd60' \
-H 'x-api-key: d44808a2785d6a...'

from gradient import WorkflowsClient

api_key = 'd44808a2785d6a...'

workflow_client = WorkflowsClient(api_key)

print(
    workflow_client.logs(job_id='wfrj67d8451ea1bf49749925755f5e61a46e'),
    workflow_client.logs(job_id='wfrjaedc275373ec4d239f3ea1af029dfd60')
)

Options

Name	Type	Attributes	Description
`--id`	string	required	Workflow ID
`--run`	string	required	Specify workload run
`--logId`	string	optional	Workflow log ID
`--line`	integer	optional	Specific line number of logs
`--limit`	integer	optional	Limit of logs to fetch
`--apiKey`	string	optional	API key to use this time only
`--optionsFile`	string	optional	Path to YAML with predefined options
`--createOptionsFile`	string	optional	Generate template options file

Response

CLI
curl
Python

+Workflows wfrjaedc275373ec4d239f3ea1af029dfd60 logs------------------------------------------------------------------------------------------------------------------------------------+
| LINE | MESSAGE                                                                                                                                                                        |
+------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| 1    | Collecting scipy==1.3.3                                                                                                                                                        |
| 2    |   Downloading https://files.pythonhosted.org/packages/54/18/d7c101d5e93b6c78dc206fcdf7bd04c1f8138a7b1a93578158fa3b132b08/scipy-1.3.3-cp36-cp36m-manylinux1_x86_64.whl (25.2MB) |
| 3    | Requirement already satisfied: numpy>=1.13.3 in /usr/local/lib/python3.6/dist-packages (from scipy==1.3.3) (1.16.4)                                                            |
| 4    | Installing collected packages: scipy                                                                                                                                           |
| 5    | Successfully installed scipy-1.3.3                                                                                                                                             |
| 6    | WARNING: You are using pip version 19.1.1, however version 21.3.1 is available.                                                                                                |
| 7    | You should consider upgrading via the 'pip install --upgrade pip' command.                                                                                                     |
| 8    | Collecting requests==2.22.0                                                                                                                                                    |
| 9    |   Downloading https://files.pythonhosted.org/packages/51/bd/23c926cd341ea6b7dd0b2a00aba99ae0f828be89d72b2190f27c11d4b7fb/requests-2.22.0-py2.py3-none-any.whl (57kB)           |
| 10   | Collecting chardet<3.1.0,>=3.0.2 (from requests==2.22.0)                                                                                                                       |
| 11   |   Downloading https://files.pythonhosted.org/packages/bc/a9/01ffebfb562e4274b6487b4bb1ddec7ca55ec7510b22e4c51f14098443b8/chardet-3.0.4-py2.py3-none-any.whl (133kB)            |
| 12   | Collecting certifi>=2017.4.17 (from requests==2.22.0)                                                                                                                          |
| 13   |   Downloading https://files.pythonhosted.org/packages/37/45/946c02767aabb873146011e665728b680884cd8fe70dde973c640e45b775/certifi-2021.10.8-py2.py3-none-any.whl (149kB)        |
| 14   | Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 (from requests==2.22.0)                                                                                                     |
| 15   |   Downloading https://files.pythonhosted.org/packages/56/aa/4ef5aa67a9a62505db124a5cb5262332d1d4153462eb8fd89c9fa41e5d92/urllib3-1.25.11-py2.py3-none-any.whl (127kB)          |
| 16   | Requirement already satisfied: idna<2.9,>=2.5 in /usr/lib/python3/dist-packages (from requests==2.22.0) (2.6)                                                                  |
| 17   | Installing collected packages: chardet, certifi, urllib3, requests                                                                                                             |
| 18   | Successfully installed certifi-2021.10.8 chardet-3.0.4 requests-2.22.0 urllib3-1.25.11                                                                                         |
| 19   | WARNING: You are using pip version 19.1.1, however version 21.3.1 is available.                                                                                                |
| 20   | You should consider upgrading via the 'pip install --upgrade pip' command.                                                                                                     |
| 21   | Collecting Pillow==6.2.1                                                                                                                                                       |
| 22   |   Downloading https://files.pythonhosted.org/packages/10/5c/0e94e689de2476c4c5e644a3bd223a1c1b9e2bdb7c510191750be74fa786/Pillow-6.2.1-cp36-cp36m-manylinux1_x86_64.whl (2.1MB) |
| 23   | Installing collected packages: Pillow                                                                                                                                          |
| 24   | Successfully installed Pillow-6.2.1                                                                                                                                            |
| 25   | WARNING: You are using pip version 19.1.1, however version 21.3.1 is available.                                                                                                |
| 26   | You should consider upgrading via the 'pip install --upgrade pip' command.                                                                                                     |
+------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+

// Response to first query
{
  "id": 1,
  "uuid": "4dbd74f4-03d1-46fc-97cb-15a76805f3a1",
  "meta": {},
  "cluster": { "id": "clg07azjl" },
  "spec": {
    "jobs": {
      "CloneRepo": {
        "uses": "git-checkout@v1",
        "with": { "url": "https://github.com/NVlabs/stylegan2.git" },
        "outputs": { "repo": { "type": "volume" } },
        "resources": { "instance-type": "C5" }
      },
      "StyleGan2": {
        "uses": "script@v1",
        "with": {
          "image": "tensorflow/tensorflow:1.14.0-gpu-py3",
          "script": "pip install scipy==1.3.3\npip install requests==2.22.0\npip install Pillow==6.2.1\ncd /inputs/repo\npython run_generator.py generate-images \\\n  --network=gdrive:networks/stylegan2-ffhq-config-f.pkl \\\n  --seeds=6600-6605 \\\n  --truncation-psi=0.5 \\\n  --result-dir=/outputs/generatedFaces"
        },
        "needs": ["CloneRepo"],
        "inputs": { "repo": "CloneRepo.outputs.repo" },
        "outputs": {
          "generatedFaces": {
            "type": "dataset",
            "with": { "ref": "dsx8kjbbkxtc5v3" }
          }
        },
        "resources": { "instance-type": "P4000" }
      }
    }
  },
  "status": {
    "phase": "SUCCEEDED",
    "logId": "wfr4dbd74f403d146fc97cb15a76805f3a1",
    "started": "2022-03-22T08:28:53.000Z",
    "finished": "2022-03-22T08:31:37.833Z",
    "jobs": {
      "CloneRepo": {
        "phase": "SUCCEEDED",
        "logId": "wfrj67d8451ea1bf49749925755f5e61a46e",
        "started": "2022-03-22T08:29:07.000Z",
        "finished": "2022-03-22T08:29:22.000Z"
      },
      "StyleGan2": {
        "phase": "SUCCEEDED",
        "logId": "wfrjaedc275373ec4d239f3ea1af029dfd60",
        "started": "2022-03-22T08:29:23.000Z",
        "finished": "2022-03-22T08:31:16.000Z",
        "outputs": {
          "generatedFaces": {
            "dataset": { "id": "demo-dataset:3w8iqvi", "isCommitted": true }
          }
        }
      }
    }
  }
}


// Response to second query

[
  {
    "jobId": "wfrj67d8451ea1bf49749925755f5e61a46e",
    "line": 1,
    "timestamp": "2022-03-22T08:29:20.836Z",
    "message": "Cloning into '/outputs/repo'...\n",
    "instanceName": {
      "String": "wf4dbd74f403d146fc97cb15a76805f3a1-3308432544",
      "Valid": true
    },
    "uuid": { "String": "", "Valid": true },
    "instanceCount": { "Int64": 0, "Valid": false }
  }
]

// Response to third query

[
  {
    "jobId": "wfrjaedc275373ec4d239f3ea1af029dfd60",
    "line": 1,
    "timestamp": "2022-03-22T08:29:42.347Z",
    "message": "Collecting scipy==1.3.3\n",
    "instanceName": {
      "String": "wf4dbd74f403d146fc97cb15a76805f3a1-1115298116",
      "Valid": true
    },
    "uuid": { "String": "", "Valid": true },
    "instanceCount": { "Int64": 0, "Valid": false }
  },
  {
    "jobId": "wfrjaedc275373ec4d239f3ea1af029dfd60",
    "line": 2,
    "timestamp": "2022-03-22T08:29:42.873Z",
    "message": "  Downloading https://files.pythonhosted.org/packages/54/18/d7c101d5e93b6c78dc206fcdf7bd04c1f8138a7b1a93578158fa3b132b08/scipy-1.3.3-cp36-cp36m-manylinux1_x86_64.whl (25.2MB)\n",
    "instanceName": {
      "String": "wf4dbd74f403d146fc97cb15a76805f3a1-1115298116",
      "Valid": true
    },
    "uuid": { "String": "", "Valid": true },
    "instanceCount": { "Int64": 0, "Valid": false }
  },
  {
    "jobId": "wfrjaedc275373ec4d239f3ea1af029dfd60",
    "line": 3,
    "timestamp": "2022-03-22T08:29:44.778Z",
    "message": "Requirement already satisfied: numpy\u003e=1.13.3 in /usr/local/lib/python3.6/dist-packages (from scipy==1.3.3) (1.16.4)\n",
    "instanceName": {
      "String": "wf4dbd74f403d146fc97cb15a76805f3a1-1115298116",
      "Valid": true
    },
    "uuid": { "String": "", "Valid": true },
    "instanceCount": { "Int64": 0, "Valid": false }
  },
  ...
]

[
  (LogRow(
    (line = 1),
    (message =
      "Cloning into '/outputs/repo'...\n"),
    (timestamp = "2022-03-22T08:29:20.836Z")
  )
]
[
  (LogRow(
    (line = 2),
    (message =
      "  Downloading https://files.pythonhosted.org/packages/54/18/d7c101d5e93b6c78dc206fcdf7bd04c1f8138a7b1a93578158fa3b132b08/scipy-1.3.3-cp36-cp36m-manylinux1_x86_64.whl (25.2MB)"),
    (timestamp = "2022-03-22T08:29:42.873Z")
  ),
  LogRow(
    (line = 3),
    (message =
      "Requirement already satisfied: numpy>=1.13.3 in /usr/local/lib/python3.6/dist-packages (from scipy==1.3.3) (1.16.4)"),
    (timestamp = "2022-03-22T08:29:44.778Z")
  ),
  ...
];

run

Run workflow spec

Examples

CLI
curl
Python

gradient workflows run \
  --id "075615ac-a036-408c-a95e-36e5b18c73aa" \
  --path "workflow-spec.yaml"

curl -X POST 'https://api.paperspace.io/workflows/075615ac-a036-408c-a95e-36e5b18c73aa/runs' \
-H 'x-api-key: d44808a2785d6a...' \
-H 'Content-Type: application/json' \
--data-raw '{
    "spec": {
        "jobs": {
            "CloneRepo": {
                "resources": {
                    "instance-type": "C5"
                },
                "outputs": {
                    "repo": {
                        "type": "volume"
                    }
                },
                "uses": "git-checkout@v1",
                "with": {
                    "url": "https://github.com/NVlabs/stylegan2.git"
                }
            },
            "StyleGan2": {
                "resources": {
                    "instance-type": "P4000"
                },
                "needs": ["CloneRepo"],
                "inputs": {
                    "repo": "CloneRepo.outputs.repo"
                },
                "outputs": {
                    "generatedFaces": {
                        "type": "dataset",
                        "with": {
                            "ref": "dsx8kjbbkxtc5v3"
                        }
                    }
                },
                "uses": "script@v1",
                "with": {
                    "script": "pip install scipy==1.3.3\npip install requests==2.22.0\npip install Pillow==6.2.1\ncd /inputs/repo\npython run_generator.py generate-images \\\n  --network=gdrive:networks/stylegan2-ffhq-config-f.pkl \\\n  --seeds=6600-6605 \\\n  --truncation-psi=0.5 \\\n  --result-dir=/outputs/generatedFaces",
                    "image": "tensorflow/tensorflow:1.14.0-gpu-py3"
                }
            }
        }
    },
    "cluster": {
        "id": "clg07azjl"
    },
    "run": "True",
    "markDefault": "False"
}'

from gradient import WorkflowsClient
import yaml

api_key = 'd44808a2785d6a...'

workflow_client = WorkflowsClient(api_key)

spec_path = "./workflow-spec.yaml"

yaml_spec = open(spec_path, 'r')
spec = yaml.safe_load(yaml_spec)

print(workflow_client.run_workflow(
    workflow_id='553d666c-2306-4bf3-a3db-9bd1275a9657',
    spec=spec,
    cluster_id='clg07azjl',
    inputs=None
))

Options

Name	Type	Attributes	Description
`--id`	string	required	Workflow ID
`--clusterId`	string	optional	Cluster ID
`--path`	string	required	Path to workflow specification
`--inputPath`	string	optional	Path to inputs
`--apiKey`	string	optional	API key to use this time only
`--optionsFile`	string	optional	Path to YAML with predefined options
`--createOptionsFile`	string	optional	Generate template options file

Response

CLI
curl
Python

Created workflow run wfr627e7d055e0146ad94677d32f9380991

{
  "id": 2,
  "uuid": "627e7d05-5e01-46ad-9467-7d32f9380991",
  "meta": {},
  "cluster": { "id": "clg07azjl" },
  "spec": {
    "jobs": {
      "CloneRepo": {
        "uses": "git-checkout@v1",
        "with": { "url": "https://github.com/NVlabs/stylegan2.git" },
        "outputs": { "repo": { "type": "volume" } },
        "resources": { "instance-type": "C5" }
      },
      "StyleGan2": {
        "uses": "script@v1",
        "with": {
          "image": "tensorflow/tensorflow:1.14.0-gpu-py3",
          "script": "pip install scipy==1.3.3\\npip install requests==2.22.0\\npip install Pillow==6.2.1\\ncd /inputs/repo\\npython run_generator.py generate-images \\\\\\n  --network=gdrive:networks/stylegan2-ffhq-config-f.pkl \\\\\\n  --seeds=6600-6605 \\\\\\n  --truncation-psi=0.5 \\\\\\n  --result-dir=/outputs/generatedFaces"
        },
        "needs": ["CloneRepo"],
        "inputs": { "repo": "CloneRepo.outputs.repo" },
        "outputs": {
          "generatedFaces": {
            "type": "dataset",
            "with": { "ref": "dsx8kjbbkxtc5v3" }
          }
        },
        "resources": { "instance-type": "P4000" }
      }
    }
  },
  "status": {
    "phase": "SUBMITTING",
    "logId": "wfr627e7d055e0146ad94677d32f9380991",
    "jobs": {
      "CloneRepo": {
        "phase": "SUBMITTING",
        "logId": "wfrj2ac144f8c3174efeac14aea8f522edaf"
      },
      "StyleGan2": {
        "phase": "SUBMITTING",
        "logId": "wfrj5634fae547234e7598741a17126a174e",
        "outputs": {
          "generatedFaces": {
            "dataset": { "id": "demo-dataset:hypjdmy", "isCommitted": false }
          }
        }
      }
    }
  }
}

[
  Workflow(
    (id = "075615ac-a036-408c-a95e-36e5b18c73aa"),
    (team_id = None),
    (project_id = None),
    (name = "New-Workflow"),
    (workflow_spec_id = None),
    (dt_deleted = None),
    (dt_created = datetime.datetime(
      2022,
      3,
      22,
      8,
      10,
      29,
      677000,
      (tzinfo = tzutc())
    )),
    (dt_modified = datetime.datetime(
      2022,
      3,
      22,
      9,
      21,
      29,
      214000,
      (tzinfo = tzutc())
    ))
  ),
];

workflows

create​

Examples​

Options​

Response​

get​

Examples​

Options​

Response​

list​

Examples​

Options​

Response​

logs​

Examples​

Options​

Response​

run​

Examples​

Options​

Response​

create

Examples

Options

Response

get

Examples

Options

Response

list

Examples

Options

Response

logs

Examples

Options

Response

run

Examples

Options

Response