datacatalog.linkedstores.pipelinejob package

Submodules

datacatalog.linkedstores.pipelinejob.exceptions module

exception datacatalog.linkedstores.pipelinejob.exceptions.JobError

Bases: Exception

A generic job error

exception datacatalog.linkedstores.pipelinejob.exceptions.JobCreateFailure

Bases: datacatalog.linkedstores.pipelinejob.exceptions.JobError

Job was not created

exception datacatalog.linkedstores.pipelinejob.exceptions.JobUpdateFailure

Bases: datacatalog.linkedstores.pipelinejob.exceptions.JobError

Job was not updated

exception datacatalog.linkedstores.pipelinejob.exceptions.DuplicateJobError(error, code=None, details=None, max_wire_version=None)

Bases: pymongo.errors.DuplicateKeyError

Job was a duplicate which is not allowed

exception datacatalog.linkedstores.pipelinejob.exceptions.UnknownPipeline

Bases: datacatalog.linkedstores.pipelinejob.exceptions.JobError

The referenced pipeline is not known

exception datacatalog.linkedstores.pipelinejob.exceptions.UnknownJob

Bases: datacatalog.linkedstores.pipelinejob.exceptions.JobError

The referenced job is not known

datacatalog.linkedstores.pipelinejob.fsm module

class datacatalog.linkedstores.pipelinejob.fsm.JobStateMachine(state='CREATED')

Bases: transitions.core.Machine

classmethod get_events()

classmethod get_states()

handle(event_name, event_opts={})

states = ['CREATED', 'RUNNING', 'FAILED', 'FINISHED', 'INDEXING', 'VALIDATING', 'VALIDATED', 'REJECTED', 'FINALIZED', 'RETIRED', 'RESET']

transitions = [{'trigger': 'create', 'source': 'CREATED', 'dest': 'CREATED'}, {'trigger': 'run', 'source': ['CREATED', 'RUNNING'], 'dest': 'RUNNING'}, {'trigger': 'update', 'source': ['CREATED', 'RUNNING', 'VALIDATING', 'INDEXING'], 'dest': '='}, {'trigger': 'resource', 'source': ['CREATED', 'RUNNING'], 'dest': '='}, {'trigger': 'fail', 'source': ['CREATED', 'RUNNING', 'VALIDATING', 'INDEXING'], 'dest': 'FAILED'}, {'trigger': 'finish', 'source': ['RUNNING', 'FINISHED'], 'dest': 'FINISHED'}, {'trigger': 'index', 'source': ['INDEXING', 'FINISHED'], 'dest': 'INDEXING'}, {'trigger': 'indexed', 'source': ['FINISHED', 'INDEXING'], 'dest': 'FINISHED'}, {'trigger': 'validate', 'source': ['FINISHED', 'VALIDATED'], 'dest': 'VALIDATING'}, {'trigger': 'validated', 'source': ['VALIDATED', 'VALIDATING'], 'dest': 'VALIDATED'}, {'trigger': 'reject', 'source': 'VALIDATING', 'dest': 'REJECTED'}, {'trigger': 'finalize', 'source': 'VALIDATED', 'dest': 'FINALIZED'}, {'trigger': 'retire', 'source': ['FAILED', 'FINALIZED', 'FINISHED', 'REJECTED', 'VALIDATING', 'VALIDATED'], 'dest': 'RETIRED'}, {'trigger': 'reset', 'source': '*', 'dest': 'RESET'}, {'trigger': 'ready', 'source': ['RESET'], 'dest': 'CREATED'}]

class datacatalog.linkedstores.pipelinejob.fsm.EventResponse(**kwargs)

Bases: attrdict.dictionary.AttrDict

PARAMS = [('last_event', True, None), ('state', True, None)]

datacatalog.linkedstores.pipelinejob.graphfsm module

datacatalog.linkedstores.pipelinejob.graphfsm.build_graph(pstate, ptitle, events=[])

datacatalog.linkedstores.pipelinejob.graphfsm.get_machine(pstate, ptitle)

datacatalog.linkedstores.pipelinejob.graphfsm.get_machine_state(pstate, ptitle, events=['create'])

datacatalog.linkedstores.pipelinejob.graphfsm.render_graph(**kwargs)

datacatalog.linkedstores.pipelinejob.job module

class datacatalog.linkedstores.pipelinejob.job.HistoryEntry(entry)

Bases: datacatalog.linkedstores.basestore.extensible.ExtensibleAttrDict

to_dict()

class datacatalog.linkedstores.pipelinejob.job.PipelineJob(job_document, agave=None)

Bases: datacatalog.linkedstores.basestore.extensible.ExtensibleAttrDict, datacatalog.linkedstores.basestore.agaveclient.DocumentAgaveClient

gethistory()

handle(event, opts={})

new(data={})

to_dict()

exception datacatalog.linkedstores.pipelinejob.job.PipelineJobError

Bases: datacatalog.linkedstores.basestore.exceptions.CatalogError

Error occured within scope of a PipelineJob

datacatalog.linkedstores.pipelinejob.mappings module

class datacatalog.linkedstores.pipelinejob.mappings.AgaveEvents

Bases: object

Maps between Agave API and Pipeline job events

agavejobs = {'ARCHIVING': 'resource', 'ARCHIVING_FAILED': 'fail', 'ARCHIVING_FINISHED': 'resource', 'CLEANING_UP': 'resource', 'CREATED': 'resource', 'DELETED': 'fail', 'FAILED': 'fail', 'FINISHED': 'finish', 'HEARTBEAT': 'update', 'KILLED': 'update', 'PAUSED': 'resource', 'PENDING': 'resource', 'PERMISSION_GRANT': 'resource', 'PERMISSION_REVOKE': 'resource', 'PROCESSING_INPUTS': 'resource', 'QUEUED': 'resource', 'RUNNING': 'run', 'STAGED': 'resource', 'STAGING_INPUTS': 'resource', 'STAGING_JOB': 'resource', 'STOPPED': 'fail', 'SUBMITTING': 'resource', 'UPDATED': 'updated'}

datacatalog.linkedstores.pipelinejob.schema module

class datacatalog.linkedstores.pipelinejob.schema.HistoryEventDocument(inheritance=True, document='pipelinejob_event.json', filters='pipelinejob_event_filters.json', **kwargs)

Bases: datacatalog.linkedstores.basestore.heritableschema.HeritableDocumentSchema

class datacatalog.linkedstores.pipelinejob.schema.JobDocument(inheritance=True, **kwargs)

Bases: datacatalog.linkedstores.basestore.heritableschema.HeritableDocumentSchema

datacatalog.linkedstores.pipelinejob.schemas module

datacatalog.linkedstores.pipelinejob.schemas.get_definitions()

datacatalog.linkedstores.pipelinejob.schemas.get_document_schemas()

datacatalog.linkedstores.pipelinejob.schemas.get_event_document_schemas()

datacatalog.linkedstores.pipelinejob.schemas.get_fsm_schema_events()

datacatalog.linkedstores.pipelinejob.schemas.get_fsm_schema_states()

datacatalog.linkedstores.pipelinejob.schemas.get_job_document_schemas()

datacatalog.linkedstores.pipelinejob.schemas.get_primitives()

datacatalog.linkedstores.pipelinejob.schemas.get_schemas()

datacatalog.linkedstores.pipelinejob.store module

class datacatalog.linkedstores.pipelinejob.store.PipelineJobStore(mongodb, config={}, session=None, agave=None, **kwargs)

Bases: datacatalog.linkedstores.basestore.agaveclient.AgaveClient, datacatalog.linkedstores.basestore.softdelete.SoftDelete, datacatalog.linkedstores.basestore.store.LinkedStore

LINK_FIELDS = ['child_of', 'acted_on', 'acted_using', 'generated_by']

Fields that should never be indexed

NEVER_INDEX_FIELDS = 'data'

create(job_document, **kwargs)

delete(job_uuid, token, force=True)

fsm_state_png(uuid)

handle(event_document, token=None, data=None, **kwargs)

history(job_uuid, limit=None, skip=None)

list_job_archive_path(job_uuid, recurse=True, directories=False, **kwargs)

Returns contents of a job’s archive_path on the job’s archive_system

Parameters:

• job_uuid (str) – UUID of the job
• recurse (bool, optional) – List recursively
• directories (bool, optional) – Include directories in response

Notes

PipelineJobStore must be initialized with a valid Agave API client

Returns:Agave-canonical absolute filenames in job.archive_path Return type:list

uuid_from_properties(job_document, **kwargs)

validate_pipeline_uuid(pipeline_uuid)

class datacatalog.linkedstores.pipelinejob.store.StoreInterface(mongodb, config={}, session=None, agave=None, **kwargs)

Bases: datacatalog.linkedstores.pipelinejob.store.PipelineJobStore

datacatalog.linkedstores.pipelinejob.utils module