diff --git a/docs/_build/doctrees/environment.pickle b/docs/_build/doctrees/environment.pickle
index d3c502ed2..cf2e7c8a1 100644
Binary files a/docs/_build/doctrees/environment.pickle and b/docs/_build/doctrees/environment.pickle differ
diff --git a/docs/_build/doctrees/tools/indexing.doctree b/docs/_build/doctrees/tools/indexing.doctree
index 2c5e8e933..855280806 100644
Binary files a/docs/_build/doctrees/tools/indexing.doctree and b/docs/_build/doctrees/tools/indexing.doctree differ
diff --git a/docs/_build/doctrees/tools/metadata.doctree b/docs/_build/doctrees/tools/metadata.doctree
index 7f3613f24..df3acba95 100644
Binary files a/docs/_build/doctrees/tools/metadata.doctree and b/docs/_build/doctrees/tools/metadata.doctree differ
diff --git a/docs/_build/html/searchindex.js b/docs/_build/html/searchindex.js
index 562f6a08b..f2cf73e58 100644
--- a/docs/_build/html/searchindex.js
+++ b/docs/_build/html/searchindex.js
@@ -1 +1 @@
-Search.setIndex({"docnames": ["auth", "file", "index", "indexing", "jobs", "metadata", "object", "query", "submission", "tools", "tools/drs_pull", "tools/indexing", "tools/metadata", "wss"], "filenames": ["auth.rst", "file.rst", "index.rst", "indexing.rst", "jobs.rst", "metadata.rst", "object.rst", "query.rst", "submission.rst", "tools.rst", "tools/drs_pull.rst", "tools/indexing.rst", "tools/metadata.rst", "wss.rst"], "titles": ["Gen3 Auth Helper", "Gen3 File Class", "Welcome to Gen3 SDK\u2019s documentation!", "Gen3 Index Class", "Gen3 Jobs Class", "Gen3 Metadata Class", "Gen3 Object Class", "Gen3 Query Class", "Gen3 Submission Class", "Gen3 Tools", "DRS Download Tools", "Indexing Tools", "Metadata Tools", "Gen3 Workspace Storage"], "terms": {"class": [0, 2, 10, 11, 13], "gen3auth": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "endpoint": [0, 1, 2, 3, 4, 5, 7, 8, 13], "none": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "refresh_fil": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "refresh_token": 0, "idp": 0, "sourc": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "base": [0, 1, 3, 4, 5, 6, 7, 8, 9, 11, 13], "authbas": 0, "us": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "request": [0, 1, 3, 5, 8, 11, 12], "implement": 0, "order": [0, 8], "support": [0, 1, 5, 8, 11], "jwt": 0, "authent": 0, "gener": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 13], "access": [0, 1, 3, 7, 10], "token": [0, 10], "from": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "provid": [0, 1, 3, 5, 7, 8, 12], "refresh": [0, 10], "file": [0, 2, 3, 4, 8, 9, 10, 11, 12, 13], "string": [0, 3, 5, 11, 13], "automat": 0, "when": [0, 3, 7, 10, 12], "thei": [0, 10], "expir": [0, 1], "paramet": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "str": [0, 1, 3, 4, 5, 7, 8, 10, 11, 12], "opt": 0, "The": [0, 1, 2, 3, 5, 8, 10, 11], "contain": [0, 2, 5, 8, 9, 10, 11, 12], "download": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 13], "json": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "web": 0, "option": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11], "work": [0, 10], "workspac": [0, 2], "default": [0, 1, 3, 7, 8, 11, 12], "env": 0, "gen3_api_kei": 0, "credenti": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "set": [0, 1, 5, 10], "includ": [0, 3], "search": [0, 2, 3], "path": [0, 1, 5, 10, 11, 13], "valu": [0, 1, 3, 5, 7, 10, 11], "doe": [0, 12], "interpret": 0, "wt": [0, 10], "an": [0, 3, 6, 8, 10, 11], "accesstoken": 0, "If": [0, 1, 7, 11, 12], "can": [0, 3, 4, 8, 11, 12], "specifi": [0, 1, 3, 11, 13], "local": [0, 13], "indic": [0, 11], "environ": 0, "fenc": [0, 1], "exampl": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "thi": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "point": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "sandbox": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "common": [0, 1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "while": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "profil": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "page": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 13], "instal": [0, 2, 11], "crdc": 0, "some": [0, 2], "arbitrari": 0, "kei": [0, 3, 5, 13], "variabl": [0, 7, 8], "rather": 0, "than": [0, 5], "pass": [0, 7, 8, 10], "argument": [0, 13], "constructor": 0, "initi": [0, 10], "follow": [0, 11], "curl": [0, 2], "data": [0, 1, 3, 5, 7, 8, 10, 11], "given": [0, 3, 4, 5, 8, 10, 12, 13], "ex": [0, 11, 12], "user": [0, 10, 12], "return": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11], "respons": [0, 1, 3, 4, 5], "under": [0, 8, 13], "index": [0, 2, 5, 9], "authz": [0, 1, 2, 3, 9, 10, 11], "map": [0, 11], "get": [0, 1, 2, 3, 4, 5, 8, 10, 11, 12, 13], "post": [0, 11], "put": 0, "delet": [0, 1, 2, 3, 5, 6, 8, 10, 11], "i": [0, 1, 2, 3, 4, 5, 8, 10, 11, 12], "els": [0, 12], "filenam": [0, 8, 10, 11, 12], "get_access_token": [0, 2], "auto": [0, 2], "within": [0, 2, 9], "5": [0, 5], "minut": 0, "get_access_token_from_wt": [0, 2], "try": 0, "fetch": 0, "namespac": [0, 12], "refresh_access_token": [0, 2], "new": [0, 3], "gen3fil": [1, 2], "auth_provid": [1, 2, 3, 4, 5, 6, 7, 8, 13], "object": [1, 2, 3, 4, 5, 7, 8, 9, 10, 11, 13], "For": [1, 5, 6, 7, 8, 9, 11], "interact": [1, 3, 4, 5, 6, 8, 13], "manag": [1, 5, 10], "featur": [1, 6], "A": [1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "servic": [1, 3, 4, 5, 6, 8, 11, 12, 13], "presign": 1, "url": [1, 2, 3, 9, 10, 11, 12, 13], "right": 1, "now": [1, 8], "instanc": [1, 3, 6, 7, 8, 9, 10], "auth": [1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "delete_fil": [1, 2], "guid": [1, 2, 3, 5, 6, 9, 11, 12], "method": [1, 7, 10], "deprec": 1, "delete_file_loc": [1, 2, 6], "instead": [1, 7, 11], "all": [1, 3, 4, 5, 6, 7, 8, 10, 11, 12], "locat": [1, 6], "store": [1, 3, 10], "remov": [1, 6, 11, 13], "its": [1, 3], "record": [1, 3, 5, 7, 8, 11, 12], "indexd": [1, 3, 6, 10, 11, 12], "uuid": [1, 3, 8], "id": [0, 1, 3, 5, 10, 11], "text": [1, 7, 8], "result": [1, 8, 10, 11], "type": [1, 3, 4, 5, 7, 8, 10, 11, 12], "get_presigned_url": [1, 2], "protocol": 1, "retriev": [1, 8, 10, 12], "give": 1, "limit": [1, 3, 5, 12], "time": [1, 8, 10, 11], "pick": 1, "avail": [1, 2, 10, 11], "queri": [1, 2, 3, 5, 8, 11, 12], "upload_fil": [1, 2], "file_nam": [1, 2, 3, 9, 10, 11], "expires_in": 1, "upload": [1, 2, 3, 8, 13], "list": [0, 1, 3, 4, 5, 7, 8, 10, 11, 13], "author": 1, "scope": [0, 1], "storag": [1, 2, 6], "s3": [1, 10, 11], "az": 1, "isn": 1, "t": [1, 5, 11], "int": [1, 3, 5, 7, 8, 10, 11, 12], "amount": [1, 9], "second": [1, 4], "sign": 1, "datetim": [1, 10], "utcnow": 1, "Be": 1, "sure": 1, "posit": [1, 7], "integ": [1, 3, 8], "also": 1, "treat": [1, 5], "max_presigned_url_ttl": 1, "configur": 1, "represent": [1, 3], "document": [1, 3], "readm": 2, "python": [2, 9, 11], "github": [2, 7], "repo": 2, "setup": 2, "inform": [2, 3, 10], "along": 2, "quickstart": 2, "script": 2, "more": [2, 5, 7, 9, 10], "detail": [2, 7, 10], "about": [2, 3], "variou": 2, "function": [2, 3, 4, 5, 9, 10, 11, 12], "mostli": 2, "docstr": 2, "code": [2, 8], "helper": 2, "gen3index": [2, 3], "async_create_record": [2, 3], "async_get_record": [2, 3], "async_get_records_from_checksum": [2, 3], "async_get_records_on_pag": [2, 3], "async_get_with_param": [2, 3], "async_query_url": [2, 3], "async_update_record": [2, 3], "create_blank": [2, 3], "create_new_vers": [2, 3], "create_record": [2, 3], "delete_record": [2, 3, 8], "get_all_record": [2, 3], "get_guids_prefix": [2, 3], "get_latest_vers": [2, 3], "get_record": [2, 3], "get_record_doc": [2, 3], "get_records_on_pag": [2, 3], "get_stat": [2, 3], "get_url": [2, 3], "get_valid_guid": [2, 3], "get_vers": [2, 3, 4, 5], "get_with_param": [2, 3], "is_healthi": [2, 3, 4, 5], "query_url": [2, 3], "update_blank": [2, 3], "update_record": [2, 3], "job": 2, "gen3job": [2, 4, 10], "async_run_job_and_wait": [2, 4], "create_job": [2, 4], "get_output": [2, 4], "get_statu": [2, 4], "list_job": [2, 4], "metadata": [2, 3, 6, 9, 11], "gen3metadata": [2, 5], "async_cr": [2, 5], "async_create_alias": [2, 5], "async_delete_alia": [2, 5], "async_delete_alias": [2, 5], "async_get": [2, 5], "async_get_alias": [2, 5], "async_upd": [2, 5], "async_update_alias": [2, 5], "batch_creat": [2, 5], "creat": [2, 3, 4, 5, 6, 8, 10, 11], "create_alias": [2, 5], "create_index_key_path": [2, 5], "delete_alia": [2, 5], "delete_alias": [2, 5], "delete_index_key_path": [2, 5], "get_alias": [2, 5], "get_index_key_path": [2, 5], "updat": [2, 3, 5, 10, 11], "update_alias": [2, 5], "gen3object": [2, 6], "delete_object": [2, 6], "gen3queri": [2, 7], "graphql_queri": [2, 7], "raw_data_download": [2, 7], "submiss": 2, "gen3submiss": [2, 8], "create_program": [2, 8], "create_project": [2, 8], "delete_nod": [2, 8], "delete_program": [2, 8], "delete_project": [2, 8], "export_nod": [2, 8], "export_record": [2, 8], "get_dictionary_al": [2, 8], "get_dictionary_nod": [2, 8], "get_graphql_schema": [2, 8], "get_program": [2, 8], "get_project_dictionari": [2, 8], "get_project_manifest": [2, 8], "get_project": [2, 8], "open_project": [2, 8], "submit_fil": [2, 8], "submit_record": [2, 8], "tool": 2, "dr": [2, 9], "downloadmanag": [2, 9, 10], "cache_hosts_wts_token": [2, 9, 10], "get_fresh_token": [2, 9, 10], "resolve_object": [2, 9, 10], "user_access": [2, 9, 10], "downloadstatu": [2, 9, 10], "statu": [2, 4, 9, 10], "start_tim": [2, 9, 10], "end_tim": [2, 9, 10], "object_id": [1, 2, 9, 10], "object_typ": [2, 9, 10], "hostnam": [2, 9, 10], "file_s": [2, 9, 10, 11], "updated_tim": [2, 9, 10], "created_tim": [2, 9, 10], "access_method": [2, 9, 10], "children": [2, 9, 10], "_manag": [2, 9, 10], "pprint": [2, 9, 10], "manifest": [2, 8, 9, 10, 11, 12], "md5sum": [2, 9, 10], "commons_url": [2, 9, 10, 11, 12], "create_object_list": [2, 9, 10], "load": [2, 9, 10], "load_manifest": [2, 9, 10], "download_drs_object": [2, 9, 10], "download_files_in_drs_manifest": [2, 9, 10], "list_access_in_drs_manifest": [2, 9, 10], "list_drs_object": [2, 9, 10], "list_files_in_drs_manifest": [2, 9, 10], "current_dir": [2, 9, 11], "indexd_record_page_s": [2, 9, 11], "max_concurrent_request": [2, 9, 11, 12], "tmp_folder": [2, 9, 11], "async_download_object_manifest": [2, 9, 11], "size": [2, 3, 9, 10, 11], "md5": [2, 3, 9, 11], "acl": [2, 3, 9, 11], "prev_guid": [2, 9, 11], "threadcontrol": [2, 9, 11], "delete_all_guid": [2, 9, 11], "index_object_manifest": [2, 9, 11], "verifi": [2, 9], "async_verify_object_manifest": [2, 9, 11], "ingest": [2, 9], "column_to_use_as_guid": [2, 9, 12], "guid_type_for_indexed_file_object": [2, 9, 12], "guid_type_for_non_indexed_file_object": [2, 9, 12], "async_ingest_metadata_manifest": [2, 9, 12], "async_query_urls_from_indexd": [2, 9, 12], "gen3wsstorag": [2, 13], "copi": [2, 13], "download_url": [2, 13], "l": [2, 13], "ls_path": [2, 13], "rm": [2, 13], "rm_path": [2, 13], "upload_url": [2, 13], "modul": [2, 10, 11], "service_loc": [3, 4, 5], "public": [3, 5], "read": [3, 5, 11], "onli": [3, 5, 7, 8, 10, 11], "necessari": [3, 5], "basic": [3, 11, 12], "cred": 3, "tupl": [0, 3, 11, 12], "async": [3, 4, 5, 9, 11, 12], "hash": [3, 11], "did": 3, "baseid": 3, "urls_metadata": 3, "version": [3, 4, 5], "_ssl": [3, 4, 5], "asynchron": [3, 4, 5], "dict": [3, 4, 5, 10, 11, 12], "eg": 3, "ab167e49d25b488939b1ede42752458b": 3, "associ": [3, 5], "made": 3, "where": [3, 5, 11, 12], "you": [3, 8, 11], "control": 3, "rbac": 3, "name": [3, 4, 8, 10, 11, 12, 13], "addit": [3, 5, 10, 11], "entri": [3, 11], "attach": [3, 5], "each": [3, 8, 10, 11], "group": 3, "previou": [3, 11], "checksum": [3, 10], "checksum_typ": 3, "match": [3, 5, 12], "param": [3, 5, 8, 10], "correspond": 3, "suppli": 3, "need": [3, 7, 10, 11], "handl": [3, 10], "hash_typ": 3, "whether": [3, 4, 5, 8, 11, 12], "ssl": [3, 4, 5], "pattern": [3, 12], "against": [3, 7, 8, 11, 12], "bodi": 3, "dictionari": [3, 4, 5, 7, 8], "format": [3, 5, 8, 11], "blank": 3, "add": [3, 5], "sinc": 3, "content": [3, 13], "immut": 3, "want": [3, 8], "chang": [3, 11], "That": 3, "field": [3, 5, 7, 11, 12], "old": 3, "ad": 3, "least": 3, "one": [3, 5, 10, 11], "suffici": 3, "note": [0, 3, 11, 12], "good": 3, "idea": 3, "number": [3, 7, 8, 11, 12], "noth": [3, 6], "dist_resolut": 3, "true": [3, 4, 5, 6, 7, 8, 10, 11, 12], "alia": [3, 5], "distribut": 3, "identifi": [3, 5, 9, 11], "boolean": 3, "we": [3, 11], "pagin": 3, "fals": [3, 5, 6, 10, 11], "start": [3, 4, 7, 8, 10], "prefix": 3, "rtype": 3, "has_vers": 3, "latest": 3, "exclud": 3, "without": [3, 5], "repres": [3, 5, 10], "info": [3, 11], "algorithm": 3, "count": 3, "valid": [3, 7], "file_st": 3, "client": [0, 3], "like": [3, 5, 9, 11, 12], "signpost": 3, "healthi": [3, 4, 5], "rev": 3, "revis": 3, "simpl": 3, "consist": 3, "mechan": 3, "exist": [1, 3, 5, 6, 9, 12], "": [1, 4, 8, 10, 11], "dispatch": 4, "job_nam": 4, "job_input": 4, "kwarg": [3, 4, 5], "wait": 4, "output": [4, 5, 11, 12], "Will": [4, 7, 10], "sleep": 4, "linear": 4, "delai": 4, "until": [4, 10], "done": 4, "1": [4, 5, 8, 10, 11], "global": [4, 5], "input": [4, 10, 11], "job_id": 4, "previous": 4, "complet": [4, 11], "bool": [4, 5, 8, 10, 11, 12], "md": [5, 7, 10, 12], "admin_endpoint_suffix": 5, "admin": [5, 11], "alias": 5, "overwrit": 5, "what": 5, "end": [5, 10], "up": [5, 9], "blob": [5, 7], "asyncron": 5, "uniqu": [1, 5], "singl": [1, 5, 8, 10], "altern": [5, 11], "merg": 5, "AND": 5, "descript": 5, "metadata_list": 5, "specif": [5, 8, 11, 12], "expect": [5, 9, 11], "anoth": 5, "databas": 5, "return_full_metadata": 5, "10": [5, 7, 11], "offset": [5, 7], "0": [5, 7, 8, 11], "use_agg_md": 5, "off": 5, "logic": [5, 12], "filter": [5, 7], "b": [5, 11], "2": [5, 11], "have": [5, 11], "ar": [5, 7, 8, 9, 10, 11], "alwai": 5, "nest": 5, "c": [5, 11], "3": [5, 11], "same": [5, 11, 13], "whose": 5, "ani": [5, 10, 11], "But": 5, "differ": 5, "must": [1, 5], "33": 5, "d": 5, "4": [5, 11], "e": [5, 10], "won": 5, "333": 5, "defin": [5, 8, 10], "api": [5, 8, 11], "just": [5, 11, 12], "max": 5, "num": 5, "OR": 5, "level": 6, "current": [6, 8, 10], "allow": [0, 6, 8, 10, 11, 12], "system": [6, 7, 8, 9], "bucket": [1, 6], "elasticsearch": 7, "query_str": 7, "execut": [7, 8, 11], "graphql": [7, 8], "query_txt": [7, 8], "syntax": 7, "see": [7, 10, 11], "http": [0, 7, 11, 12], "com": 7, "uc": 7, "cdi": 7, "guppi": 7, "master": 7, "doc": [7, 10], "data_typ": 7, "my_index": 7, "my_field": 7, "first": [7, 8], "filter_object": 7, "sort_object": 7, "verbos": [7, 8], "row": [7, 8, 11, 12], "sort": 7, "equal": 7, "respect": 7, "complex": 7, "appli": 7, "One": 7, "unaccess": 7, "regular": 7, "tier": 7, "mode": 7, "subject": [7, 8], "50": 7, "vital_statu": 7, "submitter_id": 7, "aliv": 7, "asc": 7, "sort_field": 7, "raw": [7, 11], "project_id": [7, 8], "my_program": 7, "my_project": 7, "submit": [8, 11], "export": [8, 10], "sheepdog": 8, "through": [8, 11], "peregrin": 8, "sub": 8, "program": [8, 11], "project": [8, 11], "dcf": 8, "node_nam": 8, "batch_siz": 8, "100": [8, 11], "node": 8, "how": [8, 11], "mani": [8, 11], "print": [8, 10], "progress": [8, 10], "log": [8, 10, 11, 12], "ccle": 8, "demograph": 8, "ordered_node_list": 8, "revers": 8, "graph": 8, "experi": 8, "empti": [8, 11], "uuid1": 8, "uuid2": 8, "node_typ": 8, "fileformat": 8, "which": [8, 10], "belong": 8, "either": 8, "tsv": [8, 11, 12], "screen": 8, "sampl": [8, 10], "ccle_sample_nod": 8, "d70b41b9": 8, "6f90": 8, "4714": 8, "8420": 8, "e043ab8b77b9": 8, "ccle_one_record": 8, "entir": 8, "schema": 8, "run": [8, 11], "introspect": 8, "regist": 8, "mark": 8, "open": [8, 10, 11], "mean": 8, "etc": 8, "max_tri": 8, "retri": 8, "fail": [8, 10], "chunk_siz": 8, "30": 8, "row_offset": 8, "spreadsheet": 8, "multipl": [8, 11], "csv": [8, 11, 12], "xlsx": 8, "worksheet": 8, "skip": 8, "data_spreadsheet": 8, "should": [0, 8, 11], "arrai": 8, "action": [9, 11], "These": 9, "broken": 9, "broad": 9, "categori": 9, "task": 9, "relat": 9, "persist": 9, "Such": 9, "mai": [9, 11], "involv": 9, "assign": 9, "everi": [9, 11], "alreadi": 9, "larg": 9, "most": 9, "util": 9, "capabl": 9, "make": [9, 11], "effici": 9, "main": 10, "datafil": 10, "manifest_1": 10, "my_common": 10, "org": 10, "gen3": [10, 11, 12], "my_credenti": 10, "howto": 10, "drsdownload": 10, "drs_download": 10, "download_list": 10, "show_progress": 10, "assist": 10, "minimum": 10, "interest": 10, "object_list": 10, "host": 10, "obtain": [0, 10], "It": 10, "possibl": 10, "save_directori": 10, "unpack_packag": 10, "delete_unpacked_packag": 10, "directori": [10, 11], "call": [10, 13], "case": 10, "error": [10, 11, 12], "continu": 10, "save": 10, "show": 10, "bar": 10, "disabl": 10, "unpack": 10, "packag": 10, "after": 10, "them": [10, 11], "drs_hostnam": 10, "known": 10, "otherwis": 10, "success": 10, "resolv": 10, "permiss": 10, "wai": 10, "determin": [10, 11, 12], "reason": 10, "pend": 10, "objectid": 10, "datamanag": 10, "occur": 10, "attribut": [10, 11], "drsobjecttyp": 10, "unknown": 10, "factori": 10, "popul": [10, 12], "server": 10, "requir": 10, "byte": 10, "timestamp": 10, "last": 10, "g": 10, "child": 10, "bundl": 10, "self": 10, "indent": 10, "pretti": 10, "In": 10, "similar": 10, "linux": 10, "tree": 10, "command": [10, 11], "typic": 10, "discoveri": 10, "cach": 10, "files": 10, "static": 10, "bownload": 10, "successfulli": 10, "pars": [10, 11, 12, 13], "output_dir": 10, "conveni": 10, "infil": 10, "wrapper": 10, "cli": 10, "process": 11, "coroutin": 11, "asyncio": [11, 12], "librari": 11, "comma": 11, "separ": [0, 11], "header": 11, "space": [0, 11], "download_manifest": 11, "per": [11, 12], "maximum": [11, 12], "desir": 11, "concurr": [11, 12], "across": 11, "thread": 11, "folder": 11, "place": 11, "temporari": 11, "write": 11, "safe": 11, "so": 11, "To": 11, "workaround": 11, "concat": 11, "output_filenam": [11, 12], "num_process": 11, "24": [11, 12], "input_manifest": 11, "root": [11, 12], "domain": [11, 12], "live": [11, 12], "parallel": 11, "hit": 11, "total": 11, "help": 11, "everyth": 11, "attempt": 11, "tab": 11, "session": 11, "column": [11, 12], "shown": 11, "below": 11, "255e396f": 11, "f1f8": 11, "11e9": 11, "9a07": 11, "0a80fada099c": 11, "473d83400bc1bc9dc635e334faddf33c": 11, "363455714": 11, "pdcdatastor": 11, "test1": 11, "0a80fada098c": 11, "473d83400bc1bc9dc635e334faddd33c": 11, "343434344": 11, "test2": 11, "0a80fada097c": 11, "473d83400bc1bc9dc635e334fadd433c": 11, "543434443": 11, "phs0001": 11, "phs0002": 11, "test3": 11, "0a80fada096c": 11, "test4": 11, "0a80fada010c": 11, "473d83400bc1bc9dc635e334fadde33c": 11, "test5": 11, "index_manifest": 11, "usag": 11, "py": 11, "giangb": 11, "planx": 11, "pla": 11, "net": 11, "manifest_fil": [11, 12], "path_to_manifest": 11, "replace_url": 11, "thread_num": 11, "api_kei": 11, "processed_fil": 11, "num_total_fil": 11, "synchron": 11, "warn": 11, "IF": 11, "know": 11, "THE": 11, "implic": 11, "manifest_file_delimit": [11, 12], "submit_additional_metadata_column": 11, "force_metadata_columns_even_if_empti": 11, "loop": 11, "ha": [0, 11], "password": [11, 12], "flag": 11, "replac": 11, "delimit": 11, "forc": 11, "creation": 11, "even": 11, "enabl": 11, "illustr": 11, "columna": 11, "columnb": 11, "columnc": 11, "dataa": 11, "datab": 11, "guid_exampl": 11, "md5_hash": 11, "fieldnam": 11, "There": 11, "those": 11, "abov": 11, "overrid": [11, 12], "assum": 11, "behavior": 11, "simpli": 11, "manifest_row_pars": [11, 12], "custom": 11, "import": 11, "verify_manifest": 11, "def": 11, "_get_authz_from_row": 11, "strip": 11, "parser": [11, 12], "verify_object_manifest": 11, "verif": 11, "error_nam": 11, "value_from_manifest": 11, "actual": 11, "value_from_indexd": 11, "93d9af72": 11, "b0f1": 11, "450c": 11, "a5c6": 11, "7d3d8d2083b4": 11, "dev": 11, "test": 11, "_get_acl_from_row": 11, "_get_file_name_from_row": 11, "_get_file_size_from_row": 11, "_get_guid_from_row": 11, "_get_md5_from_row": 11, "_get_urls_from_row": 11, "1664913404": [], "4600763": [], "indexd_field": [11, 12], "func_to_parse_row": [11, 12], "delimet": [11, 12], "metdata": 12, "ingest_manifest": 12, "NOT": 12, "guid_from_fil": 12, "_get_guid_for_row": 12, "indexed_file_object_guid": 12, "_query_for_associated_indexd_record_guid": 12, "guid_for_row": 12, "elsewher": 12, "metadata_sourc": 12, "7884498": [], "get_guid_from_fil": 12, "metadata_typ": 12, "dbgap": 12, "usual": 12, "fill": 12, "_guid_typ": 12, "is_indexed_file_object": 12, "lock": 12, "semaphor": 12, "semaphon": 12, "ammount": 12, "connect": 12, "wss": 13, "src_urlstr": 13, "dest_urlstr": 13, "appropri": 13, "src_w": 13, "src_wskei": 13, "dest_path": 13, "disk": 13, "w": 13, "wskei": 13, "ws_urlstr": 13, "form": 13, "src_path": 13, "dest_w": 13, "dest_wskei": 13, "allowed_data_upload_bucket": 1, "data_upload_bucket": 1, "upload_file_to_guid": [1, 2], "1664995746": [], "2094085": [], "6912842": [], "1665361686": [], "0196362": [], "4282863": [], "1665415592": [], "0841575": [], "369792": [], "download_singl": [1, 2], "1667237446": [], "310436": [], "461374": [], "1667237580": [], "6713352": [], "1667237581": [], "0283232": [], "1668452999": [], "8464384": [], "1668453000": [], "219631": [], "1668455015": [], "5573874": [], "9250507": [], "client_credenti": 0, "client_scop": 0, "client_id": 0, "client_secret": 0, "oidc": 0, "grant": 0, "openid": 0, "secret": 0, "1670019997": [], "6497164": [], "9551775": [], "1670272579": [], "3670187": [], "6539462": [], "never": 0, "hardcod": 0, "o": 0, "gen3_oidc_client_creds_secret": 0, "1671036553": [], "0874994": [], "3574216": [], "datacommon": 0, "1671044831": [], "8660681": [], "1671044832": [], "1868503": [], "1671140235": [], "0801866": [], "4101439": [], "1671140559": [], "279068": [], "6550183": [], "1673541949": [], "824122": [], "1673541950": [], "2653491": [], "1674497076": [], "8602023": [], "1674497077": [], "2165172": [], "python_subprocess_command": 11, "By": 11, "someth": 11, "maco": 11, "x": 11, "python3": 11, "1676929147": [], "3869426": [], "758984": [], "1676929361": [], "016488": [], "398911": [], "1676930607": [], "061559": [], "4525151": [], "1677279338": [11, 12], "2703023": 11, "6413171": 12}, "objects": {"gen3.auth": [[0, 0, 1, "", "Gen3Auth"]], "gen3.auth.Gen3Auth": [[0, 1, 1, "", "curl"], [0, 1, 1, "", "get_access_token"], [0, 1, 1, "", "get_access_token_from_wts"], [0, 1, 1, "", "refresh_access_token"]], "gen3.file": [[1, 0, 1, "", "Gen3File"]], "gen3.file.Gen3File": [[1, 1, 1, "", "delete_file"], [1, 1, 1, "", "delete_file_locations"], [1, 1, 1, "", "download_single"], [1, 1, 1, "", "get_presigned_url"], [1, 1, 1, "", "upload_file"], [1, 1, 1, "", "upload_file_to_guid"]], "gen3.index": [[3, 0, 1, "", "Gen3Index"]], "gen3.index.Gen3Index": [[3, 1, 1, "", "async_create_record"], [3, 1, 1, "", "async_get_record"], [3, 1, 1, "", "async_get_records_from_checksum"], [3, 1, 1, "", "async_get_records_on_page"], [3, 1, 1, "", "async_get_with_params"], [3, 1, 1, "", "async_query_urls"], [3, 1, 1, "", "async_update_record"], [3, 1, 1, "", "create_blank"], [3, 1, 1, "", "create_new_version"], [3, 1, 1, "", "create_record"], [3, 1, 1, "", "delete_record"], [3, 1, 1, "", "get"], [3, 1, 1, "", "get_all_records"], [3, 1, 1, "", "get_guids_prefix"], [3, 1, 1, "", "get_latest_version"], [3, 1, 1, "", "get_record"], [3, 1, 1, "", "get_record_doc"], [3, 1, 1, "", "get_records"], [3, 1, 1, "", "get_records_on_page"], [3, 1, 1, "", "get_stats"], [3, 1, 1, "", "get_urls"], [3, 1, 1, "", "get_valid_guids"], [3, 1, 1, "", "get_version"], [3, 1, 1, "", "get_versions"], [3, 1, 1, "", "get_with_params"], [3, 1, 1, "", "is_healthy"], [3, 1, 1, "", "query_urls"], [3, 1, 1, "", "update_blank"], [3, 1, 1, "", "update_record"]], "gen3.jobs": [[4, 0, 1, "", "Gen3Jobs"]], "gen3.jobs.Gen3Jobs": [[4, 1, 1, "", "async_run_job_and_wait"], [4, 1, 1, "", "create_job"], [4, 1, 1, "", "get_output"], [4, 1, 1, "", "get_status"], [4, 1, 1, "", "get_version"], [4, 1, 1, "", "is_healthy"], [4, 1, 1, "", "list_jobs"]], "gen3.metadata": [[5, 0, 1, "", "Gen3Metadata"]], "gen3.metadata.Gen3Metadata": [[5, 1, 1, "", "async_create"], [5, 1, 1, "", "async_create_aliases"], [5, 1, 1, "", "async_delete_alias"], [5, 1, 1, "", "async_delete_aliases"], [5, 1, 1, "", "async_get"], [5, 1, 1, "", "async_get_aliases"], [5, 1, 1, "", "async_update"], [5, 1, 1, "", "async_update_aliases"], [5, 2, 1, "", "auth_provider"], [5, 1, 1, "", "batch_create"], [5, 1, 1, "", "create"], [5, 1, 1, "", "create_aliases"], [5, 1, 1, "", "create_index_key_path"], [5, 1, 1, "", "delete"], [5, 1, 1, "", "delete_alias"], [5, 1, 1, "", "delete_aliases"], [5, 1, 1, "", "delete_index_key_path"], [5, 2, 1, "", "endpoint"], [5, 1, 1, "", "get"], [5, 1, 1, "", "get_aliases"], [5, 1, 1, "", "get_index_key_paths"], [5, 1, 1, "", "get_version"], [5, 1, 1, "", "is_healthy"], [5, 1, 1, "", "query"], [5, 1, 1, "", "update"], [5, 1, 1, "", "update_aliases"]], "gen3.object": [[6, 0, 1, "", "Gen3Object"]], "gen3.object.Gen3Object": [[6, 1, 1, "", "delete_object"]], "gen3.query": [[7, 0, 1, "", "Gen3Query"]], "gen3.query.Gen3Query": [[7, 1, 1, "", "graphql_query"], [7, 1, 1, "", "query"], [7, 1, 1, "", "raw_data_download"]], "gen3.submission": [[8, 0, 1, "", "Gen3Submission"]], "gen3.submission.Gen3Submission": [[8, 1, 1, "", "create_program"], [8, 1, 1, "", "create_project"], [8, 1, 1, "", "delete_node"], [8, 1, 1, "", "delete_nodes"], [8, 1, 1, "", "delete_program"], [8, 1, 1, "", "delete_project"], [8, 1, 1, "", "delete_record"], [8, 1, 1, "", "delete_records"], [8, 1, 1, "", "export_node"], [8, 1, 1, "", "export_record"], [8, 1, 1, "", "get_dictionary_all"], [8, 1, 1, "", "get_dictionary_node"], [8, 1, 1, "", "get_graphql_schema"], [8, 1, 1, "", "get_programs"], [8, 1, 1, "", "get_project_dictionary"], [8, 1, 1, "", "get_project_manifest"], [8, 1, 1, "", "get_projects"], [8, 1, 1, "", "open_project"], [8, 1, 1, "", "query"], [8, 1, 1, "", "submit_file"], [8, 1, 1, "", "submit_record"]], "gen3": [[9, 3, 0, "-", "tools"]], "gen3.tools.download": [[10, 3, 0, "-", "drs_download"]], "gen3.tools.download.drs_download": [[10, 0, 1, "", "DownloadManager"], [10, 0, 1, "", "DownloadStatus"], [10, 0, 1, "", "Downloadable"], [10, 0, 1, "", "Manifest"], [10, 4, 1, "", "download_drs_object"], [10, 4, 1, "", "download_files_in_drs_manifest"], [10, 4, 1, "", "list_access_in_drs_manifest"], [10, 4, 1, "", "list_drs_object"], [10, 4, 1, "", "list_files_in_drs_manifest"]], "gen3.tools.download.drs_download.DownloadManager": [[10, 1, 1, "", "cache_hosts_wts_tokens"], [10, 1, 1, "", "download"], [10, 1, 1, "", "get_fresh_token"], [10, 1, 1, "", "resolve_objects"], [10, 1, 1, "", "user_access"]], "gen3.tools.download.drs_download.DownloadStatus": [[10, 2, 1, "", "end_time"], [10, 2, 1, "", "start_time"], [10, 2, 1, "", "status"]], "gen3.tools.download.drs_download.Downloadable": [[10, 2, 1, "", "_manager"], [10, 2, 1, "", "access_methods"], [10, 2, 1, "", "children"], [10, 2, 1, "", "created_time"], [10, 1, 1, "", "download"], [10, 2, 1, "", "file_name"], [10, 2, 1, "", "file_size"], [10, 2, 1, "", "hostname"], [10, 2, 1, "", "object_id"], [10, 2, 1, "", "object_type"], [10, 1, 1, "", "pprint"], [10, 2, 1, "", "updated_time"]], "gen3.tools.download.drs_download.Manifest": [[10, 2, 1, "", "commons_url"], [10, 1, 1, "", "create_object_list"], [10, 2, 1, "", "file_name"], [10, 2, 1, "", "file_size"], [10, 1, 1, "", "load"], [10, 1, 1, "", "load_manifest"], [10, 2, 1, "", "md5sum"], [10, 2, 1, "", "object_id"]], "gen3.tools.indexing": [[11, 3, 0, "-", "download_manifest"], [11, 3, 0, "-", "index_manifest"], [11, 3, 0, "-", "verify_manifest"]], "gen3.tools.indexing.download_manifest": [[11, 2, 1, "", "CURRENT_DIR"], [11, 2, 1, "", "INDEXD_RECORD_PAGE_SIZE"], [11, 2, 1, "", "MAX_CONCURRENT_REQUESTS"], [11, 2, 1, "", "TMP_FOLDER"], [11, 4, 1, "", "async_download_object_manifest"]], "gen3.tools.indexing.index_manifest": [[11, 2, 1, "", "ACLS"], [11, 2, 1, "", "AUTHZ"], [11, 2, 1, "", "CURRENT_DIR"], [11, 2, 1, "", "GUID"], [11, 2, 1, "", "MD5"], [11, 2, 1, "", "PREV_GUID"], [11, 2, 1, "", "SIZE"], [11, 0, 1, "", "ThreadControl"], [11, 2, 1, "", "URLS"], [11, 4, 1, "", "delete_all_guids"], [11, 4, 1, "", "index_object_manifest"]], "gen3.tools.indexing.verify_manifest": [[11, 2, 1, "", "CURRENT_DIR"], [11, 2, 1, "", "MAX_CONCURRENT_REQUESTS"], [11, 4, 1, "", "async_verify_object_manifest"]], "gen3.tools.metadata": [[12, 3, 0, "-", "ingest_manifest"]], "gen3.tools.metadata.ingest_manifest": [[12, 2, 1, "", "COLUMN_TO_USE_AS_GUID"], [12, 2, 1, "", "GUID_TYPE_FOR_INDEXED_FILE_OBJECT"], [12, 2, 1, "", "GUID_TYPE_FOR_NON_INDEXED_FILE_OBJECT"], [12, 2, 1, "", "MAX_CONCURRENT_REQUESTS"], [12, 4, 1, "", "async_ingest_metadata_manifest"], [12, 4, 1, "", "async_query_urls_from_indexd"]], "gen3.wss": [[13, 0, 1, "", "Gen3WsStorage"]], "gen3.wss.Gen3WsStorage": [[13, 1, 1, "", "copy"], [13, 1, 1, "", "download"], [13, 1, 1, "", "download_url"], [13, 1, 1, "", "ls"], [13, 1, 1, "", "ls_path"], [13, 1, 1, "", "rm"], [13, 1, 1, "", "rm_path"], [13, 1, 1, "", "upload"], [13, 1, 1, "", "upload_url"]]}, "objtypes": {"0": "py:class", "1": "py:method", "2": "py:attribute", "3": "py:module", "4": "py:function"}, "objnames": {"0": ["py", "class", "Python class"], "1": ["py", "method", "Python method"], "2": ["py", "attribute", "Python attribute"], "3": ["py", "module", "Python module"], "4": ["py", "function", "Python function"]}, "titleterms": {"gen3": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 13], "auth": 0, "helper": 0, "file": 1, "class": [1, 3, 4, 5, 6, 7, 8], "welcom": 2, "sdk": 2, "": 2, "document": 2, "indic": 2, "tabl": 2, "index": [3, 11], "job": 4, "metadata": [5, 12], "object": 6, "queri": 7, "submiss": 8, "tool": [9, 10, 11, 12], "dr": 10, "download": [10, 11], "verifi": 11, "ingest": 12, "workspac": 13, "storag": 13}, "envversion": {"sphinx.domains.c": 2, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 8, "sphinx.domains.index": 1, "sphinx.domains.javascript": 2, "sphinx.domains.math": 2, "sphinx.domains.python": 3, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "sphinx.ext.viewcode": 1, "sphinx": 57}, "alltitles": {"Gen3 Auth Helper": [[0, "gen3-auth-helper"]], "Gen3 File Class": [[1, "gen3-file-class"]], "Welcome to Gen3 SDK\u2019s documentation!": [[2, "welcome-to-gen3-sdk-s-documentation"]], "Indices and tables": [[2, "indices-and-tables"]], "Gen3 Index Class": [[3, "gen3-index-class"]], "Gen3 Jobs Class": [[4, "gen3-jobs-class"]], "Gen3 Metadata Class": [[5, "gen3-metadata-class"]], "Gen3 Object Class": [[6, "gen3-object-class"]], "Gen3 Query Class": [[7, "gen3-query-class"]], "Gen3 Submission Class": [[8, "gen3-submission-class"]], "Gen3 Tools": [[9, "gen3-tools"]], "DRS Download Tools": [[10, "module-gen3.tools.download.drs_download"]], "Indexing Tools": [[11, "indexing-tools"]], "Download": [[11, "module-gen3.tools.indexing.download_manifest"]], "Index": [[11, "module-gen3.tools.indexing.index_manifest"]], "Verify": [[11, "module-gen3.tools.indexing.verify_manifest"]], "Metadata Tools": [[12, "metadata-tools"]], "Ingest": [[12, "module-gen3.tools.metadata.ingest_manifest"]], "Gen3 Workspace Storage": [[13, "gen3-workspace-storage"]]}, "indexentries": {"gen3auth (class in gen3.auth)": [[0, "gen3.auth.Gen3Auth"]], "curl() (gen3.auth.gen3auth method)": [[0, "gen3.auth.Gen3Auth.curl"]], "get_access_token() (gen3.auth.gen3auth method)": [[0, "gen3.auth.Gen3Auth.get_access_token"]], "get_access_token_from_wts() (gen3.auth.gen3auth method)": [[0, "gen3.auth.Gen3Auth.get_access_token_from_wts"]], "refresh_access_token() (gen3.auth.gen3auth method)": [[0, "gen3.auth.Gen3Auth.refresh_access_token"]], "gen3file (class in gen3.file)": [[1, "gen3.file.Gen3File"]], "delete_file() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.delete_file"]], "delete_file_locations() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.delete_file_locations"]], "download_single() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.download_single"]], "get_presigned_url() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.get_presigned_url"]], "upload_file() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.upload_file"]], "upload_file_to_guid() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.upload_file_to_guid"]], "gen3index (class in gen3.index)": [[3, "gen3.index.Gen3Index"]], "async_create_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_create_record"]], "async_get_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_get_record"]], "async_get_records_from_checksum() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_get_records_from_checksum"]], "async_get_records_on_page() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_get_records_on_page"]], "async_get_with_params() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_get_with_params"]], "async_query_urls() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_query_urls"]], "async_update_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_update_record"]], "create_blank() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.create_blank"]], "create_new_version() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.create_new_version"]], "create_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.create_record"]], "delete_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.delete_record"]], "get() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get"]], "get_all_records() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_all_records"]], "get_guids_prefix() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_guids_prefix"]], "get_latest_version() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_latest_version"]], "get_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_record"]], "get_record_doc() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_record_doc"]], "get_records() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_records"]], "get_records_on_page() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_records_on_page"]], "get_stats() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_stats"]], "get_urls() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_urls"]], "get_valid_guids() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_valid_guids"]], "get_version() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_version"]], "get_versions() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_versions"]], "get_with_params() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_with_params"]], "is_healthy() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.is_healthy"]], "query_urls() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.query_urls"]], "update_blank() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.update_blank"]], "update_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.update_record"]], "gen3jobs (class in gen3.jobs)": [[4, "gen3.jobs.Gen3Jobs"]], "async_run_job_and_wait() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.async_run_job_and_wait"]], "create_job() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.create_job"]], "get_output() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.get_output"]], "get_status() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.get_status"]], "get_version() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.get_version"]], "is_healthy() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.is_healthy"]], "list_jobs() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.list_jobs"]], "gen3metadata (class in gen3.metadata)": [[5, "gen3.metadata.Gen3Metadata"]], "async_create() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_create"]], "async_create_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_create_aliases"]], "async_delete_alias() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_delete_alias"]], "async_delete_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_delete_aliases"]], "async_get() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_get"]], "async_get_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_get_aliases"]], "async_update() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_update"]], "async_update_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_update_aliases"]], "auth_provider (gen3.metadata.gen3metadata attribute)": [[5, "gen3.metadata.Gen3Metadata.auth_provider"]], "batch_create() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.batch_create"]], "create() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.create"]], "create_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.create_aliases"]], "create_index_key_path() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.create_index_key_path"]], "delete() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.delete"]], "delete_alias() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.delete_alias"]], "delete_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.delete_aliases"]], "delete_index_key_path() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.delete_index_key_path"]], "endpoint (gen3.metadata.gen3metadata attribute)": [[5, "gen3.metadata.Gen3Metadata.endpoint"]], "get() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.get"]], "get_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.get_aliases"]], "get_index_key_paths() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.get_index_key_paths"]], "get_version() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.get_version"]], "is_healthy() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.is_healthy"]], "query() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.query"]], "update() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.update"]], "update_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.update_aliases"]], "gen3object (class in gen3.object)": [[6, "gen3.object.Gen3Object"]], "delete_object() (gen3.object.gen3object method)": [[6, "gen3.object.Gen3Object.delete_object"]], "gen3query (class in gen3.query)": [[7, "gen3.query.Gen3Query"]], "graphql_query() (gen3.query.gen3query method)": [[7, "gen3.query.Gen3Query.graphql_query"]], "query() (gen3.query.gen3query method)": [[7, "gen3.query.Gen3Query.query"]], "raw_data_download() (gen3.query.gen3query method)": [[7, "gen3.query.Gen3Query.raw_data_download"]], "gen3submission (class in gen3.submission)": [[8, "gen3.submission.Gen3Submission"]], "create_program() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.create_program"]], "create_project() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.create_project"]], "delete_node() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_node"]], "delete_nodes() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_nodes"]], "delete_program() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_program"]], "delete_project() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_project"]], "delete_record() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_record"]], "delete_records() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_records"]], "export_node() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.export_node"]], "export_record() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.export_record"]], "get_dictionary_all() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_dictionary_all"]], "get_dictionary_node() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_dictionary_node"]], "get_graphql_schema() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_graphql_schema"]], "get_programs() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_programs"]], "get_project_dictionary() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_project_dictionary"]], "get_project_manifest() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_project_manifest"]], "get_projects() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_projects"]], "open_project() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.open_project"]], "query() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.query"]], "submit_file() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.submit_file"]], "submit_record() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.submit_record"]], "gen3.tools": [[9, "module-gen3.tools"]], "module": [[9, "module-gen3.tools"], [10, "module-gen3.tools.download.drs_download"], [11, "module-gen3.tools.indexing.download_manifest"], [11, "module-gen3.tools.indexing.index_manifest"], [11, "module-gen3.tools.indexing.verify_manifest"], [12, "module-gen3.tools.metadata.ingest_manifest"]], "downloadmanager (class in gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.DownloadManager"]], "downloadstatus (class in gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.DownloadStatus"]], "downloadable (class in gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.Downloadable"]], "manifest (class in gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.Manifest"]], "_manager (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable._manager"]], "access_methods (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.access_methods"]], "cache_hosts_wts_tokens() (gen3.tools.download.drs_download.downloadmanager method)": [[10, "gen3.tools.download.drs_download.DownloadManager.cache_hosts_wts_tokens"]], "children (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.children"]], "commons_url (gen3.tools.download.drs_download.manifest attribute)": [[10, "gen3.tools.download.drs_download.Manifest.commons_url"]], "create_object_list() (gen3.tools.download.drs_download.manifest static method)": [[10, "gen3.tools.download.drs_download.Manifest.create_object_list"]], "created_time (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.created_time"]], "download() (gen3.tools.download.drs_download.downloadmanager method)": [[10, "gen3.tools.download.drs_download.DownloadManager.download"]], "download() (gen3.tools.download.drs_download.downloadable method)": [[10, "gen3.tools.download.drs_download.Downloadable.download"]], "download_drs_object() (in module gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.download_drs_object"]], "download_files_in_drs_manifest() (in module gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.download_files_in_drs_manifest"]], "end_time (gen3.tools.download.drs_download.downloadstatus attribute)": [[10, "gen3.tools.download.drs_download.DownloadStatus.end_time"]], "file_name (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.file_name"]], "file_name (gen3.tools.download.drs_download.manifest attribute)": [[10, "gen3.tools.download.drs_download.Manifest.file_name"]], "file_size (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.file_size"]], "file_size (gen3.tools.download.drs_download.manifest attribute)": [[10, "gen3.tools.download.drs_download.Manifest.file_size"]], "gen3.tools.download.drs_download": [[10, "module-gen3.tools.download.drs_download"]], "get_fresh_token() (gen3.tools.download.drs_download.downloadmanager method)": [[10, "gen3.tools.download.drs_download.DownloadManager.get_fresh_token"]], "hostname (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.hostname"]], "list_access_in_drs_manifest() (in module gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.list_access_in_drs_manifest"]], "list_drs_object() (in module gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.list_drs_object"]], "list_files_in_drs_manifest() (in module gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.list_files_in_drs_manifest"]], "load() (gen3.tools.download.drs_download.manifest static method)": [[10, "gen3.tools.download.drs_download.Manifest.load"]], "load_manifest() (gen3.tools.download.drs_download.manifest static method)": [[10, "gen3.tools.download.drs_download.Manifest.load_manifest"]], "md5sum (gen3.tools.download.drs_download.manifest attribute)": [[10, "gen3.tools.download.drs_download.Manifest.md5sum"]], "object_id (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.object_id"]], "object_id (gen3.tools.download.drs_download.manifest attribute)": [[10, "gen3.tools.download.drs_download.Manifest.object_id"]], "object_type (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.object_type"]], "pprint() (gen3.tools.download.drs_download.downloadable method)": [[10, "gen3.tools.download.drs_download.Downloadable.pprint"]], "resolve_objects() (gen3.tools.download.drs_download.downloadmanager method)": [[10, "gen3.tools.download.drs_download.DownloadManager.resolve_objects"]], "start_time (gen3.tools.download.drs_download.downloadstatus attribute)": [[10, "gen3.tools.download.drs_download.DownloadStatus.start_time"]], "status (gen3.tools.download.drs_download.downloadstatus attribute)": [[10, "gen3.tools.download.drs_download.DownloadStatus.status"]], "updated_time (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.updated_time"]], "user_access() (gen3.tools.download.drs_download.downloadmanager method)": [[10, "gen3.tools.download.drs_download.DownloadManager.user_access"]], "acls (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.ACLS"]], "authz (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.AUTHZ"]], "current_dir (in module gen3.tools.indexing.download_manifest)": [[11, "gen3.tools.indexing.download_manifest.CURRENT_DIR"]], "current_dir (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.CURRENT_DIR"]], "current_dir (in module gen3.tools.indexing.verify_manifest)": [[11, "gen3.tools.indexing.verify_manifest.CURRENT_DIR"]], "guid (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.GUID"]], "indexd_record_page_size (in module gen3.tools.indexing.download_manifest)": [[11, "gen3.tools.indexing.download_manifest.INDEXD_RECORD_PAGE_SIZE"]], "max_concurrent_requests (in module gen3.tools.indexing.download_manifest)": [[11, "gen3.tools.indexing.download_manifest.MAX_CONCURRENT_REQUESTS"]], "max_concurrent_requests (in module gen3.tools.indexing.verify_manifest)": [[11, "gen3.tools.indexing.verify_manifest.MAX_CONCURRENT_REQUESTS"]], "md5 (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.MD5"]], "prev_guid (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.PREV_GUID"]], "size (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.SIZE"]], "tmp_folder (in module gen3.tools.indexing.download_manifest)": [[11, "gen3.tools.indexing.download_manifest.TMP_FOLDER"]], "threadcontrol (class in gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.ThreadControl"]], "urls (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.URLS"]], "async_download_object_manifest() (in module gen3.tools.indexing.download_manifest)": [[11, "gen3.tools.indexing.download_manifest.async_download_object_manifest"]], "async_verify_object_manifest() (in module gen3.tools.indexing.verify_manifest)": [[11, "gen3.tools.indexing.verify_manifest.async_verify_object_manifest"]], "delete_all_guids() (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.delete_all_guids"]], "gen3.tools.indexing.download_manifest": [[11, "module-gen3.tools.indexing.download_manifest"]], "gen3.tools.indexing.index_manifest": [[11, "module-gen3.tools.indexing.index_manifest"]], "gen3.tools.indexing.verify_manifest": [[11, "module-gen3.tools.indexing.verify_manifest"]], "index_object_manifest() (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.index_object_manifest"]], "column_to_use_as_guid (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.COLUMN_TO_USE_AS_GUID"]], "guid_type_for_indexed_file_object (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.GUID_TYPE_FOR_INDEXED_FILE_OBJECT"]], "guid_type_for_non_indexed_file_object (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.GUID_TYPE_FOR_NON_INDEXED_FILE_OBJECT"]], "max_concurrent_requests (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.MAX_CONCURRENT_REQUESTS"]], "async_ingest_metadata_manifest() (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest"]], "async_query_urls_from_indexd() (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.async_query_urls_from_indexd"]], "gen3.tools.metadata.ingest_manifest": [[12, "module-gen3.tools.metadata.ingest_manifest"]], "gen3wsstorage (class in gen3.wss)": [[13, "gen3.wss.Gen3WsStorage"]], "copy() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.copy"]], "download() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.download"]], "download_url() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.download_url"]], "ls() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.ls"]], "ls_path() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.ls_path"]], "rm() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.rm"]], "rm_path() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.rm_path"]], "upload() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.upload"]], "upload_url() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.upload_url"]]}})
\ No newline at end of file
+Search.setIndex({"docnames": ["auth", "file", "index", "indexing", "jobs", "metadata", "object", "query", "submission", "tools", "tools/drs_pull", "tools/indexing", "tools/metadata", "wss"], "filenames": ["auth.rst", "file.rst", "index.rst", "indexing.rst", "jobs.rst", "metadata.rst", "object.rst", "query.rst", "submission.rst", "tools.rst", "tools/drs_pull.rst", "tools/indexing.rst", "tools/metadata.rst", "wss.rst"], "titles": ["Gen3 Auth Helper", "Gen3 File Class", "Welcome to Gen3 SDK\u2019s documentation!", "Gen3 Index Class", "Gen3 Jobs Class", "Gen3 Metadata Class", "Gen3 Object Class", "Gen3 Query Class", "Gen3 Submission Class", "Gen3 Tools", "DRS Download Tools", "Indexing Tools", "Metadata Tools", "Gen3 Workspace Storage"], "terms": {"class": [0, 2, 10, 11, 13], "gen3auth": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "endpoint": [0, 1, 2, 3, 4, 5, 7, 8, 13], "none": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "refresh_fil": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "refresh_token": 0, "idp": 0, "sourc": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "base": [0, 1, 3, 4, 5, 6, 7, 8, 9, 11, 13], "authbas": 0, "us": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "request": [0, 1, 3, 5, 8, 11, 12], "implement": 0, "order": [0, 8], "support": [0, 1, 5, 8, 11], "jwt": 0, "authent": 0, "gener": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 13], "access": [0, 1, 3, 7, 10], "token": [0, 10], "from": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "provid": [0, 1, 3, 5, 7, 8, 12], "refresh": [0, 10], "file": [0, 2, 3, 4, 8, 9, 10, 11, 12, 13], "string": [0, 3, 5, 11, 13], "automat": 0, "when": [0, 3, 7, 10, 12], "thei": [0, 10], "expir": [0, 1], "paramet": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "str": [0, 1, 3, 4, 5, 7, 8, 10, 11, 12], "opt": 0, "The": [0, 1, 2, 3, 5, 8, 10, 11], "contain": [0, 2, 5, 8, 9, 10, 11, 12], "download": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 13], "json": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "web": 0, "option": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11], "work": [0, 10], "workspac": [0, 2], "default": [0, 1, 3, 7, 8, 11, 12], "env": 0, "gen3_api_kei": 0, "credenti": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "set": [0, 1, 5, 10], "includ": [0, 3], "search": [0, 2, 3], "path": [0, 1, 5, 10, 11, 13], "valu": [0, 1, 3, 5, 7, 10, 11], "doe": [0, 12], "interpret": 0, "wt": [0, 10], "an": [0, 3, 6, 8, 10, 11], "accesstoken": 0, "If": [0, 1, 7, 11, 12], "can": [0, 3, 4, 8, 11, 12], "specifi": [0, 1, 3, 11, 13], "local": [0, 13], "indic": [0, 11], "environ": 0, "fenc": [0, 1], "exampl": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "thi": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "point": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "sandbox": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "common": [0, 1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "while": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "profil": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "page": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 13], "instal": [0, 2, 11], "crdc": 0, "some": [0, 2], "arbitrari": 0, "kei": [0, 3, 5, 13], "variabl": [0, 7, 8], "rather": 0, "than": [0, 5], "pass": [0, 7, 8, 10], "argument": [0, 13], "constructor": 0, "initi": [0, 10], "follow": [0, 11], "curl": [0, 2], "data": [0, 1, 3, 5, 7, 8, 10, 11], "given": [0, 3, 4, 5, 8, 10, 12, 13], "ex": [0, 11, 12], "user": [0, 10, 12], "return": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11], "respons": [0, 1, 3, 4, 5], "under": [0, 8, 13], "index": [0, 2, 5, 9], "authz": [0, 1, 2, 3, 9, 10, 11], "map": [0, 11], "get": [0, 1, 2, 3, 4, 5, 8, 10, 11, 12, 13], "post": [0, 11], "put": 0, "delet": [0, 1, 2, 3, 5, 6, 8, 10, 11], "i": [0, 1, 2, 3, 4, 5, 8, 10, 11, 12], "els": [0, 12], "filenam": [0, 8, 10, 11, 12], "get_access_token": [0, 2], "auto": [0, 2], "within": [0, 2, 9], "5": [0, 5], "minut": 0, "get_access_token_from_wt": [0, 2], "try": 0, "fetch": 0, "namespac": [0, 12], "refresh_access_token": [0, 2], "new": [0, 3], "gen3fil": [1, 2], "auth_provid": [1, 2, 3, 4, 5, 6, 7, 8, 13], "object": [1, 2, 3, 4, 5, 7, 8, 9, 10, 11, 13], "For": [1, 5, 6, 7, 8, 9, 11], "interact": [1, 3, 4, 5, 6, 8, 13], "manag": [1, 5, 10], "featur": [1, 6], "A": [1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "servic": [1, 3, 4, 5, 6, 8, 11, 12, 13], "presign": 1, "url": [1, 2, 3, 9, 10, 11, 12, 13], "right": 1, "now": [1, 8], "instanc": [1, 3, 6, 7, 8, 9, 10], "auth": [1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "delete_fil": [1, 2], "guid": [1, 2, 3, 5, 6, 9, 11, 12], "method": [1, 7, 10], "deprec": 1, "delete_file_loc": [1, 2, 6], "instead": [1, 7, 11], "all": [1, 3, 4, 5, 6, 7, 8, 10, 11, 12], "locat": [1, 6], "store": [1, 3, 10], "remov": [1, 6, 11, 13], "its": [1, 3], "record": [1, 3, 5, 7, 8, 11, 12], "indexd": [1, 3, 6, 10, 11, 12], "uuid": [1, 3, 8], "id": [0, 1, 3, 5, 10, 11], "text": [1, 7, 8], "result": [1, 8, 10, 11], "type": [1, 3, 4, 5, 7, 8, 10, 11, 12], "get_presigned_url": [1, 2], "protocol": 1, "retriev": [1, 8, 10, 12], "give": 1, "limit": [1, 3, 5, 12], "time": [1, 8, 10, 11], "pick": 1, "avail": [1, 2, 10, 11], "queri": [1, 2, 3, 5, 8, 11, 12], "upload_fil": [1, 2], "file_nam": [1, 2, 3, 9, 10, 11], "expires_in": 1, "upload": [1, 2, 3, 8, 13], "list": [0, 1, 3, 4, 5, 7, 8, 10, 11, 13], "author": 1, "scope": [0, 1], "storag": [1, 2, 6], "s3": [1, 10, 11], "az": 1, "isn": 1, "t": [1, 5, 11], "int": [1, 3, 5, 7, 8, 10, 11, 12], "amount": [1, 9], "second": [1, 4], "sign": 1, "datetim": [1, 10], "utcnow": 1, "Be": 1, "sure": 1, "posit": [1, 7], "integ": [1, 3, 8], "also": 1, "treat": [1, 5], "max_presigned_url_ttl": 1, "configur": 1, "represent": [1, 3], "document": [1, 3], "readm": 2, "python": [2, 9, 11], "github": [2, 7], "repo": 2, "setup": 2, "inform": [2, 3, 10], "along": 2, "quickstart": 2, "script": 2, "more": [2, 5, 7, 9, 10], "detail": [2, 7, 10], "about": [2, 3], "variou": 2, "function": [2, 3, 4, 5, 9, 10, 11, 12], "mostli": 2, "docstr": 2, "code": [2, 8], "helper": 2, "gen3index": [2, 3], "async_create_record": [2, 3], "async_get_record": [2, 3], "async_get_records_from_checksum": [2, 3], "async_get_records_on_pag": [2, 3], "async_get_with_param": [2, 3], "async_query_url": [2, 3], "async_update_record": [2, 3], "create_blank": [2, 3], "create_new_vers": [2, 3], "create_record": [2, 3], "delete_record": [2, 3, 8], "get_all_record": [2, 3], "get_guids_prefix": [2, 3], "get_latest_vers": [2, 3], "get_record": [2, 3], "get_record_doc": [2, 3], "get_records_on_pag": [2, 3], "get_stat": [2, 3], "get_url": [2, 3], "get_valid_guid": [2, 3], "get_vers": [2, 3, 4, 5], "get_with_param": [2, 3], "is_healthi": [2, 3, 4, 5], "query_url": [2, 3], "update_blank": [2, 3], "update_record": [2, 3], "job": 2, "gen3job": [2, 4, 10], "async_run_job_and_wait": [2, 4], "create_job": [2, 4], "get_output": [2, 4], "get_statu": [2, 4], "list_job": [2, 4], "metadata": [2, 3, 6, 9, 11], "gen3metadata": [2, 5], "async_cr": [2, 5], "async_create_alias": [2, 5], "async_delete_alia": [2, 5], "async_delete_alias": [2, 5], "async_get": [2, 5], "async_get_alias": [2, 5], "async_upd": [2, 5], "async_update_alias": [2, 5], "batch_creat": [2, 5], "creat": [2, 3, 4, 5, 6, 8, 10, 11], "create_alias": [2, 5], "create_index_key_path": [2, 5], "delete_alia": [2, 5], "delete_alias": [2, 5], "delete_index_key_path": [2, 5], "get_alias": [2, 5], "get_index_key_path": [2, 5], "updat": [2, 3, 5, 10, 11], "update_alias": [2, 5], "gen3object": [2, 6], "delete_object": [2, 6], "gen3queri": [2, 7], "graphql_queri": [2, 7], "raw_data_download": [2, 7], "submiss": 2, "gen3submiss": [2, 8], "create_program": [2, 8], "create_project": [2, 8], "delete_nod": [2, 8], "delete_program": [2, 8], "delete_project": [2, 8], "export_nod": [2, 8], "export_record": [2, 8], "get_dictionary_al": [2, 8], "get_dictionary_nod": [2, 8], "get_graphql_schema": [2, 8], "get_program": [2, 8], "get_project_dictionari": [2, 8], "get_project_manifest": [2, 8], "get_project": [2, 8], "open_project": [2, 8], "submit_fil": [2, 8], "submit_record": [2, 8], "tool": 2, "dr": [2, 9], "downloadmanag": [2, 9, 10], "cache_hosts_wts_token": [2, 9, 10], "get_fresh_token": [2, 9, 10], "resolve_object": [2, 9, 10], "user_access": [2, 9, 10], "downloadstatu": [2, 9, 10], "statu": [2, 4, 9, 10], "start_tim": [2, 9, 10], "end_tim": [2, 9, 10], "object_id": [1, 2, 9, 10], "object_typ": [2, 9, 10], "hostnam": [2, 9, 10], "file_s": [2, 9, 10, 11], "updated_tim": [2, 9, 10], "created_tim": [2, 9, 10], "access_method": [2, 9, 10], "children": [2, 9, 10], "_manag": [2, 9, 10], "pprint": [2, 9, 10], "manifest": [2, 8, 9, 10, 11, 12], "md5sum": [2, 9, 10], "commons_url": [2, 9, 10, 11, 12], "create_object_list": [2, 9, 10], "load": [2, 9, 10], "load_manifest": [2, 9, 10], "download_drs_object": [2, 9, 10], "download_files_in_drs_manifest": [2, 9, 10], "list_access_in_drs_manifest": [2, 9, 10], "list_drs_object": [2, 9, 10], "list_files_in_drs_manifest": [2, 9, 10], "current_dir": [2, 9, 11], "indexd_record_page_s": [2, 9, 11], "max_concurrent_request": [2, 9, 11, 12], "tmp_folder": [2, 9, 11], "async_download_object_manifest": [2, 9, 11], "size": [2, 3, 9, 10, 11], "md5": [2, 3, 9, 11], "acl": [2, 3, 9, 11], "prev_guid": [2, 9, 11], "threadcontrol": [2, 9, 11], "delete_all_guid": [2, 9, 11], "index_object_manifest": [2, 9, 11], "verifi": [2, 9], "async_verify_object_manifest": [2, 9, 11], "ingest": [2, 9], "column_to_use_as_guid": [2, 9, 12], "guid_type_for_indexed_file_object": [2, 9, 12], "guid_type_for_non_indexed_file_object": [2, 9, 12], "async_ingest_metadata_manifest": [2, 9, 12], "async_query_urls_from_indexd": [2, 9, 12], "gen3wsstorag": [2, 13], "copi": [2, 13], "download_url": [2, 13], "l": [2, 13], "ls_path": [2, 13], "rm": [2, 13], "rm_path": [2, 13], "upload_url": [2, 13], "modul": [2, 10, 11], "service_loc": [3, 4, 5], "public": [3, 5], "read": [3, 5, 11], "onli": [3, 5, 7, 8, 10, 11], "necessari": [3, 5], "basic": [3, 11, 12], "cred": 3, "tupl": [0, 3, 11, 12], "async": [3, 4, 5, 9, 11, 12], "hash": [3, 11], "did": 3, "baseid": 3, "urls_metadata": 3, "version": [3, 4, 5], "_ssl": [3, 4, 5], "asynchron": [3, 4, 5], "dict": [3, 4, 5, 10, 11, 12], "eg": 3, "ab167e49d25b488939b1ede42752458b": 3, "associ": [3, 5], "made": 3, "where": [3, 5, 11, 12], "you": [3, 8, 11], "control": 3, "rbac": 3, "name": [3, 4, 8, 10, 11, 12, 13], "addit": [3, 5, 10, 11], "entri": [3, 11], "attach": [3, 5], "each": [3, 8, 10, 11], "group": 3, "previou": [3, 11], "checksum": [3, 10], "checksum_typ": 3, "match": [3, 5, 12], "param": [3, 5, 8, 10], "correspond": 3, "suppli": 3, "need": [3, 7, 10, 11], "handl": [3, 10], "hash_typ": 3, "whether": [3, 4, 5, 8, 11, 12], "ssl": [3, 4, 5], "pattern": [3, 12], "against": [3, 7, 8, 11, 12], "bodi": 3, "dictionari": [3, 4, 5, 7, 8], "format": [3, 5, 8, 11], "blank": 3, "add": [3, 5], "sinc": 3, "content": [3, 13], "immut": 3, "want": [3, 8], "chang": [3, 11], "That": 3, "field": [3, 5, 7, 11, 12], "old": 3, "ad": 3, "least": 3, "one": [3, 5, 10, 11], "suffici": 3, "note": [0, 3, 11, 12], "good": 3, "idea": 3, "number": [3, 7, 8, 11, 12], "noth": [3, 6], "dist_resolut": 3, "true": [3, 4, 5, 6, 7, 8, 10, 11, 12], "alia": [3, 5], "distribut": 3, "identifi": [3, 5, 9, 11], "boolean": 3, "we": [3, 11], "pagin": 3, "fals": [3, 5, 6, 10, 11], "start": [3, 4, 7, 8, 10], "prefix": 3, "rtype": 3, "has_vers": 3, "latest": 3, "exclud": 3, "without": [3, 5], "repres": [3, 5, 10], "info": [3, 11], "algorithm": 3, "count": 3, "valid": [3, 7], "file_st": 3, "client": [0, 3], "like": [3, 5, 9, 11, 12], "signpost": 3, "healthi": [3, 4, 5], "rev": 3, "revis": 3, "simpl": 3, "consist": 3, "mechan": 3, "exist": [1, 3, 5, 6, 9, 12], "": [1, 4, 8, 10, 11], "dispatch": 4, "job_nam": 4, "job_input": 4, "kwarg": [3, 4, 5], "wait": 4, "output": [4, 5, 11, 12], "Will": [4, 7, 10], "sleep": 4, "linear": 4, "delai": 4, "until": [4, 10], "done": 4, "1": [4, 5, 8, 10, 11], "global": [4, 5], "input": [4, 10, 11], "job_id": 4, "previous": 4, "complet": [4, 11], "bool": [4, 5, 8, 10, 11, 12], "md": [5, 7, 10, 12], "admin_endpoint_suffix": 5, "admin": [5, 11], "alias": 5, "overwrit": 5, "what": 5, "end": [5, 10], "up": [5, 9], "blob": [5, 7], "asyncron": 5, "uniqu": [1, 5], "singl": [1, 5, 8, 10], "altern": [5, 11], "merg": 5, "AND": 5, "descript": 5, "metadata_list": 5, "specif": [5, 8, 11, 12], "expect": [5, 9, 11], "anoth": 5, "databas": 5, "return_full_metadata": 5, "10": [5, 7, 11], "offset": [5, 7], "0": [5, 7, 8, 11], "use_agg_md": 5, "off": 5, "logic": [5, 12], "filter": [5, 7], "b": [5, 11], "2": [5, 11], "have": [5, 11], "ar": [5, 7, 8, 9, 10, 11], "alwai": 5, "nest": 5, "c": [5, 11], "3": [5, 11], "same": [5, 11, 13], "whose": 5, "ani": [5, 10, 11], "But": 5, "differ": 5, "must": [1, 5], "33": 5, "d": 5, "4": [5, 11], "e": [5, 10], "won": 5, "333": 5, "defin": [5, 8, 10], "api": [5, 8, 11], "just": [5, 11, 12], "max": 5, "num": 5, "OR": 5, "level": 6, "current": [6, 8, 10], "allow": [0, 6, 8, 10, 11, 12], "system": [6, 7, 8, 9], "bucket": [1, 6], "elasticsearch": 7, "query_str": 7, "execut": [7, 8, 11], "graphql": [7, 8], "query_txt": [7, 8], "syntax": 7, "see": [7, 10, 11], "http": [0, 7, 11, 12], "com": 7, "uc": 7, "cdi": 7, "guppi": 7, "master": 7, "doc": [7, 10], "data_typ": 7, "my_index": 7, "my_field": 7, "first": [7, 8], "filter_object": 7, "sort_object": 7, "verbos": [7, 8], "row": [7, 8, 11, 12], "sort": 7, "equal": 7, "respect": 7, "complex": 7, "appli": 7, "One": 7, "unaccess": 7, "regular": 7, "tier": 7, "mode": 7, "subject": [7, 8], "50": 7, "vital_statu": 7, "submitter_id": 7, "aliv": 7, "asc": 7, "sort_field": 7, "raw": [7, 11], "project_id": [7, 8], "my_program": 7, "my_project": 7, "submit": [8, 11], "export": [8, 10], "sheepdog": 8, "through": [8, 11], "peregrin": 8, "sub": 8, "program": [8, 11], "project": [8, 11], "dcf": 8, "node_nam": 8, "batch_siz": 8, "100": [8, 11], "node": 8, "how": [8, 11], "mani": [8, 11], "print": [8, 10], "progress": [8, 10], "log": [8, 10, 11, 12], "ccle": 8, "demograph": 8, "ordered_node_list": 8, "revers": 8, "graph": 8, "experi": 8, "empti": [8, 11], "uuid1": 8, "uuid2": 8, "node_typ": 8, "fileformat": 8, "which": [8, 10], "belong": 8, "either": 8, "tsv": [8, 11, 12], "screen": 8, "sampl": [8, 10], "ccle_sample_nod": 8, "d70b41b9": 8, "6f90": 8, "4714": 8, "8420": 8, "e043ab8b77b9": 8, "ccle_one_record": 8, "entir": 8, "schema": 8, "run": [8, 11], "introspect": 8, "regist": 8, "mark": 8, "open": [8, 10, 11], "mean": 8, "etc": 8, "max_tri": 8, "retri": 8, "fail": [8, 10], "chunk_siz": 8, "30": 8, "row_offset": 8, "spreadsheet": 8, "multipl": [8, 11], "csv": [8, 11, 12], "xlsx": 8, "worksheet": 8, "skip": 8, "data_spreadsheet": 8, "should": [0, 8, 11], "arrai": 8, "action": [9, 11], "These": 9, "broken": 9, "broad": 9, "categori": 9, "task": 9, "relat": 9, "persist": 9, "Such": 9, "mai": [9, 11], "involv": 9, "assign": 9, "everi": [9, 11], "alreadi": 9, "larg": 9, "most": 9, "util": 9, "capabl": 9, "make": [9, 11], "effici": 9, "main": 10, "datafil": 10, "manifest_1": 10, "my_common": 10, "org": 10, "gen3": [10, 11, 12], "my_credenti": 10, "howto": 10, "drsdownload": 10, "drs_download": 10, "download_list": 10, "show_progress": 10, "assist": 10, "minimum": 10, "interest": 10, "object_list": 10, "host": 10, "obtain": [0, 10], "It": 10, "possibl": 10, "save_directori": 10, "unpack_packag": 10, "delete_unpacked_packag": 10, "directori": [10, 11], "call": [10, 13], "case": 10, "error": [10, 11, 12], "continu": 10, "save": 10, "show": 10, "bar": 10, "disabl": 10, "unpack": 10, "packag": 10, "after": 10, "them": [10, 11], "drs_hostnam": 10, "known": 10, "otherwis": 10, "success": 10, "resolv": 10, "permiss": 10, "wai": 10, "determin": [10, 11, 12], "reason": 10, "pend": 10, "objectid": 10, "datamanag": 10, "occur": 10, "attribut": [10, 11], "drsobjecttyp": 10, "unknown": 10, "factori": 10, "popul": [10, 12], "server": 10, "requir": 10, "byte": 10, "timestamp": 10, "last": 10, "g": 10, "child": 10, "bundl": 10, "self": 10, "indent": 10, "pretti": 10, "In": 10, "similar": 10, "linux": 10, "tree": 10, "command": [10, 11], "typic": 10, "discoveri": 10, "cach": 10, "files": 10, "static": 10, "bownload": 10, "successfulli": 10, "pars": [10, 11, 12, 13], "output_dir": 10, "conveni": 10, "infil": 10, "wrapper": 10, "cli": 10, "process": 11, "coroutin": 11, "asyncio": [11, 12], "librari": 11, "comma": 11, "separ": [0, 11], "header": 11, "space": [0, 11], "download_manifest": 11, "per": [11, 12], "maximum": [11, 12], "desir": 11, "concurr": [11, 12], "across": 11, "thread": 11, "folder": 11, "place": 11, "temporari": 11, "write": 11, "safe": 11, "so": 11, "To": 11, "workaround": 11, "concat": 11, "output_filenam": [11, 12], "num_process": 11, "24": [11, 12], "input_manifest": 11, "root": [11, 12], "domain": [11, 12], "live": [11, 12], "parallel": 11, "hit": 11, "total": 11, "help": 11, "everyth": 11, "attempt": 11, "tab": 11, "session": 11, "column": [11, 12], "shown": 11, "below": 11, "255e396f": 11, "f1f8": 11, "11e9": 11, "9a07": 11, "0a80fada099c": 11, "473d83400bc1bc9dc635e334faddf33c": 11, "363455714": 11, "pdcdatastor": 11, "test1": 11, "0a80fada098c": 11, "473d83400bc1bc9dc635e334faddd33c": 11, "343434344": 11, "test2": 11, "0a80fada097c": 11, "473d83400bc1bc9dc635e334fadd433c": 11, "543434443": 11, "phs0001": 11, "phs0002": 11, "test3": 11, "0a80fada096c": 11, "test4": 11, "0a80fada010c": 11, "473d83400bc1bc9dc635e334fadde33c": 11, "test5": 11, "index_manifest": 11, "usag": 11, "py": 11, "giangb": 11, "planx": 11, "pla": 11, "net": 11, "manifest_fil": [11, 12], "path_to_manifest": 11, "replace_url": 11, "thread_num": 11, "api_kei": 11, "processed_fil": 11, "num_total_fil": 11, "synchron": 11, "warn": 11, "IF": 11, "know": 11, "THE": 11, "implic": 11, "manifest_file_delimit": [11, 12], "submit_additional_metadata_column": 11, "force_metadata_columns_even_if_empti": 11, "loop": 11, "ha": [0, 11], "password": [11, 12], "flag": 11, "replac": 11, "delimit": 11, "forc": 11, "creation": 11, "even": 11, "enabl": 11, "illustr": 11, "columna": 11, "columnb": 11, "columnc": 11, "dataa": 11, "datab": 11, "guid_exampl": 11, "md5_hash": 11, "fieldnam": 11, "There": 11, "those": 11, "abov": 11, "overrid": [11, 12], "assum": 11, "behavior": 11, "simpli": 11, "manifest_row_pars": [11, 12], "custom": 11, "import": 11, "verify_manifest": 11, "def": 11, "_get_authz_from_row": 11, "strip": 11, "parser": [11, 12], "verify_object_manifest": 11, "verif": 11, "error_nam": 11, "value_from_manifest": 11, "actual": 11, "value_from_indexd": 11, "93d9af72": 11, "b0f1": 11, "450c": 11, "a5c6": 11, "7d3d8d2083b4": 11, "dev": 11, "test": 11, "_get_acl_from_row": 11, "_get_file_name_from_row": 11, "_get_file_size_from_row": 11, "_get_guid_from_row": 11, "_get_md5_from_row": 11, "_get_urls_from_row": 11, "1664913404": [], "4600763": [], "indexd_field": [11, 12], "func_to_parse_row": [11, 12], "delimet": [11, 12], "metdata": 12, "ingest_manifest": 12, "NOT": 12, "guid_from_fil": 12, "_get_guid_for_row": 12, "indexed_file_object_guid": 12, "_query_for_associated_indexd_record_guid": 12, "guid_for_row": 12, "elsewher": 12, "metadata_sourc": 12, "7884498": [], "get_guid_from_fil": 12, "metadata_typ": 12, "dbgap": 12, "usual": 12, "fill": 12, "_guid_typ": 12, "is_indexed_file_object": 12, "lock": 12, "semaphor": 12, "semaphon": 12, "ammount": 12, "connect": 12, "wss": 13, "src_urlstr": 13, "dest_urlstr": 13, "appropri": 13, "src_w": 13, "src_wskei": 13, "dest_path": 13, "disk": 13, "w": 13, "wskei": 13, "ws_urlstr": 13, "form": 13, "src_path": 13, "dest_w": 13, "dest_wskei": 13, "allowed_data_upload_bucket": 1, "data_upload_bucket": 1, "upload_file_to_guid": [1, 2], "1664995746": [], "2094085": [], "6912842": [], "1665361686": [], "0196362": [], "4282863": [], "1665415592": [], "0841575": [], "369792": [], "download_singl": [1, 2], "1667237446": [], "310436": [], "461374": [], "1667237580": [], "6713352": [], "1667237581": [], "0283232": [], "1668452999": [], "8464384": [], "1668453000": [], "219631": [], "1668455015": [], "5573874": [], "9250507": [], "client_credenti": 0, "client_scop": 0, "client_id": 0, "client_secret": 0, "oidc": 0, "grant": 0, "openid": 0, "secret": 0, "1670019997": [], "6497164": [], "9551775": [], "1670272579": [], "3670187": [], "6539462": [], "never": 0, "hardcod": 0, "o": 0, "gen3_oidc_client_creds_secret": 0, "1671036553": [], "0874994": [], "3574216": [], "datacommon": 0, "1671044831": [], "8660681": [], "1671044832": [], "1868503": [], "1671140235": [], "0801866": [], "4101439": [], "1671140559": [], "279068": [], "6550183": [], "1673541949": [], "824122": [], "1673541950": [], "2653491": [], "1674497076": [], "8602023": [], "1674497077": [], "2165172": [], "python_subprocess_command": 11, "By": 11, "someth": 11, "maco": 11, "x": 11, "python3": 11, "1676929147": [], "3869426": [], "758984": [], "1676929361": [], "016488": [], "398911": [], "1676930607": [], "061559": [], "4525151": [], "1677279338": [], "2703023": [], "6413171": [], "1679334258": [11, 12], "6011407": 11, "9486785": 12}, "objects": {"gen3.auth": [[0, 0, 1, "", "Gen3Auth"]], "gen3.auth.Gen3Auth": [[0, 1, 1, "", "curl"], [0, 1, 1, "", "get_access_token"], [0, 1, 1, "", "get_access_token_from_wts"], [0, 1, 1, "", "refresh_access_token"]], "gen3.file": [[1, 0, 1, "", "Gen3File"]], "gen3.file.Gen3File": [[1, 1, 1, "", "delete_file"], [1, 1, 1, "", "delete_file_locations"], [1, 1, 1, "", "download_single"], [1, 1, 1, "", "get_presigned_url"], [1, 1, 1, "", "upload_file"], [1, 1, 1, "", "upload_file_to_guid"]], "gen3.index": [[3, 0, 1, "", "Gen3Index"]], "gen3.index.Gen3Index": [[3, 1, 1, "", "async_create_record"], [3, 1, 1, "", "async_get_record"], [3, 1, 1, "", "async_get_records_from_checksum"], [3, 1, 1, "", "async_get_records_on_page"], [3, 1, 1, "", "async_get_with_params"], [3, 1, 1, "", "async_query_urls"], [3, 1, 1, "", "async_update_record"], [3, 1, 1, "", "create_blank"], [3, 1, 1, "", "create_new_version"], [3, 1, 1, "", "create_record"], [3, 1, 1, "", "delete_record"], [3, 1, 1, "", "get"], [3, 1, 1, "", "get_all_records"], [3, 1, 1, "", "get_guids_prefix"], [3, 1, 1, "", "get_latest_version"], [3, 1, 1, "", "get_record"], [3, 1, 1, "", "get_record_doc"], [3, 1, 1, "", "get_records"], [3, 1, 1, "", "get_records_on_page"], [3, 1, 1, "", "get_stats"], [3, 1, 1, "", "get_urls"], [3, 1, 1, "", "get_valid_guids"], [3, 1, 1, "", "get_version"], [3, 1, 1, "", "get_versions"], [3, 1, 1, "", "get_with_params"], [3, 1, 1, "", "is_healthy"], [3, 1, 1, "", "query_urls"], [3, 1, 1, "", "update_blank"], [3, 1, 1, "", "update_record"]], "gen3.jobs": [[4, 0, 1, "", "Gen3Jobs"]], "gen3.jobs.Gen3Jobs": [[4, 1, 1, "", "async_run_job_and_wait"], [4, 1, 1, "", "create_job"], [4, 1, 1, "", "get_output"], [4, 1, 1, "", "get_status"], [4, 1, 1, "", "get_version"], [4, 1, 1, "", "is_healthy"], [4, 1, 1, "", "list_jobs"]], "gen3.metadata": [[5, 0, 1, "", "Gen3Metadata"]], "gen3.metadata.Gen3Metadata": [[5, 1, 1, "", "async_create"], [5, 1, 1, "", "async_create_aliases"], [5, 1, 1, "", "async_delete_alias"], [5, 1, 1, "", "async_delete_aliases"], [5, 1, 1, "", "async_get"], [5, 1, 1, "", "async_get_aliases"], [5, 1, 1, "", "async_update"], [5, 1, 1, "", "async_update_aliases"], [5, 2, 1, "", "auth_provider"], [5, 1, 1, "", "batch_create"], [5, 1, 1, "", "create"], [5, 1, 1, "", "create_aliases"], [5, 1, 1, "", "create_index_key_path"], [5, 1, 1, "", "delete"], [5, 1, 1, "", "delete_alias"], [5, 1, 1, "", "delete_aliases"], [5, 1, 1, "", "delete_index_key_path"], [5, 2, 1, "", "endpoint"], [5, 1, 1, "", "get"], [5, 1, 1, "", "get_aliases"], [5, 1, 1, "", "get_index_key_paths"], [5, 1, 1, "", "get_version"], [5, 1, 1, "", "is_healthy"], [5, 1, 1, "", "query"], [5, 1, 1, "", "update"], [5, 1, 1, "", "update_aliases"]], "gen3.object": [[6, 0, 1, "", "Gen3Object"]], "gen3.object.Gen3Object": [[6, 1, 1, "", "delete_object"]], "gen3.query": [[7, 0, 1, "", "Gen3Query"]], "gen3.query.Gen3Query": [[7, 1, 1, "", "graphql_query"], [7, 1, 1, "", "query"], [7, 1, 1, "", "raw_data_download"]], "gen3.submission": [[8, 0, 1, "", "Gen3Submission"]], "gen3.submission.Gen3Submission": [[8, 1, 1, "", "create_program"], [8, 1, 1, "", "create_project"], [8, 1, 1, "", "delete_node"], [8, 1, 1, "", "delete_nodes"], [8, 1, 1, "", "delete_program"], [8, 1, 1, "", "delete_project"], [8, 1, 1, "", "delete_record"], [8, 1, 1, "", "delete_records"], [8, 1, 1, "", "export_node"], [8, 1, 1, "", "export_record"], [8, 1, 1, "", "get_dictionary_all"], [8, 1, 1, "", "get_dictionary_node"], [8, 1, 1, "", "get_graphql_schema"], [8, 1, 1, "", "get_programs"], [8, 1, 1, "", "get_project_dictionary"], [8, 1, 1, "", "get_project_manifest"], [8, 1, 1, "", "get_projects"], [8, 1, 1, "", "open_project"], [8, 1, 1, "", "query"], [8, 1, 1, "", "submit_file"], [8, 1, 1, "", "submit_record"]], "gen3": [[9, 3, 0, "-", "tools"]], "gen3.tools.download": [[10, 3, 0, "-", "drs_download"]], "gen3.tools.download.drs_download": [[10, 0, 1, "", "DownloadManager"], [10, 0, 1, "", "DownloadStatus"], [10, 0, 1, "", "Downloadable"], [10, 0, 1, "", "Manifest"], [10, 4, 1, "", "download_drs_object"], [10, 4, 1, "", "download_files_in_drs_manifest"], [10, 4, 1, "", "list_access_in_drs_manifest"], [10, 4, 1, "", "list_drs_object"], [10, 4, 1, "", "list_files_in_drs_manifest"]], "gen3.tools.download.drs_download.DownloadManager": [[10, 1, 1, "", "cache_hosts_wts_tokens"], [10, 1, 1, "", "download"], [10, 1, 1, "", "get_fresh_token"], [10, 1, 1, "", "resolve_objects"], [10, 1, 1, "", "user_access"]], "gen3.tools.download.drs_download.DownloadStatus": [[10, 2, 1, "", "end_time"], [10, 2, 1, "", "start_time"], [10, 2, 1, "", "status"]], "gen3.tools.download.drs_download.Downloadable": [[10, 2, 1, "", "_manager"], [10, 2, 1, "", "access_methods"], [10, 2, 1, "", "children"], [10, 2, 1, "", "created_time"], [10, 1, 1, "", "download"], [10, 2, 1, "", "file_name"], [10, 2, 1, "", "file_size"], [10, 2, 1, "", "hostname"], [10, 2, 1, "", "object_id"], [10, 2, 1, "", "object_type"], [10, 1, 1, "", "pprint"], [10, 2, 1, "", "updated_time"]], "gen3.tools.download.drs_download.Manifest": [[10, 2, 1, "", "commons_url"], [10, 1, 1, "", "create_object_list"], [10, 2, 1, "", "file_name"], [10, 2, 1, "", "file_size"], [10, 1, 1, "", "load"], [10, 1, 1, "", "load_manifest"], [10, 2, 1, "", "md5sum"], [10, 2, 1, "", "object_id"]], "gen3.tools.indexing": [[11, 3, 0, "-", "download_manifest"], [11, 3, 0, "-", "index_manifest"], [11, 3, 0, "-", "verify_manifest"]], "gen3.tools.indexing.download_manifest": [[11, 2, 1, "", "CURRENT_DIR"], [11, 2, 1, "", "INDEXD_RECORD_PAGE_SIZE"], [11, 2, 1, "", "MAX_CONCURRENT_REQUESTS"], [11, 2, 1, "", "TMP_FOLDER"], [11, 4, 1, "", "async_download_object_manifest"]], "gen3.tools.indexing.index_manifest": [[11, 2, 1, "", "ACLS"], [11, 2, 1, "", "AUTHZ"], [11, 2, 1, "", "CURRENT_DIR"], [11, 2, 1, "", "GUID"], [11, 2, 1, "", "MD5"], [11, 2, 1, "", "PREV_GUID"], [11, 2, 1, "", "SIZE"], [11, 0, 1, "", "ThreadControl"], [11, 2, 1, "", "URLS"], [11, 4, 1, "", "delete_all_guids"], [11, 4, 1, "", "index_object_manifest"]], "gen3.tools.indexing.verify_manifest": [[11, 2, 1, "", "CURRENT_DIR"], [11, 2, 1, "", "MAX_CONCURRENT_REQUESTS"], [11, 4, 1, "", "async_verify_object_manifest"]], "gen3.tools.metadata": [[12, 3, 0, "-", "ingest_manifest"]], "gen3.tools.metadata.ingest_manifest": [[12, 2, 1, "", "COLUMN_TO_USE_AS_GUID"], [12, 2, 1, "", "GUID_TYPE_FOR_INDEXED_FILE_OBJECT"], [12, 2, 1, "", "GUID_TYPE_FOR_NON_INDEXED_FILE_OBJECT"], [12, 2, 1, "", "MAX_CONCURRENT_REQUESTS"], [12, 4, 1, "", "async_ingest_metadata_manifest"], [12, 4, 1, "", "async_query_urls_from_indexd"]], "gen3.wss": [[13, 0, 1, "", "Gen3WsStorage"]], "gen3.wss.Gen3WsStorage": [[13, 1, 1, "", "copy"], [13, 1, 1, "", "download"], [13, 1, 1, "", "download_url"], [13, 1, 1, "", "ls"], [13, 1, 1, "", "ls_path"], [13, 1, 1, "", "rm"], [13, 1, 1, "", "rm_path"], [13, 1, 1, "", "upload"], [13, 1, 1, "", "upload_url"]]}, "objtypes": {"0": "py:class", "1": "py:method", "2": "py:attribute", "3": "py:module", "4": "py:function"}, "objnames": {"0": ["py", "class", "Python class"], "1": ["py", "method", "Python method"], "2": ["py", "attribute", "Python attribute"], "3": ["py", "module", "Python module"], "4": ["py", "function", "Python function"]}, "titleterms": {"gen3": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 13], "auth": 0, "helper": 0, "file": 1, "class": [1, 3, 4, 5, 6, 7, 8], "welcom": 2, "sdk": 2, "": 2, "document": 2, "indic": 2, "tabl": 2, "index": [3, 11], "job": 4, "metadata": [5, 12], "object": 6, "queri": 7, "submiss": 8, "tool": [9, 10, 11, 12], "dr": 10, "download": [10, 11], "verifi": 11, "ingest": 12, "workspac": 13, "storag": 13}, "envversion": {"sphinx.domains.c": 2, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 8, "sphinx.domains.index": 1, "sphinx.domains.javascript": 2, "sphinx.domains.math": 2, "sphinx.domains.python": 3, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "sphinx.ext.viewcode": 1, "sphinx": 57}, "alltitles": {"Gen3 Auth Helper": [[0, "gen3-auth-helper"]], "Gen3 File Class": [[1, "gen3-file-class"]], "Welcome to Gen3 SDK\u2019s documentation!": [[2, "welcome-to-gen3-sdk-s-documentation"]], "Indices and tables": [[2, "indices-and-tables"]], "Gen3 Index Class": [[3, "gen3-index-class"]], "Gen3 Jobs Class": [[4, "gen3-jobs-class"]], "Gen3 Metadata Class": [[5, "gen3-metadata-class"]], "Gen3 Object Class": [[6, "gen3-object-class"]], "Gen3 Query Class": [[7, "gen3-query-class"]], "Gen3 Submission Class": [[8, "gen3-submission-class"]], "Gen3 Tools": [[9, "gen3-tools"]], "DRS Download Tools": [[10, "module-gen3.tools.download.drs_download"]], "Indexing Tools": [[11, "indexing-tools"]], "Download": [[11, "module-gen3.tools.indexing.download_manifest"]], "Index": [[11, "module-gen3.tools.indexing.index_manifest"]], "Verify": [[11, "module-gen3.tools.indexing.verify_manifest"]], "Metadata Tools": [[12, "metadata-tools"]], "Ingest": [[12, "module-gen3.tools.metadata.ingest_manifest"]], "Gen3 Workspace Storage": [[13, "gen3-workspace-storage"]]}, "indexentries": {"gen3auth (class in gen3.auth)": [[0, "gen3.auth.Gen3Auth"]], "curl() (gen3.auth.gen3auth method)": [[0, "gen3.auth.Gen3Auth.curl"]], "get_access_token() (gen3.auth.gen3auth method)": [[0, "gen3.auth.Gen3Auth.get_access_token"]], "get_access_token_from_wts() (gen3.auth.gen3auth method)": [[0, "gen3.auth.Gen3Auth.get_access_token_from_wts"]], "refresh_access_token() (gen3.auth.gen3auth method)": [[0, "gen3.auth.Gen3Auth.refresh_access_token"]], "gen3file (class in gen3.file)": [[1, "gen3.file.Gen3File"]], "delete_file() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.delete_file"]], "delete_file_locations() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.delete_file_locations"]], "download_single() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.download_single"]], "get_presigned_url() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.get_presigned_url"]], "upload_file() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.upload_file"]], "upload_file_to_guid() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.upload_file_to_guid"]], "gen3index (class in gen3.index)": [[3, "gen3.index.Gen3Index"]], "async_create_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_create_record"]], "async_get_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_get_record"]], "async_get_records_from_checksum() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_get_records_from_checksum"]], "async_get_records_on_page() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_get_records_on_page"]], "async_get_with_params() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_get_with_params"]], "async_query_urls() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_query_urls"]], "async_update_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_update_record"]], "create_blank() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.create_blank"]], "create_new_version() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.create_new_version"]], "create_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.create_record"]], "delete_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.delete_record"]], "get() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get"]], "get_all_records() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_all_records"]], "get_guids_prefix() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_guids_prefix"]], "get_latest_version() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_latest_version"]], "get_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_record"]], "get_record_doc() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_record_doc"]], "get_records() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_records"]], "get_records_on_page() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_records_on_page"]], "get_stats() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_stats"]], "get_urls() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_urls"]], "get_valid_guids() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_valid_guids"]], "get_version() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_version"]], "get_versions() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_versions"]], "get_with_params() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_with_params"]], "is_healthy() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.is_healthy"]], "query_urls() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.query_urls"]], "update_blank() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.update_blank"]], "update_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.update_record"]], "gen3jobs (class in gen3.jobs)": [[4, "gen3.jobs.Gen3Jobs"]], "async_run_job_and_wait() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.async_run_job_and_wait"]], "create_job() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.create_job"]], "get_output() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.get_output"]], "get_status() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.get_status"]], "get_version() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.get_version"]], "is_healthy() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.is_healthy"]], "list_jobs() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.list_jobs"]], "gen3metadata (class in gen3.metadata)": [[5, "gen3.metadata.Gen3Metadata"]], "async_create() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_create"]], "async_create_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_create_aliases"]], "async_delete_alias() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_delete_alias"]], "async_delete_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_delete_aliases"]], "async_get() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_get"]], "async_get_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_get_aliases"]], "async_update() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_update"]], "async_update_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_update_aliases"]], "auth_provider (gen3.metadata.gen3metadata attribute)": [[5, "gen3.metadata.Gen3Metadata.auth_provider"]], "batch_create() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.batch_create"]], "create() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.create"]], "create_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.create_aliases"]], "create_index_key_path() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.create_index_key_path"]], "delete() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.delete"]], "delete_alias() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.delete_alias"]], "delete_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.delete_aliases"]], "delete_index_key_path() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.delete_index_key_path"]], "endpoint (gen3.metadata.gen3metadata attribute)": [[5, "gen3.metadata.Gen3Metadata.endpoint"]], "get() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.get"]], "get_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.get_aliases"]], "get_index_key_paths() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.get_index_key_paths"]], "get_version() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.get_version"]], "is_healthy() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.is_healthy"]], "query() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.query"]], "update() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.update"]], "update_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.update_aliases"]], "gen3object (class in gen3.object)": [[6, "gen3.object.Gen3Object"]], "delete_object() (gen3.object.gen3object method)": [[6, "gen3.object.Gen3Object.delete_object"]], "gen3query (class in gen3.query)": [[7, "gen3.query.Gen3Query"]], "graphql_query() (gen3.query.gen3query method)": [[7, "gen3.query.Gen3Query.graphql_query"]], "query() (gen3.query.gen3query method)": [[7, "gen3.query.Gen3Query.query"]], "raw_data_download() (gen3.query.gen3query method)": [[7, "gen3.query.Gen3Query.raw_data_download"]], "gen3submission (class in gen3.submission)": [[8, "gen3.submission.Gen3Submission"]], "create_program() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.create_program"]], "create_project() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.create_project"]], "delete_node() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_node"]], "delete_nodes() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_nodes"]], "delete_program() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_program"]], "delete_project() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_project"]], "delete_record() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_record"]], "delete_records() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_records"]], "export_node() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.export_node"]], "export_record() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.export_record"]], "get_dictionary_all() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_dictionary_all"]], "get_dictionary_node() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_dictionary_node"]], "get_graphql_schema() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_graphql_schema"]], "get_programs() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_programs"]], "get_project_dictionary() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_project_dictionary"]], "get_project_manifest() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_project_manifest"]], "get_projects() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_projects"]], "open_project() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.open_project"]], "query() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.query"]], "submit_file() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.submit_file"]], "submit_record() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.submit_record"]], "gen3.tools": [[9, "module-gen3.tools"]], "module": [[9, "module-gen3.tools"], [10, "module-gen3.tools.download.drs_download"], [11, "module-gen3.tools.indexing.download_manifest"], [11, "module-gen3.tools.indexing.index_manifest"], [11, "module-gen3.tools.indexing.verify_manifest"], [12, "module-gen3.tools.metadata.ingest_manifest"]], "downloadmanager (class in gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.DownloadManager"]], "downloadstatus (class in gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.DownloadStatus"]], "downloadable (class in gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.Downloadable"]], "manifest (class in gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.Manifest"]], "_manager (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable._manager"]], "access_methods (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.access_methods"]], "cache_hosts_wts_tokens() (gen3.tools.download.drs_download.downloadmanager method)": [[10, "gen3.tools.download.drs_download.DownloadManager.cache_hosts_wts_tokens"]], "children (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.children"]], "commons_url (gen3.tools.download.drs_download.manifest attribute)": [[10, "gen3.tools.download.drs_download.Manifest.commons_url"]], "create_object_list() (gen3.tools.download.drs_download.manifest static method)": [[10, "gen3.tools.download.drs_download.Manifest.create_object_list"]], "created_time (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.created_time"]], "download() (gen3.tools.download.drs_download.downloadmanager method)": [[10, "gen3.tools.download.drs_download.DownloadManager.download"]], "download() (gen3.tools.download.drs_download.downloadable method)": [[10, "gen3.tools.download.drs_download.Downloadable.download"]], "download_drs_object() (in module gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.download_drs_object"]], "download_files_in_drs_manifest() (in module gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.download_files_in_drs_manifest"]], "end_time (gen3.tools.download.drs_download.downloadstatus attribute)": [[10, "gen3.tools.download.drs_download.DownloadStatus.end_time"]], "file_name (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.file_name"]], "file_name (gen3.tools.download.drs_download.manifest attribute)": [[10, "gen3.tools.download.drs_download.Manifest.file_name"]], "file_size (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.file_size"]], "file_size (gen3.tools.download.drs_download.manifest attribute)": [[10, "gen3.tools.download.drs_download.Manifest.file_size"]], "gen3.tools.download.drs_download": [[10, "module-gen3.tools.download.drs_download"]], "get_fresh_token() (gen3.tools.download.drs_download.downloadmanager method)": [[10, "gen3.tools.download.drs_download.DownloadManager.get_fresh_token"]], "hostname (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.hostname"]], "list_access_in_drs_manifest() (in module gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.list_access_in_drs_manifest"]], "list_drs_object() (in module gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.list_drs_object"]], "list_files_in_drs_manifest() (in module gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.list_files_in_drs_manifest"]], "load() (gen3.tools.download.drs_download.manifest static method)": [[10, "gen3.tools.download.drs_download.Manifest.load"]], "load_manifest() (gen3.tools.download.drs_download.manifest static method)": [[10, "gen3.tools.download.drs_download.Manifest.load_manifest"]], "md5sum (gen3.tools.download.drs_download.manifest attribute)": [[10, "gen3.tools.download.drs_download.Manifest.md5sum"]], "object_id (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.object_id"]], "object_id (gen3.tools.download.drs_download.manifest attribute)": [[10, "gen3.tools.download.drs_download.Manifest.object_id"]], "object_type (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.object_type"]], "pprint() (gen3.tools.download.drs_download.downloadable method)": [[10, "gen3.tools.download.drs_download.Downloadable.pprint"]], "resolve_objects() (gen3.tools.download.drs_download.downloadmanager method)": [[10, "gen3.tools.download.drs_download.DownloadManager.resolve_objects"]], "start_time (gen3.tools.download.drs_download.downloadstatus attribute)": [[10, "gen3.tools.download.drs_download.DownloadStatus.start_time"]], "status (gen3.tools.download.drs_download.downloadstatus attribute)": [[10, "gen3.tools.download.drs_download.DownloadStatus.status"]], "updated_time (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.updated_time"]], "user_access() (gen3.tools.download.drs_download.downloadmanager method)": [[10, "gen3.tools.download.drs_download.DownloadManager.user_access"]], "acls (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.ACLS"]], "authz (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.AUTHZ"]], "current_dir (in module gen3.tools.indexing.download_manifest)": [[11, "gen3.tools.indexing.download_manifest.CURRENT_DIR"]], "current_dir (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.CURRENT_DIR"]], "current_dir (in module gen3.tools.indexing.verify_manifest)": [[11, "gen3.tools.indexing.verify_manifest.CURRENT_DIR"]], "guid (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.GUID"]], "indexd_record_page_size (in module gen3.tools.indexing.download_manifest)": [[11, "gen3.tools.indexing.download_manifest.INDEXD_RECORD_PAGE_SIZE"]], "max_concurrent_requests (in module gen3.tools.indexing.download_manifest)": [[11, "gen3.tools.indexing.download_manifest.MAX_CONCURRENT_REQUESTS"]], "max_concurrent_requests (in module gen3.tools.indexing.verify_manifest)": [[11, "gen3.tools.indexing.verify_manifest.MAX_CONCURRENT_REQUESTS"]], "md5 (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.MD5"]], "prev_guid (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.PREV_GUID"]], "size (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.SIZE"]], "tmp_folder (in module gen3.tools.indexing.download_manifest)": [[11, "gen3.tools.indexing.download_manifest.TMP_FOLDER"]], "threadcontrol (class in gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.ThreadControl"]], "urls (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.URLS"]], "async_download_object_manifest() (in module gen3.tools.indexing.download_manifest)": [[11, "gen3.tools.indexing.download_manifest.async_download_object_manifest"]], "async_verify_object_manifest() (in module gen3.tools.indexing.verify_manifest)": [[11, "gen3.tools.indexing.verify_manifest.async_verify_object_manifest"]], "delete_all_guids() (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.delete_all_guids"]], "gen3.tools.indexing.download_manifest": [[11, "module-gen3.tools.indexing.download_manifest"]], "gen3.tools.indexing.index_manifest": [[11, "module-gen3.tools.indexing.index_manifest"]], "gen3.tools.indexing.verify_manifest": [[11, "module-gen3.tools.indexing.verify_manifest"]], "index_object_manifest() (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.index_object_manifest"]], "column_to_use_as_guid (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.COLUMN_TO_USE_AS_GUID"]], "guid_type_for_indexed_file_object (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.GUID_TYPE_FOR_INDEXED_FILE_OBJECT"]], "guid_type_for_non_indexed_file_object (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.GUID_TYPE_FOR_NON_INDEXED_FILE_OBJECT"]], "max_concurrent_requests (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.MAX_CONCURRENT_REQUESTS"]], "async_ingest_metadata_manifest() (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest"]], "async_query_urls_from_indexd() (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.async_query_urls_from_indexd"]], "gen3.tools.metadata.ingest_manifest": [[12, "module-gen3.tools.metadata.ingest_manifest"]], "gen3wsstorage (class in gen3.wss)": [[13, "gen3.wss.Gen3WsStorage"]], "copy() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.copy"]], "download() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.download"]], "download_url() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.download_url"]], "ls() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.ls"]], "ls_path() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.ls_path"]], "rm() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.rm"]], "rm_path() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.rm_path"]], "upload() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.upload"]], "upload_url() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.upload_url"]]}})
\ No newline at end of file
diff --git a/docs/_build/html/tools/indexing.html b/docs/_build/html/tools/indexing.html
index 57515b283..74d996659 100644
--- a/docs/_build/html/tools/indexing.html
+++ b/docs/_build/html/tools/indexing.html
@@ -381,7 +381,7 @@ <h1>Indexing Tools<a class="headerlink" href="#indexing-tools" title="Permalink
 
 <dl class="py function">
 <dt class="sig sig-object py" id="gen3.tools.indexing.verify_manifest.async_verify_object_manifest">
-<em class="property"><span class="k"><span class="pre">async</span></span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">gen3.tools.indexing.verify_manifest.</span></span><span class="sig-name descname"><span class="pre">async_verify_object_manifest</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">commons_url</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_file</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">max_concurrent_requests=24</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_row_parsers={'acl':</span> <span class="pre">&lt;function</span> <span class="pre">_get_acl_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'authz':</span> <span class="pre">&lt;function</span> <span class="pre">_get_authz_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'file_name':</span> <span class="pre">&lt;function</span> <span class="pre">_get_file_name_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'file_size':</span> <span class="pre">&lt;function</span> <span class="pre">_get_file_size_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'guid':</span> <span class="pre">&lt;function</span> <span class="pre">_get_guid_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'md5':</span> <span class="pre">&lt;function</span> <span class="pre">_get_md5_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'urls':</span> <span class="pre">&lt;function</span> <span class="pre">_get_urls_from_row&gt;}</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_file_delimiter=None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">output_filename='verify-manifest-errors-1677279338.2703023.log'</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/gen3/tools/indexing/verify_manifest.html#async_verify_object_manifest"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#gen3.tools.indexing.verify_manifest.async_verify_object_manifest" title="Permalink to this definition">¶</a></dt>
+<em class="property"><span class="k"><span class="pre">async</span></span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">gen3.tools.indexing.verify_manifest.</span></span><span class="sig-name descname"><span class="pre">async_verify_object_manifest</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">commons_url</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_file</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">max_concurrent_requests=24</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_row_parsers={'acl':</span> <span class="pre">&lt;function</span> <span class="pre">_get_acl_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'authz':</span> <span class="pre">&lt;function</span> <span class="pre">_get_authz_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'file_name':</span> <span class="pre">&lt;function</span> <span class="pre">_get_file_name_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'file_size':</span> <span class="pre">&lt;function</span> <span class="pre">_get_file_size_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'guid':</span> <span class="pre">&lt;function</span> <span class="pre">_get_guid_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'md5':</span> <span class="pre">&lt;function</span> <span class="pre">_get_md5_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'urls':</span> <span class="pre">&lt;function</span> <span class="pre">_get_urls_from_row&gt;}</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_file_delimiter=None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">output_filename='verify-manifest-errors-1679334258.6011407.log'</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/gen3/tools/indexing/verify_manifest.html#async_verify_object_manifest"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#gen3.tools.indexing.verify_manifest.async_verify_object_manifest" title="Permalink to this definition">¶</a></dt>
 <dd><p>Verify all file object records into a manifest csv</p>
 <dl class="field-list simple">
 <dt class="field-odd">Parameters<span class="colon">:</span></dt>
diff --git a/docs/_build/html/tools/metadata.html b/docs/_build/html/tools/metadata.html
index c1bbe9e00..e5cf31575 100644
--- a/docs/_build/html/tools/metadata.html
+++ b/docs/_build/html/tools/metadata.html
@@ -102,7 +102,7 @@ <h1>Metadata Tools<a class="headerlink" href="#metadata-tools" title="Permalink
 
 <dl class="py function">
 <dt class="sig sig-object py" id="gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest">
-<em class="property"><span class="k"><span class="pre">async</span></span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">gen3.tools.metadata.ingest_manifest.</span></span><span class="sig-name descname"><span class="pre">async_ingest_metadata_manifest</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">commons_url</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_file</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">metadata_source</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">auth=None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">max_concurrent_requests=24</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_row_parsers={'guid_for_row':</span> <span class="pre">&lt;function</span> <span class="pre">_get_guid_for_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'indexed_file_object_guid':</span> <span class="pre">&lt;function</span> <span class="pre">_query_for_associated_indexd_record_guid&gt;}</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_file_delimiter=None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">output_filename='ingest-metadata-manifest-errors-1677279338.6413171.log'</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">get_guid_from_file=True</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">metadata_type=None</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/gen3/tools/metadata/ingest_manifest.html#async_ingest_metadata_manifest"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest" title="Permalink to this definition">¶</a></dt>
+<em class="property"><span class="k"><span class="pre">async</span></span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">gen3.tools.metadata.ingest_manifest.</span></span><span class="sig-name descname"><span class="pre">async_ingest_metadata_manifest</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">commons_url</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_file</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">metadata_source</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">auth=None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">max_concurrent_requests=24</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_row_parsers={'guid_for_row':</span> <span class="pre">&lt;function</span> <span class="pre">_get_guid_for_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'indexed_file_object_guid':</span> <span class="pre">&lt;function</span> <span class="pre">_query_for_associated_indexd_record_guid&gt;}</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_file_delimiter=None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">output_filename='ingest-metadata-manifest-errors-1679334258.9486785.log'</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">get_guid_from_file=True</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">metadata_type=None</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/gen3/tools/metadata/ingest_manifest.html#async_ingest_metadata_manifest"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest" title="Permalink to this definition">¶</a></dt>
 <dd><p>Ingest all metadata records into a manifest csv</p>
 <dl class="field-list simple">
 <dt class="field-odd">Parameters<span class="colon">:</span></dt>
diff --git a/gen3/tools/file_utils.py b/gen3/tools/file_utils.py
new file mode 100644
index 000000000..f21f6bb7a
--- /dev/null
+++ b/gen3/tools/file_utils.py
@@ -0,0 +1,430 @@
+import os
+import csv
+
+from typing import Union
+from cdislogging import get_logger
+
+logging = get_logger("__name__")
+
+
+def _get_directory_file_paths(directory: str):
+    """
+    List all file paths in a directory.
+    Args:
+        directory(str):
+            Path of the directory containing the input manifests. All of the manifests contained in directory are assumed to be in a delimiter-separated values (DSV) format, and that there are no other non-DSV files in directory.
+    Returns:
+        List of files in directory
+    """
+
+    logging.info(f"Iterating over files in {directory} directory")
+
+    files = []
+    if os.path.exists(directory):
+        directory_list = os.listdir(directory)
+    else:
+        raise NotADirectoryError()
+    if directory_list:
+        for file in sorted(directory_list):
+            files.append(os.path.join(directory, file))
+
+        files.sort()
+    else:
+        raise RuntimeError(f"Directory {directory} is empty")
+    return files
+
+
+def _delimiter(file_name: str):
+    """
+    Determine delimiter of DSV file from filename, non-DSV files not accepted.
+    Args:
+        file_name(str):
+            Filename or file path
+    Returns:
+        DSV file delimiter
+    """
+
+    ext = file_name.lower()[-3:]
+    delimiter = ""
+    if ext == "tsv":
+        delimiter = "\t"
+    elif "csv":
+        delimiter = ","
+    elif "txt":
+        delimiter = "."
+    else:
+        raise ValueError(
+            "Please check you are providing file with an appropriate extension - TSV, CSV, or TXT. Filenames without an extension will error."
+        )
+
+    return delimiter
+
+
+def get_headers(file_name: str):
+    """
+    Get headers from DSV file
+    Args:
+        file_name(str):
+            Filename or file path
+    Returns:
+        List of headers
+    """
+
+    logging.info(f"Collecting headers from {file_name}...")
+
+    delimiter = _delimiter(file_name)
+    headers = []
+    if delimiter == "txt":
+        file = open(file_name, "r")
+        lines = file.readlines()
+        headers.append(lines[0])
+    else:
+        with open(file_name) as process_file:
+            file_reader = csv.reader(process_file, delimiter=delimiter)
+            headers = next(file_reader)
+
+    return headers
+
+
+def file_to_list(file_name: str, has_headers=False):
+    """
+    Collect records from DSV file
+    Args:
+        file_name(str):
+            Filename or file path
+        has_headers(bool):
+            True when headers present
+    Returns:
+        List of lists (rows) from input file
+    """
+
+    logging.info(f"Collecting records from {file_name}...")
+
+    delimiter = _delimiter(file_name)
+    records = []
+    if delimiter == "txt":
+        file = open(file_name, "r")
+        lines = file.readlines()
+        if has_headers:
+            lines.pop(0)
+        for line in lines:
+            records.append(line)
+    else:
+        with open(file_name) as process_file:
+            file_reader = csv.reader(process_file, delimiter=delimiter)
+            if has_headers:
+                next(file_reader)
+            for row in file_reader:
+                records.append(row)
+
+    return records
+
+
+def move_columns(
+    reordered_headers: list,
+    file_name: str,
+):
+    """
+    Move columns in list to match header order
+    Args:
+        reordered_headers(list):
+            ""
+        file_name(str):
+            Filename or file path
+    Returns:
+        List of lists (rows) from input file, with modified order based on given headers
+    """
+
+    original_headers = get_headers(file_name)
+    if reordered_headers != original_headers and sorted(reordered_headers) == sorted(
+        original_headers
+    ):
+        relational_indices = []
+        for header in reordered_headers:
+            relational_indices.append(original_headers.index(header))
+
+        original_file_list = file_to_list(file_name, True)
+        reordered_file_list = []
+        for row in original_file_list:
+            reordered_row = []
+            for column in relational_indices:
+                reordered_row.append(row[column])
+            reordered_file_list.append(reordered_row)
+
+    elif reordered_headers != original_headers:
+        raise AssertionError("Columns do not have matching headers")
+
+
+def write(records: list, output_file_name: str):
+    """
+    Write records to DSV file
+    Args:
+        records(list):
+            List of rows, each row a list itself (list of lists)
+        output_file_name(str):
+            Filename or file path, expect file extension in the filename to determine file type
+    Returns:
+        None
+    """
+
+    logging.info(f"Writing to {output_file_name}...")
+
+    delimiter = _delimiter(output_file_name)
+    with open(output_file_name, "w") as output:
+        output_writer = csv.writer(output, delimiter=delimiter)
+        for record in records:
+            output_writer.writerow(record)
+
+
+def convert_type(file_name: str, new_type: str, has_headers=False):
+    """
+    Convert file to different extension type
+    Args:
+        file_name(str):
+            Filename or file path
+        new_type(str):
+            Name of DSV type file is being converted to
+        has_headers(bool):
+            True when headers present
+    Returns:
+        None
+    """
+
+    logging.info(f"Converting {file_name} to {new_type}...")
+
+    records = file_to_list(file_name, has_headers)
+    if has_headers:
+        headers = get_headers(file_name)
+        records.insert(0, headers)
+    new_file_name = f"{file_name[:-3:]}{new_type.lower()}"
+    write(records, new_file_name)
+
+
+def merge_files(files: Union[str, list], output_file_name: str, has_headers=False):
+    """
+    Merge multiple files into one file
+    Args:
+        files(str | list):
+            List of filepaths, or string directory path
+        output_file_name(str):
+            Name of file to be output
+        has_headers(bool):
+            True when headers present
+    Returns:
+        None
+    """
+
+    logging.info(f"Merging files: {files}...")
+
+    if type(files) is str:
+        files = _get_directory_file_paths(files)
+
+    all_lists = []
+    for file in files:
+        all_lists.append(file_to_list(file, True))
+    all_records = [item for list_ in all_lists for item in list_]
+    if has_headers:
+        headers = get_headers(files[0])
+        all_records.insert(0, headers)
+
+    write(all_records, output_file_name)
+
+
+def split_by_headers(
+    file_name: str,
+):
+    """
+    Split records by header and write into separate files
+    Args:
+        file_name(str):
+            Filename or file path
+    Returns:
+        None
+    """
+    logging.info(f"Splitting {file_name}...")
+
+    records = file_to_list(file_name, has_headers=True)
+    headers = get_headers(file_name)
+    all_chunks = []
+    for i in range(len(headers)):
+        chunk = []
+        chunk.append(headers[i])
+        for record in records:
+            chunk.append(record[i])
+        all_chunks.append(chunk)
+
+    for i in range(len(all_chunks)):
+        ext = file_name[-3::]
+        output_file_name = f"{file_name[:-4:]}_{headers[i]}.{ext}"
+        write(all_chunks[i], output_file_name)
+
+
+def chunk(
+    file_name: str,
+    chunk_size: int,
+    has_headers=False,
+):
+    """
+    Chunk records into files of input size
+    Args:
+        file_name(str):
+            Filename or file path
+        chunk_size(int):
+            Size of desired segment
+        has_headers(bool):
+            True when headers present
+    Returns:
+        None
+    """
+    logging.info(f"Chunking records into {chunk_size} size...")
+
+    records = file_to_list(file_name, has_headers)
+    all_chunks = []
+    chunk = []
+    marker = 0
+    for record in records:
+        if marker != chunk_size:
+            chunk.append(record)
+            marker += 1
+        else:
+            all_chunks.append(chunk)
+            chunk = []
+            chunk.append(record)
+            marker = 1
+    if len(chunk) > 0:
+        all_chunks.append(chunk)
+
+    if has_headers:
+        headers = get_headers(file_name)
+    for i in range(len(all_chunks)):
+        if has_headers:
+            all_chunks[i].insert(0, headers)
+        ext = file_name[-3::]
+        output_file_name = f"{file_name[:-4:]}_{i}.{ext}"
+        write(all_chunks[i], output_file_name)
+
+
+def file_difference(file_one: str, file_two: str, has_headers=False, strict=False):
+    """
+    Take difference between two files, files must have matching column headers
+    Args:
+        file_one(str):
+            Filename or file path
+        file_two(str):
+            Filename or file path
+        has_headers(bool):
+            True when headers present
+        strict(bool):
+            When true, will look for exact match; else, will match lower and uppercase strings and match array of different order
+    Returns:
+        List of lists, returns rows that contain at least one difference
+    """
+
+    logging.info(f"Taking differences between {file_one} & {file_two}...")
+
+    list_one = file_to_list(file_one, has_headers)
+
+    if has_headers:
+        file_one_headers = get_headers(file_one)
+        file_two_headers = get_headers(file_two)
+        if file_one_headers != file_two_headers and sorted(file_one_headers) == sorted(
+            file_two_headers
+        ):
+            list_two = move_columns(file_one_headers, file_two)
+        elif file_one_headers != file_two_headers:
+            raise AssertionError("Columns do not have matching headers")
+        else:
+            list_two = file_to_list(file_two, has_headers)
+    else:
+        list_two = file_to_list(file_two, has_headers)
+
+    if not strict:
+        for row in list_one:
+            for column in row:
+                column.lower()
+                str(sorted(eval(column)))
+        for row in list_two:
+            for column in row:
+                column.lower()
+                str(sorted(eval(column)))
+
+    return [row for row in list_one if row not in list_two] + [
+        row for row in list_two if row not in list_one
+    ]
+
+
+def file_intersection(file_one: str, file_two: str, has_headers=False, strict=False):
+    """
+    Take intersection between two files, files must have matching column headers
+    Args:
+        file_one(str):
+            Filename or file path
+        file_two(str):
+            Filename or file path
+        has_headers(bool):
+            True when headers present
+        strict(bool):
+            When true, will look for exact match; else, will match lower and uppercase strings and match array of different order
+    Returns:
+        List of lists, returns rows that contain matches between all columns
+    """
+
+    logging.info(f"Taking intersection between {file_one} & {file_two}...")
+
+    list_one = file_to_list(file_one, has_headers)
+
+    if has_headers:
+        file_one_headers = get_headers(file_one)
+        file_two_headers = get_headers(file_two)
+        if file_one_headers != file_two_headers and sorted(file_one_headers) == sorted(
+            file_two_headers
+        ):
+            list_two = move_columns(file_one_headers, file_two)
+        elif file_one_headers != file_two_headers:
+            raise AssertionError("Columns do not have matching headers")
+        else:
+            list_two = file_to_list(file_two, has_headers)
+    else:
+        list_two = file_to_list(file_two, has_headers)
+
+    if not strict:
+        for row in list_one:
+            for column in row:
+                column.lower()
+                str(sorted(eval(column)))
+        for row in list_two:
+            for column in row:
+                column.lower()
+                str(sorted(eval(column)))
+
+    return [row for row in list_one if row in list_two]
+
+
+# TODO
+# def file_subset(
+#     file_name: str,
+#     criteria: str,
+#     column: str,
+#     has_headers=False
+# ):
+#     """
+#     Find all records that match a given criteria within a specified column
+#     Args:
+#         file_name(str):
+#             Filename or file path
+#         criteria(str):
+#             Expression to match (for example: >5, ==3, ==this_guid)
+#         column(str | int):
+#             If has_headers, str name of column to search for criteria; else, int index of column
+#         has_headers(bool):
+#             True when headers present
+#     Returns:
+#         List of lists, returns rows that match criteria
+#     """
+#     logging.info(f"Finding subset of records that match {criteria} for {column}...")
+
+#     records = file_to_list(file_name, has_headers)
+#     subset = []
+#     # do stuff here
+
+#     return subset