From 8788f6386f1fa552487779dabc7d906b9a93bf9d Mon Sep 17 00:00:00 2001 From: idelcano Date: Tue, 4 Aug 2020 18:46:07 +0200 Subject: [PATCH 1/3] Added config file --- .../export_import_datavalues/config.json | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 DHIS2/metadata_manipulation/export_import_datavalues/config.json diff --git a/DHIS2/metadata_manipulation/export_import_datavalues/config.json b/DHIS2/metadata_manipulation/export_import_datavalues/config.json new file mode 100644 index 00000000..1260ec18 --- /dev/null +++ b/DHIS2/metadata_manipulation/export_import_datavalues/config.json @@ -0,0 +1,28 @@ +{ + "server_config": + { + "local_temp_folder": "/home/user/keep_data", + "api_remote": { + "url": "https://export_from_this_server", + "username": "user", + "password": "pass" + }, + "api_local": { + "url": "https://import_in_this_server", + "username": "user", + "password": "pass" + } + }, + "data": [ + { + "type": "dataset", + "ou_uid": "H8RixfF8ugH", + "dataset_uid": "Wxl8hxuOsIQ", + "startdate": "2014-04-01", + "enddate": "2019-04-30", + "removeuids": [ + "Lkjp0MFLDov" + ] + } + ] +} \ No newline at end of file From 6ac4cbf59e6dcf710ff6dc8f91507275b4253088 Mon Sep 17 00:00:00 2001 From: idelcano Date: Tue, 4 Aug 2020 18:46:24 +0200 Subject: [PATCH 2/3] Added script --- .../export_import_datavalues/keep_data.py | 124 ++++++++++++++++++ 1 file changed, 124 insertions(+) create mode 100644 DHIS2/metadata_manipulation/export_import_datavalues/keep_data.py diff --git a/DHIS2/metadata_manipulation/export_import_datavalues/keep_data.py b/DHIS2/metadata_manipulation/export_import_datavalues/keep_data.py new file mode 100644 index 00000000..2f3b701b --- /dev/null +++ b/DHIS2/metadata_manipulation/export_import_datavalues/keep_data.py @@ -0,0 +1,124 @@ +import json +import sys + +import requests + +import dhis2api + + +def main(): + cfg = get_config("config.json") + export_keep_data(cfg['server_config']['api_remote'], cfg['server_config']['local_temp_folder'], cfg['data']) + import_keep_data(cfg['server_config']['api_local'], cfg['server_config']['local_temp_folder'], cfg['data']) + + +def get_config(fname): + "Return dict with the options read from configuration file" + print('Reading from config file %s ...' % fname) + try: + with open(fname) as f: + config = json.load(f) + except (AssertionError, IOError, ValueError) as e: + sys.exit('Error reading config file %s: %s' % (fname, e)) + return config + + +def import_keep_data(cfg, folder, keep_data): + for data in keep_data: + if data['type'] == 'dataset': + file = folder + "/" + data['dataset_uid']+data['ou_uid']+".json" + response = import_data_set(cfg, file, data) + return response + + +def remove_uids(json, uids): + elements_new = [] + if 'dataValues' in json: + for element in json['dataValues']: + remove = remove_datalements(element, uids) + if not remove: + elements_new.append(element) + return elements_new + + +def remove_datalements(element, removable_ids): + if isinstance(element, list): + for child in element[:]: + if isinstance(child, dict) and "dataElement" in child.keys() and child["dataElement"] in removable_ids: + return True + elif isinstance(element, dict) and "dataElement" in element.keys() and element["dataElement"] in removable_ids: + return True + return False + + +def export_keep_data(api_local, folder, keep_data): + for data in keep_data: + if data['type'] == 'dataset': + values = export_data_set(data['startdate'],data['enddate'], data['ou_uid'], data['dataset_uid'], api_local) + if data['removeuids'] is not None: + values =remove_uids(values, data['removeuids']) + file = folder + "/" + data['dataset_uid']+data['ou_uid']+".json" + + with open(file, 'wt', encoding="utf-8") as fout: + datavalues = json.dumps({"dataValues":values}) + fout.write(datavalues) + + +def export_data_set(start_date, end_date, ou_uid, dataset_uid, cfg): + api = dhis2api.Dhis2Api(cfg['url'], cfg['username'], cfg['password']) + + wait_for_server(api) + print('Exporting dataset: ou %s program %s' % (ou_uid, dataset_uid)) + + if not dataset_uid or not ou_uid: + print('No uid provided - Cancelling export dataset') + return [] + + post_content = 'dataElementIdScheme=UID&orgUnitIdScheme=UID&includeDeleted' \ + '=false&children=true&categoryOptionComboIdScheme=CODE&start' \ + 'Date=%s&endDate=%s&orgUnit=%s&dataSet=%s' \ + % (start_date, end_date, ou_uid, dataset_uid) + + response = api.get("/dataValueSets.json", post_content) + + return response + + +def import_data_set(cfg, file, data): + print('Importing dataset: ou %s program %s' % (data['ou_uid'], data['dataset_uid'])) + + api = dhis2api.Dhis2Api(cfg['url'], cfg['username'], cfg['password']) + + wait_for_server(api) + + file_to_import = json.load(open(file)) + response = api.post('/dataValueSets.json?dataElementIdScheme=UID&dryRun=false&idScheme=CODE' \ + '&orgUnitIdScheme=UID&preheatCache=false&skipExistingCheck=false' \ + '&strategy=NEW_AND_UPDATES&format=json&async=true', params={ + }, payload=file_to_import) + + if response.get('status') == 'OK': + print(data['dataset_uid'] + "-" + data['ou_uid'] + ".json imported") + else: + print(data['dataset_uid'] + "-" + data['ou_uid'] + ".json failed") + return response + + +def wait_for_server(api, delay=90, timeout=900): + "Sleep until server is ready to accept requests" + print('Check active API: %s' % api.api_url) + import time + time.sleep(delay) # in case tomcat is still starting + start_time = time.time() + while True: + try: + api.get('/me') + break + except requests.exceptions.ConnectionError: + if time.time() - start_time > timeout: + raise RuntimeError('Timeout: could not connect to the API') + time.sleep(1) + + +if __name__ == '__main__': + main() \ No newline at end of file From 1ff2163b973d46372a429994a67b64c326f0d4b8 Mon Sep 17 00:00:00 2001 From: idelcano Date: Tue, 4 Aug 2020 18:46:37 +0200 Subject: [PATCH 3/3] Added dhis2api symbolic link --- DHIS2/metadata_manipulation/export_import_datavalues/dhis2api.py | 1 + 1 file changed, 1 insertion(+) create mode 120000 DHIS2/metadata_manipulation/export_import_datavalues/dhis2api.py diff --git a/DHIS2/metadata_manipulation/export_import_datavalues/dhis2api.py b/DHIS2/metadata_manipulation/export_import_datavalues/dhis2api.py new file mode 120000 index 00000000..0dfc8fcb --- /dev/null +++ b/DHIS2/metadata_manipulation/export_import_datavalues/dhis2api.py @@ -0,0 +1 @@ +../../cloner/dhis2api.py \ No newline at end of file