From c05a4da462f2623590f3a03c547702d16baa7a62 Mon Sep 17 00:00:00 2001 From: v01dma1n Date: Thu, 24 Sep 2015 08:30:29 -0400 Subject: [PATCH 1/2] Converted to Python 3, added csvonly data format --- gcexport.py | 138 +++++++++++++++++++++++++++------------------------- 1 file changed, 71 insertions(+), 67 deletions(-) diff --git a/gcexport.py b/gcexport.py index fcea013..51e2d09 100755 --- a/gcexport.py +++ b/gcexport.py @@ -9,11 +9,14 @@ See README.md for more information. Usage: python gcexport.py [how_many] [format] [directory] how_many - number of recent activities to download, or "all" (default: 1) - format - export format; can be "gpx," "tcx," or "original" (default: gpx) + format - export format; can be "gpx," "tcx," "csvonly," (activity list) or "original" (default: gpx) directory - the directory to export to (default: "YYYY-MM-DD_garmin_connect_export") + +2015-09-23 v01dma1n: converted to Python 3, added csvonly data format + """ -from urllib import urlencode +from urllib.parse import urlencode from datetime import datetime from getpass import getpass from sys import argv @@ -22,7 +25,7 @@ from os import mkdir from xml.dom.minidom import parseString -import urllib2, cookielib, json +import urllib.request, urllib.error, urllib.parse, http.cookiejar, json from fileinput import filename if len(argv) > 4: @@ -36,22 +39,22 @@ if len(argv) > 2: data_format = argv[2].lower() - if data_format != 'gpx' and data_format != 'tcx' and data_format != 'original': - raise Exception('Format can only be "gpx," "tcx," or "original."') + if data_format != 'gpx' and data_format != 'tcx' and data_format != 'original' and data_format != 'csvonly': + raise Exception('Format can only be "gpx," "tcx," "csvonly," or "original."') else: data_format = 'gpx' -cookie_jar = cookielib.CookieJar() -opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookie_jar)) +cookie_jar = http.cookiejar.CookieJar() +opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cookie_jar)) # url is a string, post is a dictionary of POST parameters, headers is a dictionary of headers. def http_req(url, post=None, headers={}): - request = urllib2.Request(url) + request = urllib.request.Request(url) request.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 5.2; rv:2.0.1) Gecko/20100101 Firefox/4.0.1') # Tell Garmin we're some supported browser. - for header_key, header_value in headers.iteritems(): + for header_key, header_value in headers.items(): request.add_header(header_key, header_value) if post: - post = urlencode(post) # Convert dictionary to POST parameter string. + post = urlencode(post).encode('utf-8') # Convert dictionary to POST parameter string. response = opener.open(request, data=post) # This line may throw a urllib2.HTTPError. # N.B. urllib2 will follow any 302 redirects. Also, the "open" call above may throw a urllib2.HTTPError which is checked for below. @@ -60,15 +63,16 @@ def http_req(url, post=None, headers={}): return response.read() -print 'Welcome to Garmin Connect Exporter!' +print('Welcome to Garmin Connect Exporter!') # Create directory for data files. if isdir(activities_directory): - print 'Warning: Output directory already exists. Will skip already-downloaded files and append to the CSV file.' + print('Warning: Output directory already exists. Will skip already-downloaded files and append to the CSV file.') -username = raw_input('Username: ') +username = input('Username: ') password = getpass() + # Maximum number of activities you can request at once. Set and enforced by Garmin. limit_maximum = 100 @@ -140,8 +144,7 @@ def http_req(url, post=None, headers={}): search_params = {'start': total_downloaded, 'limit': num_to_download} # Query Garmin Connect result = http_req(url_gc_search + urlencode(search_params)) - json_results = json.loads(result) # TODO: Catch possible exceptions here. - + json_results = json.loads(result.decode('utf-8')) # TODO: Catch possible exceptions here. search = json_results['results']['search'] @@ -157,54 +160,55 @@ def http_req(url, post=None, headers={}): # Process each activity. for a in activities: # Display which entry we're working on. - print 'Garmin Connect activity: [' + a['activity']['activityId'] + ']', - print a['activity']['beginTimestamp']['display'] + ':', - print a['activity']['activityName']['value'] - - if data_format == 'gpx': - filename = activities_directory + '/activity_' + a['activity']['activityId'] + '.gpx' - download_url = url_gc_gpx_activity + a['activity']['activityId'] + '?full=true' - file_mode = 'w' - elif data_format == 'tcx': - filename = activities_directory + '/activity_' + a['activity']['activityId'] + '.tcx' - download_url = url_gc_tcx_activity + a['activity']['activityId'] + '?full=true' - file_mode = 'w' - else: - filename = activities_directory + '/activity_' + a['activity']['activityId'] + '.zip' - download_url = url_gc_original_activity + a['activity']['activityId'] - file_mode = 'wb' - - if isfile(filename): - print '\tData file already exists; skipping...' - continue - - # Download the data file from Garmin Connect. - # If the download fails (e.g., due to timeout), this script will die, but nothing - # will have been written to disk about this activity, so just running it again - # should pick up where it left off. - print '\tDownloading file...', - - try: - data = http_req(download_url) - except urllib2.HTTPError as e: - # Handle expected (though unfortunate) error codes; die on unexpected ones. - if e.code == 500 and data_format == 'tcx': - # Garmin will give an internal server error (HTTP 500) when downloading TCX files if the original was a manual GPX upload. - # Writing an empty file prevents this file from being redownloaded, similar to the way GPX files are saved even when there are no tracks. - # One could be generated here, but that's a bit much. Use the GPX format if you want actual data in every file, as I believe Garmin provides a GPX file for every activity. - print 'Writing empty file since Garmin did not generate a TCX file for this activity...', - data = '' - elif e.code == 404 and data_format == 'original': - # For manual activities (i.e., entered in online without a file upload), there is no original file. - # Write an empty file to prevent redownloading it. - print 'Writing empty file since there was no original activity data...', - data = '' + print('Garmin Connect activity: [' + a['activity']['activityId'] + ']', end=' ') + print(a['activity']['beginTimestamp']['display'] + ':', end=' ') + print(a['activity']['activityName']['value']) + + if data_format != 'csvonly': + if data_format == 'gpx': + filename = activities_directory + '/activity_' + a['activity']['activityId'] + '.gpx' + download_url = url_gc_gpx_activity + a['activity']['activityId'] + '?full=true' + file_mode = 'wb' + elif data_format == 'tcx': + filename = activities_directory + '/activity_' + a['activity']['activityId'] + '.tcx' + download_url = url_gc_tcx_activity + a['activity']['activityId'] + '?full=true' + file_mode = 'w' else: - raise Exception('Failed. Got an unexpected HTTP error (' + str(e.code) + ').') - - save_file = open(filename, file_mode) - save_file.write(data) - save_file.close() + filename = activities_directory + '/activity_' + a['activity']['activityId'] + '.zip' + download_url = url_gc_original_activity + a['activity']['activityId'] + file_mode = 'wb' + + if isfile(filename): + print('\tData file already exists; skipping...') + continue + + # Download the data file from Garmin Connect. + # If the download fails (e.g., due to timeout), this script will die, but nothing + # will have been written to disk about this activity, so just running it again + # should pick up where it left off. + print('\tDownloading file...', end=' ') + + try: + data = http_req(download_url) + except urllib.error.HTTPError as e: + # Handle expected (though unfortunate) error codes; die on unexpected ones. + if e.code == 500 and data_format == 'tcx': + # Garmin will give an internal server error (HTTP 500) when downloading TCX files if the original was a manual GPX upload. + # Writing an empty file prevents this file from being redownloaded, similar to the way GPX files are saved even when there are no tracks. + # One could be generated here, but that's a bit much. Use the GPX format if you want actual data in every file, as I believe Garmin provides a GPX file for every activity. + print('Writing empty file since Garmin did not generate a TCX file for this activity...', end=' ') + data = '' + elif e.code == 404 and data_format == 'original': + # For manual activities (i.e., entered in online without a file upload), there is no original file. + # Write an empty file to prevent redownloading it. + print('Writing empty file since there was no original activity data...', end=' ') + data = '' + else: + raise Exception('Failed. Got an unexpected HTTP error (' + str(e.code) + ').') + + save_file = open(filename, file_mode) + save_file.write(data) + save_file.close() # Write stats to CSV. empty_record = '"",' @@ -254,7 +258,7 @@ def http_req(url, post=None, headers={}): csv_record += empty_record if 'lossElevation' not in a['activity'] else '"' + a['activity']['lossElevation']['value'].replace('"', '""') + '"' csv_record += '\n' - csv_file.write(csv_record.encode('utf8')) + csv_file.write(csv_record) if data_format == 'gpx': # Validate GPX data. If we have an activity without GPS data (e.g., running on a treadmill), @@ -264,15 +268,15 @@ def http_req(url, post=None, headers={}): gpx_data_exists = len(gpx.getElementsByTagName('trkpt')) > 0 if gpx_data_exists: - print 'Done. GPX data saved.' + print('Done. GPX data saved.') else: - print 'Done. No track points found.' + print('Done. No track points found.') else: # TODO: Consider validating other formats. - print 'Done.' + print('Done.') total_downloaded += num_to_download # End while loop for multiple chunks. csv_file.close() -print 'Done!' +print('Done!') From 5b2c48f3d05d388c1c9c47b2112d5f091b48c54c Mon Sep 17 00:00:00 2001 From: IrekRybark Date: Wed, 6 Apr 2016 13:09:33 -0400 Subject: [PATCH 2/2] Refactored column parsing --- .gitignore | 2 +- gcexport.py | 408 ++++++++++++++++++++++++++++------------------------ 2 files changed, 221 insertions(+), 189 deletions(-) diff --git a/.gitignore b/.gitignore index 5dc43a8..689698a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,3 @@ - +*.config .DS_Store extras/ diff --git a/gcexport.py b/gcexport.py index 51e2d09..027aa8d 100755 --- a/gcexport.py +++ b/gcexport.py @@ -6,101 +6,119 @@ Date: April 28, 2015 Description: Use this script to export your fitness data from Garmin Connect. - See README.md for more information. + See README.md for more information. Usage: python gcexport.py [how_many] [format] [directory] - how_many - number of recent activities to download, or "all" (default: 1) - format - export format; can be "gpx," "tcx," "csvonly," (activity list) or "original" (default: gpx) - directory - the directory to export to (default: "YYYY-MM-DD_garmin_connect_export") + how_many - number of recent activities to download, or "all" (default: 1) + format - export format; can be "gpx," "tcx," "csvonly," (activity list) or "original" (default: gpx) + directory - the directory to export to (default: "YYYY-MM-DD_garmin_connect_export") 2015-09-23 v01dma1n: converted to Python 3, added csvonly data format - +2016-04-06 Irek Rybark: split device name and device version fields; device version appended as last column """ from urllib.parse import urlencode from datetime import datetime +from configparser import ConfigParser from getpass import getpass from sys import argv from os.path import isdir from os.path import isfile from os import mkdir from xml.dom.minidom import parseString - import urllib.request, urllib.error, urllib.parse, http.cookiejar, json from fileinput import filename if len(argv) > 4: - raise Exception('Too many arguments.') + raise Exception('Too many arguments.') if len(argv) > 3: - activities_directory = argv[3] + activities_directory = argv[3] else: - current_date = datetime.now().strftime('%Y-%m-%d') - activities_directory = './' + current_date + '_garmin_connect_export' + current_date = datetime.now().strftime('%Y-%m-%d') + activities_directory = './' + current_date + '_garmin_connect_export' if len(argv) > 2: - data_format = argv[2].lower() - if data_format != 'gpx' and data_format != 'tcx' and data_format != 'original' and data_format != 'csvonly': - raise Exception('Format can only be "gpx," "tcx," "csvonly," or "original."') + data_format = argv[2].lower() + if data_format != 'gpx' and data_format != 'tcx' and data_format != 'original' and data_format != 'csvonly': + raise Exception('Format can only be "gpx," "tcx," "csvonly," or "original."') else: - data_format = 'gpx' + data_format = 'gpx' + + cookie_jar = http.cookiejar.CookieJar() opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cookie_jar)) + # url is a string, post is a dictionary of POST parameters, headers is a dictionary of headers. def http_req(url, post=None, headers={}): - request = urllib.request.Request(url) - request.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 5.2; rv:2.0.1) Gecko/20100101 Firefox/4.0.1') # Tell Garmin we're some supported browser. - for header_key, header_value in headers.items(): - request.add_header(header_key, header_value) - if post: - post = urlencode(post).encode('utf-8') # Convert dictionary to POST parameter string. - response = opener.open(request, data=post) # This line may throw a urllib2.HTTPError. + request = urllib.request.Request(url) + request.add_header('User-Agent', + 'Mozilla/5.0 (Windows NT 5.2; rv:2.0.1) Gecko/20100101 Firefox/4.0.1') # Tell Garmin we're some supported browser. + for header_key, header_value in headers.items(): + request.add_header(header_key, header_value) + if post: + post = urlencode(post).encode('utf-8') # Convert dictionary to POST parameter string. + response = opener.open(request, data=post) # This line may throw a urllib2.HTTPError. + + # N.B. urllib2 will follow any 302 redirects. Also, the "open" call above may throw a urllib2.HTTPError which is checked for below. + if response.getcode() != 200: + raise Exception('Bad return code (' + response.getcode() + ') for: ' + url) - # N.B. urllib2 will follow any 302 redirects. Also, the "open" call above may throw a urllib2.HTTPError which is checked for below. - if response.getcode() != 200: - raise Exception('Bad return code (' + response.getcode() + ') for: ' + url) + return response.read() - return response.read() print('Welcome to Garmin Connect Exporter!') # Create directory for data files. if isdir(activities_directory): - print('Warning: Output directory already exists. Will skip already-downloaded files and append to the CSV file.') - -username = input('Username: ') -password = getpass() + print('Warning: Output directory already exists. Will skip already-downloaded files and append to the CSV file.') + +username = '' +password = '' +# try to read config... +config = ConfigParser() +try: + config.read('gcexport.config') + username = config['login']['username'] + password = config['login']['password'] +except Exception: + pass +# ...otherwise prompt +if username == '' or password == '': + username = input('Username: ') + password = getpass() # Maximum number of activities you can request at once. Set and enforced by Garmin. limit_maximum = 100 # URLs for various services. -url_gc_login = 'https://sso.garmin.com/sso/login?service=https%3A%2F%2Fconnect.garmin.com%2Fpost-auth%2Flogin&webhost=olaxpw-connect04&source=https%3A%2F%2Fconnect.garmin.com%2Fen-US%2Fsignin&redirectAfterAccountLoginUrl=https%3A%2F%2Fconnect.garmin.com%2Fpost-auth%2Flogin&redirectAfterAccountCreationUrl=https%3A%2F%2Fconnect.garmin.com%2Fpost-auth%2Flogin&gauthHost=https%3A%2F%2Fsso.garmin.com%2Fsso&locale=en_US&id=gauth-widget&cssUrl=https%3A%2F%2Fstatic.garmincdn.com%2Fcom.garmin.connect%2Fui%2Fcss%2Fgauth-custom-v1.1-min.css&clientId=GarminConnect&rememberMeShown=true&rememberMeChecked=false&createAccountShown=true&openCreateAccount=false&usernameShown=false&displayNameShown=false&consumeServiceTicket=false&initialFocus=true&embedWidget=false&generateExtraServiceTicket=false' +url_gc_login = 'https://sso.garmin.com/sso/login?service=https%3A%2F%2Fconnect.garmin.com%2Fpost-auth%2Flogin&webhost=olaxpw-connect04&source=https%3A%2F%2Fconnect.garmin.com%2Fen-US%2Fsignin&redirectAfterAccountLoginUrl=https%3A%2F%2Fconnect.garmin.com%2Fpost-auth%2Flogin&redirectAfterAccountCreationUrl=https%3A%2F%2Fconnect.garmin.com%2Fpost-auth%2Flogin&gauthHost=https%3A%2F%2Fsso.garmin.com%2Fsso&locale=en_US&id=gauth-widget&cssUrl=https%3A%2F%2Fstatic.garmincdn.com%2Fcom.garmin.connect%2Fui%2Fcss%2Fgauth-custom-v1.1-min.css&clientId=GarminConnect&rememberMeShown=true&rememberMeChecked=false&createAccountShown=true&openCreateAccount=false&usernameShown=false&displayNameShown=false&consumeServiceTicket=false&initialFocus=true&embedWidget=false&generateExtraServiceTicket=false' url_gc_post_auth = 'https://connect.garmin.com/post-auth/login?' -url_gc_search = 'http://connect.garmin.com/proxy/activity-search-service-1.0/json/activities?' -url_gc_gpx_activity = 'http://connect.garmin.com/proxy/activity-service-1.1/gpx/activity/' -url_gc_tcx_activity = 'http://connect.garmin.com/proxy/activity-service-1.1/tcx/activity/' -url_gc_original_activity = 'http://connect.garmin.com/proxy/download-service/files/activity/' +url_gc_search = 'http://connect.garmin.com/proxy/activity-search-service-1.0/json/activities?' +url_gc_gpx_activity = 'http://connect.garmin.com/proxy/activity-service-1.1/gpx/activity/' +url_gc_tcx_activity = 'http://connect.garmin.com/proxy/activity-service-1.1/tcx/activity/' +url_gc_original_activity = 'http://connect.garmin.com/proxy/download-service/files/activity/' # Initially, we need to get a valid session cookie, so we pull the login page. http_req(url_gc_login) # Now we'll actually login. -post_data = {'username': username, 'password': password, 'embed': 'true', 'lt': 'e1s1', '_eventId': 'submit', 'displayNameRequired': 'false'} # Fields that are passed in a typical Garmin login. +post_data = {'username': username, 'password': password, 'embed': 'true', 'lt': 'e1s1', '_eventId': 'submit', + 'displayNameRequired': 'false'} # Fields that are passed in a typical Garmin login. http_req(url_gc_login, post_data) # Get the key. # TODO: Can we do this without iterating? login_ticket = None for cookie in cookie_jar: - if cookie.name == 'CASTGC': - login_ticket = cookie.value - break + if cookie.name == 'CASTGC': + login_ticket = cookie.value + break if not login_ticket: - raise Exception('Did not get a ticket cookie. Cannot log in. Did you enter the correct username and password?') + raise Exception('Did not get a ticket cookie. Cannot log in. Did you enter the correct username and password?') # Chop of 'TGT-' off the beginning, prepend 'ST-0'. login_ticket = 'ST-0' + login_ticket[4:] @@ -109,7 +127,7 @@ def http_req(url, post=None, headers={}): # We should be logged in now. if not isdir(activities_directory): - mkdir(activities_directory) + mkdir(activities_directory) csv_filename = activities_directory + '/activities.csv' csv_existed = isfile(csv_filename) @@ -118,163 +136,177 @@ def http_req(url, post=None, headers={}): # Write header to CSV file if not csv_existed: - csv_file.write('Activity ID,Activity Name,Description,Begin Timestamp,Begin Timestamp (Raw Milliseconds),End Timestamp,End Timestamp (Raw Milliseconds),Device,Activity Parent,Activity Type,Event Type,Activity Time Zone,Max. Elevation,Max. Elevation (Raw),Begin Latitude (Decimal Degrees Raw),Begin Longitude (Decimal Degrees Raw),End Latitude (Decimal Degrees Raw),End Longitude (Decimal Degrees Raw),Average Moving Speed,Average Moving Speed (Raw),Max. Heart Rate (bpm),Average Heart Rate (bpm),Max. Speed,Max. Speed (Raw),Calories,Calories (Raw),Duration (h:m:s),Duration (Raw Seconds),Moving Duration (h:m:s),Moving Duration (Raw Seconds),Average Speed,Average Speed (Raw),Distance,Distance (Raw),Max. Heart Rate (bpm),Min. Elevation,Min. Elevation (Raw),Elevation Gain,Elevation Gain (Raw),Elevation Loss,Elevation Loss (Raw)\n') + csv_file.write( + 'Activity ID,Activity Name,Description,Begin Timestamp,Begin Timestamp (Raw Milliseconds),End Timestamp,End Timestamp (Raw Milliseconds),Device,Activity Parent,Activity Type,Event Type,Activity Time Zone,Max. Elevation,Max. Elevation (Raw),Begin Latitude (Decimal Degrees Raw),Begin Longitude (Decimal Degrees Raw),End Latitude (Decimal Degrees Raw),End Longitude (Decimal Degrees Raw),Average Moving Speed,Average Moving Speed (Raw),Max. Heart Rate (bpm),Average Heart Rate (bpm),Max. Speed,Max. Speed (Raw),Calories,Calories (Raw),Duration (h:m:s),Duration (Raw Seconds),Moving Duration (h:m:s),Moving Duration (Raw Seconds),Average Speed,Average Speed (Raw),Distance,Distance (Raw),Max. Heart Rate (bpm),Min. Elevation,Min. Elevation (Raw),Elevation Gain,Elevation Gain (Raw),Elevation Loss,Elevation Loss (Raw)\n') download_all = False if len(argv) > 1 and argv[1].isdigit(): - total_to_download = int(argv[1]) + total_to_download = int(argv[1]) elif len(argv) > 1 and argv[1] == 'all': - # If the user wants to download all activities, first download one, - # then the result of that request will tell us how many are available - # so we will modify the variables then. - total_to_download = 1 - download_all = True + # If the user wants to download all activities, first download one, + # then the result of that request will tell us how many are available + # so we will modify the variables then. + total_to_download = 1 + download_all = True else: - total_to_download = 1 + total_to_download = 1 total_downloaded = 0 # This while loop will download data from the server in multiple chunks, if necessary. while total_downloaded < total_to_download: - # Maximum of 100... 400 return status if over 100. So download 100 or whatever remains if less than 100. - if total_to_download - total_downloaded > 100: - num_to_download = 100 - else: - num_to_download = total_to_download - total_downloaded - - search_params = {'start': total_downloaded, 'limit': num_to_download} - # Query Garmin Connect - result = http_req(url_gc_search + urlencode(search_params)) - json_results = json.loads(result.decode('utf-8')) # TODO: Catch possible exceptions here. - - search = json_results['results']['search'] - - if download_all: - # Modify total_to_download based on how many activities the server reports. - total_to_download = int(search['totalFound']) - # Do it only once. - download_all = False - - # Pull out just the list of activities. - activities = json_results['results']['activities'] - - # Process each activity. - for a in activities: - # Display which entry we're working on. - print('Garmin Connect activity: [' + a['activity']['activityId'] + ']', end=' ') - print(a['activity']['beginTimestamp']['display'] + ':', end=' ') - print(a['activity']['activityName']['value']) - - if data_format != 'csvonly': - if data_format == 'gpx': - filename = activities_directory + '/activity_' + a['activity']['activityId'] + '.gpx' - download_url = url_gc_gpx_activity + a['activity']['activityId'] + '?full=true' - file_mode = 'wb' - elif data_format == 'tcx': - filename = activities_directory + '/activity_' + a['activity']['activityId'] + '.tcx' - download_url = url_gc_tcx_activity + a['activity']['activityId'] + '?full=true' - file_mode = 'w' - else: - filename = activities_directory + '/activity_' + a['activity']['activityId'] + '.zip' - download_url = url_gc_original_activity + a['activity']['activityId'] - file_mode = 'wb' - - if isfile(filename): - print('\tData file already exists; skipping...') - continue - - # Download the data file from Garmin Connect. - # If the download fails (e.g., due to timeout), this script will die, but nothing - # will have been written to disk about this activity, so just running it again - # should pick up where it left off. - print('\tDownloading file...', end=' ') - - try: - data = http_req(download_url) - except urllib.error.HTTPError as e: - # Handle expected (though unfortunate) error codes; die on unexpected ones. - if e.code == 500 and data_format == 'tcx': - # Garmin will give an internal server error (HTTP 500) when downloading TCX files if the original was a manual GPX upload. - # Writing an empty file prevents this file from being redownloaded, similar to the way GPX files are saved even when there are no tracks. - # One could be generated here, but that's a bit much. Use the GPX format if you want actual data in every file, as I believe Garmin provides a GPX file for every activity. - print('Writing empty file since Garmin did not generate a TCX file for this activity...', end=' ') - data = '' - elif e.code == 404 and data_format == 'original': - # For manual activities (i.e., entered in online without a file upload), there is no original file. - # Write an empty file to prevent redownloading it. - print('Writing empty file since there was no original activity data...', end=' ') - data = '' - else: - raise Exception('Failed. Got an unexpected HTTP error (' + str(e.code) + ').') - - save_file = open(filename, file_mode) - save_file.write(data) - save_file.close() - - # Write stats to CSV. - empty_record = '"",' - - csv_record = '' - - csv_record += empty_record if 'activityId' not in a['activity'] else '"' + a['activity']['activityId'].replace('"', '""') + '",' - csv_record += empty_record if 'activityName' not in a['activity'] else '"' + a['activity']['activityName']['value'].replace('"', '""') + '",' - csv_record += empty_record if 'activityDescription' not in a['activity'] else '"' + a['activity']['activityDescription']['value'].replace('"', '""') + '",' - csv_record += empty_record if 'beginTimestamp' not in a['activity'] else '"' + a['activity']['beginTimestamp']['display'].replace('"', '""') + '",' - csv_record += empty_record if 'beginTimestamp' not in a['activity'] else '"' + a['activity']['beginTimestamp']['millis'].replace('"', '""') + '",' - csv_record += empty_record if 'endTimestamp' not in a['activity'] else '"' + a['activity']['endTimestamp']['display'].replace('"', '""') + '",' - csv_record += empty_record if 'endTimestamp' not in a['activity'] else '"' + a['activity']['endTimestamp']['millis'].replace('"', '""') + '",' - csv_record += empty_record if 'device' not in a['activity'] else '"' + a['activity']['device']['display'].replace('"', '""') + ' ' + a['activity']['device']['version'].replace('"', '""') + '",' - csv_record += empty_record if 'activityType' not in a['activity'] else '"' + a['activity']['activityType']['parent']['display'].replace('"', '""') + '",' - csv_record += empty_record if 'activityType' not in a['activity'] else '"' + a['activity']['activityType']['display'].replace('"', '""') + '",' - csv_record += empty_record if 'eventType' not in a['activity'] else '"' + a['activity']['eventType']['display'].replace('"', '""') + '",' - csv_record += empty_record if 'activityTimeZone' not in a['activity'] else '"' + a['activity']['activityTimeZone']['display'].replace('"', '""') + '",' - csv_record += empty_record if 'maxElevation' not in a['activity'] else '"' + a['activity']['maxElevation']['withUnit'].replace('"', '""') + '",' - csv_record += empty_record if 'maxElevation' not in a['activity'] else '"' + a['activity']['maxElevation']['value'].replace('"', '""') + '",' - csv_record += empty_record if 'beginLatitude' not in a['activity'] else '"' + a['activity']['beginLatitude']['value'].replace('"', '""') + '",' - csv_record += empty_record if 'beginLongitude' not in a['activity'] else '"' + a['activity']['beginLongitude']['value'].replace('"', '""') + '",' - csv_record += empty_record if 'endLatitude' not in a['activity'] else '"' + a['activity']['endLatitude']['value'].replace('"', '""') + '",' - csv_record += empty_record if 'endLongitude' not in a['activity'] else '"' + a['activity']['endLongitude']['value'].replace('"', '""') + '",' - csv_record += empty_record if 'weightedMeanMovingSpeed' not in a['activity'] else '"' + a['activity']['weightedMeanMovingSpeed']['display'].replace('"', '""') + '",' # The units vary between Minutes per Mile and mph, but withUnit always displays "Minutes per Mile" - csv_record += empty_record if 'weightedMeanMovingSpeed' not in a['activity'] else '"' + a['activity']['weightedMeanMovingSpeed']['value'].replace('"', '""') + '",' - csv_record += empty_record if 'maxHeartRate' not in a['activity'] else '"' + a['activity']['maxHeartRate']['display'].replace('"', '""') + '",' - csv_record += empty_record if 'weightedMeanHeartRate' not in a['activity'] else '"' + a['activity']['weightedMeanHeartRate']['display'].replace('"', '""') + '",' - csv_record += empty_record if 'maxSpeed' not in a['activity'] else '"' + a['activity']['maxSpeed']['display'].replace('"', '""') + '",' # The units vary between Minutes per Mile and mph, but withUnit always displays "Minutes per Mile" - csv_record += empty_record if 'maxSpeed' not in a['activity'] else '"' + a['activity']['maxSpeed']['value'].replace('"', '""') + '",' - csv_record += empty_record if 'sumEnergy' not in a['activity'] else '"' + a['activity']['sumEnergy']['display'].replace('"', '""') + '",' - csv_record += empty_record if 'sumEnergy' not in a['activity'] else '"' + a['activity']['sumEnergy']['value'].replace('"', '""') + '",' - csv_record += empty_record if 'sumElapsedDuration' not in a['activity'] else '"' + a['activity']['sumElapsedDuration']['display'].replace('"', '""') + '",' - csv_record += empty_record if 'sumElapsedDuration' not in a['activity'] else '"' + a['activity']['sumElapsedDuration']['value'].replace('"', '""') + '",' - csv_record += empty_record if 'sumMovingDuration' not in a['activity'] else '"' + a['activity']['sumMovingDuration']['display'].replace('"', '""') + '",' - csv_record += empty_record if 'sumMovingDuration' not in a['activity'] else '"' + a['activity']['sumMovingDuration']['value'].replace('"', '""') + '",' - csv_record += empty_record if 'weightedMeanSpeed' not in a['activity'] else '"' + a['activity']['weightedMeanSpeed']['withUnit'].replace('"', '""') + '",' - csv_record += empty_record if 'weightedMeanSpeed' not in a['activity'] else '"' + a['activity']['weightedMeanSpeed']['value'].replace('"', '""') + '",' - csv_record += empty_record if 'sumDistance' not in a['activity'] else '"' + a['activity']['sumDistance']['withUnit'].replace('"', '""') + '",' - csv_record += empty_record if 'sumDistance' not in a['activity'] else '"' + a['activity']['sumDistance']['value'].replace('"', '""') + '",' - csv_record += empty_record if 'minHeartRate' not in a['activity'] else '"' + a['activity']['minHeartRate']['display'].replace('"', '""') + '",' - csv_record += empty_record if 'maxElevation' not in a['activity'] else '"' + a['activity']['maxElevation']['withUnit'].replace('"', '""') + '",' - csv_record += empty_record if 'maxElevation' not in a['activity'] else '"' + a['activity']['maxElevation']['value'].replace('"', '""') + '",' - csv_record += empty_record if 'gainElevation' not in a['activity'] else '"' + a['activity']['gainElevation']['withUnit'].replace('"', '""') + '",' - csv_record += empty_record if 'gainElevation' not in a['activity'] else '"' + a['activity']['gainElevation']['value'].replace('"', '""') + '",' - csv_record += empty_record if 'lossElevation' not in a['activity'] else '"' + a['activity']['lossElevation']['withUnit'].replace('"', '""') + '",' - csv_record += empty_record if 'lossElevation' not in a['activity'] else '"' + a['activity']['lossElevation']['value'].replace('"', '""') + '"' - csv_record += '\n' - - csv_file.write(csv_record) - - if data_format == 'gpx': - # Validate GPX data. If we have an activity without GPS data (e.g., running on a treadmill), - # Garmin Connect still kicks out a GPX, but there is only activity information, no GPS data. - # N.B. You can omit the XML parse (and the associated log messages) to speed things up. - gpx = parseString(data) - gpx_data_exists = len(gpx.getElementsByTagName('trkpt')) > 0 - - if gpx_data_exists: - print('Done. GPX data saved.') - else: - print('Done. No track points found.') - else: - # TODO: Consider validating other formats. - print('Done.') - total_downloaded += num_to_download + # Maximum of 100... 400 return status if over 100. So download 100 or whatever remains if less than 100. + if total_to_download - total_downloaded > 100: + num_to_download = 100 + else: + num_to_download = total_to_download - total_downloaded + + search_params = {'start': total_downloaded, 'limit': num_to_download} + # Query Garmin Connect + result = http_req(url_gc_search + urlencode(search_params)) + json_results = json.loads(result.decode('utf-8')) # TODO: Catch possible exceptions here. + + search = json_results['results']['search'] + + if download_all: + # Modify total_to_download based on how many activities the server reports. + total_to_download = int(search['totalFound']) + # Do it only once. + download_all = False + + # Pull out just the list of activities. + activities = json_results['results']['activities'] + + # Process each activity. + for a in activities: + # Display which entry we're working on. + print('Garmin Connect activity: [' + a['activity']['activityId'] + ']', end=' ') + print(a['activity']['beginTimestamp']['display'] + ':', end=' ') + print(a['activity']['activityName']['value']) + + if data_format != 'csvonly': + if data_format == 'gpx': + filename = activities_directory + '/activity_' + a['activity']['activityId'] + '.gpx' + download_url = url_gc_gpx_activity + a['activity']['activityId'] + '?full=true' + file_mode = 'wb' + elif data_format == 'tcx': + filename = activities_directory + '/activity_' + a['activity']['activityId'] + '.tcx' + download_url = url_gc_tcx_activity + a['activity']['activityId'] + '?full=true' + file_mode = 'w' + else: + filename = activities_directory + '/activity_' + a['activity']['activityId'] + '.zip' + download_url = url_gc_original_activity + a['activity']['activityId'] + file_mode = 'wb' + + if isfile(filename): + print('\tData file already exists; skipping...') + continue + + # Download the data file from Garmin Connect. + # If the download fails (e.g., due to timeout), this script will die, but nothing + # will have been written to disk about this activity, so just running it again + # should pick up where it left off. + print('\tDownloading file...', end=' ') + + try: + data = http_req(download_url) + except urllib.error.HTTPError as e: + # Handle expected (though unfortunate) error codes; die on unexpected ones. + if e.code == 500 and data_format == 'tcx': + # Garmin will give an internal server error (HTTP 500) when downloading TCX files if the original was a manual GPX upload. + # Writing an empty file prevents this file from being redownloaded, similar to the way GPX files are saved even when there are no tracks. + # One could be generated here, but that's a bit much. Use the GPX format if you want actual data in every file, as I believe Garmin provides a GPX file for every activity. + print('Writing empty file since Garmin did not generate a TCX file for this activity...', end=' ') + data = '' + elif e.code == 404 and data_format == 'original': + # For manual activities (i.e., entered in online without a file upload), there is no original file. + # Write an empty file to prevent redownloading it. + print('Writing empty file since there was no original activity data...', end=' ') + data = '' + else: + raise Exception('Failed. Got an unexpected HTTP error (' + str(e.code) + ').') + + save_file = open(filename, file_mode) + save_file.write(data) + save_file.close() + + # Write stats to CSV. + csv_record = '' + + def append_col(field_name, field_val1="", field_val2=""): + global csv_record + if field_name not in a['activity']: + csv_record += '"",' + else: + if field_val1 != "": + if field_val2 != "": + fval = a['activity'][field_name][field_val1][field_val2] + else: + fval = a['activity'][field_name][field_val1] + else: + fval = a['activity'][field_name] + csv_record += '"' + fval.replace('"', '""') + '",' + + append_col('activityId') + append_col('activityName', 'value') + append_col('activityDescription', 'value') + append_col('beginTimestamp', 'display') + append_col('beginTimestamp', 'millis') + append_col('endTimestamp', 'display') + append_col('endTimestamp', 'millis') + append_col('device', 'display') + append_col('activityType', 'parent', 'display') + append_col('activityType', 'display') + append_col('eventType', 'display') + append_col('activityTimeZone', 'display') + append_col('maxElevation', 'withUnit') + append_col('maxElevation', 'value') + append_col('beginLatitude', 'value') + append_col('beginLongitude', 'value') + append_col('endLatitude', 'value') + append_col('endLongitude', 'value') + append_col('weightedMeanMovingSpeed', 'display') + append_col('weightedMeanMovingSpeed', 'value') + append_col('maxHeartRate', 'display') + append_col('weightedMeanHeartRate', 'display') + append_col('maxSpeed', 'display') + append_col('maxSpeed', 'value') + append_col('sumEnergy', 'display') + append_col('sumEnergy', 'value') + append_col('sumElapsedDuration', 'display') + append_col('sumElapsedDuration', 'value') + append_col('sumMovingDuration', 'display') + append_col('sumMovingDuration', 'value') + append_col('weightedMeanSpeed', 'withUnit') + append_col('weightedMeanSpeed', 'value') + append_col('sumDistance', 'withUnit') + append_col('sumDistance', 'value') + append_col('minHeartRate', 'display') + append_col('maxElevation', 'withUnit') + append_col('maxElevation', 'value') + append_col('gainElevation', 'withUnit') + append_col('gainElevation', 'value') + append_col('lossElevation', 'withUnit') + append_col('lossElevation', 'value') + append_col('device', 'version') + csv_record += '\n' + + csv_file.write(csv_record) + + if data_format == 'gpx': + # Validate GPX data. If we have an activity without GPS data (e.g., running on a treadmill), + # Garmin Connect still kicks out a GPX, but there is only activity information, no GPS data. + # N.B. You can omit the XML parse (and the associated log messages) to speed things up. + gpx = parseString(data) + gpx_data_exists = len(gpx.getElementsByTagName('trkpt')) > 0 + + if gpx_data_exists: + print('Done. GPX data saved.') + else: + print('Done. No track points found.') + else: + # TODO: Consider validating other formats. + print('Done.') + total_downloaded += num_to_download # End while loop for multiple chunks. csv_file.close()