From 3521d509d2528890ee3a9dbddaddb9328138ef87 Mon Sep 17 00:00:00 2001 From: Maxim Laptop Date: Thu, 24 Dec 2015 14:29:17 +0100 Subject: [PATCH 1/6] Added GarminHandler class Rewritten gcexport.py to use the GarminHandler No support yet for downloading tcx/gpx/original files. --- GarminHandler.py | 171 ++++++++++++++++++++ gcexport.py | 408 ++++++++++++++++++++--------------------------- 2 files changed, 348 insertions(+), 231 deletions(-) create mode 100644 GarminHandler.py diff --git a/GarminHandler.py b/GarminHandler.py new file mode 100644 index 0000000..1aae3da --- /dev/null +++ b/GarminHandler.py @@ -0,0 +1,171 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +""" +Created on Fri Oct 23 20:14:46 2015 + +@author: Maxim +""" +from urllib import urlencode #somehow not in the urllib2 package +import urllib2, cookielib, json +from ActivityJSON import ActivityJSON + +class GarminHandler( object ): + ## Global Constants + # URLs for various services. + URL_LOGIN = 'https://sso.garmin.com/sso/login?service=https%3A%2F%2Fconnect.garmin.com%2Fpost-auth%2Flogin&webhost=olaxpw-connect04&source=https%3A%2F%2Fconnect.garmin.com%2Fen-US%2Fsignin&redirectAfterAccountLoginUrl=https%3A%2F%2Fconnect.garmin.com%2Fpost-auth%2Flogin&redirectAfterAccountCreationUrl=https%3A%2F%2Fconnect.garmin.com%2Fpost-auth%2Flogin&gauthHost=https%3A%2F%2Fsso.garmin.com%2Fsso&locale=en_US&id=gauth-widget&cssUrl=https%3A%2F%2Fstatic.garmincdn.com%2Fcom.garmin.connect%2Fui%2Fcss%2Fgauth-custom-v1.1-min.css&clientId=GarminConnect&rememberMeShown=true&rememberMeChecked=false&createAccountShown=true&openCreateAccount=false&usernameShown=false&displayNameShown=false&consumeServiceTicket=false&initialFocus=true&embedWidget=false&generateExtraServiceTicket=false' + URL_POST_AUTH = 'https://connect.garmin.com/post-auth/login?' + URL_SEARCH = 'http://connect.garmin.com/proxy/activity-search-service-1.0/json/activities?' + URL_GPX_ACTIVITY = 'http://connect.garmin.com/proxy/activity-service-1.1/gpx/activity/%s?full=true' + URL_TCX_ACTIVITY = 'http://connect.garmin.com/proxy/activity-service-1.1/tcx/activity/%s?full=true' + URL_ORIGINAL_ACTIVITY = 'http://connect.garmin.com/proxy/download-service/files/activity/' + JSON_DOWNLOAD_LIMIT = 100 # Maximum number of activities to request at once. 100 is the maximum set and enforced by Garmin. + #JSON_DOWNLOAD_LIMIT = 10 # but 10 is faster if few activities to retrieve. + + def __init__( self, username, password ): + self.opener = None + # You must be logged in to use the class + self.login( username, password ) + + def login( self, username, password ): + """ Returns True if logged in, raises error if not.""" + # Initially, we need to get a valid session cookie, so we pull the login page. + cookie_jar = cookielib.CookieJar() + self.opener = urllib2.build_opener( urllib2.HTTPCookieProcessor(cookie_jar) ) + http_req( self.opener, self.URL_LOGIN ) + + # Now we'll actually login. Post data with Fields that are passed in a typical Garmin login. + post_data = {'username': username, 'password': password, + 'embed': 'true', 'lt': 'e1s1', '_eventId': 'submit', 'displayNameRequired': 'false'} + http_req ( self.opener, self.URL_LOGIN, post_data ) + + # Get the key. + # TODO: Can we do this without iterating? + login_ticket = None + for cookie in cookie_jar: + if cookie.name == 'CASTGC': + login_ticket = cookie.value + break + + if not login_ticket: + raise Exception('Did not get a ticket cookie. Cannot log in. Did you enter the correct username and password?') + + # Post Authorize. Chop of 'TGT-' off the beginning, prepend 'ST-0'. + login_ticket = 'ST-0' + login_ticket[4:] + http_req( self.opener, self.URL_POST_AUTH + 'ticket=' + login_ticket) + + #TODO: extra check whether indeed logged in. + return True + + def activitiesGenerator( self, limit = None, reversed = False ): + """ Yields the json as dict for every activity found, + either from new to old or reversed. """ + + # Prevent downloading too large chunks (saves time) + if limit and limit < self.JSON_DOWNLOAD_LIMIT: + max_chunk_size = limit + else: + max_chunk_size = self.JSON_DOWNLOAD_LIMIT + + # Determine index to start at + if reversed: + # Download one activity. Result will contain how many activities + # there are in total + url = self.URL_SEARCH + urlencode({'start': 0, 'limit': 1}) + result = http_req(self.opener, url ) + json_results = json.loads(result) + n_activities = int( json_results['results']['search']['totalFound']) + # Start + start_index = n_activities - max_chunk_size + if start_index < 0: #Negative index gives problems + start_index = 0 + else: + start_index = 0 + + # Download data in multiple chunks of *max_chunk_size* activities + total_downloaded = 0 + downloaded_chunk_size = max_chunk_size #initialize + while downloaded_chunk_size >= max_chunk_size: # If downloaded chunk smaller, all activities are retrieved. + # Query Garmin Connect + search_params = {'start': start_index, 'limit': max_chunk_size} + url = self.URL_SEARCH + urlencode(search_params) + try: + result = http_req(self.opener, url ) + json_results = json.loads(result) + except urllib2.HTTPError as e: + raise Exception('Failed to retrieve json of activities. (' + str(e) + ').') + + # Pull out just the list of activities. + activities = json_results['results']['activities'] + downloaded_chunk_size = len(activities) + + if reversed: + activities = activities[::-1] #reverse + + for activity in activities: + activity_details = activity['activity'] + yield activity_details + + total_downloaded += 1 + # Stop if limit is reached + if total_downloaded == limit: + raise StopIteration + + # Increment start index + if reversed: + if start_index - max_chunk_size < 0: # Negative start is not allowed + max_chunk_size = start_index # Next batch will be up to last start_index + start_index = 0 + else: + start_index -= max_chunk_size #Backwards + else: + start_index += max_chunk_size #Forwards + + def getNewRuns( self, existing_ids ): + """ Iterate until an existing activiity is found. + Returns list of new activities. """ + + activities = self.activitiesGenerator() + for activity in activities: + act_id = activityDict.getID( activity ) + if act_id in existing_ids: + break + + if activityDict.isRun( activity ): + yield activity + + def downloadTCXbyID( self, activity_id ): + """ Returns content of TCX """ + # TODO: Finish and test this function + download_url = self.URL_TCX_ACTIVITY % activity_id + + try: + data = http_req( self.opener, download_url ) + except urllib2.HTTPError as e: + # Handle expected (though unfortunate) error codes; die on unexpected ones. + if e.code == 500: + # Garmin will give an internal server error (HTTP 500) when downloading TCX files if the original was a manual GPX upload. + # One could be generated here, but that's a bit much. Use the GPX format if you want actual data in every file, as I believe Garmin provides a GPX file for every activity. + print 'Writing empty file since Garmin did not generate a TCX file for this activity...', + data = '' + else: + raise Exception('Failed. Got an unexpected HTTP error (' + str(e.code) + ').') + + return data +## End of Class ## + +##Tools +def http_req(opener, url, post=None, headers={}): + """ url is a string, post is a dictionary of POST parameters, headers is a dictionary of headers. """ + request = urllib2.Request(url) + request.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 5.2; rv:2.0.1) Gecko/20100101 Firefox/4.0.1') # Tell Garmin we're some supported browser. + for header_key, header_value in headers.iteritems(): + request.add_header(header_key, header_value) + if post: + post = urlencode(post) # Convert dictionary to POST parameter string. + response = opener.open(request, data=post) # This line may throw a urllib2.HTTPError. + + # N.B. urllib2 will follow any 302 redirects. Also, the "open" call above may throw a urllib2.HTTPError which is checked for below. + if response.getcode() != 200: + raise Exception('Bad return code (' + response.getcode() + ') for: ' + url) + + return response.read() \ No newline at end of file diff --git a/gcexport.py b/gcexport.py index 2c6fec7..3f032a5 100755 --- a/gcexport.py +++ b/gcexport.py @@ -25,6 +25,9 @@ import argparse import zipfile +from GarminHandler import GarminHandler +from ActivityJSON import ActivityJSON + script_version = '1.0.0' current_date = datetime.now().strftime('%Y-%m-%d') activities_directory = './' + current_date + '_garmin_connect_export' @@ -50,30 +53,21 @@ help="if downloading ZIP files (format: 'original'), unzip the file and removes the ZIP file", action="store_true") +parser.add_argument('-r', '--reverse', + help="start with oldest activity (otherwise starts with newest)", + action="store_true") + args = parser.parse_args() if args.version: print argv[0] + ", version " + script_version exit(0) -cookie_jar = cookielib.CookieJar() -opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookie_jar)) - -# url is a string, post is a dictionary of POST parameters, headers is a dictionary of headers. -def http_req(url, post=None, headers={}): - request = urllib2.Request(url) - request.add_header('User-Agent', 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/1337 Safari/537.36') # Tell Garmin we're some supported browser. - for header_key, header_value in headers.iteritems(): - request.add_header(header_key, header_value) - if post: - post = urlencode(post) # Convert dictionary to POST parameter string. - response = opener.open(request, data=post) # This line may throw a urllib2.HTTPError. - - # N.B. urllib2 will follow any 302 redirects. Also, the "open" call above may throw a urllib2.HTTPError which is checked for below. - if response.getcode() != 200: - raise Exception('Bad return code (' + response.getcode() + ') for: ' + url) - - return response.read() +# Convert the count to integer or empty if all. +if args.count == 'all': + total_to_download = None +else: + total_to_download = int(args.count) print 'Welcome to Garmin Connect Exporter!' @@ -84,41 +78,9 @@ def http_req(url, post=None, headers={}): username = args.username if args.username else raw_input('Username: ') password = args.password if args.password else getpass() -# Maximum number of activities you can request at once. Set and enforced by Garmin. -limit_maximum = 100 - -# URLs for various services. -url_gc_login = 'https://sso.garmin.com/sso/login?service=https%3A%2F%2Fconnect.garmin.com%2Fpost-auth%2Flogin&webhost=olaxpw-connect04&source=https%3A%2F%2Fconnect.garmin.com%2Fen-US%2Fsignin&redirectAfterAccountLoginUrl=https%3A%2F%2Fconnect.garmin.com%2Fpost-auth%2Flogin&redirectAfterAccountCreationUrl=https%3A%2F%2Fconnect.garmin.com%2Fpost-auth%2Flogin&gauthHost=https%3A%2F%2Fsso.garmin.com%2Fsso&locale=en_US&id=gauth-widget&cssUrl=https%3A%2F%2Fstatic.garmincdn.com%2Fcom.garmin.connect%2Fui%2Fcss%2Fgauth-custom-v1.1-min.css&clientId=GarminConnect&rememberMeShown=true&rememberMeChecked=false&createAccountShown=true&openCreateAccount=false&usernameShown=false&displayNameShown=false&consumeServiceTicket=false&initialFocus=true&embedWidget=false&generateExtraServiceTicket=false' -url_gc_post_auth = 'https://connect.garmin.com/post-auth/login?' -url_gc_search = 'http://connect.garmin.com/proxy/activity-search-service-1.0/json/activities?' -url_gc_gpx_activity = 'http://connect.garmin.com/proxy/activity-service-1.1/gpx/activity/' -url_gc_tcx_activity = 'http://connect.garmin.com/proxy/activity-service-1.1/tcx/activity/' -url_gc_original_activity = 'http://connect.garmin.com/proxy/download-service/files/activity/' +# Login and initialize the handler. Raises exception if login failed. +garmin_handler = GarminHandler( username, password ) -# Initially, we need to get a valid session cookie, so we pull the login page. -http_req(url_gc_login) - -# Now we'll actually login. -post_data = {'username': username, 'password': password, 'embed': 'true', 'lt': 'e1s1', '_eventId': 'submit', 'displayNameRequired': 'false'} # Fields that are passed in a typical Garmin login. -http_req(url_gc_login, post_data) - -# Get the key. -# TODO: Can we do this without iterating? -login_ticket = None -for cookie in cookie_jar: - if cookie.name == 'CASTGC': - login_ticket = cookie.value - break - -if not login_ticket: - raise Exception('Did not get a ticket cookie. Cannot log in. Did you enter the correct username and password?') - -# Chop of 'TGT-' off the beginning, prepend 'ST-0'. -login_ticket = 'ST-0' + login_ticket[4:] - -http_req(url_gc_post_auth + 'ticket=' + login_ticket) - -# We should be logged in now. if not isdir(args.directory): mkdir(args.directory) @@ -131,185 +93,169 @@ def http_req(url, post=None, headers={}): if not csv_existed: csv_file.write('Activity ID,Activity Name,Description,Begin Timestamp,Begin Timestamp (Raw Milliseconds),End Timestamp,End Timestamp (Raw Milliseconds),Device,Activity Parent,Activity Type,Event Type,Activity Time Zone,Max. Elevation,Max. Elevation (Raw),Begin Latitude (Decimal Degrees Raw),Begin Longitude (Decimal Degrees Raw),End Latitude (Decimal Degrees Raw),End Longitude (Decimal Degrees Raw),Average Moving Speed,Average Moving Speed (Raw),Max. Heart Rate (bpm),Average Heart Rate (bpm),Max. Speed,Max. Speed (Raw),Calories,Calories (Raw),Duration (h:m:s),Duration (Raw Seconds),Moving Duration (h:m:s),Moving Duration (Raw Seconds),Average Speed,Average Speed (Raw),Distance,Distance (Raw),Max. Heart Rate (bpm),Min. Elevation,Min. Elevation (Raw),Elevation Gain,Elevation Gain (Raw),Elevation Loss,Elevation Loss (Raw)\n') -download_all = False -if args.count == 'all': - # If the user wants to download all activities, first download one, - # then the result of that request will tell us how many are available - # so we will modify the variables then. - total_to_download = 1 - download_all = True -else: - total_to_download = int(args.count) -total_downloaded = 0 - -# This while loop will download data from the server in multiple chunks, if necessary. -while total_downloaded < total_to_download: - # Maximum of 100... 400 return status if over 100. So download 100 or whatever remains if less than 100. - if total_to_download - total_downloaded > 100: - num_to_download = 100 - else: - num_to_download = total_to_download - total_downloaded - - search_params = {'start': total_downloaded, 'limit': num_to_download} - # Query Garmin Connect - result = http_req(url_gc_search + urlencode(search_params)) - json_results = json.loads(result) # TODO: Catch possible exceptions here. - - - search = json_results['results']['search'] - - if download_all: - # Modify total_to_download based on how many activities the server reports. - total_to_download = int(search['totalFound']) - # Do it only once. - download_all = False - - # Pull out just the list of activities. - activities = json_results['results']['activities'] - - # Process each activity. - for a in activities: - # Display which entry we're working on. - print 'Garmin Connect activity: [' + a['activity']['activityId'] + ']', - print a['activity']['activityName']['value'] - print '\t' + a['activity']['beginTimestamp']['display'] + ',', - if 'sumElapsedDuration' in a['activity']: - print a['activity']['sumElapsedDuration']['display'] + ',', - else: - print '??:??:??,', - if 'sumDistance' in a['activity']: - print a['activity']['sumDistance']['withUnit'] - else: - print '0.00 Miles' - - if args.format == 'gpx': - data_filename = args.directory + '/activity_' + a['activity']['activityId'] + '.gpx' - download_url = url_gc_gpx_activity + a['activity']['activityId'] + '?full=true' - file_mode = 'w' - elif args.format == 'tcx': - data_filename = args.directory + '/activity_' + a['activity']['activityId'] + '.tcx' - download_url = url_gc_tcx_activity + a['activity']['activityId'] + '?full=true' - file_mode = 'w' - elif args.format == 'original': - data_filename = args.directory + '/activity_' + a['activity']['activityId'] + '.zip' - fit_filename = args.directory + '/' + a['activity']['activityId'] + '.fit' - download_url = url_gc_original_activity + a['activity']['activityId'] - file_mode = 'wb' - else: - raise Exception('Unrecognized format.') - - if isfile(data_filename): - print '\tData file already exists; skipping...' - continue - if args.format == 'original' and isfile(fit_filename): # Regardless of unzip setting, don't redownload if the ZIP or FIT file exists. - print '\tFIT data file already exists; skipping...' - continue - - # Download the data file from Garmin Connect. - # If the download fails (e.g., due to timeout), this script will die, but nothing - # will have been written to disk about this activity, so just running it again - # should pick up where it left off. - print '\tDownloading file...', - - try: - data = http_req(download_url) - except urllib2.HTTPError as e: - # Handle expected (though unfortunate) error codes; die on unexpected ones. - if e.code == 500 and args.format == 'tcx': - # Garmin will give an internal server error (HTTP 500) when downloading TCX files if the original was a manual GPX upload. - # Writing an empty file prevents this file from being redownloaded, similar to the way GPX files are saved even when there are no tracks. - # One could be generated here, but that's a bit much. Use the GPX format if you want actual data in every file, - # as I believe Garmin provides a GPX file for every activity. - print 'Writing empty file since Garmin did not generate a TCX file for this activity...', - data = '' - elif e.code == 404 and args.format == 'original': - # For manual activities (i.e., entered in online without a file upload), there is no original file. - # Write an empty file to prevent redownloading it. - print 'Writing empty file since there was no original activity data...', - data = '' - else: - raise Exception('Failed. Got an unexpected HTTP error (' + str(e.code) + ').') - - save_file = open(data_filename, file_mode) - save_file.write(data) - save_file.close() - - # Write stats to CSV. - empty_record = '"",' - - csv_record = '' - - csv_record += empty_record if 'activityId' not in a['activity'] else '"' + a['activity']['activityId'].replace('"', '""') + '",' - csv_record += empty_record if 'activityName' not in a['activity'] else '"' + a['activity']['activityName']['value'].replace('"', '""') + '",' - csv_record += empty_record if 'activityDescription' not in a['activity'] else '"' + a['activity']['activityDescription']['value'].replace('"', '""') + '",' - csv_record += empty_record if 'beginTimestamp' not in a['activity'] else '"' + a['activity']['beginTimestamp']['display'].replace('"', '""') + '",' - csv_record += empty_record if 'beginTimestamp' not in a['activity'] else '"' + a['activity']['beginTimestamp']['millis'].replace('"', '""') + '",' - csv_record += empty_record if 'endTimestamp' not in a['activity'] else '"' + a['activity']['endTimestamp']['display'].replace('"', '""') + '",' - csv_record += empty_record if 'endTimestamp' not in a['activity'] else '"' + a['activity']['endTimestamp']['millis'].replace('"', '""') + '",' - csv_record += empty_record if 'device' not in a['activity'] else '"' + a['activity']['device']['display'].replace('"', '""') + ' ' + a['activity']['device']['version'].replace('"', '""') + '",' - csv_record += empty_record if 'activityType' not in a['activity'] else '"' + a['activity']['activityType']['parent']['display'].replace('"', '""') + '",' - csv_record += empty_record if 'activityType' not in a['activity'] else '"' + a['activity']['activityType']['display'].replace('"', '""') + '",' - csv_record += empty_record if 'eventType' not in a['activity'] else '"' + a['activity']['eventType']['display'].replace('"', '""') + '",' - csv_record += empty_record if 'activityTimeZone' not in a['activity'] else '"' + a['activity']['activityTimeZone']['display'].replace('"', '""') + '",' - csv_record += empty_record if 'maxElevation' not in a['activity'] else '"' + a['activity']['maxElevation']['withUnit'].replace('"', '""') + '",' - csv_record += empty_record if 'maxElevation' not in a['activity'] else '"' + a['activity']['maxElevation']['value'].replace('"', '""') + '",' - csv_record += empty_record if 'beginLatitude' not in a['activity'] else '"' + a['activity']['beginLatitude']['value'].replace('"', '""') + '",' - csv_record += empty_record if 'beginLongitude' not in a['activity'] else '"' + a['activity']['beginLongitude']['value'].replace('"', '""') + '",' - csv_record += empty_record if 'endLatitude' not in a['activity'] else '"' + a['activity']['endLatitude']['value'].replace('"', '""') + '",' - csv_record += empty_record if 'endLongitude' not in a['activity'] else '"' + a['activity']['endLongitude']['value'].replace('"', '""') + '",' - csv_record += empty_record if 'weightedMeanMovingSpeed' not in a['activity'] else '"' + a['activity']['weightedMeanMovingSpeed']['display'].replace('"', '""') + '",' # The units vary between Minutes per Mile and mph, but withUnit always displays "Minutes per Mile" - csv_record += empty_record if 'weightedMeanMovingSpeed' not in a['activity'] else '"' + a['activity']['weightedMeanMovingSpeed']['value'].replace('"', '""') + '",' - csv_record += empty_record if 'maxHeartRate' not in a['activity'] else '"' + a['activity']['maxHeartRate']['display'].replace('"', '""') + '",' - csv_record += empty_record if 'weightedMeanHeartRate' not in a['activity'] else '"' + a['activity']['weightedMeanHeartRate']['display'].replace('"', '""') + '",' - csv_record += empty_record if 'maxSpeed' not in a['activity'] else '"' + a['activity']['maxSpeed']['display'].replace('"', '""') + '",' # The units vary between Minutes per Mile and mph, but withUnit always displays "Minutes per Mile" - csv_record += empty_record if 'maxSpeed' not in a['activity'] else '"' + a['activity']['maxSpeed']['value'].replace('"', '""') + '",' - csv_record += empty_record if 'sumEnergy' not in a['activity'] else '"' + a['activity']['sumEnergy']['display'].replace('"', '""') + '",' - csv_record += empty_record if 'sumEnergy' not in a['activity'] else '"' + a['activity']['sumEnergy']['value'].replace('"', '""') + '",' - csv_record += empty_record if 'sumElapsedDuration' not in a['activity'] else '"' + a['activity']['sumElapsedDuration']['display'].replace('"', '""') + '",' - csv_record += empty_record if 'sumElapsedDuration' not in a['activity'] else '"' + a['activity']['sumElapsedDuration']['value'].replace('"', '""') + '",' - csv_record += empty_record if 'sumMovingDuration' not in a['activity'] else '"' + a['activity']['sumMovingDuration']['display'].replace('"', '""') + '",' - csv_record += empty_record if 'sumMovingDuration' not in a['activity'] else '"' + a['activity']['sumMovingDuration']['value'].replace('"', '""') + '",' - csv_record += empty_record if 'weightedMeanSpeed' not in a['activity'] else '"' + a['activity']['weightedMeanSpeed']['withUnit'].replace('"', '""') + '",' - csv_record += empty_record if 'weightedMeanSpeed' not in a['activity'] else '"' + a['activity']['weightedMeanSpeed']['value'].replace('"', '""') + '",' - csv_record += empty_record if 'sumDistance' not in a['activity'] else '"' + a['activity']['sumDistance']['withUnit'].replace('"', '""') + '",' - csv_record += empty_record if 'sumDistance' not in a['activity'] else '"' + a['activity']['sumDistance']['value'].replace('"', '""') + '",' - csv_record += empty_record if 'minHeartRate' not in a['activity'] else '"' + a['activity']['minHeartRate']['display'].replace('"', '""') + '",' - csv_record += empty_record if 'maxElevation' not in a['activity'] else '"' + a['activity']['maxElevation']['withUnit'].replace('"', '""') + '",' - csv_record += empty_record if 'maxElevation' not in a['activity'] else '"' + a['activity']['maxElevation']['value'].replace('"', '""') + '",' - csv_record += empty_record if 'gainElevation' not in a['activity'] else '"' + a['activity']['gainElevation']['withUnit'].replace('"', '""') + '",' - csv_record += empty_record if 'gainElevation' not in a['activity'] else '"' + a['activity']['gainElevation']['value'].replace('"', '""') + '",' - csv_record += empty_record if 'lossElevation' not in a['activity'] else '"' + a['activity']['lossElevation']['withUnit'].replace('"', '""') + '",' - csv_record += empty_record if 'lossElevation' not in a['activity'] else '"' + a['activity']['lossElevation']['value'].replace('"', '""') + '"' - csv_record += '\n' - - csv_file.write(csv_record.encode('utf8')) - - if args.format == 'gpx': - # Validate GPX data. If we have an activity without GPS data (e.g., running on a treadmill), - # Garmin Connect still kicks out a GPX, but there is only activity information, no GPS data. - # N.B. You can omit the XML parse (and the associated log messages) to speed things up. - gpx = parseString(data) - gpx_data_exists = len(gpx.getElementsByTagName('trkpt')) > 0 - - if gpx_data_exists: - print 'Done. GPX data saved.' - else: - print 'Done. No track points found.' - elif args.format == 'original': - if args.unzip and data_filename[-3:].lower() == 'zip': # Even manual upload of a GPX file is zipped, but we'll validate the extension. - print "Unzipping and removing original files...", - zip_file = open(data_filename, 'rb') - z = zipfile.ZipFile(zip_file) - for name in z.namelist(): - z.extract(name, args.directory) - zip_file.close() - remove(data_filename) - print 'Done.' - else: - # TODO: Consider validating other formats. - print 'Done.' - total_downloaded += num_to_download -# End while loop for multiple chunks. +# Create generator for activities. Generates activities until specified number of activities are retrieved. +# Activity is a dictionary object of the json. (without the redundant first 'activity' key) +activities_generator = garmin_handler.activitiesGenerator( limit = total_to_download, reversed = args.reverse ) + +for a in activities_generator: + # Display which entry we're working on. + print 'Garmin Connect activity: [' + a['activityId'] + ']', + print a['activityName']['value'] + print '\t' + a['beginTimestamp']['display'] + ',', + if 'sumElapsedDuration' in a: + print a['sumElapsedDuration']['display'] + ',', + else: + print '??:??:??,', + if 'sumDistance' in a: + print a['sumDistance']['withUnit'] + else: + print '0.00 Miles' + + # TODO MM: save gpx/tcx/original file + # if args.format == 'gpx': + # data_filename = args.directory + '/activity_' + a['activityId'] + '.gpx' + # download_url = url_gc_gpx_activity + a['activityId'] + '?full=true' + # file_mode = 'w' + # elif args.format == 'tcx': + # data_filename = args.directory + '/activity_' + a['activityId'] + '.tcx' + # download_url = url_gc_tcx_activity + a['activityId'] + '?full=true' + # file_mode = 'w' + # elif args.format == 'original': + # data_filename = args.directory + '/activity_' + a['activityId'] + '.zip' + # fit_filename = args.directory + '/' + a['activityId'] + '.fit' + # download_url = url_gc_original_activity + a['activityId'] + # file_mode = 'wb' + # else: + # raise Exception('Unrecognized format.') + + # if isfile(data_filename): + # print '\tData file already exists; skipping...' + # continue + # if args.format == 'original' and isfile(fit_filename): # Regardless of unzip setting, don't redownload if the ZIP or FIT file exists. + # print '\tFIT data file already exists; skipping...' + # continue + + # # Download the data file from Garmin Connect. + # # If the download fails (e.g., due to timeout), this script will die, but nothing + # # will have been written to disk about this activity, so just running it again + # # should pick up where it left off. + # print '\tDownloading file...', + + # try: + # data = http_req(download_url) + # except urllib2.HTTPError as e: + # # Handle expected (though unfortunate) error codes; die on unexpected ones. + # if e.code == 500 and args.format == 'tcx': + # # Garmin will give an internal server error (HTTP 500) when downloading TCX files if the original was a manual GPX upload. + # # Writing an empty file prevents this file from being redownloaded, similar to the way GPX files are saved even when there are no tracks. + # # One could be generated here, but that's a bit much. Use the GPX format if you want actual data in every file, + # # as I believe Garmin provides a GPX file for every activity. + # print 'Writing empty file since Garmin did not generate a TCX file for this activity...', + # data = '' + # elif e.code == 404 and args.format == 'original': + # # For manual activities (i.e., entered in online without a file upload), there is no original file. + # # Write an empty file to prevent redownloading it. + # print 'Writing empty file since there was no original activity data...', + # data = '' + # else: + # raise Exception('Failed. Got an unexpected HTTP error (' + str(e.code) + ').') + + # save_file = open(data_filename, file_mode) + # save_file.write(data) + # save_file.close() + + # Write stats to CSV. + + empty_record = '"",' + + csv_record = '' + csv_record += empty_record if 'activityId' not in a else '"' + a['activityId'].replace('"', '""') + '",' + csv_record += empty_record if 'activityName' not in a else '"' + a['activityName']['value'].replace('"', '""') + '",' + csv_record += empty_record if 'activityDescription' not in a else '"' + a['activityDescription']['value'].replace('"', '""') + '",' + csv_record += empty_record if 'beginTimestamp' not in a else '"' + a['beginTimestamp']['display'].replace('"', '""') + '",' + csv_record += empty_record if 'beginTimestamp' not in a else '"' + a['beginTimestamp']['millis'].replace('"', '""') + '",' + csv_record += empty_record if 'endTimestamp' not in a else '"' + a['endTimestamp']['display'].replace('"', '""') + '",' + csv_record += empty_record if 'endTimestamp' not in a else '"' + a['endTimestamp']['millis'].replace('"', '""') + '",' + csv_record += empty_record if 'device' not in a else '"' + a['device']['display'].replace('"', '""') + ' ' + a['device']['version'].replace('"', '""') + '",' + csv_record += empty_record if 'activityType' not in a else '"' + a['activityType']['parent']['display'].replace('"', '""') + '",' + csv_record += empty_record if 'activityType' not in a else '"' + a['activityType']['display'].replace('"', '""') + '",' + csv_record += empty_record if 'eventType' not in a else '"' + a['eventType']['display'].replace('"', '""') + '",' + csv_record += empty_record if 'activityTimeZone' not in a else '"' + a['activityTimeZone']['display'].replace('"', '""') + '",' + csv_record += empty_record if 'maxElevation' not in a else '"' + a['maxElevation']['withUnit'].replace('"', '""') + '",' + csv_record += empty_record if 'maxElevation' not in a else '"' + a['maxElevation']['value'].replace('"', '""') + '",' + csv_record += empty_record if 'beginLatitude' not in a else '"' + a['beginLatitude']['value'].replace('"', '""') + '",' + csv_record += empty_record if 'beginLongitude' not in a else '"' + a['beginLongitude']['value'].replace('"', '""') + '",' + csv_record += empty_record if 'endLatitude' not in a else '"' + a['endLatitude']['value'].replace('"', '""') + '",' + csv_record += empty_record if 'endLongitude' not in a else '"' + a['endLongitude']['value'].replace('"', '""') + '",' + csv_record += empty_record if 'weightedMeanMovingSpeed' not in a else '"' + a['weightedMeanMovingSpeed']['display'].replace('"', '""') + '",' # The units vary between Minutes per Mile and mph, but withUnit always displays "Minutes per Mile" + csv_record += empty_record if 'weightedMeanMovingSpeed' not in a else '"' + a['weightedMeanMovingSpeed']['value'].replace('"', '""') + '",' + csv_record += empty_record if 'maxHeartRate' not in a else '"' + a['maxHeartRate']['display'].replace('"', '""') + '",' + csv_record += empty_record if 'weightedMeanHeartRate' not in a else '"' + a['weightedMeanHeartRate']['display'].replace('"', '""') + '",' + csv_record += empty_record if 'maxSpeed' not in a else '"' + a['maxSpeed']['display'].replace('"', '""') + '",' # The units vary between Minutes per Mile and mph, but withUnit always displays "Minutes per Mile" + csv_record += empty_record if 'maxSpeed' not in a else '"' + a['maxSpeed']['value'].replace('"', '""') + '",' + csv_record += empty_record if 'sumEnergy' not in a else '"' + a['sumEnergy']['display'].replace('"', '""') + '",' + csv_record += empty_record if 'sumEnergy' not in a else '"' + a['sumEnergy']['value'].replace('"', '""') + '",' + csv_record += empty_record if 'sumElapsedDuration' not in a else '"' + a['sumElapsedDuration']['display'].replace('"', '""') + '",' + csv_record += empty_record if 'sumElapsedDuration' not in a else '"' + a['sumElapsedDuration']['value'].replace('"', '""') + '",' + csv_record += empty_record if 'sumMovingDuration' not in a else '"' + a['sumMovingDuration']['display'].replace('"', '""') + '",' + csv_record += empty_record if 'sumMovingDuration' not in a else '"' + a['sumMovingDuration']['value'].replace('"', '""') + '",' + csv_record += empty_record if 'weightedMeanSpeed' not in a else '"' + a['weightedMeanSpeed']['withUnit'].replace('"', '""') + '",' + csv_record += empty_record if 'weightedMeanSpeed' not in a else '"' + a['weightedMeanSpeed']['value'].replace('"', '""') + '",' + csv_record += empty_record if 'sumDistance' not in a else '"' + a['sumDistance']['withUnit'].replace('"', '""') + '",' + csv_record += empty_record if 'sumDistance' not in a else '"' + a['sumDistance']['value'].replace('"', '""') + '",' + csv_record += empty_record if 'minHeartRate' not in a else '"' + a['minHeartRate']['display'].replace('"', '""') + '",' + csv_record += empty_record if 'maxElevation' not in a else '"' + a['maxElevation']['withUnit'].replace('"', '""') + '",' + csv_record += empty_record if 'maxElevation' not in a else '"' + a['maxElevation']['value'].replace('"', '""') + '",' + csv_record += empty_record if 'gainElevation' not in a else '"' + a['gainElevation']['withUnit'].replace('"', '""') + '",' + csv_record += empty_record if 'gainElevation' not in a else '"' + a['gainElevation']['value'].replace('"', '""') + '",' + csv_record += empty_record if 'lossElevation' not in a else '"' + a['lossElevation']['withUnit'].replace('"', '""') + '",' + csv_record += empty_record if 'lossElevation' not in a else '"' + a['lossElevation']['value'].replace('"', '""') + '"' + csv_record += '\n' + + csv_file.write(csv_record.encode('utf8')) + + # TODO MM replace csv creation thing by: + # activity_obj = ActivityJSON( activity_dict ) + # csv_record = "%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s" % ( + # activity_obj.getID(), + # activity_obj.getName(), + # activity_obj.getCategory(), + # activity_obj.getDistance(), + # activity_obj.getDuration(), + # activity_obj.getComment(), + # activity_obj.getDate(), #datetime object + # activity_obj.getStartTime(), + # activity_obj.getBpmMax(), + # activity_obj.getBpmAvg(), + # activity_obj.getLatitude(), + # activity_obj.getLongitude() + # ) + + # TODO MM file validation? + # if args.format == 'gpx': + # # Validate GPX data. If we have an activity without GPS data (e.g., running on a treadmill), + # # Garmin Connect still kicks out a GPX, but there is only activity information, no GPS data. + # # N.B. You can omit the XML parse (and the associated log messages) to speed things up. + # gpx = parseString(data) + # gpx_data_exists = len(gpx.getElementsByTagName('trkpt')) > 0 + + # if gpx_data_exists: + # print 'Done. GPX data saved.' + # else: + # print 'Done. No track points found.' + # elif args.format == 'original': + # if args.unzip and data_filename[-3:].lower() == 'zip': # Even manual upload of a GPX file is zipped, but we'll validate the extension. + # print "Unzipping and removing original files...", + # zip_file = open(data_filename, 'rb') + # z = zipfile.ZipFile(zip_file) + # for name in z.namelist(): + # z.extract(name, args.directory) + # zip_file.close() + # remove(data_filename) + # print 'Done.' + # else: + # # TODO: Consider validating other formats. + # print 'Done.' csv_file.close() From 1c65bee25eb47fe1dff27ad1fae7da2773eee9c4 Mon Sep 17 00:00:00 2001 From: Maxim Laptop Date: Thu, 24 Dec 2015 15:47:33 +0100 Subject: [PATCH 2/6] Added tcx/gpx/original and csv data download functionality. --- GarminHandler.py | 51 +++++++++++++------ gcexport.py | 127 +++++++++++++++++++---------------------------- 2 files changed, 88 insertions(+), 90 deletions(-) diff --git a/GarminHandler.py b/GarminHandler.py index 1aae3da..fee38a8 100644 --- a/GarminHandler.py +++ b/GarminHandler.py @@ -15,9 +15,10 @@ class GarminHandler( object ): URL_LOGIN = 'https://sso.garmin.com/sso/login?service=https%3A%2F%2Fconnect.garmin.com%2Fpost-auth%2Flogin&webhost=olaxpw-connect04&source=https%3A%2F%2Fconnect.garmin.com%2Fen-US%2Fsignin&redirectAfterAccountLoginUrl=https%3A%2F%2Fconnect.garmin.com%2Fpost-auth%2Flogin&redirectAfterAccountCreationUrl=https%3A%2F%2Fconnect.garmin.com%2Fpost-auth%2Flogin&gauthHost=https%3A%2F%2Fsso.garmin.com%2Fsso&locale=en_US&id=gauth-widget&cssUrl=https%3A%2F%2Fstatic.garmincdn.com%2Fcom.garmin.connect%2Fui%2Fcss%2Fgauth-custom-v1.1-min.css&clientId=GarminConnect&rememberMeShown=true&rememberMeChecked=false&createAccountShown=true&openCreateAccount=false&usernameShown=false&displayNameShown=false&consumeServiceTicket=false&initialFocus=true&embedWidget=false&generateExtraServiceTicket=false' URL_POST_AUTH = 'https://connect.garmin.com/post-auth/login?' URL_SEARCH = 'http://connect.garmin.com/proxy/activity-search-service-1.0/json/activities?' - URL_GPX_ACTIVITY = 'http://connect.garmin.com/proxy/activity-service-1.1/gpx/activity/%s?full=true' - URL_TCX_ACTIVITY = 'http://connect.garmin.com/proxy/activity-service-1.1/tcx/activity/%s?full=true' - URL_ORIGINAL_ACTIVITY = 'http://connect.garmin.com/proxy/download-service/files/activity/' + URL_GPX_ACTIVITY = 'http://connect.garmin.com/proxy/activity-service-1.1/gpx/activity/%s?full=true' + URL_TCX_ACTIVITY = 'http://connect.garmin.com/proxy/activity-service-1.1/tcx/activity/%s?full=true' + URL_ZIP_ACTIVITY = 'http://connect.garmin.com/proxy/download-service/files/activity/%s' + URL_CSV_ACTIVITY = 'http://connect.garmin.com/csvExporter/%s.csv' JSON_DOWNLOAD_LIMIT = 100 # Maximum number of activities to request at once. 100 is the maximum set and enforced by Garmin. #JSON_DOWNLOAD_LIMIT = 10 # but 10 is faster if few activities to retrieve. @@ -125,19 +126,35 @@ def getNewRuns( self, existing_ids ): Returns list of new activities. """ activities = self.activitiesGenerator() - for activity in activities: - act_id = activityDict.getID( activity ) + for activity_dict in activities: + act = ActivityJSON( activity_dict ) + + act_id = act.getID() if act_id in existing_ids: break - if activityDict.isRun( activity ): - yield activity + if act.isRun(): + yield activity_dict + + def getFileByID( self, activity_id, fileformat = 'tcx' ): + """ Downloads and returns data of given activity """ - def downloadTCXbyID( self, activity_id ): - """ Returns content of TCX """ - # TODO: Finish and test this function - download_url = self.URL_TCX_ACTIVITY % activity_id + if fileformat == 'tcx': + download_url = self.URL_TCX_ACTIVITY % activity_id + + elif fileformat == 'gpx': + download_url = self.URL_GPX_ACTIVITY % activity_id + + elif fileformat == 'original': + download_url = self.URL_ZIP_ACTIVITY % activity_id + + elif fileformat == 'csv': #lap data + download_url = self.URL_CSV_ACTIVITY % activity_id + + else: + raise Exception('Unrecognized download file format. Supported: tcx,gpx,original and csv') + # Download try: data = http_req( self.opener, download_url ) except urllib2.HTTPError as e: @@ -145,15 +162,21 @@ def downloadTCXbyID( self, activity_id ): if e.code == 500: # Garmin will give an internal server error (HTTP 500) when downloading TCX files if the original was a manual GPX upload. # One could be generated here, but that's a bit much. Use the GPX format if you want actual data in every file, as I believe Garmin provides a GPX file for every activity. - print 'Writing empty file since Garmin did not generate a TCX file for this activity...', + print 'Returning empty file since Garmin did not generate a TCX file for this activity...' + data = '' + elif e.code == 404: + # For manual activities (i.e., entered in online without a file upload), there is no original file. + # Write an empty file to prevent redownloading it. + print 'Returning empty file since there was no original activity data...', data = '' else: raise Exception('Failed. Got an unexpected HTTP error (' + str(e.code) + ').') + + return data - return data ## End of Class ## -##Tools +## Tools ## def http_req(opener, url, post=None, headers={}): """ url is a string, post is a dictionary of POST parameters, headers is a dictionary of headers. """ request = urllib2.Request(url) diff --git a/gcexport.py b/gcexport.py index 3f032a5..d32e1f1 100755 --- a/gcexport.py +++ b/gcexport.py @@ -111,58 +111,31 @@ else: print '0.00 Miles' - # TODO MM: save gpx/tcx/original file - # if args.format == 'gpx': - # data_filename = args.directory + '/activity_' + a['activityId'] + '.gpx' - # download_url = url_gc_gpx_activity + a['activityId'] + '?full=true' - # file_mode = 'w' - # elif args.format == 'tcx': - # data_filename = args.directory + '/activity_' + a['activityId'] + '.tcx' - # download_url = url_gc_tcx_activity + a['activityId'] + '?full=true' - # file_mode = 'w' - # elif args.format == 'original': - # data_filename = args.directory + '/activity_' + a['activityId'] + '.zip' - # fit_filename = args.directory + '/' + a['activityId'] + '.fit' - # download_url = url_gc_original_activity + a['activityId'] - # file_mode = 'wb' - # else: - # raise Exception('Unrecognized format.') - - # if isfile(data_filename): - # print '\tData file already exists; skipping...' - # continue - # if args.format == 'original' and isfile(fit_filename): # Regardless of unzip setting, don't redownload if the ZIP or FIT file exists. - # print '\tFIT data file already exists; skipping...' - # continue - - # # Download the data file from Garmin Connect. - # # If the download fails (e.g., due to timeout), this script will die, but nothing - # # will have been written to disk about this activity, so just running it again - # # should pick up where it left off. - # print '\tDownloading file...', - - # try: - # data = http_req(download_url) - # except urllib2.HTTPError as e: - # # Handle expected (though unfortunate) error codes; die on unexpected ones. - # if e.code == 500 and args.format == 'tcx': - # # Garmin will give an internal server error (HTTP 500) when downloading TCX files if the original was a manual GPX upload. - # # Writing an empty file prevents this file from being redownloaded, similar to the way GPX files are saved even when there are no tracks. - # # One could be generated here, but that's a bit much. Use the GPX format if you want actual data in every file, - # # as I believe Garmin provides a GPX file for every activity. - # print 'Writing empty file since Garmin did not generate a TCX file for this activity...', - # data = '' - # elif e.code == 404 and args.format == 'original': - # # For manual activities (i.e., entered in online without a file upload), there is no original file. - # # Write an empty file to prevent redownloading it. - # print 'Writing empty file since there was no original activity data...', - # data = '' - # else: - # raise Exception('Failed. Got an unexpected HTTP error (' + str(e.code) + ').') - - # save_file = open(data_filename, file_mode) - # save_file.write(data) - # save_file.close() + # Download the data file from Garmin Connect. + # If the download fails (e.g., due to timeout), this script will die, but nothing + # will have been written to disk about this activity, so just running it again + # should pick up where it left off. + print '\tDownloading file...' + data = garmin_handler.getFileByID( a['activityId'], args.format ) + + if args.format == 'original': + data_filename = "%s/activity_%s.%s" % (args.directory, a['activityId'], 'zip') + fit_filename = args.directory + '/' + a['activityId'] + '.fit' + file_mode = 'wb' + else: + data_filename = "%s/activity_%s.%s" % (args.directory, a['activityId'], args.format) + file_mode = 'w' + + if isfile(data_filename): + print '\tData file already exists; skipping...' + continue + if args.format == 'original' and isfile(fit_filename): # Regardless of unzip setting, don't redownload if the ZIP or FIT file exists. + print '\tFIT data file already exists; skipping...' + continue + + save_file = open(data_filename, file_mode) + save_file.write(data) + save_file.close() # Write stats to CSV. @@ -232,31 +205,33 @@ # ) # TODO MM file validation? - # if args.format == 'gpx': - # # Validate GPX data. If we have an activity without GPS data (e.g., running on a treadmill), - # # Garmin Connect still kicks out a GPX, but there is only activity information, no GPS data. - # # N.B. You can omit the XML parse (and the associated log messages) to speed things up. - # gpx = parseString(data) - # gpx_data_exists = len(gpx.getElementsByTagName('trkpt')) > 0 - - # if gpx_data_exists: - # print 'Done. GPX data saved.' - # else: - # print 'Done. No track points found.' - # elif args.format == 'original': - # if args.unzip and data_filename[-3:].lower() == 'zip': # Even manual upload of a GPX file is zipped, but we'll validate the extension. - # print "Unzipping and removing original files...", - # zip_file = open(data_filename, 'rb') - # z = zipfile.ZipFile(zip_file) - # for name in z.namelist(): - # z.extract(name, args.directory) - # zip_file.close() - # remove(data_filename) - # print 'Done.' - # else: - # # TODO: Consider validating other formats. - # print 'Done.' + # Validate data. 24-12-2015: is this needed? + if args.format == 'gpx': + # Validate GPX data. If we have an activity without GPS data (e.g., running on a treadmill), + # Garmin Connect still kicks out a GPX, but there is only activity information, no GPS data. + # N.B. You can omit the XML parse (and the associated log messages) to speed things up. + gpx = parseString(data) + gpx_data_exists = len(gpx.getElementsByTagName('trkpt')) > 0 + + if gpx_data_exists: + print 'Done. GPX data saved.' + else: + print 'Done. No track points found.' + elif args.format == 'original': + if args.unzip and data_filename[-3:].lower() == 'zip': # Even manual upload of a GPX file is zipped, but we'll validate the extension. + print "Unzipping and removing original files...", + zip_file = open(data_filename, 'rb') + z = zipfile.ZipFile(zip_file) + for name in z.namelist(): + z.extract(name, args.directory) + zip_file.close() + remove(data_filename) + print 'Done.' + else: + # TODO: Consider validating other formats. + print 'Done.' + csv_file.close() print 'Done!' From 82565f6da8482f993f5f631adf4546c9c9ca7a39 Mon Sep 17 00:00:00 2001 From: Maxim Laptop Date: Thu, 24 Dec 2015 16:05:16 +0100 Subject: [PATCH 3/6] Added ActivityJSON to handle the activity json dicts --- ActivityJSON.py | 101 +++++++++++++++++++++++++++++++++++++++++++++++ GarminHandler.py | 2 +- 2 files changed, 102 insertions(+), 1 deletion(-) create mode 100644 ActivityJSON.py diff --git a/ActivityJSON.py b/ActivityJSON.py new file mode 100644 index 0000000..f8cef38 --- /dev/null +++ b/ActivityJSON.py @@ -0,0 +1,101 @@ +# -*- coding: utf-8 -*- +""" +Created on Sun Oct 25 09:48:32 2015 + +@author: Maxim +""" +from datetime import datetime +import re + +class ActivityJSON(object): + """ + Handles for the dictionary representation of the json activity. + Initialize with: activity = ActivityJSON( json_dict ) + Then selected details can be retrieved with: activity.get... + """ + + def __init__( self, json_dict ): + self.json_dict = json_dict + + def getID( self ): + return self.json_dict['activityId'] + + def getName( self ): + return self.json_dict['activityName']['value'] + + def getCategory( self ): + """ The 'general' type of an activity, disregarding the subtype. e.g. running, cycling, swimming, hiking... """ + return self.json_dict['activityType']['type']['key'] + + def isRun( self ): + if self.getCategory() == 'running': + return True + else: + return False + + def getDistance( self ): + parent = self.json_dict['sumDistance'] + + distance = float( parent['value'] ) + unit = parent['uom'] + if unit != 'kilometer': + raise Exception("Distance has the wrong unit: '%s'" % unit) + + return distance + + def getDuration( self ): + parent = self.json_dict['sumMovingDuration'] + + time = float( parent['value'] ) + unit = parent['uom'] + if unit != 'second': + raise Exception("Time has the wrong unit: '%s'" % unit) + + return time + + def getComment( self ): + return self.json_dict['activityDescription']['value'] #TODO remove end of lines + + def getDate( self ): + """ Returns datetime object """ + #NOTE: date also available in milliseconds ('millis', UTC) + date_yyyymmdd = self.json_dict['beginTimestamp']['value'] + date = datetime.strptime(date_yyyymmdd,"%Y-%m-%d") + return date + + def getStartTime( self ): + """ Returns string 'hh:mm' """ + full_date = self.json_dict['beginTimestamp']['display'] # 'Thu, 2015 Oct 22 17:19' + match = re.search( r'\d{2}:\d{2}', full_date ) # Get the time hh:mm + return match.group() + + def getBpmMax( self ): + if 'maxHeartRate' in self.json_dict: + parent = self.json_dict['maxHeartRate'] + return float( parent['value'] ) #Assume uom is always bpm + else: + return None + + def getBpmAvg( self ): + if 'weightedMeanHeartRate' in self.json_dict: + parent = self.json_dict['weightedMeanHeartRate'] + return float( parent['value'] ) #Assume uom is always bpm + else: + return None + + def getLatitude( self ): + if 'beginLatitude' in self.json_dict: + parent = self.json_dict['beginLatitude'] + return float( parent['value'] ) #Always in decimal degrees + else: + return None + + def getLongitude( self ): + if 'beginLongitude' in self.json_dict: + parent = self.json_dict['beginLongitude'] + return float( parent['value'] ) #Always in decimal degrees + else: + return None + + + \ No newline at end of file diff --git a/GarminHandler.py b/GarminHandler.py index fee38a8..76ab0c2 100644 --- a/GarminHandler.py +++ b/GarminHandler.py @@ -173,7 +173,7 @@ def getFileByID( self, activity_id, fileformat = 'tcx' ): raise Exception('Failed. Got an unexpected HTTP error (' + str(e.code) + ').') return data - + ## End of Class ## ## Tools ## From a1d87afab36efed62b246ed8ca6dc024d104934d Mon Sep 17 00:00:00 2001 From: Maxim Moinat Date: Wed, 2 Nov 2016 08:22:48 +0100 Subject: [PATCH 4/6] Fixed connection. Upgraded api url to new version (1.2/1.3) and added two post authorization requests --- GarminHandler.py | 106 ++++++++------- old/garmin-connect-export.php | 245 ---------------------------------- 2 files changed, 60 insertions(+), 291 deletions(-) delete mode 100644 old/garmin-connect-export.php diff --git a/GarminHandler.py b/GarminHandler.py index 76ab0c2..14a5c1d 100644 --- a/GarminHandler.py +++ b/GarminHandler.py @@ -6,39 +6,44 @@ @author: Maxim """ from urllib import urlencode #somehow not in the urllib2 package -import urllib2, cookielib, json +import urllib2, cookielib, json, re from ActivityJSON import ActivityJSON class GarminHandler( object ): ## Global Constants # URLs for various services. - URL_LOGIN = 'https://sso.garmin.com/sso/login?service=https%3A%2F%2Fconnect.garmin.com%2Fpost-auth%2Flogin&webhost=olaxpw-connect04&source=https%3A%2F%2Fconnect.garmin.com%2Fen-US%2Fsignin&redirectAfterAccountLoginUrl=https%3A%2F%2Fconnect.garmin.com%2Fpost-auth%2Flogin&redirectAfterAccountCreationUrl=https%3A%2F%2Fconnect.garmin.com%2Fpost-auth%2Flogin&gauthHost=https%3A%2F%2Fsso.garmin.com%2Fsso&locale=en_US&id=gauth-widget&cssUrl=https%3A%2F%2Fstatic.garmincdn.com%2Fcom.garmin.connect%2Fui%2Fcss%2Fgauth-custom-v1.1-min.css&clientId=GarminConnect&rememberMeShown=true&rememberMeChecked=false&createAccountShown=true&openCreateAccount=false&usernameShown=false&displayNameShown=false&consumeServiceTicket=false&initialFocus=true&embedWidget=false&generateExtraServiceTicket=false' - URL_POST_AUTH = 'https://connect.garmin.com/post-auth/login?' - URL_SEARCH = 'http://connect.garmin.com/proxy/activity-search-service-1.0/json/activities?' - URL_GPX_ACTIVITY = 'http://connect.garmin.com/proxy/activity-service-1.1/gpx/activity/%s?full=true' - URL_TCX_ACTIVITY = 'http://connect.garmin.com/proxy/activity-service-1.1/tcx/activity/%s?full=true' + URL_LOGIN = 'https://sso.garmin.com/sso/login?service=https%3A%2F%2Fconnect.garmin.com%2Fpost-auth%2Flogin&webhost=olaxpw-connect04&source=https%3A%2F%2Fconnect.garmin.com%2Fen-US%2Fsignin&redirectAfterAccountLoginUrl=https%3A%2F%2Fconnect.garmin.com%2Fpost-auth%2Flogin&redirectAfterAccountCreationUrl=https%3A%2F%2Fconnect.garmin.com%2Fpost-auth%2Flogin&gauthHost=https%3A%2F%2Fsso.garmin.com%2Fsso&locale=en_US&id=gauth-widget&cssUrl=https%3A%2F%2Fstatic.garmincdn.com%2Fcom.garmin.connect%2Fui%2Fcss%2Fgauth-custom-v1.1-min.css&clientId=GarminConnect&rememberMeShown=true&rememberMeChecked=false&createAccountShown=true&openCreateAccount=false&usernameShown=false&displayNameShown=false&consumeServiceTicket=false&initialFocus=true&embedWidget=false&generateExtraServiceTicket=false' + URL_POST_AUTH = 'https://connect.garmin.com/post-auth/login?' + URL_POST_AUTH2 = 'http://connect.garmin.com/modern' + URL_POST_AUTH3 = 'https://connect.garmin.com/legacy/session' + # Documentation: + # https://connect.garmin.com/proxy/activity-search-service-1.2/ + # https://connect.garmin.com/proxy/activity-service-1.3/ + URL_SEARCH = 'http://connect.garmin.com/proxy/activity-search-service-1.2/json/activities?' + URL_GPX_ACTIVITY = 'http://connect.garmin.com/proxy/activity-service-1.3/gpx/activity/%s?full=true' + URL_TCX_ACTIVITY = 'http://connect.garmin.com/proxy/activity-service-1.3/tcx/activity/%s?full=true' URL_ZIP_ACTIVITY = 'http://connect.garmin.com/proxy/download-service/files/activity/%s' URL_CSV_ACTIVITY = 'http://connect.garmin.com/csvExporter/%s.csv' - JSON_DOWNLOAD_LIMIT = 100 # Maximum number of activities to request at once. 100 is the maximum set and enforced by Garmin. + JSON_DOWNLOAD_LIMIT = 100 # Maximum number of activities to request at once. 100 is the maximum set and enforced by Garmin. #JSON_DOWNLOAD_LIMIT = 10 # but 10 is faster if few activities to retrieve. - + def __init__( self, username, password ): self.opener = None - # You must be logged in to use the class + # You must be logged in to use the class self.login( username, password ) - + def login( self, username, password ): """ Returns True if logged in, raises error if not.""" # Initially, we need to get a valid session cookie, so we pull the login page. cookie_jar = cookielib.CookieJar() self.opener = urllib2.build_opener( urllib2.HTTPCookieProcessor(cookie_jar) ) http_req( self.opener, self.URL_LOGIN ) - + # Now we'll actually login. Post data with Fields that are passed in a typical Garmin login. - post_data = {'username': username, 'password': password, - 'embed': 'true', 'lt': 'e1s1', '_eventId': 'submit', 'displayNameRequired': 'false'} + post_data = {'username': username, 'password': password, + 'embed': 'true', 'lt': 'e1s1', '_eventId': 'submit', 'displayNameRequired': 'false'} http_req ( self.opener, self.URL_LOGIN, post_data ) - + # Get the key. # TODO: Can we do this without iterating? login_ticket = None @@ -46,27 +51,35 @@ def login( self, username, password ): if cookie.name == 'CASTGC': login_ticket = cookie.value break - + if not login_ticket: raise Exception('Did not get a ticket cookie. Cannot log in. Did you enter the correct username and password?') - + # Post Authorize. Chop of 'TGT-' off the beginning, prepend 'ST-0'. login_ticket = 'ST-0' + login_ticket[4:] - http_req( self.opener, self.URL_POST_AUTH + 'ticket=' + login_ticket) - - #TODO: extra check whether indeed logged in. - return True - + login_response = http_req( self.opener, self.URL_POST_AUTH + 'ticket=' + login_ticket ) + http_req( self.opener, self.URL_POST_AUTH2 ) + http_req( self.opener, self.URL_POST_AUTH3 ) + + # Extra check whether indeed logged in. + res = re.search(r'fullName.+?:(.+?),', login_response) + if not res: + print ('Not logged in, post-authorization probably went wrong.') + return False + + print( 'Logged in to account of %s' % res.group(1).strip('\\"') ) + self.logged_in = True + def activitiesGenerator( self, limit = None, reversed = False ): - """ Yields the json as dict for every activity found, + """ Yields the json as dict for every activity found, either from new to old or reversed. """ - + # Prevent downloading too large chunks (saves time) if limit and limit < self.JSON_DOWNLOAD_LIMIT: max_chunk_size = limit else: max_chunk_size = self.JSON_DOWNLOAD_LIMIT - + # Determine index to start at if reversed: # Download one activity. Result will contain how many activities @@ -81,7 +94,7 @@ def activitiesGenerator( self, limit = None, reversed = False ): start_index = 0 else: start_index = 0 - + # Download data in multiple chunks of *max_chunk_size* activities total_downloaded = 0 downloaded_chunk_size = max_chunk_size #initialize @@ -89,28 +102,29 @@ def activitiesGenerator( self, limit = None, reversed = False ): # Query Garmin Connect search_params = {'start': start_index, 'limit': max_chunk_size} url = self.URL_SEARCH + urlencode(search_params) + try: result = http_req(self.opener, url ) json_results = json.loads(result) except urllib2.HTTPError as e: raise Exception('Failed to retrieve json of activities. (' + str(e) + ').') - + # Pull out just the list of activities. activities = json_results['results']['activities'] downloaded_chunk_size = len(activities) - + if reversed: activities = activities[::-1] #reverse - + for activity in activities: activity_details = activity['activity'] yield activity_details - - total_downloaded += 1 + + total_downloaded += 1 # Stop if limit is reached if total_downloaded == limit: raise StopIteration - + # Increment start index if reversed: if start_index - max_chunk_size < 0: # Negative start is not allowed @@ -120,40 +134,40 @@ def activitiesGenerator( self, limit = None, reversed = False ): start_index -= max_chunk_size #Backwards else: start_index += max_chunk_size #Forwards - + def getNewRuns( self, existing_ids ): - """ Iterate until an existing activiity is found. + """ Iterate until an existing activiity is found. Returns list of new activities. """ - + activities = self.activitiesGenerator() for activity_dict in activities: act = ActivityJSON( activity_dict ) - + act_id = act.getID() if act_id in existing_ids: break - + if act.isRun(): yield activity_dict - + def getFileByID( self, activity_id, fileformat = 'tcx' ): """ Downloads and returns data of given activity """ - + if fileformat == 'tcx': download_url = self.URL_TCX_ACTIVITY % activity_id - + elif fileformat == 'gpx': download_url = self.URL_GPX_ACTIVITY % activity_id - + elif fileformat == 'original': download_url = self.URL_ZIP_ACTIVITY % activity_id - + elif fileformat == 'csv': #lap data download_url = self.URL_CSV_ACTIVITY % activity_id - + else: raise Exception('Unrecognized download file format. Supported: tcx,gpx,original and csv') - + # Download try: data = http_req( self.opener, download_url ) @@ -171,8 +185,8 @@ def getFileByID( self, activity_id, fileformat = 'tcx' ): data = '' else: raise Exception('Failed. Got an unexpected HTTP error (' + str(e.code) + ').') - - return data + + return data ## End of Class ## @@ -191,4 +205,4 @@ def http_req(opener, url, post=None, headers={}): if response.getcode() != 200: raise Exception('Bad return code (' + response.getcode() + ') for: ' + url) - return response.read() \ No newline at end of file + return response.read() diff --git a/old/garmin-connect-export.php b/old/garmin-connect-export.php deleted file mode 100644 index ab55c3d..0000000 --- a/old/garmin-connect-export.php +++ /dev/null @@ -1,245 +0,0 @@ -#!/usr/bin/php - 1 && ( is_numeric( $argv[1] ) ) ) { - $total_to_download = $argv[1]; -} else if ( $argc > 1 && strcasecmp($argv[1], "all") == 0 ) { - // If the user wants to download all activities, first download one, - // then the result of that request will tell us how many are available - // so we will modify the variables then. - $total_to_download = 1; - $download_all = true; -} else { - $total_to_download = 1; -} -$total_downloaded = 0; - -// This while loop will download data from the server in multiple chunks, if necessary -while( $total_downloaded < $total_to_download ) { - $num_to_download = ($total_to_download - $total_downloaded > 100) ? 100 : ($total_to_download - $total_downloaded); // Maximum of 100... 400 return status if over 100. So download 100 or whatever remains if less than 100. - - // Query Garmin Connect - $search_opts = array( - 'start' => $total_downloaded, - 'limit' => $num_to_download - ); - - $result = curl( $urlGCSearch . http_build_query( $search_opts ) ); - $json = json_decode( $result ); - - if ( ! $json ) { - echo "Error: "; - switch(json_last_error()) { - case JSON_ERROR_DEPTH: - echo ' - Maximum stack depth exceeded'; - break; - case JSON_ERROR_CTRL_CHAR: - echo ' - Unexpected control character found'; - break; - case JSON_ERROR_SYNTAX: - echo ' - Syntax error, malformed JSON'; - break; - } - echo PHP_EOL; - var_dump( $result ); - die(); - } - - $search = $json->{'results'}->{'search'}; - - if ( $download_all ) { - // Modify $total_to_download based on how many activities the server reports - $total_to_download = intval( $search->{'totalFound'} ); - // Do it only once - $download_all = false; - } - - // Pull out just the list of activities - $activities = $json->{'results'}->{'activities'}; - - // Process each activity. - foreach ( $activities as $a ) { - // Display which entry we're working on. - print "Garmin Connect activity: [" . $a->{'activity'}->{'activityId'} . "] "; - print $a->{'activity'}->{'beginTimestamp'}->{'display'} . ": "; - print $a->{'activity'}->{'activityName'}->{'value'} . "\n"; - - // Write data to CSV - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'activityId'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'activityName'}->{'value'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'activityDescription'}->{'value'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'beginTimestamp'}->{'display'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'beginTimestamp'}->{'millis'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'endTimestamp'}->{'display'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'endTimestamp'}->{'millis'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'device'}->{'display'} . " " . $a->{'activity'}->{'device'}->{'version'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'activityType'}->{'parent'}->{'display'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'activityType'}->{'display'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'eventType'}->{'display'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'activityTimeZone'}->{'display'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'maxElevation'}->{'withUnit'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'maxElevation'}->{'value'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'beginLatitude'}->{'value'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'beginLongitude'}->{'value'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'endLatitude'}->{'value'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'endLongitude'}->{'value'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'weightedMeanMovingSpeed'}->{'display'}) . "\"," ); // The units vary between Minutes per Mile and mph, but withUnit always displays "Minutes per Mile" - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'weightedMeanMovingSpeed'}->{'value'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'maxHeartRate'}->{'display'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'weightedMeanHeartRate'}->{'display'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'maxSpeed'}->{'display'}) . "\"," ); // The units vary between Minutes per Mile and mph, but withUnit always displays "Minutes per Mile" - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'maxSpeed'}->{'value'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'sumEnergy'}->{'display'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'sumEnergy'}->{'value'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'sumElapsedDuration'}->{'display'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'sumElapsedDuration'}->{'value'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'sumMovingDuration'}->{'display'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'sumMovingDuration'}->{'value'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'weightedMeanSpeed'}->{'withUnit'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'weightedMeanSpeed'}->{'value'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'sumDistance'}->{'withUnit'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'sumDistance'}->{'value'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'minHeartRate'}->{'display'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'maxElevation'}->{'withUnit'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'maxElevation'}->{'value'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'gainElevation'}->{'withUnit'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'gainElevation'}->{'value'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'lossElevation'}->{'withUnit'}) . "\"," ); - fwrite( $csv_file, "\"" . str_replace("\"", "\"\"", $a->{'activity'}->{'lossElevation'}->{'value'}) . "\""); - fwrite( $csv_file, "\n"); - - // Download the GPX file from Garmin Connect - // TODO: Consider using TCX files? Does Garmin Connect include heart rate data in TCX downloads? - print "\tDownloading GPX file... "; - - $gpx_filename = $activities_directory . '/activity_' . $a->{'activity'}->{'activityId'} . '.gpx'; - $save_file = fopen( $gpx_filename, 'w+' ); - $curl_opts = array( - CURLOPT_FILE => $save_file - ); - curl( $urlGCActivity . $a->{'activity'}->{'activityId'} . '?full=true', array(), array(), $curl_opts ); - fclose( $save_file ); - - // Validate the GPX data. If we have an activity without GPS data (e.g. running on a treadmill), - // Garmin Connect still kicks out a GPX, but there is only activity information, no GPS data. - $gpx = simplexml_load_file( $gpx_filename, 'SimpleXMLElement', LIBXML_NOCDATA ); - $gpxdataexists = ( count( $gpx->trk->trkseg->trkpt ) > 0); - - if ( $gpxdataexists ) { - print "Done. GPX data saved.\n"; - } else { - print "Done. No track points found.\n"; - } - } - - $total_downloaded += $num_to_download; - -// End while loop for multiple chunks -} - -fclose($csv_file); - -print "Done!\n\n"; -// End - -function curl( $url, $post = array(), $head = array(), $opts = array() ) -{ - $cookie_file = '/tmp/cookies.txt'; - $ch = curl_init(); - - //curl_setopt( $ch, CURLOPT_VERBOSE, 1 ); - curl_setopt( $ch, CURLOPT_URL, $url ); - curl_setopt( $ch, CURLOPT_RETURNTRANSFER, 1 ); - curl_setopt( $ch, CURLOPT_ENCODING, "gzip" ); - curl_setopt( $ch, CURLOPT_COOKIEFILE, $cookie_file ); - curl_setopt( $ch, CURLOPT_COOKIEJAR, $cookie_file ); - curl_setopt( $ch, CURLOPT_FOLLOWLOCATION, 1 ); - - foreach ( $opts as $k => $v ) { - curl_setopt( $ch, $k, $v ); - } - - if ( count( $post ) > 0 ) { - // POST mode - curl_setopt( $ch, CURLOPT_POST, 1 ); - curl_setopt( $ch, CURLOPT_POSTFIELDS, $post ); - } - else { - curl_setopt( $ch, CURLOPT_HTTPHEADER, $head ); - curl_setopt( $ch, CURLOPT_CRLF, 1 ); - } - - $success = curl_exec( $ch ); - - if ( curl_errno( $ch ) !== 0 ) { - throw new Exception( sprintf( '%s: CURL Error %d: %s', __CLASS__, curl_errno( $ch ), curl_error( $ch ) ) ); - } - - if ( curl_getinfo( $ch, CURLINFO_HTTP_CODE ) !== 200 ) { - if ( curl_getinfo( $ch, CURLINFO_HTTP_CODE ) !== 201 ) { - throw new Exception( sprintf( 'Bad return code(%1$d) for: %2$s', curl_getinfo( $ch, CURLINFO_HTTP_CODE ), $url ) ); - } - } - - curl_close( $ch ); - return $success; -} - -?> From 853aa62952b1c99f73c8e0a57ceaaf37a8e2ae21 Mon Sep 17 00:00:00 2001 From: Maxim Moinat Date: Wed, 2 Nov 2016 09:18:28 +0100 Subject: [PATCH 5/6] Download urls broken. GPX and TCX give 500 and CSV empty file. Zip works --- GarminHandler.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/GarminHandler.py b/GarminHandler.py index 14a5c1d..68a4a14 100644 --- a/GarminHandler.py +++ b/GarminHandler.py @@ -20,10 +20,10 @@ class GarminHandler( object ): # https://connect.garmin.com/proxy/activity-search-service-1.2/ # https://connect.garmin.com/proxy/activity-service-1.3/ URL_SEARCH = 'http://connect.garmin.com/proxy/activity-search-service-1.2/json/activities?' - URL_GPX_ACTIVITY = 'http://connect.garmin.com/proxy/activity-service-1.3/gpx/activity/%s?full=true' - URL_TCX_ACTIVITY = 'http://connect.garmin.com/proxy/activity-service-1.3/tcx/activity/%s?full=true' - URL_ZIP_ACTIVITY = 'http://connect.garmin.com/proxy/download-service/files/activity/%s' - URL_CSV_ACTIVITY = 'http://connect.garmin.com/csvExporter/%s.csv' + URL_GPX_ACTIVITY = 'https://connect.garmin.com/modern/proxy/download-service/export/gpx/activity/%s' + URL_TCX_ACTIVITY = 'https://connect.garmin.com/modern/proxy/download-service/export/tcx/activity/%s' + URL_CSV_ACTIVITY = 'https://connect.garmin.com/modern/proxy/download-service/export/csv/activity/%s' + URL_ZIP_ACTIVITY = 'https://connect.garmin.com/modern/proxy/download-service/files/activity/%s' JSON_DOWNLOAD_LIMIT = 100 # Maximum number of activities to request at once. 100 is the maximum set and enforced by Garmin. #JSON_DOWNLOAD_LIMIT = 10 # but 10 is faster if few activities to retrieve. From b419d8d80796da6c1fb51c888ac7d2394527ec73 Mon Sep 17 00:00:00 2001 From: Maxim Moinat Date: Wed, 2 Nov 2016 09:55:12 +0100 Subject: [PATCH 6/6] Cleaned login procedure --- GarminHandler.py | 51 +++++++++++++++++++++++++++++------------------- __init__.py | 0 2 files changed, 31 insertions(+), 20 deletions(-) create mode 100644 __init__.py diff --git a/GarminHandler.py b/GarminHandler.py index 68a4a14..ecb484a 100644 --- a/GarminHandler.py +++ b/GarminHandler.py @@ -16,7 +16,8 @@ class GarminHandler( object ): URL_POST_AUTH = 'https://connect.garmin.com/post-auth/login?' URL_POST_AUTH2 = 'http://connect.garmin.com/modern' URL_POST_AUTH3 = 'https://connect.garmin.com/legacy/session' - # Documentation: + + # Documentation API: # https://connect.garmin.com/proxy/activity-search-service-1.2/ # https://connect.garmin.com/proxy/activity-service-1.3/ URL_SEARCH = 'http://connect.garmin.com/proxy/activity-search-service-1.2/json/activities?' @@ -24,13 +25,13 @@ class GarminHandler( object ): URL_TCX_ACTIVITY = 'https://connect.garmin.com/modern/proxy/download-service/export/tcx/activity/%s' URL_CSV_ACTIVITY = 'https://connect.garmin.com/modern/proxy/download-service/export/csv/activity/%s' URL_ZIP_ACTIVITY = 'https://connect.garmin.com/modern/proxy/download-service/files/activity/%s' - JSON_DOWNLOAD_LIMIT = 100 # Maximum number of activities to request at once. 100 is the maximum set and enforced by Garmin. - #JSON_DOWNLOAD_LIMIT = 10 # but 10 is faster if few activities to retrieve. - def __init__( self, username, password ): + # Maximum number of activities to request at once. 100 is the maximum set and enforced by Garmin + JSON_DOWNLOAD_LIMIT = 100 # 10 is faster if few activities to retrieve. + + def __init__( self ): self.opener = None - # You must be logged in to use the class - self.login( username, password ) + self.logged_in = False def login( self, username, password ): """ Returns True if logged in, raises error if not.""" @@ -45,35 +46,45 @@ def login( self, username, password ): http_req ( self.opener, self.URL_LOGIN, post_data ) # Get the key. - # TODO: Can we do this without iterating? - login_ticket = None - for cookie in cookie_jar: - if cookie.name == 'CASTGC': - login_ticket = cookie.value - break - - if not login_ticket: + try: + login_ticket = filter(lambda x: x.name == 'CASTGC', cookie_jar)[0].value + except: raise Exception('Did not get a ticket cookie. Cannot log in. Did you enter the correct username and password?') + # Post Authorize. + login_response = self._postAuthorize( login_ticket ) + + # Extra check that account name can be retrieved + account_name = self._getAccountName(login_response) + if not account_name: + print ('Not logged in, post-authorization probably went wrong.') + return False + + print( 'Logged in to account of %s' % account_name ) + self.logged_in = True + + def _postAuthorize( self, login_ticket ): # Post Authorize. Chop of 'TGT-' off the beginning, prepend 'ST-0'. login_ticket = 'ST-0' + login_ticket[4:] login_response = http_req( self.opener, self.URL_POST_AUTH + 'ticket=' + login_ticket ) + # Additional post-authorization 02-11-2016 http_req( self.opener, self.URL_POST_AUTH2 ) http_req( self.opener, self.URL_POST_AUTH3 ) + return login_response - # Extra check whether indeed logged in. - res = re.search(r'fullName.+?:(.+?),', login_response) + def _getAccountName( self, post_login_response ): + res = re.search(r'fullName.+?:(.+?),', post_login_response) if not res: - print ('Not logged in, post-authorization probably went wrong.') return False - - print( 'Logged in to account of %s' % res.group(1).strip('\\"') ) - self.logged_in = True + return res.group(1).strip( '\\"' ) def activitiesGenerator( self, limit = None, reversed = False ): """ Yields the json as dict for every activity found, either from new to old or reversed. """ + if not self.logged_in: + raise Exception('Please login first with .login(,)') + # Prevent downloading too large chunks (saves time) if limit and limit < self.JSON_DOWNLOAD_LIMIT: max_chunk_size = limit diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..e69de29