Skip to content

Remove 10k per directory limitation #24

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
dsnopek wants to merge 3 commits into
base: master
Choose a base branch
from
Open

Remove 10k per directory limitation #24

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
db7be74
list_directory() will now request the next set of objects if it gets …
dsnopek Sep 4, 2011
10947d3
fixed marker problem when the last item is in a subdirectory
ocamler Jun 3, 2012
988f00b
Merge pull request #1 from ocamler/master
dsnopek Jun 4, 2012
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 27 additions & 4 deletions cloudfsapi.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,9 @@
#include <stdarg.h>
#include <stdlib.h>
#include <sys/stat.h>
#ifdef __linux__
#include <alloca.h>
#endif
#include <pthread.h>
#include <time.h>
#include <unistd.h>
Expand All @@ -16,6 +18,9 @@

#define REQUEST_RETRIES 4

// defined by Rackspace
#define MAX_RESULTS_PER_REQUEST 10000

static char storage_url[MAX_URL_SIZE];
static char storage_token[MAX_HEADER_SIZE];
static pthread_mutex_t pool_mut;
Expand Down Expand Up @@ -230,13 +235,12 @@ int object_truncate(const char *path, off_t size)
return (response >= 200 && response < 300);
}

int list_directory(const char *path, dir_entry **dir_list)
int list_directory_internal(const char *path, dir_entry **dir_list)
{
char container[MAX_PATH_SIZE * 3] = "";
char object[MAX_PATH_SIZE] = "";
int response = 0;
int retval = 0;
*dir_list = NULL;
int retval = -1;
xmlNode *onode = NULL, *anode = NULL, *text_node = NULL;
xmlParserCtxtPtr xmlctx = xmlCreatePushParserCtxt(NULL, NULL, "", 0, NULL);
if (!strcmp(path, "") || !strcmp(path, "/"))
Expand All @@ -254,10 +258,16 @@ int list_directory(const char *path, dir_entry **dir_list)
curl_free(encoded_container);
curl_free(encoded_object);
}
if (*dir_list != NULL) {
strcat(container, "&marker=");
strcat(container, (*dir_list)->marker);
}
printf("%s\n", container);
response = send_request("GET", container, NULL, xmlctx, NULL);
xmlParseChunk(xmlctx, "", 0, 1);
if (xmlctx->wellFormed && response >= 200 && response < 300)
{
retval = 0;
xmlNode *root_element = xmlDocGetRootElement(xmlctx->myDoc);
for (onode = root_element->children; onode; onode = onode->next)
if ((onode->type == XML_ELEMENT_NODE) &&
Expand All @@ -280,6 +290,7 @@ int list_directory(const char *path, dir_entry **dir_list)
de->name = strdup(strrchr(content, '/')+1);
else
de->name = strdup(content);
de->marker = strdup(content);
if (asprintf(&(de->full_name), "%s/%s", path, de->name) < 0)
de->full_name = NULL;
}
Expand All @@ -304,14 +315,26 @@ int list_directory(const char *path, dir_entry **dir_list)
(strstr(de->content_type, "application/directory") != NULL));
de->next = *dir_list;
*dir_list = de;
retval++;
}
retval = 1;
}
xmlFreeDoc(xmlctx->myDoc);
xmlFreeParserCtxt(xmlctx);
return retval;
}

int list_directory(const char *path, dir_entry **dir_list)
{
int retval;
*dir_list = NULL;

do {
retval = list_directory_internal(path, dir_list);
} while(retval == MAX_RESULTS_PER_REQUEST);

return retval == -1 ? 0 : 1;
}

void free_dir_list(dir_entry *dir_list)
{
while (dir_list)
Expand Down
1 change: 1 addition & 0 deletions cloudfsapi.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ typedef struct dir_entry
time_t last_modified;
int isdir;
struct dir_entry *next;
char *marker;
} dir_entry;

int object_read_fp(const char *path, FILE *fp);
Expand Down