diff --git a/tests/providers/cloudfiles/fixtures.py b/tests/providers/cloudfiles/fixtures.py new file mode 100644 index 000000000..de8788b6c --- /dev/null +++ b/tests/providers/cloudfiles/fixtures.py @@ -0,0 +1,243 @@ +import os +import io +import time +import json +from unittest import mock + +import pytest +import aiohttp +import aiohttpretty + +from waterbutler.core import streams +from waterbutler.providers.cloudfiles import CloudFilesProvider + + +@pytest.fixture +def auth(): + return { + 'name': 'cat', + 'email': 'cat@cat.com', + } + + +@pytest.fixture +def credentials(): + return { + 'username': 'prince', + 'token': 'revolutionary', + 'region': 'iad', + } + + +@pytest.fixture +def settings(): + return {'container': 'purple rain'} + + +@pytest.fixture +def provider(auth, credentials, settings): + return CloudFilesProvider(auth, credentials, settings) + + +@pytest.fixture +def token(auth_json): + return auth_json['access']['token']['id'] + + +@pytest.fixture +def endpoint(auth_json): + return auth_json['access']['serviceCatalog'][0]['endpoints'][0]['publicURL'] + + +@pytest.fixture +def temp_url_key(): + return 'temporary beret' + + +@pytest.fixture +def mock_auth(auth_json): + aiohttpretty.register_json_uri( + 'POST', + settings.AUTH_URL, + body=auth_json, + ) + + +@pytest.fixture +def mock_temp_key(endpoint, temp_url_key): + aiohttpretty.register_uri( + 'HEAD', + endpoint, + status=204, + headers={'X-Account-Meta-Temp-URL-Key': temp_url_key}, + ) + + +@pytest.fixture +def mock_time(monkeypatch): + mock_time = mock.Mock() + mock_time.return_value = 10 + monkeypatch.setattr(time, 'time', mock_time) + + +@pytest.fixture +def connected_provider(provider, token, endpoint, temp_url_key, mock_time): + provider.token = token + provider.endpoint = endpoint + provider.temp_url_key = temp_url_key.encode() + return provider + + +@pytest.fixture +def file_content(): + return b'sleepy' + + +@pytest.fixture +def file_like(file_content): + return io.BytesIO(file_content) + + +@pytest.fixture +def file_stream(file_like): + return streams.FileStreamReader(file_like) + + +@pytest.fixture +def folder_root_empty(): + return [] + + +@pytest.fixture +def container_header_metadata_with_verision_location(): + with open(os.path.join(os.path.dirname(__file__), 'fixtures/fixtures.json'), 'r') as fp: + return json.load(fp)['container_header_metadata_with_verision_location'] + + +@pytest.fixture +def container_header_metadata_without_verision_location(): + with open(os.path.join(os.path.dirname(__file__), 'fixtures/fixtures.json'), 'r') as fp: + return json.load(fp)['container_header_metadata_without_verision_location'] + + +@pytest.fixture +def file_metadata(): + with open(os.path.join(os.path.dirname(__file__), 'fixtures/fixtures.json'), 'r') as fp: + return json.load(fp)['file_metadata'] + + +@pytest.fixture +def folder_root(): + with open(os.path.join(os.path.dirname(__file__), 'fixtures/fixtures.json'), 'r') as fp: + return json.load(fp)['folder_root'] + + +@pytest.fixture +def folder_root_level1_level2(): + with open(os.path.join(os.path.dirname(__file__), 'fixtures/fixtures.json'), 'r') as fp: + return json.load(fp)['folder_root_level1_level2'] + + +@pytest.fixture +def folder_root_level1(): + with open(os.path.join(os.path.dirname(__file__), 'fixtures/fixtures.json'), 'r') as fp: + return json.load(fp)['folder_root_level1'] + + +@pytest.fixture +def file_header_metadata(): + with open(os.path.join(os.path.dirname(__file__), 'fixtures/fixtures.json'), 'r') as fp: + return json.load(fp)['file_header_metadata'] + + +@pytest.fixture +def auth_json(): + with open(os.path.join(os.path.dirname(__file__), 'fixtures/fixtures.json'), 'r') as fp: + return json.load(fp)['auth_json'] + + +@pytest.fixture +def folder_root(): + with open(os.path.join(os.path.dirname(__file__), 'fixtures/fixtures.json'), 'r') as fp: + return json.load(fp)['folder_root'] + + +@pytest.fixture +def revision_list(): + with open(os.path.join(os.path.dirname(__file__), 'fixtures/fixtures.json'), 'r') as fp: + return json.load(fp)['revision_list'] + + +@pytest.fixture +def file_root_level1_level2_file2_txt(): + return aiohttp.multidict.CIMultiDict([ + ('ORIGIN', 'https://mycloud.rackspace.com'), + ('CONTENT-LENGTH', '216945'), + ('ACCEPT-RANGES', 'bytes'), + ('LAST-MODIFIED', 'Mon, 22 Dec 2014 19:01:02 GMT'), + ('ETAG', '44325d4f13b09f3769ede09d7c20a82c'), + ('X-TIMESTAMP', '1419274861.04433'), + ('CONTENT-TYPE', 'text/plain'), + ('X-TRANS-ID', 'tx836375d817a34b558756a-0054987deeiad3'), + ('DATE', 'Mon, 22 Dec 2014 20:24:14 GMT') + ]) + + +@pytest.fixture +def folder_root_level1_empty(): + return aiohttp.multidict.CIMultiDict([ + ('ORIGIN', 'https://mycloud.rackspace.com'), + ('CONTENT-LENGTH', '0'), + ('ACCEPT-RANGES', 'bytes'), + ('LAST-MODIFIED', 'Mon, 22 Dec 2014 18:58:56 GMT'), + ('ETAG', 'd41d8cd98f00b204e9800998ecf8427e'), + ('X-TIMESTAMP', '1419274735.03160'), + ('CONTENT-TYPE', 'application/directory'), + ('X-TRANS-ID', 'txd78273e328fc4ba3a98e3-0054987eeeiad3'), + ('DATE', 'Mon, 22 Dec 2014 20:28:30 GMT') + ]) + + +@pytest.fixture +def file_root_similar(): + return aiohttp.multidict.CIMultiDict([ + ('ORIGIN', 'https://mycloud.rackspace.com'), + ('CONTENT-LENGTH', '190'), + ('ACCEPT-RANGES', 'bytes'), + ('LAST-MODIFIED', 'Fri, 19 Dec 2014 23:22:24 GMT'), + ('ETAG', 'edfa12d00b779b4b37b81fe5b61b2b3f'), + ('X-TIMESTAMP', '1419031343.23224'), + ('CONTENT-TYPE', 'application/x-www-form-urlencoded;charset=utf-8'), + ('X-TRANS-ID', 'tx7cfeef941f244807aec37-005498754diad3'), + ('DATE', 'Mon, 22 Dec 2014 19:47:25 GMT') + ]) + + +@pytest.fixture +def file_root_similar_name(): + return aiohttp.multidict.CIMultiDict([ + ('ORIGIN', 'https://mycloud.rackspace.com'), + ('CONTENT-LENGTH', '190'), + ('ACCEPT-RANGES', 'bytes'), + ('LAST-MODIFIED', 'Mon, 22 Dec 2014 19:07:12 GMT'), + ('ETAG', 'edfa12d00b779b4b37b81fe5b61b2b3f'), + ('X-TIMESTAMP', '1419275231.66160'), + ('CONTENT-TYPE', 'application/x-www-form-urlencoded;charset=utf-8'), + ('X-TRANS-ID', 'tx438cbb32b5344d63b267c-0054987f3biad3'), + ('DATE', 'Mon, 22 Dec 2014 20:29:47 GMT') + ]) + + +@pytest.fixture +def file_header_metadata_txt(): + return aiohttp.multidict.CIMultiDict([ + ('ORIGIN', 'https://mycloud.rackspace.com'), + ('CONTENT-LENGTH', '216945'), + ('ACCEPT-RANGES', 'bytes'), + ('LAST-MODIFIED', 'Mon, 22 Dec 2014 19:01:02 GMT'), + ('ETAG', '44325d4f13b09f3769ede09d7c20a82c'), + ('X-TIMESTAMP', '1419274861.04433'), + ('CONTENT-TYPE', 'text/plain'), + ('X-TRANS-ID', 'tx836375d817a34b558756a-0054987deeiad3'), + ('DATE', 'Mon, 22 Dec 2014 20:24:14 GMT') + ]) diff --git a/tests/providers/cloudfiles/fixtures/fixtures.json b/tests/providers/cloudfiles/fixtures/fixtures.json new file mode 100644 index 000000000..2e313777b --- /dev/null +++ b/tests/providers/cloudfiles/fixtures/fixtures.json @@ -0,0 +1,181 @@ +{ + "container_header_metadata_with_verision_location":{ + "ACCEPT-RANGES":"bytes", + "CONTENT-LENGTH":"0", + "CONTENT-TYPE":"application/json; charset=utf-8", + "DATE":"Thu, 12 Oct 2017 16:13:04 GMT", + "X-CONTAINER-BYTES-USED":"90", + "X-CONTAINER-META-ACCESS-CONTROL-EXPOSE-HEADERS":"etag location x-timestamp x-trans-id", + "X-CONTAINER-META-ACCESS-LOG-DELIVERY":"false", + "X-CONTAINER-OBJECT-COUNT":"2", + "X-STORAGE-POLICY":"Policy-0", + "X-TIMESTAMP":"1506696708.03681", + "X-TRANS-ID":"txffaa4b0a06984dd0bbf27-0059df9490iad3", + "X-VERSIONS-LOCATION":"versions-container" + }, + "container_header_metadata_without_verision_location":{ + "ACCEPT-RANGES":"bytes", + "CONTENT-LENGTH":"0", + "CONTENT-TYPE":"application/json; charset=utf-8", + "DATE":"Thu, 12 Oct 2017 16:13:04 GMT", + "X-CONTAINER-BYTES-USED":"90", + "X-CONTAINER-META-ACCESS-CONTROL-EXPOSE-HEADERS":"etag location x-timestamp x-trans-id", + "X-CONTAINER-META-ACCESS-LOG-DELIVERY":"false", + "X-CONTAINER-OBJECT-COUNT":"2", + "X-STORAGE-POLICY":"Policy-0", + "X-TIMESTAMP":"1506696708.03681", + "X-TRANS-ID":"txffaa4b0a06984dd0bbf27-0059df9490iad3" + }, + "file_metadata":{ + "last_modified":"2014-12-19T23:22:14.728640", + "content_type":"application/x-www-form-urlencoded;charset=utf-8", + "hash":"edfa12d00b779b4b37b81fe5b61b2b3f", + "name":"similar.file", + "bytes":190 + }, + "file_header_metadata":{ + "CONTENT-LENGTH":"90", + "ACCEPT-RANGES":"bytes", + "LAST-MODIFIED":"Wed, 11 Oct 2017 21:37:55 GMT", + "ETAG":"8a839ea73aaa78718e27e025bdc2c767", + "X-TIMESTAMP":"1507757874.70544", + "CONTENT-TYPE":"application/octet-stream", + "X-TRANS-ID":"txae77ecf20a83452ebe2c0-0059dfa57aiad3", + "DATE":"Thu, 12 Oct 2017 17:25:14 GMT" + }, + "folder_root":[ + { + "last_modified":"2014-12-19T22:08:23.006360", + "content_type":"application/directory", + "hash":"d41d8cd98f00b204e9800998ecf8427e", + "name":"level1", + "bytes":0 + }, + { + "subdir":"level1/" + }, + { + "last_modified":"2014-12-19T23:22:23.232240", + "content_type":"application/x-www-form-urlencoded;charset=utf-8", + "hash":"edfa12d00b779b4b37b81fe5b61b2b3f", + "name":"similar", + "bytes":190 + }, + { + "last_modified":"2014-12-19T23:22:14.728640", + "content_type":"application/x-www-form-urlencoded;charset=utf-8", + "hash":"edfa12d00b779b4b37b81fe5b61b2b3f", + "name":"similar.file", + "bytes":190 + }, + { + "last_modified":"2014-12-19T23:20:16.718860", + "content_type":"application/directory", + "hash":"d41d8cd98f00b204e9800998ecf8427e", + "name":"level1_empty", + "bytes":0 + } + ], + "folder_root_level1":[ + { + "last_modified":"2014-12-19T22:08:26.958830", + "content_type":"application/directory", + "hash":"d41d8cd98f00b204e9800998ecf8427e", + "name":"level1/level2", + "bytes":0 + }, + { + "subdir":"level1/level2/" + } + ], + "folder_root_level1_level2":[ + { + "name":"level1/level2/file2.txt", + "content_type":"application/x-www-form-urlencoded;charset=utf-8", + "last_modified":"2014-12-19T23:25:22.497420", + "bytes":1365336, + "hash":"ebc8cdd3f712fd39476fb921d43aca1a" + } + ], + "auth_json":{ + "access":{ + "serviceCatalog":[ + { + "name":"cloudFiles", + "type":"object-store", + "endpoints":[ + { + "publicURL":"https://fakestorage", + "internalURL":"https://internal_fake_storage", + "region":"IAD", + "tenantId":"someid_123456" + } + ] + } + ], + "token":{ + "RAX-AUTH:authenticatedBy":[ + "APIKEY" + ], + "tenant":{ + "name":"12345", + "id":"12345" + }, + "id":"2322f6b2322f4dbfa69802baf50b0832", + "expires":"2014-12-17T09:12:26.069Z" + }, + "user":{ + "name":"osf-production", + "roles":[ + { + "name":"object-store:admin", + "id":"10000256", + "description":"Object Store Admin Role for Account User" + }, + { + "name":"compute:default", + "description":"A Role that allows a user access to keystone Service methods", + "id":"6", + "tenantId":"12345" + }, + { + "name":"object-store:default", + "description":"A Role that allows a user access to keystone Service methods", + "id":"5", + "tenantId":"some_id_12345" + }, + { + "name":"identity:default", + "id":"2", + "description":"Default Role." + } + ], + "id":"secret", + "RAX-AUTH:defaultRegion":"IAD" + } + } + }, + "revision_list":[ + { + "hash":"8a839ea73aaa78718e27e025bdc2c767", + "bytes":90, + "name":"007123.csv/1507756317.92019", + "content_type":"application/octet-stream", + "last_modified":"2017-10-11T21:24:43.459520" + }, + { + "hash":"cacef99009078d6fbf994dd18aac5658", + "bytes":90, + "name":"007123.csv/1507757083.60055", + "content_type":"application/octet-stream", + "last_modified":"2017-10-11T21:31:34.386170" + }, + { + "hash":"63e4d56ff3b8a3bf4981f071dac1522e", + "bytes":90, + "name":"007123.csv/1507757494.53144", + "content_type":"application/octet-stream", + "last_modified":"2017-10-11T21:37:54.645380" + } + ] +} \ No newline at end of file diff --git a/tests/providers/cloudfiles/test_metadata.py b/tests/providers/cloudfiles/test_metadata.py index b69e44cc3..5904116e3 100644 --- a/tests/providers/cloudfiles/test_metadata.py +++ b/tests/providers/cloudfiles/test_metadata.py @@ -1,36 +1,18 @@ import pytest -import aiohttp from waterbutler.core.path import WaterButlerPath -from waterbutler.providers.cloudfiles.metadata import (CloudFilesFileMetadata, - CloudFilesHeaderMetadata, - CloudFilesFolderMetadata) - - -@pytest.fixture -def file_header_metadata_txt(): - return aiohttp.multidict.CIMultiDict([ - ('ORIGIN', 'https://mycloud.rackspace.com'), - ('CONTENT-LENGTH', '216945'), - ('ACCEPT-RANGES', 'bytes'), - ('LAST-MODIFIED', 'Mon, 22 Dec 2014 19:01:02 GMT'), - ('ETAG', '44325d4f13b09f3769ede09d7c20a82c'), - ('X-TIMESTAMP', '1419274861.04433'), - ('CONTENT-TYPE', 'text/plain'), - ('X-TRANS-ID', 'tx836375d817a34b558756a-0054987deeiad3'), - ('DATE', 'Mon, 22 Dec 2014 20:24:14 GMT') - ]) - - -@pytest.fixture -def file_metadata(): - return { - 'last_modified': '2014-12-19T23:22:14.728640', - 'content_type': 'application/x-www-form-urlencoded;charset=utf-8', - 'hash': 'edfa12d00b779b4b37b81fe5b61b2b3f', - 'name': 'similar.file', - 'bytes': 190 - } +from waterbutler.providers.cloudfiles.metadata import ( + CloudFilesFileMetadata, + CloudFilesHeaderMetadata, + CloudFilesFolderMetadata, + CloudFilesRevisonMetadata +) + +from tests.providers.cloudfiles.fixtures import ( + file_header_metadata_txt, + file_metadata, + revision_list +) class TestCloudfilesMetadata: @@ -38,7 +20,7 @@ class TestCloudfilesMetadata: def test_header_metadata(self, file_header_metadata_txt): path = WaterButlerPath('/file.txt') - data = CloudFilesHeaderMetadata(file_header_metadata_txt, path.path) + data = CloudFilesHeaderMetadata(file_header_metadata_txt, path) assert data.name == 'file.txt' assert data.path == '/file.txt' assert data.provider == 'cloudfiles' @@ -242,3 +224,13 @@ def test_folder_metadata(self): 'new_folder': ('http://localhost:7777/v1/resources/' 'cn42d/providers/cloudfiles/level1/?kind=folder') } + + def test_revision_metadata(self, revision_list): + data = CloudFilesRevisonMetadata(revision_list[0]) + + assert data.version_identifier == 'revision' + assert data.name == '007123.csv/1507756317.92019' + assert data.version == '007123.csv/1507756317.92019' + assert data.size == 90 + assert data.content_type == 'application/octet-stream' + assert data.modified == '2017-10-11T21:24:43.459520' diff --git a/tests/providers/cloudfiles/test_provider.py b/tests/providers/cloudfiles/test_provider.py index da8bb55a5..6e688282c 100644 --- a/tests/providers/cloudfiles/test_provider.py +++ b/tests/providers/cloudfiles/test_provider.py @@ -1,331 +1,48 @@ -import io -import os import json -import time import hashlib -import functools -from unittest import mock import furl import pytest import aiohttp import aiohttpretty -import aiohttp.multidict -from waterbutler.core import streams +from tests.utils import MockCoroutine from waterbutler.core import exceptions from waterbutler.core.path import WaterButlerPath -from waterbutler.providers.cloudfiles import CloudFilesProvider from waterbutler.providers.cloudfiles import settings as cloud_settings - -@pytest.fixture -def auth(): - return { - 'name': 'cat', - 'email': 'cat@cat.com', - } - - -@pytest.fixture -def credentials(): - return { - 'username': 'prince', - 'token': 'revolutionary', - 'region': 'iad', - } - - -@pytest.fixture -def settings(): - return {'container': 'purple rain'} - - -@pytest.fixture -def provider(auth, credentials, settings): - return CloudFilesProvider(auth, credentials, settings) - - -@pytest.fixture -def auth_json(): - return { - "access": { - "serviceCatalog": [ - { - "name": "cloudFiles", - "type": "object-store", - "endpoints": [ - { - "publicURL": "https://fakestorage", - "internalURL": "https://internal_fake_storage", - "region": "IAD", - "tenantId": "someid_123456" - }, - ] - } - ], - "token": { - "RAX-AUTH:authenticatedBy": [ - "APIKEY" - ], - "tenant": { - "name": "12345", - "id": "12345" - }, - "id": "2322f6b2322f4dbfa69802baf50b0832", - "expires": "2014-12-17T09:12:26.069Z" - }, - "user": { - "name": "osf-production", - "roles": [ - { - "name": "object-store:admin", - "id": "10000256", - "description": "Object Store Admin Role for Account User" - }, - { - "name": "compute:default", - "description": "A Role that allows a user access to keystone Service methods", - "id": "6", - "tenantId": "12345" - }, - { - "name": "object-store:default", - "description": "A Role that allows a user access to keystone Service methods", - "id": "5", - "tenantId": "some_id_12345" - }, - { - "name": "identity:default", - "id": "2", - "description": "Default Role." - } - ], - "id": "secret", - "RAX-AUTH:defaultRegion": "IAD" - } - } - } - - -@pytest.fixture -def token(auth_json): - return auth_json['access']['token']['id'] - - -@pytest.fixture -def endpoint(auth_json): - return auth_json['access']['serviceCatalog'][0]['endpoints'][0]['publicURL'] - - -@pytest.fixture -def temp_url_key(): - return 'temporary beret' - - -@pytest.fixture -def mock_auth(auth_json): - aiohttpretty.register_json_uri( - 'POST', - settings.AUTH_URL, - body=auth_json, - ) - - -@pytest.fixture -def mock_temp_key(endpoint, temp_url_key): - aiohttpretty.register_uri( - 'HEAD', - endpoint, - status=204, - headers={'X-Account-Meta-Temp-URL-Key': temp_url_key}, - ) - - -@pytest.fixture -def mock_time(monkeypatch): - mock_time = mock.Mock() - mock_time.return_value = 10 - monkeypatch.setattr(time, 'time', mock_time) - - -@pytest.fixture -def connected_provider(provider, token, endpoint, temp_url_key, mock_time): - provider.token = token - provider.endpoint = endpoint - provider.temp_url_key = temp_url_key.encode() - return provider - - -@pytest.fixture -def file_content(): - return b'sleepy' - - -@pytest.fixture -def file_like(file_content): - return io.BytesIO(file_content) - - -@pytest.fixture -def file_stream(file_like): - return streams.FileStreamReader(file_like) - - -@pytest.fixture -def file_metadata(): - return aiohttp.multidict.CIMultiDict([ - ('LAST-MODIFIED', 'Thu, 25 Dec 2014 02:54:35 GMT'), - ('CONTENT-LENGTH', '0'), - ('ETAG', 'edfa12d00b779b4b37b81fe5b61b2b3f'), - ('CONTENT-TYPE', 'text/html; charset=UTF-8'), - ('X-TRANS-ID', 'txf876a4b088e3451d94442-00549b7c6aiad3'), - ('DATE', 'Thu, 25 Dec 2014 02:54:34 GMT') - ]) - - -# Metadata Test Scenarios -# / (folder_root_empty) -# / (folder_root) -# /level1/ (folder_root_level1) -# /level1/level2/ (folder_root_level1_level2) -# /level1/level2/file2.file - (file_root_level1_level2_file2_txt) -# /level1_empty/ (folder_root_level1_empty) -# /similar (file_similar) -# /similar.name (file_similar_name) -# /does_not_exist (404) -# /does_not_exist/ (404) - - -@pytest.fixture -def folder_root_empty(): - return [] - - -@pytest.fixture -def folder_root(): - return [ - { - 'last_modified': '2014-12-19T22:08:23.006360', - 'content_type': 'application/directory', - 'hash': 'd41d8cd98f00b204e9800998ecf8427e', - 'name': 'level1', - 'bytes': 0 - }, - { - 'subdir': 'level1/' - }, - { - 'last_modified': '2014-12-19T23:22:23.232240', - 'content_type': 'application/x-www-form-urlencoded;charset=utf-8', - 'hash': 'edfa12d00b779b4b37b81fe5b61b2b3f', - 'name': 'similar', - 'bytes': 190 - }, - { - 'last_modified': '2014-12-19T23:22:14.728640', - 'content_type': 'application/x-www-form-urlencoded;charset=utf-8', - 'hash': 'edfa12d00b779b4b37b81fe5b61b2b3f', - 'name': 'similar.file', - 'bytes': 190 - }, - { - 'last_modified': '2014-12-19T23:20:16.718860', - 'content_type': 'application/directory', - 'hash': 'd41d8cd98f00b204e9800998ecf8427e', - 'name': 'level1_empty', - 'bytes': 0 - } - ] - - -@pytest.fixture -def folder_root_level1(): - return [ - { - 'last_modified': '2014-12-19T22:08:26.958830', - 'content_type': 'application/directory', - 'hash': 'd41d8cd98f00b204e9800998ecf8427e', - 'name': 'level1/level2', - 'bytes': 0 - }, - { - 'subdir': 'level1/level2/' - } - ] - - -@pytest.fixture -def folder_root_level1_level2(): - return [ - { - 'name': 'level1/level2/file2.txt', - 'content_type': 'application/x-www-form-urlencoded;charset=utf-8', - 'last_modified': '2014-12-19T23:25:22.497420', - 'bytes': 1365336, - 'hash': 'ebc8cdd3f712fd39476fb921d43aca1a' - } - ] - - -@pytest.fixture -def file_root_level1_level2_file2_txt(): - return aiohttp.multidict.CIMultiDict([ - ('ORIGIN', 'https://mycloud.rackspace.com'), - ('CONTENT-LENGTH', '216945'), - ('ACCEPT-RANGES', 'bytes'), - ('LAST-MODIFIED', 'Mon, 22 Dec 2014 19:01:02 GMT'), - ('ETAG', '44325d4f13b09f3769ede09d7c20a82c'), - ('X-TIMESTAMP', '1419274861.04433'), - ('CONTENT-TYPE', 'text/plain'), - ('X-TRANS-ID', 'tx836375d817a34b558756a-0054987deeiad3'), - ('DATE', 'Mon, 22 Dec 2014 20:24:14 GMT') - ]) - - -@pytest.fixture -def folder_root_level1_empty(): - return aiohttp.multidict.CIMultiDict([ - ('ORIGIN', 'https://mycloud.rackspace.com'), - ('CONTENT-LENGTH', '0'), - ('ACCEPT-RANGES', 'bytes'), - ('LAST-MODIFIED', 'Mon, 22 Dec 2014 18:58:56 GMT'), - ('ETAG', 'd41d8cd98f00b204e9800998ecf8427e'), - ('X-TIMESTAMP', '1419274735.03160'), - ('CONTENT-TYPE', 'application/directory'), - ('X-TRANS-ID', 'txd78273e328fc4ba3a98e3-0054987eeeiad3'), - ('DATE', 'Mon, 22 Dec 2014 20:28:30 GMT') - ]) - - -@pytest.fixture -def file_root_similar(): - return aiohttp.multidict.CIMultiDict([ - ('ORIGIN', 'https://mycloud.rackspace.com'), - ('CONTENT-LENGTH', '190'), - ('ACCEPT-RANGES', 'bytes'), - ('LAST-MODIFIED', 'Fri, 19 Dec 2014 23:22:24 GMT'), - ('ETAG', 'edfa12d00b779b4b37b81fe5b61b2b3f'), - ('X-TIMESTAMP', '1419031343.23224'), - ('CONTENT-TYPE', 'application/x-www-form-urlencoded;charset=utf-8'), - ('X-TRANS-ID', 'tx7cfeef941f244807aec37-005498754diad3'), - ('DATE', 'Mon, 22 Dec 2014 19:47:25 GMT') - ]) - - -@pytest.fixture -def file_root_similar_name(): - return aiohttp.multidict.CIMultiDict([ - ('ORIGIN', 'https://mycloud.rackspace.com'), - ('CONTENT-LENGTH', '190'), - ('ACCEPT-RANGES', 'bytes'), - ('LAST-MODIFIED', 'Mon, 22 Dec 2014 19:07:12 GMT'), - ('ETAG', 'edfa12d00b779b4b37b81fe5b61b2b3f'), - ('X-TIMESTAMP', '1419275231.66160'), - ('CONTENT-TYPE', 'application/x-www-form-urlencoded;charset=utf-8'), - ('X-TRANS-ID', 'tx438cbb32b5344d63b267c-0054987f3biad3'), - ('DATE', 'Mon, 22 Dec 2014 20:29:47 GMT') - ]) +from waterbutler.providers.cloudfiles.metadata import (CloudFilesHeaderMetadata, + CloudFilesRevisonMetadata) + +from tests.providers.cloudfiles.fixtures import ( + auth, + settings, + credentials, + token, + endpoint, + temp_url_key, + mock_time, + file_content, + file_stream, + file_like, + mock_temp_key, + provider, + connected_provider, + file_metadata, + auth_json, + folder_root, + folder_root_empty, + folder_root_level1_empty, + file_root_similar_name, + file_root_similar, + file_header_metadata, + folder_root_level1, + folder_root_level1_level2, + file_root_level1_level2_file2_txt, + container_header_metadata_with_verision_location, + container_header_metadata_without_verision_location, + revision_list +) class TestCRUD: @@ -344,6 +61,30 @@ async def test_download(self, connected_provider): assert content == body + @pytest.mark.asyncio + @pytest.mark.aiohttpretty + async def test_download_revision(self, + connected_provider, + container_header_metadata_with_verision_location): + body = b'dearly-beloved' + path = WaterButlerPath('/lets-go-crazy') + url = connected_provider.sign_url(path) + aiohttpretty.register_uri('GET', url, body=body, auto_length=True) + + container_url = connected_provider.build_url('') + aiohttpretty.register_uri('HEAD', + container_url, + headers=container_header_metadata_with_verision_location) + + version_name = '{:03x}'.format(len(path.name)) + path.name + '/' + revision_url = connected_provider.build_url(version_name, container='versions-container', ) + aiohttpretty.register_uri('GET', revision_url, body=body, auto_length=True) + + result = await connected_provider.download(path, version=version_name) + content = await result.read() + + assert content == body + @pytest.mark.asyncio @pytest.mark.aiohttpretty async def test_download_accept_url(self, connected_provider): @@ -373,21 +114,26 @@ async def test_download_not_found(self, connected_provider): @pytest.mark.asyncio @pytest.mark.aiohttpretty - async def test_upload(self, connected_provider, file_content, file_stream, file_metadata): - path = WaterButlerPath('/foo.bar') + async def test_upload(self, + connected_provider, + file_content, + file_stream, + file_header_metadata): + path = WaterButlerPath('/similar.file') content_md5 = hashlib.md5(file_content).hexdigest() metadata_url = connected_provider.build_url(path.path) url = connected_provider.sign_url(path, 'PUT') - aiohttpretty.register_uri( - 'HEAD', - metadata_url, - responses=[ - {'status': 404}, - {'headers': file_metadata}, - ] - ) - aiohttpretty.register_uri('PUT', url, status=200, + aiohttpretty.register_uri('HEAD', + metadata_url, + responses=[ + {'status': 404}, + {'headers': file_header_metadata} + ] + ) + + aiohttpretty.register_uri('PUT', url, status=201, headers={'ETag': '"{}"'.format(content_md5)}) + metadata, created = await connected_provider.upload(file_stream, path) assert created is True @@ -395,23 +141,31 @@ async def test_upload(self, connected_provider, file_content, file_stream, file_ assert aiohttpretty.has_call(method='PUT', uri=url) assert aiohttpretty.has_call(method='HEAD', uri=metadata_url) + @pytest.mark.asyncio + @pytest.mark.aiohttpretty + async def test_create_folder(self, connected_provider, file_header_metadata): + path = WaterButlerPath('/foo/', folder=True) + metadata_url = connected_provider.build_url(path.path) + url = connected_provider.sign_url(path, 'PUT') + aiohttpretty.register_uri('PUT', url, status=201) + aiohttpretty.register_uri('HEAD', metadata_url, headers=file_header_metadata) + + metadata = await connected_provider.create_folder(path) + + assert metadata.kind == 'folder' + assert aiohttpretty.has_call(method='PUT', uri=url) + assert aiohttpretty.has_call(method='HEAD', uri=metadata_url) + @pytest.mark.asyncio @pytest.mark.aiohttpretty async def test_upload_check_none(self, connected_provider, - file_content, file_stream, file_metadata): - path = WaterButlerPath('/foo.bar') + file_content, file_stream, file_header_metadata): + path = WaterButlerPath('/similar.file') content_md5 = hashlib.md5(file_content).hexdigest() metadata_url = connected_provider.build_url(path.path) url = connected_provider.sign_url(path, 'PUT') - aiohttpretty.register_uri( - 'HEAD', - metadata_url, - responses=[ - {'status': 404}, - {'headers': file_metadata}, - ] - ) - aiohttpretty.register_uri('PUT', url, status=200, + aiohttpretty.register_uri('HEAD', metadata_url, status=404, headers=file_header_metadata) + aiohttpretty.register_uri('PUT', url, status=201, headers={'ETag': '"{}"'.format(content_md5)}) metadata, created = await connected_provider.upload( file_stream, path, check_created=False, fetch_metadata=False) @@ -422,19 +176,16 @@ async def test_upload_check_none(self, connected_provider, @pytest.mark.asyncio @pytest.mark.aiohttpretty - async def test_upload_checksum_mismatch(self, connected_provider, file_stream, file_metadata): - path = WaterButlerPath('/foo.bar') + async def test_upload_checksum_mismatch(self, + connected_provider, + file_stream, + file_header_metadata): + path = WaterButlerPath('/similar.file') metadata_url = connected_provider.build_url(path.path) url = connected_provider.sign_url(path, 'PUT') - aiohttpretty.register_uri( - 'HEAD', - metadata_url, - responses=[ - {'status': 404}, - {'headers': file_metadata}, - ] - ) - aiohttpretty.register_uri('PUT', url, status=200, headers={'ETag': '"Bogus MD5"'}) + aiohttpretty.register_uri('HEAD', metadata_url, status=404, headers=file_header_metadata) + + aiohttpretty.register_uri('PUT', url, status=201, headers={'ETag': '"Bogus MD5"'}) with pytest.raises(exceptions.UploadChecksumMismatchError): await connected_provider.upload(file_stream, path) @@ -442,30 +193,57 @@ async def test_upload_checksum_mismatch(self, connected_provider, file_stream, f assert aiohttpretty.has_call(method='PUT', uri=url) assert aiohttpretty.has_call(method='HEAD', uri=metadata_url) - # @pytest.mark.asyncio - # @pytest.mark.aiohttpretty - # async def test_delete_folder(self, connected_provider, folder_root_empty, file_metadata): - # # This test will probably fail on a live - # # version of the provider because build_url is called wrong. - # # Will comment out parts of this test till that is fixed. - # path = WaterButlerPath('/delete/') - # query = {'prefix': path.path} - # url = connected_provider.build_url('', **query) - # body = json.dumps(folder_root_empty).encode('utf-8') + @pytest.mark.asyncio + @pytest.mark.aiohttpretty + async def test_delete_folder(self, connected_provider, folder_root_empty, file_header_metadata): + + path = WaterButlerPath('/delete/') + query = {'prefix': path.path, 'delimiter': '/'} + url = connected_provider.build_url('', **query) + body = json.dumps(folder_root_empty).encode('utf-8') + + delete_query = {'bulk-delete': ''} + delete_url_folder = connected_provider.build_url(path.name, **delete_query) + delete_url_content = connected_provider.build_url('', **delete_query) + + file_url = connected_provider.build_url(path.path) + aiohttpretty.register_uri('GET', url, body=body) + aiohttpretty.register_uri('HEAD', file_url, headers=file_header_metadata) + + aiohttpretty.register_uri('DELETE', delete_url_content, status=200) + aiohttpretty.register_uri('DELETE', delete_url_folder, status=204) + + await connected_provider.delete(path) + + assert aiohttpretty.has_call(method='DELETE', uri=delete_url_folder) + + @pytest.mark.asyncio + @pytest.mark.aiohttpretty + async def test_delete_root(self, connected_provider, folder_root_empty, file_header_metadata): + + path = WaterButlerPath('/') + query = {'prefix': path.path, 'delimiter': '/'} + url = connected_provider.build_url('', **query) + body = json.dumps(folder_root_empty).encode('utf-8') + + delete_query = {'bulk-delete': ''} + delete_url_folder = connected_provider.build_url(path.name, **delete_query) + delete_url_content = connected_provider.build_url('', **delete_query) - # delete_query = {'bulk-delete': ''} - # delete_url = connected_provider.build_url('', **delete_query) + file_url = connected_provider.build_url(path.path) + aiohttpretty.register_uri('GET', url, body=body) + aiohttpretty.register_uri('HEAD', file_url, headers=file_header_metadata) - # file_url = connected_provider.build_url(path.path) + aiohttpretty.register_uri('DELETE', delete_url_content, status=200) - # aiohttpretty.register_uri('GET', url, body=body) - # aiohttpretty.register_uri('HEAD', file_url, headers=file_metadata) + with pytest.raises(exceptions.DeleteError) as exc: + await connected_provider.delete(path) - # aiohttpretty.register_uri('DELETE', delete_url) + assert exc.value.message == 'query argument confirm_delete=1 is required for' \ + ' deleting the entire root contents.' - # await connected_provider.delete(path) + await connected_provider.delete(path, confirm_delete=1) - # assert aiohttpretty.has_call(method='DELETE', uri=delete_url) @pytest.mark.asyncio @pytest.mark.aiohttpretty @@ -479,20 +257,77 @@ async def test_delete_file(self, connected_provider): @pytest.mark.asyncio @pytest.mark.aiohttpretty - async def test_intra_copy(self, connected_provider, file_metadata): + async def test_intra_copy(self, connected_provider, file_header_metadata): src_path = WaterButlerPath('/delete.file') dest_path = WaterButlerPath('/folder1/delete.file') dest_url = connected_provider.build_url(dest_path.path) - aiohttpretty.register_uri('HEAD', dest_url, headers=file_metadata) + aiohttpretty.register_uri('HEAD', dest_url, headers=file_header_metadata) aiohttpretty.register_uri('PUT', dest_url, status=201) result = await connected_provider.intra_copy(connected_provider, src_path, dest_path) assert result[0].path == '/folder1/delete.file' assert result[0].name == 'delete.file' - assert result[0].etag == 'edfa12d00b779b4b37b81fe5b61b2b3f' + assert result[0].etag == '8a839ea73aaa78718e27e025bdc2c767' + + +class TestRevisions: + + @pytest.mark.asyncio + @pytest.mark.aiohttpretty + async def test_revisions(self, + connected_provider, + container_header_metadata_with_verision_location, + revision_list, + file_header_metadata): + + path = WaterButlerPath('/file.txt') + + container_url = connected_provider.build_url('') + aiohttpretty.register_uri('HEAD', + container_url, + headers=container_header_metadata_with_verision_location) + + query = {'prefix': '{:03x}'.format(len(path.name)) + path.name + '/'} + revision_url = connected_provider.build_url('', container='versions-container', **query) + aiohttpretty.register_json_uri('GET', revision_url, body=revision_list) + + metadata_url = connected_provider.build_url(path.path) + aiohttpretty.register_uri('HEAD', metadata_url, status=200, headers=file_header_metadata) + + result = await connected_provider.revisions(path) + + assert type(result) == list + assert len(result) == 4 + assert type(result[0]) == CloudFilesRevisonMetadata + assert result[0].name == 'file.txt' + assert result[1].name == '007123.csv/1507756317.92019' + + @pytest.mark.asyncio + @pytest.mark.aiohttpretty + async def test_revision_metadata(self, + connected_provider, + container_header_metadata_with_verision_location, + file_header_metadata): + + path = WaterButlerPath('/file.txt') + container_url = connected_provider.build_url('') + aiohttpretty.register_uri('HEAD', container_url, + headers=container_header_metadata_with_verision_location) + + version_name = '{:03x}'.format(len(path.name)) + path.name + '/' + revision_url = connected_provider.build_url(version_name + '1507756317.92019', + container='versions-container') + aiohttpretty.register_json_uri('HEAD', revision_url, body=file_header_metadata) + + result = await connected_provider.metadata(path, version=version_name + '1507756317.92019') + + assert type(result) == CloudFilesHeaderMetadata + assert result.name == 'file.txt' + assert result.path == '/file.txt' + assert result.kind == 'file' class TestMetadata: @@ -532,6 +367,19 @@ async def test_metadata_folder_root(self, connected_provider, folder_root): assert result[3].path == '/level1_empty/' assert result[3].kind == 'folder' + @pytest.mark.asyncio + @pytest.mark.aiohttpretty + async def test_metadata_404(self, connected_provider): + path = WaterButlerPath('/level1/') + url = connected_provider.build_url('', prefix=path.path, delimiter='/') + aiohttpretty.register_uri('GET', url, status=200, body=b'') + connected_provider._metadata_item = MockCoroutine(return_value=None) + + with pytest.raises(exceptions.MetadataError) as exc: + await connected_provider.metadata(path) + + assert exc.value.message == "'/level1/' could not be found." + @pytest.mark.asyncio @pytest.mark.aiohttpretty async def test_metadata_folder_root_level1(self, connected_provider, folder_root_level1): @@ -624,9 +472,11 @@ async def test_metadata_file_does_not_exist(self, connected_provider): path = WaterButlerPath('/does_not.exist') url = connected_provider.build_url(path.path) aiohttpretty.register_uri('HEAD', url, status=404) - with pytest.raises(exceptions.MetadataError): + with pytest.raises(exceptions.MetadataError) as exc: await connected_provider.metadata(path) + assert exc.value.message == "'/does_not.exist' could not be found." + @pytest.mark.asyncio @pytest.mark.aiohttpretty async def test_metadata_folder_does_not_exist(self, connected_provider): @@ -636,19 +486,10 @@ async def test_metadata_folder_does_not_exist(self, connected_provider): file_url = connected_provider.build_url(path.path.rstrip('/')) aiohttpretty.register_uri('GET', folder_url, status=200, body=folder_body) aiohttpretty.register_uri('HEAD', file_url, status=404) - with pytest.raises(exceptions.MetadataError): + with pytest.raises(exceptions.MetadataError) as exc: await connected_provider.metadata(path) - @pytest.mark.asyncio - @pytest.mark.aiohttpretty - async def test_metadata_file_bad_content_type(self, connected_provider, file_metadata): - item = file_metadata - item['Content-Type'] = 'application/directory' - path = WaterButlerPath('/does_not.exist') - url = connected_provider.build_url(path.path) - aiohttpretty.register_uri('HEAD', url, headers=item) - with pytest.raises(exceptions.MetadataError): - await connected_provider.metadata(path) + assert exc.value.message == "'/does_not_exist/' could not be found." class TestV1ValidatePath: @@ -674,6 +515,28 @@ async def test_ensure_connection(self, provider, auth_json, mock_temp_key): await provider._ensure_connection() assert aiohttpretty.has_call(method='POST', uri=token_url) + @pytest.mark.asyncio + @pytest.mark.aiohttpretty + async def test_no_version_location(self, + connected_provider, + container_header_metadata_without_verision_location): + + path = WaterButlerPath('/file.txt') + + container_url = connected_provider.build_url('') + aiohttpretty.register_uri('HEAD', + container_url, + headers=container_header_metadata_without_verision_location) + + with pytest.raises(exceptions.MetadataError) as exc: + await connected_provider.revisions(path) + + assert exc.value.message == 'The your container does not have a defined version location.' \ + ' To set a version location and store file versions follow' \ + ' the instructions here: https://developer.rackspace.com/' \ + 'docs/cloud-files/v1/use-cases/' \ + 'additional-object-services-information/#object-versioning' + @pytest.mark.asyncio @pytest.mark.aiohttpretty async def test_ensure_connection_not_public(self, provider, auth_json, temp_url_key): diff --git a/waterbutler/core/provider.py b/waterbutler/core/provider.py index ba443e33f..559f9a46a 100644 --- a/waterbutler/core/provider.py +++ b/waterbutler/core/provider.py @@ -323,7 +323,6 @@ async def _folder_file_op(self, """ assert src_path.is_dir, 'src_path must be a directory' assert asyncio.iscoroutinefunction(func), 'func must be a coroutine' - try: await dest_provider.delete(dest_path) created = False diff --git a/waterbutler/providers/cloudfiles/metadata.py b/waterbutler/providers/cloudfiles/metadata.py index aa13c1ed9..90b124304 100644 --- a/waterbutler/providers/cloudfiles/metadata.py +++ b/waterbutler/providers/cloudfiles/metadata.py @@ -55,21 +55,33 @@ def __init__(self, raw, path): super().__init__(raw) self._path = path + def to_revision(self): + revison_dict = {'bytes': self.size, + 'name': self.name, + 'last_modified': self.modified, + 'content_type': self.content_type} + + return CloudFilesRevisonMetadata(revison_dict) + + @property + def kind(self): + return 'folder' if self._path.is_dir else 'file' + @property def name(self): - return os.path.split(self._path)[1] + return self._path.name @property def path(self): - return self.build_path(self._path) + return self._path.materialized_path @property def size(self): - return int(self.raw['Content-Length']) + return int(self.raw['CONTENT-LENGTH']) @property def modified(self): - return self.raw['Last-Modified'] + return self.raw['LAST-MODIFIED'] @property def created_utc(self): @@ -77,17 +89,17 @@ def created_utc(self): @property def content_type(self): - return self.raw['Content-Type'] + return self.raw['CONTENT-TYPE'] @property def etag(self): - return self.raw['etag'] + return self.raw['ETAG'].replace('"', '') @property def extra(self): return { 'hashes': { - 'md5': self.raw['etag'].replace('"', ''), + 'md5': self.raw['ETAG'].replace('"', '') }, } @@ -101,3 +113,30 @@ def name(self): @property def path(self): return self.build_path(self.raw['subdir']) + + +class CloudFilesRevisonMetadata(metadata.BaseFileRevisionMetadata): + + @property + def version_identifier(self): + return 'revision' + + @property + def version(self): + return self.raw['name'] + + @property + def modified(self): + return self.raw['last_modified'] + + @property + def size(self): + return self.raw['bytes'] + + @property + def name(self): + return self.raw['name'] + + @property + def content_type(self): + return self.raw['content_type'] diff --git a/waterbutler/providers/cloudfiles/provider.py b/waterbutler/providers/cloudfiles/provider.py index fa0248571..912d17719 100644 --- a/waterbutler/providers/cloudfiles/provider.py +++ b/waterbutler/providers/cloudfiles/provider.py @@ -2,21 +2,27 @@ import hmac import json import time +import typing +from typing import List, Union, Tuple import asyncio import hashlib import functools +from urllib.parse import unquote import furl -from waterbutler.core import streams +from waterbutler.core.streams import ResponseStreamReader, HashStreamWriter from waterbutler.core import provider from waterbutler.core import exceptions from waterbutler.core.path import WaterButlerPath from waterbutler.providers.cloudfiles import settings -from waterbutler.providers.cloudfiles.metadata import CloudFilesFileMetadata -from waterbutler.providers.cloudfiles.metadata import CloudFilesFolderMetadata -from waterbutler.providers.cloudfiles.metadata import CloudFilesHeaderMetadata +from waterbutler.providers.cloudfiles.metadata import ( + CloudFilesFileMetadata, + CloudFilesFolderMetadata, + CloudFilesHeaderMetadata, + CloudFilesRevisonMetadata +) def ensure_connection(func): @@ -25,7 +31,7 @@ def ensure_connection(func): @functools.wraps(func) async def wrapped(self, *args, **kwargs): await self._ensure_connection() - return (await func(self, *args, **kwargs)) + return await func(self, *args, **kwargs) return wrapped @@ -49,10 +55,10 @@ def __init__(self, auth, credentials, settings): self.use_public = self.settings.get('use_public', True) self.metrics.add('region', self.region) - async def validate_v1_path(self, path, **kwargs): + async def validate_v1_path(self, path: str, **kwargs): return await self.validate_path(path, **kwargs) - async def validate_path(self, path, **kwargs): + async def validate_path(self, path: str, **kwargs): return WaterButlerPath(path) @property @@ -72,42 +78,54 @@ async def intra_copy(self, dest_provider, source_path, dest_path): headers={ 'X-Copy-From': os.path.join(self.container, source_path.path) }, - expects=(201, ), + expects=(201,), throws=exceptions.IntraCopyError, ) await resp.release() return (await dest_provider.metadata(dest_path)), not exists @ensure_connection - async def download(self, path, accept_url=False, range=None, **kwargs): - """Returns a ResponseStreamReader (Stream) for the specified path - :param str path: Path to the object you want to download - :param dict \*\*kwargs: Additional arguments that are ignored - :rtype str: - :rtype ResponseStreamReader: - :raises: exceptions.DownloadError - """ + async def download(self, + path: WaterButlerPath, + accept_url: bool=False, + request_range: tuple=None, + version: str=None, + revision: str=None, + displayName: str=None, + **kwargs) -> ResponseStreamReader: + """Returns a ResponseStreamReader (Stream) for the specified path """ self.metrics.add('download.accept_url', accept_url) if accept_url: parsed_url = furl.furl(self.sign_url(path, endpoint=self.public_endpoint)) - parsed_url.args['filename'] = kwargs.get('displayName') or path.name + parsed_url.args['filename'] = displayName or path.name return parsed_url.url + version = revision or version + if version: + return await self._download_revision(request_range, version) + resp = await self.make_request( 'GET', functools.partial(self.sign_url, path), - range=range, + range=request_range, expects=(200, 206), throws=exceptions.DownloadError, ) - return streams.ResponseStreamReader(resp) + return ResponseStreamReader(resp) @ensure_connection - async def upload(self, stream, path, check_created=True, fetch_metadata=True, **kwargs): + async def upload(self, + stream: ResponseStreamReader, + path: WaterButlerPath, + check_created: bool=True, + fetch_metadata: bool=True, + **kwargs) -> Tuple[CloudFilesHeaderMetadata, bool]: """Uploads the given stream to CloudFiles :param ResponseStreamReader stream: The stream to put to CloudFiles :param str path: The full path of the object to upload to/into - :rtype ResponseStreamReader: + :param bool check_created: This checks if uploaded file already exists + :param bool fetch_metadata: If true upload will return metadata + :rtype (dict/None, bool): """ if check_created: created = not (await self.exists(path)) @@ -115,13 +133,13 @@ async def upload(self, stream, path, check_created=True, fetch_metadata=True, ** created = None self.metrics.add('upload.check_created', check_created) - stream.add_writer('md5', streams.HashStreamWriter(hashlib.md5)) + stream.add_writer('md5', HashStreamWriter(hashlib.md5)) resp = await self.make_request( 'PUT', functools.partial(self.sign_url, path, 'PUT'), data=stream, headers={'Content-Length': str(stream.size)}, - expects=(200, 201), + expects=(201,), throws=exceptions.UploadError, ) await resp.release() @@ -138,61 +156,84 @@ async def upload(self, stream, path, check_created=True, fetch_metadata=True, ** return metadata, created @ensure_connection - async def delete(self, path, **kwargs): - """Deletes the key at the specified path - :param str path: The path of the key to delete - :rtype ResponseStreamReader: - """ - if path.is_dir: - metadata = await self.metadata(path, recursive=True) - - delete_files = [ - os.path.join('/', self.container, path.child(item['name']).path) - for item in metadata - ] - - delete_files.append(os.path.join('/', self.container, path.path)) - - query = {'bulk-delete': ''} - resp = await self.make_request( - 'DELETE', - functools.partial(self.build_url, **query), - data='\n'.join(delete_files), - expects=(200, ), - throws=exceptions.DeleteError, - headers={ - 'Content-Type': 'text/plain', - }, + async def delete(self, path: WaterButlerPath, confirm_delete: int=0) -> None: + """Deletes the key at the specified path.""" + + if path.is_root and confirm_delete != 1: + raise exceptions.DeleteError( + 'query argument confirm_delete=1 is required for' + ' deleting the entire root contents.', + code=400 ) + + if path.is_dir: + await self._delete_folder(path) else: - resp = await self.make_request( - 'DELETE', - functools.partial(self.build_url, path.path), - expects=(204, ), - throws=exceptions.DeleteError, - ) + await self._delete_item(path) + + @ensure_connection + async def _delete_folder(self, path: WaterButlerPath) -> None: + """Folders must be emptied of all contents before they can be deleted""" + + metadata = await self._metadata_folder(path) + + delete_files = [] + for item in metadata: + if item.kind == 'folder': + await self._delete_folder(path.from_metadata(item)) + else: + delete_files.append(os.path.join('/', self.container, path.child(item.name).path)) + + query = {'bulk-delete': ''} + resp = await self.make_request( + 'DELETE', + functools.partial(self.build_url, '', **query), + data='\n'.join(delete_files), + expects=(200,), + throws=exceptions.DeleteError, + headers={ + 'Content-Type': 'text/plain', + }, + ) await resp.release() + if not path.is_root: # deleting root here would destory the container + await self._delete_item(path) + @ensure_connection - async def metadata(self, path, recursive=False, **kwargs): - """Get Metadata about the requested file or folder - :param str path: The path to a key or folder - :rtype dict: - :rtype list: - """ + async def _delete_item(self, path: WaterButlerPath) -> None: + + resp = await self.make_request( + 'DELETE', + functools.partial(self.build_url, path.path), + expects=(204, ), + throws=exceptions.DeleteError, + ) + await resp.release() + + @ensure_connection + async def metadata(self, path: WaterButlerPath, + version: str=None, + revision: str=None, + **kwargs) -> Union[CloudFilesHeaderMetadata, List]: + """Get Metadata about the requested file or the metadata of a folder's contents""" + if path.is_dir: - return (await self._metadata_folder(path, recursive=recursive, **kwargs)) + return await self._metadata_folder(path) + elif version or revision: + return await self._metadata_revision(path, version, revision) else: - return (await self._metadata_file(path, **kwargs)) - - def build_url(self, path, _endpoint=None, **query): - """Build the url for the specified object - :param args segments: URI segments - :param kwargs query: Query parameters - :rtype str: - """ + return await self._metadata_item(path) + + def build_url(self, + path: str, + _endpoint: str=None, + container: str=None, + **query) -> str: + """Build the url for the specified object.""" endpoint = _endpoint or self.endpoint - return provider.build_url(endpoint, self.container, *path.split('/'), **query) + container = container or self.container + return provider.build_url(endpoint, container, *path.split('/'), **query) def can_duplicate_names(self): return False @@ -203,35 +244,36 @@ def can_intra_copy(self, dest_provider, path=None): def can_intra_move(self, dest_provider, path=None): return type(self) == type(dest_provider) and not getattr(path, 'is_dir', False) - def sign_url(self, path, method='GET', endpoint=None, seconds=settings.TEMP_URL_SECS): - """Sign a temp url for the specified stream - :param str stream: The requested stream's path - :param CloudFilesPath path: A path to a file/folder - :param str method: The HTTP method used to access the returned url - :param int seconds: Time for the url to live - :rtype str: - """ + def sign_url(self, + path: WaterButlerPath, + method: str='GET', + endpoint: str=None, + seconds: int=settings.TEMP_URL_SECS) -> str: + """Sign a temp url for the specified stream""" + method = method.upper() expires = str(int(time.time() + seconds)) url = furl.furl(self.build_url(path.path, _endpoint=endpoint)) - body = '\n'.join([method, expires, str(url.path)]).encode() - signature = hmac.new(self.temp_url_key, body, hashlib.sha1).hexdigest() - + body = '\n'.join([method, expires]) + body += '\n' + str(url.path) + body_data = unquote(body).encode() + signature = hmac.new(self.temp_url_key, body_data, hashlib.sha1).hexdigest() url.args.update({ 'temp_url_sig': signature, 'temp_url_expires': expires, }) - return url.url + + return unquote(str(url.url)) async def make_request(self, *args, **kwargs): try: - return (await super().make_request(*args, **kwargs)) + return await super().make_request(*args, **kwargs) except exceptions.ProviderError as e: if e.code != 408: raise await asyncio.sleep(1) - return (await super().make_request(*args, **kwargs)) + return await super().make_request(*args, **kwargs) async def _ensure_connection(self): """Defines token, endpoint and temp_url_key if they are not already defined @@ -259,24 +301,19 @@ async def _ensure_connection(self): except KeyError: raise exceptions.ProviderError('No temp url key is available', code=503) - def _extract_endpoints(self, data): - """Pulls both the public and internal cloudfiles urls, - returned respectively, from the return of tokens - Very optimized. - :param dict data: The json response from the token endpoint - :rtype (str, str): - """ + def _extract_endpoints(self, data: dict): + """Pulls both the public and internal cloudfiles urls, returned respectively, from the + return of tokens Very optimized.""" for service in reversed(data['access']['serviceCatalog']): if service['name'].lower() == 'cloudfiles': for region in service['endpoints']: if region['region'].lower() == self.region.lower(): return region['publicURL'], region['internalURL'] - async def _get_token(self): + async def _get_token(self) -> dict: """Fetches an access token from cloudfiles for actual api requests Returns the entire json response from the tokens endpoint Notably containing our token and proper endpoint to send requests to - :rtype dict: """ resp = await self.make_request( 'POST', @@ -294,43 +331,31 @@ async def _get_token(self): }, expects=(200, ), ) - data = await resp.json() - return data + return await resp.json() - async def _metadata_file(self, path, is_folder=False, **kwargs): - """Get Metadata about the requested file - :param str path: The path to a key - :rtype dict: - :rtype list: - """ + async def _metadata_item(self, path: WaterButlerPath) -> CloudFilesHeaderMetadata: + """Get Metadata about the requested file or folder""" resp = await self.make_request( 'HEAD', functools.partial(self.build_url, path.path), - expects=(200, ), + expects=(200, 404), throws=exceptions.MetadataError, ) - await resp.release() - if (resp.headers['Content-Type'] == 'application/directory' and not is_folder): + if resp.status == 404: raise exceptions.MetadataError( - 'Could not retrieve file \'{0}\''.format(str(path)), + '\'{}\' could not be found.'.format(str(path)), code=404, ) - return CloudFilesHeaderMetadata(resp.headers, path.path) + return CloudFilesHeaderMetadata(dict(resp.headers), path) - async def _metadata_folder(self, path, recursive=False, **kwargs): - """Get Metadata about the requested folder - :param str path: The path to a folder - :rtype dict: - :rtype list: - """ + async def _metadata_folder(self, path: WaterButlerPath) -> \ + List[Union[CloudFilesFolderMetadata, CloudFilesFileMetadata]]: + """Get Metadata about the contents of requested folder""" # prefix must be blank when searching the root of the container - query = {'prefix': path.path} - self.metrics.add('metadata.folder.is_recursive', True if recursive else False) - if not recursive: - query.update({'delimiter': '/'}) + query = {'prefix': path.path, 'delimiter': '/'} resp = await self.make_request( 'GET', functools.partial(self.build_url, '', **query), @@ -339,14 +364,15 @@ async def _metadata_folder(self, path, recursive=False, **kwargs): ) data = await resp.json() - # no data and the provider path is not root, we are left with either a file or a directory marker + # no data and the provider path is not root, we are left with either a file or a directory + # marker if not data and not path.is_root: - # Convert the parent path into a directory marker (file) and check for an empty folder - dir_marker = path.parent.child(path.name, folder=False) - metadata = await self._metadata_file(dir_marker, is_folder=True, **kwargs) + # Convert the parent path into a directory marker (item) and check for an empty folder + dir_marker = path.parent.child(path.name, folder=path.is_dir) + metadata = await self._metadata_item(dir_marker) if not metadata: raise exceptions.MetadataError( - 'Could not retrieve folder \'{0}\''.format(str(path)), + '\'{0}\' could not be found.'.format(str(path)), code=404, ) @@ -361,13 +387,110 @@ async def _metadata_folder(self, path, recursive=False, **kwargs): break return [ - self._serialize_folder_metadata(item) + self._serialize_metadata(item) for item in data ] - def _serialize_folder_metadata(self, data): + def _serialize_metadata(self, data: dict) -> typing.Union[CloudFilesFolderMetadata, + CloudFilesFileMetadata]: if data.get('subdir'): return CloudFilesFolderMetadata(data) elif data['content_type'] == 'application/directory': return CloudFilesFolderMetadata({'subdir': data['name'] + '/'}) return CloudFilesFileMetadata(data) + + @ensure_connection + async def create_folder(self, path: WaterButlerPath, **kwargs) -> CloudFilesHeaderMetadata: + """Create a folder in the current provider at `path`. Returns a `BaseFolderMetadata` object + if successful. May throw a 409 Conflict if a directory with the same name already exists. + Enpoint information can be found here: + https://developer.rackspace.com/docs/cloud-files/v1/general-api-info/pseudo-hierarchical-folders-and-directories/ + """ + + resp = await self.make_request( + 'PUT', + functools.partial(self.sign_url, path, 'PUT'), + expects=(200, 201), + throws=exceptions.CreateFolderError, + headers={'Content-Type': 'application/directory'} + ) + await resp.release() + return await self._metadata_item(path) + + @ensure_connection + async def _get_version_location(self) -> str: + resp = await self.make_request( + 'HEAD', + functools.partial(self.build_url, ''), + expects=(200, 204), + throws=exceptions.MetadataError, + ) + await resp.release() + + try: + return resp.headers['X-VERSIONS-LOCATION'] + except KeyError: + raise exceptions.MetadataError('The your container does not have a defined version' + ' location. To set a version location and store file ' + 'versions follow the instructions here: ' + 'https://developer.rackspace.com/docs/cloud-files/v1/' + 'use-cases/additional-object-services-information/' + '#object-versioning') + + @ensure_connection + async def revisions(self, path: WaterButlerPath, **kwarg) -> List[CloudFilesRevisonMetadata]: + """Get past versions of the requested file from special user designated version container, + if the user hasn't designated a version_location container it raises an infomative error + message. The revision endpoint also doesn't return the current version so that is added to + the revision list after other revisions are returned. More info about versioning with Cloud + Files here: + + https://developer.rackspace.com/docs/cloud-files/v1/use-cases/additional-object-services-information/#object-versioning + + :param str path: The path to a key + :rtype list: + """ + + version_location = await self._get_version_location() + + query = {'prefix': '{:03x}'.format(len(path.name)) + path.name + '/'} + resp = await self.make_request( + 'GET', + functools.partial(self.build_url, '', container=version_location, **query), + expects=(200, 204), + throws=exceptions.MetadataError, + ) + json_resp = await resp.json() + current = (await self.metadata(path)).to_revision() + return [current] + [CloudFilesRevisonMetadata(revision_data) for revision_data in json_resp] + + @ensure_connection + async def _metadata_revision(self, + path: WaterButlerPath, + version: str=None, + revision: str=None) -> CloudFilesHeaderMetadata: + version_location = await self._get_version_location() + + resp = await self.make_request( + 'HEAD', + functools.partial(self.build_url, version or revision, container=version_location), + expects=(200, ), + throws=exceptions.MetadataError, + ) + await resp.release() + + return CloudFilesHeaderMetadata(resp.headers, path) + + async def _download_revision(self, request_range: tuple, version: str) -> ResponseStreamReader: + + version_location = await self._get_version_location() + + resp = await self.make_request( + 'GET', + functools.partial(self.build_url, version, container=version_location), + range=request_range, + expects=(200, 206), + throws=exceptions.DownloadError + ) + + return ResponseStreamReader(resp)