diff --git a/tests/core/fixtures.py b/tests/core/fixtures.py new file mode 100644 index 000000000..acc276069 --- /dev/null +++ b/tests/core/fixtures.py @@ -0,0 +1,206 @@ +import time +from http import HTTPStatus +from unittest import mock + +import pytest + +from waterbutler.core.path import WaterButlerPath +from waterbutler.core.log_payload import LogPayload + +from tests.utils import MockCoroutine +from tests.providers.osfstorage.fixtures import (auth, file_path, file_lineage, provider, + file_metadata_object, file_metadata) + + +@pytest.fixture +def log_payload(file_metadata_object, file_path, provider): + return LogPayload('guid0', provider, file_metadata_object, file_path) + + +@pytest.fixture +def callback_log_payload_move(): + return { + 'auth': { + 'callback_url': 'fakecallback.com', + 'id': 'cat', + 'name': 'cat', + 'email': 'cat@cat.com' + }, + 'time': 70, + 'action': 'move', + 'source': { + 'materialized': WaterButlerPath('/doc.rst', prepend=None), + 'path': '/59a9b628b7d1c903ab5a8f52', + 'kind': 'file', + 'extra': { + 'checkout': None, + 'downloads': 0, + 'guid': None, + 'hashes': { + 'sha256': '043be9ff919762f0dc36fff0222cd90c753ce28b39feb52112be9360c476ef88', + 'md5': 'eb3f7cc15ba7b6effb2186284185c5cf' + }, + 'version': 1 + }, + 'nid': 'guid0', + 'etag': 'eccd2270585257f4b48d8493bed863c01cf0b6dc0bb590101407c9b5e10b8e08', + 'contentType': None, + 'created_utc': '2017-09-01T19:34:00.175741+00:00', + 'provider': 'osfstorage', + 'modified': '2017-09-01T19:34:00.175741+00:00', + 'modified_utc': '2017-09-01T19:34:00.175741+00:00', + 'name': 'doc.rst', + 'size': 5596, + 'resource': 'guid0' + }, + 'errors': [], + 'destination': { + 'materialized': WaterButlerPath('/doc.rst', prepend=None), + 'path': '/59a9b628b7d1c903ab5a8f52', + 'kind': 'file', + 'extra': { + 'checkout': None, + 'downloads': 0, + 'guid': None, + 'hashes': { + 'sha256': '043be9ff919762f0dc36fff0222cd90c753ce28b39feb52112be9360c476ef88', + 'md5': 'eb3f7cc15ba7b6effb2186284185c5cf' + }, + 'version': 1 + }, + 'nid': 'guid0', + 'etag': 'eccd2270585257f4b48d8493bed863c01cf0b6dc0bb590101407c9b5e10b8e08', + 'contentType': None, 'created_utc': '2017-09-01T19:34:00.175741+00:00', + 'provider': 'osfstorage', + 'modified': '2017-09-01T19:34:00.175741+00:00', + 'modified_utc': '2017-09-01T19:34:00.175741+00:00', + 'name': 'doc.rst', + 'size': 5596, + 'resource': 'guid0' + } + } + + +@pytest.fixture +def callback_log_payload_copy(): + return { + 'auth': { + 'callback_url': 'fakecallback.com', + 'id': 'cat', + 'name': 'cat', + 'email': 'cat@cat.com' + }, + 'time': 70, + 'action': 'copy', + 'source': { + 'materialized': WaterButlerPath('/doc.rst', prepend=None), + 'path': '/59a9b628b7d1c903ab5a8f52', + 'kind': 'file', + 'extra': { + 'checkout': None, + 'downloads': 0, + 'guid': None, + 'hashes': { + 'sha256': '043be9ff919762f0dc36fff0222cd90c753ce28b39feb52112be9360c476ef88', + 'md5': 'eb3f7cc15ba7b6effb2186284185c5cf' + }, + 'version': 1 + }, + 'nid': 'guid0', + 'etag': 'eccd2270585257f4b48d8493bed863c01cf0b6dc0bb590101407c9b5e10b8e08', + 'contentType': None, + 'created_utc': '2017-09-01T19:34:00.175741+00:00', + 'provider': 'osfstorage', + 'modified': '2017-09-01T19:34:00.175741+00:00', + 'modified_utc': '2017-09-01T19:34:00.175741+00:00', + 'name': 'doc.rst', + 'size': 5596, + 'resource': 'guid0' + }, + 'errors': [], + 'destination': { + 'materialized': WaterButlerPath('/doc.rst', prepend=None), + 'path': '/59a9b628b7d1c903ab5a8f52', + 'kind': 'file', + 'extra': { + 'checkout': None, + 'downloads': 0, + 'guid': None, + 'hashes': { + 'sha256': '043be9ff919762f0dc36fff0222cd90c753ce28b39feb52112be9360c476ef88', + 'md5': 'eb3f7cc15ba7b6effb2186284185c5cf' + }, + 'version': 1 + }, + 'nid': 'guid0', + 'etag': 'eccd2270585257f4b48d8493bed863c01cf0b6dc0bb590101407c9b5e10b8e08', + 'contentType': None, 'created_utc': '2017-09-01T19:34:00.175741+00:00', + 'provider': 'osfstorage', + 'modified': '2017-09-01T19:34:00.175741+00:00', + 'modified_utc': '2017-09-01T19:34:00.175741+00:00', + 'name': 'doc.rst', + 'size': 5596, + 'resource': 'guid0' + } + } + + +@pytest.fixture +def callback_log_payload_upload(): + return { + 'auth': { + 'id': 'cat', + 'email': 'cat@cat.com', + 'name': 'cat', + 'callback_url': 'fakecallback.com' + }, + 'errors': [], + 'time': 70, + 'action': 'upload', + 'provider': 'osfstorage', + 'metadata': { + 'kind': 'file', + 'name': 'doc.rst', + 'resource': 'guid0', + 'modified_utc': '2017-09-01T19:34:00.175741+00:00', + 'created_utc': '2017-09-01T19:34:00.175741+00:00', + 'provider': 'osfstorage', + 'modified': '2017-09-01T19:34:00.175741+00:00', + 'size': 5596, + 'path': '/59a9b628b7d1c903ab5a8f52', + 'etag': 'eccd2270585257f4b48d8493bed863c01cf0b6dc0bb590101407c9b5e10b8e08', + 'materialized': WaterButlerPath('/doc.rst', prepend=None), + 'extra': { + 'downloads': 0, + 'guid': None, + 'hashes': { + 'sha256': '043be9ff919762f0dc36fff0222cd90c753ce28b39feb52112be9360c476ef88', + 'md5': 'eb3f7cc15ba7b6effb2186284185c5cf'}, + 'checkout': None, + 'version': 1 + }, + 'contentType': None, + 'nid': 'guid0'} + } + + +@pytest.fixture +def mock_time(monkeypatch): + mock_time = mock.Mock() + mock_time.return_value = 10 + monkeypatch.setattr(time, 'time', mock_time) + + +@pytest.fixture +def mock_signed_request(): + return MockCoroutine(return_value=MockResponse()) + + +class MockResponse: + status = HTTPStatus.OK + read = MockCoroutine(return_value=b'{"status": "success"}') + + +class MockBadResponse: + status = HTTPStatus.INTERNAL_SERVER_ERROR + read = MockCoroutine(return_value=b'{"status": "failure"}') diff --git a/tests/core/test_remote_logging.py b/tests/core/test_remote_logging.py index 88ca8bd42..4ad891a77 100644 --- a/tests/core/test_remote_logging.py +++ b/tests/core/test_remote_logging.py @@ -1,6 +1,17 @@ +from unittest import mock + import pytest from waterbutler.core import remote_logging +from waterbutler.core.log_payload import LogPayload +from waterbutler.core.remote_logging import log_to_callback + +from tests.core.fixtures import (MockBadResponse, log_payload, mock_time, + mock_signed_request, callback_log_payload_upload, + callback_log_payload_move, callback_log_payload_copy) +from tests.providers.osfstorage.fixtures import (auth, credentials, provider, + settings, file_path, file_lineage, + file_metadata, file_metadata_object) class TestScrubPayloadForKeen: @@ -74,3 +85,87 @@ def test_max_iteration(self): 'key-test': 'value2', 'key-test-1': 'value3' } + + +class TestLogPayLoad: + + def test_log_payload(self, log_payload, file_metadata_object, file_path, provider): + assert log_payload.resource == 'guid0' + assert log_payload.provider == provider + assert log_payload.metadata == file_metadata_object + assert log_payload.path == file_path + + with pytest.raises(Exception) as exc: + LogPayload('guid0', 'osfstorage') + assert exc.message == 'Log payload needs either a path or metadata.' + + +class TestLogToCallback: + + @pytest.mark.asyncio + async def test_log_to_callback_no_logging(self): + assert (await log_to_callback('download_file')) is None + assert (await log_to_callback('download_zip')) is None + assert (await log_to_callback('metadata')) is None + + @pytest.mark.asyncio + async def test_log_to_callback_move( + self, + log_payload, + callback_log_payload_move, + mock_signed_request, + mock_time + ): + with mock.patch('waterbutler.core.utils.send_signed_request', mock_signed_request): + await log_to_callback('move', source=log_payload, destination=log_payload) + mock_signed_request.assert_called_with( + 'PUT', + log_payload.auth['callback_url'], + callback_log_payload_move + ) + + @pytest.mark.asyncio + async def test_log_to_callback_copy( + self, + log_payload, + callback_log_payload_copy, + mock_signed_request, + mock_time + ): + with mock.patch('waterbutler.core.utils.send_signed_request', mock_signed_request): + await log_to_callback('copy', source=log_payload, destination=log_payload) + mock_signed_request.assert_called_with( + 'PUT', + log_payload.auth['callback_url'], + callback_log_payload_copy + ) + + @pytest.mark.asyncio + async def test_log_to_callback_upload( + self, + log_payload, + callback_log_payload_upload, + mock_signed_request, + mock_time + ): + with mock.patch('waterbutler.core.utils.send_signed_request', mock_signed_request): + await log_to_callback('upload', source=log_payload, destination=log_payload) + mock_signed_request.assert_called_with( + 'PUT', + log_payload.auth['callback_url'], + callback_log_payload_upload + ) + + # TODO: should we fix or skip this? This test never passes for me locally but always takes a long time. + @pytest.mark.skipif( + reason="This test takes too much time because it has 5 retries before " + "throwing the desired exception, it should take around 50-60 seconds" + ) + @pytest.mark.asyncio + async def test_log_to_callback_throws_exception(self, mock_signed_request): + with mock.patch('waterbutler.core.utils.send_signed_request', mock_signed_request): + with pytest.raises(Exception) as exc: + await log_to_callback('upload') + expected_message = 'Callback for upload request failed with {},' \ + ' got {{"status": "failure"}}'.format(MockBadResponse()) + assert exc.message == expected_message diff --git a/tests/providers/osfstorage/fixtures.py b/tests/providers/osfstorage/fixtures.py index bc3d9243d..354857987 100644 --- a/tests/providers/osfstorage/fixtures.py +++ b/tests/providers/osfstorage/fixtures.py @@ -22,6 +22,7 @@ def auth(): 'id': 'cat', 'name': 'cat', 'email': 'cat@cat.com', + 'callback_url': 'fakecallback.com', } diff --git a/tests/server/api/v1/fixtures.py b/tests/server/api/v1/fixtures.py new file mode 100644 index 000000000..c10b55dc4 --- /dev/null +++ b/tests/server/api/v1/fixtures.py @@ -0,0 +1,111 @@ +import time +import asyncio +from unittest import mock + +import pytest +from tornado.httputil import HTTPServerRequest +from tornado.http1connection import HTTP1ConnectionParameters + +from waterbutler.server.app import make_app +from waterbutler.core.path import WaterButlerPath +from waterbutler.core.log_payload import LogPayload +from waterbutler.server.api.v1.provider import ProviderHandler + +from tests.utils import MockProvider, MockFileMetadata +from tests.providers.osfstorage.fixtures import (auth, provider, file_metadata_object, + file_metadata, file_path, file_lineage) + + +@pytest.yield_fixture +def event_loop(): + """Create an instance of the default event loop for each test case.""" + policy = asyncio.get_event_loop_policy() + res = policy.new_event_loop() + asyncio.set_event_loop(res) + res._close = res.close + res.close = lambda: None + + yield res + + res._close() + + +@pytest.fixture +def http_request(): + mocked_http_request = HTTPServerRequest( + uri='/v1/resources/test/providers/test/path/mock', + method='GET' + ) + mocked_http_request.headers['User-Agent'] = 'test' + mocked_http_request.connection = HTTP1ConnectionParameters() + mocked_http_request.connection.set_close_callback = mock.Mock() + mocked_http_request.request_time = mock.Mock(return_value=10) + + return mocked_http_request + + +@pytest.fixture +def log_payload(): + return LogPayload('test', MockProvider(), path=WaterButlerPath('/test_path')) + + +@pytest.fixture +def mock_time(monkeypatch): + mocked_time = mock.Mock() + mocked_time.return_value = 10 + monkeypatch.setattr(time, 'time', mocked_time) + + +@pytest.fixture +def handler(http_request): + mocked_handler = ProviderHandler(make_app(True), http_request) + mocked_handler.path = WaterButlerPath('/test_path') + + mocked_handler.provider = MockProvider() + mocked_handler.resource = 'test_source_resource' + mocked_handler.metadata = MockFileMetadata() + + mocked_handler.dest_provider = MockProvider() + mocked_handler.dest_resource = 'test_dest_resource' + mocked_handler.dest_meta = MockFileMetadata() + + return mocked_handler + + +@pytest.fixture +def source_payload(handler): + return LogPayload(handler.resource, handler.provider, path=handler.path) + + +@pytest.fixture +def destination_payload(handler): + return LogPayload(handler.dest_resource, handler.provider, metadata=handler.dest_meta) + + +@pytest.fixture +def payload_path(handler): + return LogPayload(handler.resource, handler.provider, path=handler.path) + + +@pytest.fixture +def payload_metadata(handler): + return LogPayload(handler.resource, handler.provider, metadata=handler.metadata) + + +@pytest.fixture +def serialized_request(): + return { + 'request': { + 'url': 'http://127.0.0.1/v1/resources/test/providers/test/path/mock', + 'method': 'GET', + 'headers': {}, + 'time': 10 + }, + 'tech': { + 'ua': 'test', + 'ip': None + }, + 'referrer': { + 'url': None + }, + } diff --git a/tests/server/api/v1/test_handler.py b/tests/server/api/v1/test_handler.py new file mode 100644 index 000000000..fdead56ce --- /dev/null +++ b/tests/server/api/v1/test_handler.py @@ -0,0 +1,91 @@ +from unittest import mock + +import pytest + +from tests.utils import HandlerTestCase, MockProvider, MockFileMetadata, MockCoroutine +from tests.server.api.v1.fixtures import (handler, mock_time, http_request, + source_payload, destination_payload, + log_payload, payload_metadata, + payload_path, serialized_request) + + +class TestSendHook: + + @pytest.mark.parametrize('action', ['move', 'copy']) + @mock.patch('waterbutler.core.remote_logging.log_file_action') + def test_send_hook_cant_intra_move_copy(self, mocked_log_file_action, handler, action): + assert handler._send_hook(action) is None + mocked_log_file_action.assert_not_called() + + @pytest.mark.parametrize('action', ['move', 'copy']) + @mock.patch('waterbutler.core.remote_logging.log_file_action') + def test_send_hook_can_intra_move_copy( + self, + mocked_log_file_action, + handler, + action, + source_payload, + destination_payload, + serialized_request, + event_loop + ): + setattr(handler.provider, 'can_intra_' + action, mock.Mock(return_value=True)) + assert handler._send_hook(action) is None + mocked_log_file_action.assert_called_once_with( + action, + api_version='v1', + bytes_downloaded=0, + bytes_uploaded=0, + source=source_payload, + destination=destination_payload, + request=serialized_request + ) + + @pytest.mark.parametrize("action", ['create', 'create_folder', 'update']) + @mock.patch('waterbutler.core.remote_logging.log_file_action') + def test_send_hook_always_send_metadata( + self, + mocked_log_file_action, + handler, + action, + payload_metadata, + serialized_request, + event_loop + ): + assert handler._send_hook(action) is None + mocked_log_file_action.assert_called_once_with( + action, + api_version='v1', + bytes_downloaded=0, + bytes_uploaded=0, + source=payload_metadata, + destination=None, + request=serialized_request + ) + + @pytest.mark.parametrize("action", ['delete', 'download_file', 'download_zip', 'metadata']) + @mock.patch('waterbutler.core.remote_logging.log_file_action') + def test_send_hook_always_send_path( + self, + mocked_log_file_action, + handler, + action, + source_payload, + serialized_request, + event_loop + ): + assert handler._send_hook(action) is None + mocked_log_file_action.assert_called_once_with( + action, + api_version='v1', + bytes_downloaded=0, + bytes_uploaded=0, + source=source_payload, + destination=None, + request=serialized_request + ) + + @mock.patch('waterbutler.core.remote_logging.log_file_action') + def test_send_hook_invalid_action(self, mocked_log_file_action, handler, event_loop): + assert handler._send_hook('invalid_action') is None + mocked_log_file_action.assert_not_called() diff --git a/waterbutler/core/log_payload.py b/waterbutler/core/log_payload.py index b95e0a267..a9946feef 100644 --- a/waterbutler/core/log_payload.py +++ b/waterbutler/core/log_payload.py @@ -57,6 +57,10 @@ def serialize(self): return payload + def __eq__(self, other: object) -> bool: + # This is allows for easy comparisons via unit tests. + return isinstance(other, LogPayload) and self.serialize() == other.serialize() + @property def auth(self): """The auth object for the entity. Contains the callback_url.""" diff --git a/waterbutler/core/remote_logging.py b/waterbutler/core/remote_logging.py index e537f8d16..4eee7edcd 100644 --- a/waterbutler/core/remote_logging.py +++ b/waterbutler/core/remote_logging.py @@ -19,7 +19,7 @@ @utils.async_retry(retries=5, backoff=5) async def log_to_callback(action, source=None, destination=None, start_time=None, errors=[]): """PUT a logging payload back to the callback given by the auth provider.""" - if action in ('download_file', 'download_zip'): + if action in ('download_file', 'download_zip', 'metadata'): logger.debug('Not logging for {} action'.format(action)) return @@ -41,10 +41,6 @@ async def log_to_callback(action, source=None, destination=None, start_time=None log_payload['metadata'] = source.serialize() log_payload['provider'] = log_payload['metadata']['provider'] - if action in ('download_file', 'download_zip'): - logger.info('Not logging for {} action'.format(action)) - return - resp = await utils.send_signed_request('PUT', auth['callback_url'], log_payload) resp_data = await resp.read() diff --git a/waterbutler/server/api/v1/provider/__init__.py b/waterbutler/server/api/v1/provider/__init__.py index d293d87a4..c8dc80b7f 100644 --- a/waterbutler/server/api/v1/provider/__init__.py +++ b/waterbutler/server/api/v1/provider/__init__.py @@ -150,12 +150,10 @@ def on_finish(self): if any((method in ('HEAD', 'OPTIONS'), status == 202, status > 302, status < 200)): return - if method == 'GET' and 'meta' in self.request.query_arguments: - return - # Done here just because method is defined action = { - 'GET': lambda: 'download_file' if self.path.is_file else 'download_zip', + 'GET': lambda: 'metadata' if 'meta' in self.request.query_arguments else ( + 'download_file' if self.path.is_file else 'download_zip'), 'PUT': lambda: ('create' if self.target_path.is_file else 'create_folder') if status == 201 else 'update', 'POST': lambda: 'move' if self.json['action'] == 'rename' else self.json['action'], 'DELETE': lambda: 'delete' @@ -180,7 +178,7 @@ def _send_hook(self, action): ) elif action in ('create', 'create_folder', 'update'): source = LogPayload(self.resource, self.provider, metadata=self.metadata) - elif action in ('delete', 'download_file', 'download_zip'): + elif action in ('delete', 'download_file', 'download_zip', 'metadata'): source = LogPayload(self.resource, self.provider, path=self.path) else: return