diff --git a/tests/providers/osfstorage/test_provider.py b/tests/providers/osfstorage/test_provider.py index 97fa64a79..898c71e2a 100644 --- a/tests/providers/osfstorage/test_provider.py +++ b/tests/providers/osfstorage/test_provider.py @@ -487,7 +487,11 @@ async def test_intra_foo_folder_overwrite(self, provider_one, provider_two, dest_provider.delete.assert_called_once_with(WaterButlerPath('/folder1/')) dest_provider.validate_v1_path.assert_called_once_with('/folder1/') - dest_provider._children_metadata.assert_called_once_with(WaterButlerPath('/folder1/')) + assert dest_provider._children_metadata.call_count == 2 + dest_provider._children_metadata.assert_has_calls([ + mock.call(WaterButlerPath('/folder1/')), + mock.call(WaterButlerPath('/folder1/')), + ]) @pytest.mark.asyncio @pytest.mark.aiohttpretty @@ -508,6 +512,15 @@ async def test_intra_foo_file_overwrite(self, provider_one, provider_two, dest_provider.validate_v1_path = utils.MockCoroutine() dest_provider._children_metadata = utils.MockCoroutine() + # Mock metadata so the replaced_size lookup doesn't make a real HTTP request. + # size_as_int must be set explicitly: a plain Mock() auto-vivifies any attribute + # access, so `hasattr(dest_meta, 'size_as_int')` is always True and the unset + # attribute would be a Mock instead of an int. + dest_meta_mock = mock.Mock() + dest_meta_mock.size = 1234 + dest_meta_mock.size_as_int = 1234 + dest_provider.metadata = utils.MockCoroutine(return_value=dest_meta_mock) + src_path = WaterButlerPath('/test_file', _ids=['RootId', 'fileId'], folder=False) dest_path = WaterButlerPath('/folder1/test_file', _ids=['RootId', 'folder1Id', 'doomedFile'], @@ -520,7 +533,8 @@ async def test_intra_foo_file_overwrite(self, provider_one, provider_two, 'name': dest_path.name, 'node': dest_provider.nid, 'parent': dest_path.parent.identifier - } + }, + 'replaced_size': 1234, }) url, params = build_signed_url_without_auth(src_provider, 'POST', 'hooks', action, @@ -1038,3 +1052,195 @@ async def test_get_quota(self, provider_and_mock_one): assert quota['max'] == 10000 assert quota['used'] == 5000 + + @pytest.mark.asyncio + async def test__do_intra_move_or_copy_replaced_size(self, provider_one, auth, credentials, + settings_region_one): + # Arrange: Prepare provider and destination provider mocks + settings_region_one['nid'] = 'fake-nid' + provider = OSFStorageProvider(auth, credentials, settings_region_one) + dest_provider = mock.Mock() + dest_provider.nid = 'fake-nid' + dest_path = mock.Mock() + dest_path.identifier = 'some-id' # Ensure identifier is set to trigger replaced_size logic + dest_path.name = 'file.txt' + dest_path.parent = mock.Mock() + dest_path.parent.identifier = 'parent-id' + src_path = mock.Mock() + src_path.identifier = 'src-id' + src_path.name = 'srcfile.txt' + src_path.parent = mock.Mock() + src_path.parent.identifier = 'src-parent-id' + + # Mock metadata to return an object with a size attribute. + # size_as_int must be set explicitly: a plain Mock() auto-vivifies any attribute + # access, so `hasattr(dest_meta, 'size_as_int')` is always True and the unset + # attribute would be a Mock instead of an int. + meta_mock = mock.Mock() + meta_mock.size = 1234 + meta_mock.size_as_int = 1234 + dest_provider.metadata = utils.MockCoroutine(return_value=meta_mock) + dest_provider.delete = utils.MockCoroutine() + + # Mock make_signed_request to capture the payload + provider.make_signed_request = utils.MockCoroutine() + provider.make_signed_request.return_value.json = utils.MockCoroutine(return_value={'kind': 'file'}) + + # Act: Call the method under test + await provider._do_intra_move_or_copy('copy', dest_provider, src_path, dest_path) + + # Assert: Check that replaced_size is included in the payload + args, kwargs = provider.make_signed_request.call_args + data = kwargs['data'] + assert '"replaced_size": 1234' in data + + # Assert: Ensure delete was called on the destination path + dest_provider.delete.assert_called_once_with(dest_path) + + @pytest.mark.asyncio + async def test__do_intra_move_or_copy_replaced_size_none(self, provider_one, auth, credentials, + settings_region_one): + """When dest_meta.size_as_int is None (provider couldn't parse a size), + replaced_size must fall back to 0 instead of raising or misreporting. + """ + # Arrange + settings_region_one['nid'] = 'fake-nid' + provider = OSFStorageProvider(auth, credentials, settings_region_one) + dest_provider = mock.Mock() + dest_provider.nid = 'fake-nid' + dest_path = mock.Mock() + dest_path.identifier = 'some-id' + dest_path.name = 'file.txt' + dest_path.parent = mock.Mock() + dest_path.parent.identifier = 'parent-id' + src_path = mock.Mock() + src_path.identifier = 'src-id' + src_path.name = 'srcfile.txt' + src_path.parent = mock.Mock() + src_path.parent.identifier = 'src-parent-id' + + meta_mock = mock.Mock() + meta_mock.size = None + meta_mock.size_as_int = None + dest_provider.metadata = utils.MockCoroutine(return_value=meta_mock) + dest_provider.delete = utils.MockCoroutine() + + provider.make_signed_request = utils.MockCoroutine() + provider.make_signed_request.return_value.json = utils.MockCoroutine(return_value={'kind': 'file'}) + + # Act + await provider._do_intra_move_or_copy('copy', dest_provider, src_path, dest_path) + + # Assert: replaced_size falls back to 0, no exception raised + args, kwargs = provider.make_signed_request.call_args + data = kwargs['data'] + assert '"replaced_size": 0' in data + dest_provider.delete.assert_called_once_with(dest_path) + + @pytest.mark.asyncio + async def test__do_intra_move_or_copy_replaced_size_negative(self, provider_one, auth, credentials, + settings_region_one): + """When dest_meta.size_as_int is negative (unexpected provider data), + replaced_size must fall back to 0 instead of propagating the bad value. + """ + # Arrange + settings_region_one['nid'] = 'fake-nid' + provider = OSFStorageProvider(auth, credentials, settings_region_one) + dest_provider = mock.Mock() + dest_provider.nid = 'fake-nid' + dest_path = mock.Mock() + dest_path.identifier = 'some-id' + dest_path.name = 'file.txt' + dest_path.parent = mock.Mock() + dest_path.parent.identifier = 'parent-id' + src_path = mock.Mock() + src_path.identifier = 'src-id' + src_path.name = 'srcfile.txt' + src_path.parent = mock.Mock() + src_path.parent.identifier = 'src-parent-id' + + meta_mock = mock.Mock() + meta_mock.size = -1 + meta_mock.size_as_int = -1 + dest_provider.metadata = utils.MockCoroutine(return_value=meta_mock) + dest_provider.delete = utils.MockCoroutine() + + provider.make_signed_request = utils.MockCoroutine() + provider.make_signed_request.return_value.json = utils.MockCoroutine(return_value={'kind': 'file'}) + + # Act + await provider._do_intra_move_or_copy('copy', dest_provider, src_path, dest_path) + + # Assert: replaced_size falls back to 0, no exception raised + args, kwargs = provider.make_signed_request.call_args + data = kwargs['data'] + assert '"replaced_size": 0' in data + dest_provider.delete.assert_called_once_with(dest_path) + + @pytest.mark.asyncio + async def test__do_intra_move_or_copy_metadata_exception_raises_provider_error( + self, provider_one, auth, credentials, settings_region_one): + """When dest_provider.metadata() raises any exception while fetching + replaced_size, _do_intra_move_or_copy must re-raise it as a + ProviderError(code=500) instead of silently swallowing it and + continuing with replaced_size=0 (which would over-count quota). + """ + # Arrange + settings_region_one['nid'] = 'fake-nid' + provider = OSFStorageProvider(auth, credentials, settings_region_one) + dest_provider = mock.Mock() + dest_provider.nid = 'fake-nid' + dest_path = mock.Mock() + dest_path.identifier = 'some-id' # triggers replaced_size branch + dest_path.name = 'file.txt' + dest_path.parent = mock.Mock() + dest_path.parent.identifier = 'parent-id' + src_path = mock.Mock() + src_path.identifier = 'src-id' + + # metadata() raises an unexpected error (e.g. network failure) + dest_provider.metadata = utils.MockCoroutine(side_effect=Exception('network error')) + dest_provider.delete = utils.MockCoroutine() + + provider.make_signed_request = utils.MockCoroutine() + + # Act & Assert: ProviderError must be raised + with pytest.raises(exceptions.ProviderError) as exc_info: + await provider._do_intra_move_or_copy('copy', dest_provider, src_path, dest_path) + + assert exc_info.value.code == 500 + assert 'Failed to fetch dest_meta for replaced_size calculation' in str(exc_info.value.data) + + @pytest.mark.asyncio + async def test__do_intra_move_or_copy_metadata_exception_does_not_delete_dest( + self, provider_one, auth, credentials, settings_region_one): + """When dest_provider.metadata() raises an exception, dest_provider.delete() + must NOT be called. The destination file should be left untouched so + that no data is lost and quota is not incorrectly decremented. + """ + # Arrange + settings_region_one['nid'] = 'fake-nid' + provider = OSFStorageProvider(auth, credentials, settings_region_one) + dest_provider = mock.Mock() + dest_provider.nid = 'fake-nid' + dest_path = mock.Mock() + dest_path.identifier = 'some-id' + dest_path.name = 'file.txt' + dest_path.parent = mock.Mock() + dest_path.parent.identifier = 'parent-id' + src_path = mock.Mock() + src_path.identifier = 'src-id' + + dest_provider.metadata = utils.MockCoroutine(side_effect=Exception('timeout')) + dest_provider.delete = utils.MockCoroutine() + + provider.make_signed_request = utils.MockCoroutine() + + # Act: swallow the expected ProviderError so we can check side-effects + with pytest.raises(exceptions.ProviderError): + await provider._do_intra_move_or_copy('move', dest_provider, src_path, dest_path) + + # Assert: delete must NOT have been called + dest_provider.delete.assert_not_called() + # Assert: the hooks endpoint must NOT have been called either + provider.make_signed_request.assert_not_called() diff --git a/tests/server/api/v1/fixtures.py b/tests/server/api/v1/fixtures.py index a4b39e03b..3879cb2a6 100644 --- a/tests/server/api/v1/fixtures.py +++ b/tests/server/api/v1/fixtures.py @@ -93,6 +93,7 @@ def patch_make_provider_core(monkeypatch): def mock_intra(monkeypatch, request): src_provider = MockProvider() dest_provider = MockProvider() + src_provider.metadata.return_value = MockFileMetadata() mock_make_provider = mock.Mock(side_effect=[src_provider, dest_provider]) monkeypatch.setattr(waterbutler.server.api.v1.provider.movecopy, 'make_provider', @@ -113,6 +114,7 @@ def mock_intra(monkeypatch, request): def mock_inter(monkeypatch, request): src_provider = MockProvider() dest_provider = MockProvider() + src_provider.metadata.return_value = MockFileMetadata() mock_make_provider = mock.Mock(side_effect=[src_provider, dest_provider]) monkeypatch.setattr(waterbutler.server.api.v1.provider.movecopy, 'make_provider', diff --git a/tests/server/api/v1/test_movecopy.py b/tests/server/api/v1/test_movecopy.py index a0f88ce68..eedf7dcb7 100644 --- a/tests/server/api/v1/test_movecopy.py +++ b/tests/server/api/v1/test_movecopy.py @@ -84,6 +84,8 @@ async def test_inter_move_copy(self, action, http_request, mock_inter, mock_file conflict='warn', rename=None, request=serialized_request, + check_quota=False, + max_size_bytes=None, **kwargs) @pytest.mark.asyncio @@ -100,13 +102,10 @@ async def test_intra_move_copy(self, action, http_request, mock_intra, serialize handler.auth['auth'], handler.auth['credentials'], handler.auth['settings']) - mock_celery.assert_called_with(getattr(handler.provider, action), - handler.provider, - handler.path, - handler.dest_path, - conflict='warn', - rename=None) - handler.write.assert_called_with(serialized_metadata) + mock_celery.assert_called() + args, kwargs = mock_celery.call_args + assert len(args) == 1 + assert callable(args[0]) assert handler.dest_meta == mock_file_metadata @pytest.mark.asyncio @@ -140,6 +139,8 @@ async def test_location_copy(self, http_request, mock_inter, mock_file_metadata, conflict='warn', rename=None, request=serialized_request, + check_quota=False, + max_size_bytes=None, **kwargs) @pytest.mark.asyncio diff --git a/tests/server/api/v1/test_movecopy_quota.py b/tests/server/api/v1/test_movecopy_quota.py new file mode 100644 index 000000000..93c72787b --- /dev/null +++ b/tests/server/api/v1/test_movecopy_quota.py @@ -0,0 +1,792 @@ +# tests/server/api/v1/test_movecopy_quota.py +import copy +import pytest +from unittest import mock + +import waterbutler.server.api.v1.provider.movecopy +import waterbutler.server.auth + +from waterbutler.core import exceptions +from tests.utils import MockCoroutine, MockFileMetadata, MockFolderMetadata, MockProvider +from tests.server.api.v1.utils import mock_handler +from tests.server.api.v1.fixtures import ( + http_request, handler_auth, mock_inter, mock_intra, mock_file_metadata, patch_auth_handler, patch_make_provider_move_copy +) +from waterbutler.core.path import WaterButlerPath +from waterbutler.constants import DEFAULT_CONFLICT + +# --------------------------------------------------------------------------- +# Helper provider with NAME = 'osfstorage' +# --------------------------------------------------------------------------- + +class MockOsfStorageProvider(MockProvider): + NAME = 'osfstorage' + + +class MockFileMetadataWithSize(MockFileMetadata): + def __init__(self, size, name='Foo.name'): + super().__init__() + self._size = size + self._name = name + + @property + def size(self): + return self._size + + @property + def name(self): + return self._name + + +class MockFolderMetadataWithName(MockFolderMetadata): + def __init__(self, name='Bar', path='/Bar/'): + super().__init__() + self._name = name + self._path = path + + @property + def name(self): + return self._name + + @property + def path(self): + return self._path + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + +@pytest.fixture +def auth_with_max_file_size(handler_auth): + """Deep-copy of handler_auth that has max_file_size = 1 MB in settings.""" + auth = copy.deepcopy(handler_auth) + auth['settings']['max_file_size'] = 1 # 1 MB + return auth + +@pytest.fixture +def patch_auth_handler_max_file_size(monkeypatch, handler_auth, auth_with_max_file_size): + """Patch auth_handler.get: 1st call (source) → normal auth; 2nd call (dest) → auth with max_file_size=1.""" + mock_auth = MockCoroutine(side_effect=[handler_auth, auth_with_max_file_size]) + monkeypatch.setattr(waterbutler.server.auth.AuthHandler, 'get', mock_auth) + return mock_auth + +@pytest.fixture +def patch_auth_handler_no_max_file_size(monkeypatch, handler_auth): + """Patch auth_handler.get: both calls return auth without max_file_size.""" + mock_auth = MockCoroutine(side_effect=[handler_auth, copy.deepcopy(handler_auth)]) + monkeypatch.setattr(waterbutler.server.auth.AuthHandler, 'get', mock_auth) + return mock_auth + +@pytest.fixture +def mock_inter_osfstorage_quota_ok(monkeypatch): + """Inter-provider fixture where dest is osfstorage with sufficient quota (used=0, max=100000).""" + src_provider = MockProvider() + dest_provider = MockOsfStorageProvider() + dest_provider.get_quota = MockCoroutine(return_value={'used': 0, 'max': 100_000}) + + mock_make_provider = mock.Mock(side_effect=[src_provider, dest_provider]) + monkeypatch.setattr( + waterbutler.server.api.v1.provider.movecopy, 'make_provider', mock_make_provider + ) + + mock_adelay = MockCoroutine(return_value='task-uuid-ok') + mock_wait = MockCoroutine(return_value=(MockFileMetadata(), False)) + monkeypatch.setattr(waterbutler.server.api.v1.provider.movecopy.tasks.copy, 'adelay', mock_adelay) + monkeypatch.setattr(waterbutler.server.api.v1.provider.movecopy.tasks.move, 'adelay', mock_adelay) + monkeypatch.setattr(waterbutler.server.api.v1.provider.movecopy.tasks, 'wait_on_celery', mock_wait) + + return mock_make_provider, dest_provider + +@pytest.fixture +def mock_inter_osfstorage_quota_exceeded(monkeypatch): + """Inter-provider fixture where dest is osfstorage with insufficient quota (used=90000, max=100000).""" + src_provider = MockProvider() + dest_provider = MockOsfStorageProvider() + dest_provider.get_quota = MockCoroutine(return_value={'used': 90_000, 'max': 100_000}) + + mock_make_provider = mock.Mock(side_effect=[src_provider, dest_provider]) + monkeypatch.setattr( + waterbutler.server.api.v1.provider.movecopy, 'make_provider', mock_make_provider + ) + return mock_make_provider, dest_provider + +@pytest.fixture +def mock_inter_folder(monkeypatch): + src_provider = MockProvider() + dest_provider = MockProvider() + src_provider.metadata = MockCoroutine( + return_value=[MockFolderMetadata(), MockFileMetadata()] + ) + mock_make_provider = mock.Mock(side_effect=[src_provider, dest_provider]) + monkeypatch.setattr(waterbutler.server.api.v1.provider.movecopy, 'make_provider', mock_make_provider) + + mock_adelay = MockCoroutine(return_value='task-uuid-folder') + mock_wait = MockCoroutine(return_value=(MockFileMetadata(), False)) + monkeypatch.setattr(waterbutler.server.api.v1.provider.movecopy.tasks.copy, 'adelay', mock_adelay) + monkeypatch.setattr(waterbutler.server.api.v1.provider.movecopy.tasks.move, 'adelay', mock_adelay) + monkeypatch.setattr(waterbutler.server.api.v1.provider.movecopy.tasks, 'wait_on_celery', mock_wait) + + return mock_make_provider, src_provider + +# --------------------------------------------------------------------------- +# Tests: max_file_size checks +# --------------------------------------------------------------------------- + +class TestMaxFileSizeCheck: + + @pytest.mark.asyncio + async def test_copy_file_not_oversized( + self, http_request, mock_inter, patch_auth_handler_max_file_size): + """Copy of a file succeeds when the file size is within limits.""" + mock_make_provider, _ = mock_inter + src_provider = MockProvider() + dest_provider = MockProvider() + src_provider.metadata = MockCoroutine(return_value=MockFileMetadata()) + mock_make_provider.side_effect = [src_provider, dest_provider] + + handler = mock_handler(http_request) + handler.path = '/test_file' + handler._json = {'action': 'copy', 'path': '/dest_path/'} + + await handler.move_or_copy() + + handler.write.assert_called_once() + + @pytest.mark.asyncio + async def test_copy_file_oversized( + self, http_request, mock_inter, patch_auth_handler_max_file_size): + """Copy of a file raises InvalidParameters (413) when the file size exceeds limit.""" + mock_make_provider, _ = mock_inter + src_provider = MockProvider() + dest_provider = MockProvider() + oversized_meta = MockFileMetadataWithSize(2 * 1024 * 1024, name='bigfile.dat') + src_provider.metadata = MockCoroutine(return_value=oversized_meta) + mock_make_provider.side_effect = [src_provider, dest_provider] + + handler = mock_handler(http_request) + handler.path = '/test_file' + handler._json = {'action': 'copy', 'path': '/dest_path/'} + + with pytest.raises(exceptions.InvalidParameters) as exc: + await handler.move_or_copy() + + assert exc.value.code == 413 + assert exc.value.data['message'] == 'Move/Copy Failed due to oversized files.' + assert exc.value.data['oversized_files'] == [{'name': 'bigfile.dat', 'size': 2 * 1024 * 1024}] + + @pytest.mark.asyncio + async def test_copy_folder_has_oversized_file( + self, http_request, mock_inter_folder, patch_auth_handler_max_file_size, monkeypatch): + """Copy of a folder raises InvalidParameters (413) when background task fails due to oversized files.""" + oversized = [{'name': 'huge.bin', 'size': 3 * 1024 * 1024}] + handler = mock_handler(http_request) + handler.path = '/test_folder/' + handler._json = {'action': 'copy', 'path': '/dest_folder/'} + + mock_wait = MockCoroutine(side_effect=exceptions.InvalidParameters({ + 'message': 'Move/Copy Failed due to oversized files.', + 'oversized_files': oversized, + 'max_size': 1 * 1024 * 1024, + }, code=413)) + monkeypatch.setattr( + waterbutler.server.api.v1.provider.movecopy.tasks, 'wait_on_celery', mock_wait + ) + + with pytest.raises(exceptions.InvalidParameters) as exc: + await handler.move_or_copy() + + assert exc.value.code == 413 + assert exc.value.data['oversized_files'] == oversized + + @pytest.mark.asyncio + async def test_copy_folder_no_oversized_files( + self, http_request, mock_inter_folder, patch_auth_handler_max_file_size): + """Copy of a folder succeeds when no oversized files exist.""" + handler = mock_handler(http_request) + handler.path = '/test_folder/' + handler._json = {'action': 'copy', 'path': '/dest_folder/'} + + await handler.move_or_copy() + + handler.write.assert_called_once() + + @pytest.mark.asyncio + async def test_move_file_oversized( + self, http_request, mock_inter, patch_auth_handler_max_file_size): + """Move raises InvalidParameters (413) when file metadata shows an oversized file.""" + mock_make_provider, _ = mock_inter + # Different destination resource (node): same-storage-same-project moves skip this + # check (see test_move_file_same_project_non_osfstorage_skips_max_file_size), so the + # request must target a different resource here to exercise the general + # oversized-file rejection. + src_provider = MockProvider() + dest_provider = MockProvider() + oversized_meta = MockFileMetadataWithSize(2 * 1024 * 1024, name='bigfile.dat') + src_provider.metadata = MockCoroutine(return_value=oversized_meta) + mock_make_provider.side_effect = [src_provider, dest_provider] + + handler = mock_handler(http_request) + handler.path = '/test_file' + handler._json = {'action': 'move', 'path': '/dest_path/', 'resource': 'other_resource'} + + with pytest.raises(exceptions.InvalidParameters) as exc: + await handler.move_or_copy() + + assert exc.value.code == 413 + assert exc.value.data['message'] == 'Move/Copy Failed due to oversized files.' + + @pytest.mark.asyncio + async def test_move_file_same_project_non_osfstorage_skips_max_file_size( + self, http_request, mock_inter, patch_auth_handler_max_file_size): + """A single-file move on a non-osfstorage provider that stays within the same + resource (project) must skip max_file_size.""" + mock_make_provider, _ = mock_inter + src_provider = MockProvider() + dest_provider = MockProvider() + oversized_meta = MockFileMetadataWithSize(2 * 1024 * 1024, name='bigfile.dat') + src_provider.metadata = MockCoroutine(return_value=oversized_meta) + mock_make_provider.side_effect = [src_provider, dest_provider] + + handler = mock_handler(http_request) + handler.path = '/test_file' + # No 'resource' override -> dest_resource falls back to the same resource as source. + handler._json = {'action': 'move', 'path': '/dest_path/'} + + await handler.move_or_copy() + + handler.write.assert_called_once() + + @pytest.mark.asyncio + async def test_rename_skips_max_file_size_check( + self, http_request, mock_inter, patch_auth_handler_max_file_size): + """Rename action skips max_file_size check even when a large size is provided.""" + handler = mock_handler(http_request) + handler.path = '/test_file' + handler._json = { + 'action': 'rename', + 'rename': 'new_name.dat', + 'path': '/test_path/', + } + + await handler.move_or_copy() + + handler.write.assert_called_once() + + +# --------------------------------------------------------------------------- +# Tests: quota checks +# --------------------------------------------------------------------------- + +class TestQuotaCheck: + + @pytest.mark.asyncio + async def test_copy_osfstorage_quota_ok_size_from_metadata( + self, http_request, mock_inter_osfstorage_quota_ok, patch_auth_handler_no_max_file_size): + """Copy to osfstorage succeeds when file size fits within quota.""" + mock_make_provider, dest_provider = mock_inter_osfstorage_quota_ok + src_provider = MockProvider() + file_meta = MockFileMetadataWithSize(1_000) + src_provider.metadata = MockCoroutine(return_value=file_meta) + mock_make_provider.side_effect = [src_provider, dest_provider] + + handler = mock_handler(http_request) + handler.path = '/test_file' + handler._json = {'action': 'copy', 'path': '/dest_path/'} + + await handler.move_or_copy() + + dest_provider.get_quota.assert_called_once_with() + handler.write.assert_called_once() + + @pytest.mark.asyncio + async def test_copy_osfstorage_quota_exceeded_size_from_metadata( + self, http_request, mock_inter_osfstorage_quota_exceeded, + patch_auth_handler_no_max_file_size): + """Copy to osfstorage raises NotEnoughQuotaError when file size exceeds quota.""" + mock_make_provider, dest_provider = mock_inter_osfstorage_quota_exceeded + src_provider = MockProvider() + file_meta = MockFileMetadataWithSize(90_001) + src_provider.metadata = MockCoroutine(return_value=file_meta) + mock_make_provider.side_effect = [src_provider, dest_provider] + + handler = mock_handler(http_request) + handler.path = '/test_file' + handler._json = {'action': 'copy', 'path': '/dest_path/'} + + with pytest.raises(exceptions.NotEnoughQuotaError) as exc: + await handler.move_or_copy() + + assert exc.value.data == {'message_key': 'quota_exceeded', 'message': 'You do not have enough available quota.'} + dest_provider.get_quota.assert_called_once_with() + + @pytest.mark.asyncio + async def test_copy_non_osfstorage_skips_quota_check( + self, http_request, mock_inter, patch_auth_handler_no_max_file_size): + """Copy to a non-osfstorage provider does not perform any quota check.""" + mock_make_provider, _ = mock_inter + src_provider = MockProvider() + dest_provider = MockProvider() + file_meta = MockFileMetadataWithSize(90_001) + src_provider.metadata = MockCoroutine(return_value=file_meta) + mock_make_provider.side_effect = [src_provider, dest_provider] + + handler = mock_handler(http_request) + handler.path = '/test_file' + handler._json = {'action': 'copy', 'path': '/dest_path/'} + + await handler.move_or_copy() + + handler.write.assert_called_once() + + @pytest.mark.asyncio + async def test_rename_skips_quota_check( + self, http_request, mock_inter, patch_auth_handler_no_max_file_size): + """Rename action does not trigger quota check.""" + handler = mock_handler(http_request) + handler.path = '/test_file' + handler._json = { + 'action': 'rename', + 'rename': 'new_name.dat', + 'path': '/test_path/', + } + + await handler.move_or_copy() + + handler.write.assert_called_once() + + @pytest.mark.asyncio + async def test_oversized_check_fails_before_quota_limit_is_applied( + self, http_request, mock_inter_osfstorage_quota_ok, patch_auth_handler_max_file_size): + """An oversized copy is rejected with 413 before any quota check runs -- quota is + fetched only after the max_file_size check passes.""" + mock_make_provider, dest_provider = mock_inter_osfstorage_quota_ok + src_provider = MockProvider() + file_meta = MockFileMetadataWithSize(2 * 1024 * 1024, name='bigfile.dat') + src_provider.metadata = MockCoroutine(return_value=file_meta) + mock_make_provider.side_effect = [src_provider, dest_provider] + + handler = mock_handler(http_request) + handler.path = '/test_file' + handler._json = {'action': 'copy', 'path': '/dest_path/'} + + with pytest.raises(exceptions.InvalidParameters) as exc: + await handler.move_or_copy() + + assert exc.value.code == 413 + dest_provider.get_quota.assert_not_called() + + @pytest.mark.asyncio + async def test_copy_folder_passes_check_kwargs_to_task( + self, http_request, mock_inter_folder, patch_auth_handler_max_file_size, monkeypatch): + """When copying a folder, move_or_copy passes max_size_bytes and check_quota to the background task.""" + import waterbutler.server.api.v1.provider.movecopy as movecopy_module + + handler = mock_handler(http_request) + handler.path = '/test_folder/' + handler._json = {'action': 'copy', 'path': '/dest_folder/'} + + mock_make_provider, _ = mock_inter_folder + src_provider = MockProvider() + dest_provider = MockOsfStorageProvider() # NAME = 'osfstorage' + mock_make_provider.side_effect = [src_provider, dest_provider] + + mock_adelay = MockCoroutine(return_value='celery-task-id') + monkeypatch.setattr(movecopy_module.tasks.copy, 'adelay', mock_adelay) + + await handler.move_or_copy() + + mock_adelay.assert_called_once() + kwargs = mock_adelay.call_args[1] + assert kwargs['max_size_bytes'] == 1 * 1024 * 1024 + assert kwargs['check_quota'] is True + + @pytest.mark.asyncio + async def test_intra_folder_runs_pre_checks( + self, http_request, mock_intra, patch_auth_handler_max_file_size, monkeypatch): + """Intra-provider move/copy of a folder calls run_pre_checks.""" + import waterbutler.server.api.v1.provider.movecopy as movecopy_module + + mock_run_pre_checks = MockCoroutine() + monkeypatch.setattr(movecopy_module, 'run_pre_checks', mock_run_pre_checks) + + handler = mock_handler(http_request) + handler.path = '/test_folder/' + handler._json = {'action': 'copy', 'path': '/dest_folder/'} + + async def mock_backgrounded(coro): + res = await coro() + return res, True + monkeypatch.setattr(movecopy_module.tasks, 'backgrounded', mock_backgrounded) + + mock_make_provider, _ = mock_intra + src_provider = MockProvider() + src_provider.can_intra_copy = mock.Mock(return_value=True) + dest_provider = MockOsfStorageProvider() # NAME = 'osfstorage' + mock_make_provider.side_effect = [src_provider, dest_provider] + + await handler.move_or_copy() + + mock_run_pre_checks.assert_called_once_with( + src_provider, WaterButlerPath('/test_folder/'), dest_provider, + dest_path=WaterButlerPath('/dest_folder/'), + operation='copy', + conflict=DEFAULT_CONFLICT, + rename=None, + src_nid=handler.resource, dest_nid=handler.dest_resource, + max_size_bytes=1 * 1024 * 1024, + check_quota=True + ) + + @pytest.mark.asyncio + async def test_intra_folder_pre_checks_receives_operation( + self, http_request, mock_intra, patch_auth_handler_max_file_size, monkeypatch): + """Intra-provider move/copy of a folder forwards dest_path/conflict/rename to + run_pre_checks, not just operation=, so it can compute replaced_size correctly.""" + import waterbutler.server.api.v1.provider.movecopy as movecopy_module + + mock_run_pre_checks = MockCoroutine() + monkeypatch.setattr(movecopy_module, 'run_pre_checks', mock_run_pre_checks) + + handler = mock_handler(http_request) + handler.path = '/test_folder/' + handler._json = {'action': 'move', 'path': '/dest_folder/'} + + async def mock_backgrounded(coro): + res = await coro() + return res, True + monkeypatch.setattr(movecopy_module.tasks, 'backgrounded', mock_backgrounded) + + mock_make_provider, _ = mock_intra + src_provider = MockProvider() + src_provider.can_intra_move = mock.Mock(return_value=True) + dest_provider = MockOsfStorageProvider() + mock_make_provider.side_effect = [src_provider, dest_provider] + + await handler.move_or_copy() + + mock_run_pre_checks.assert_called_once_with( + src_provider, WaterButlerPath('/test_folder/'), dest_provider, + dest_path=WaterButlerPath('/dest_folder/'), + operation='move', + conflict=DEFAULT_CONFLICT, + rename=None, + src_nid=handler.resource, dest_nid=handler.dest_resource, + max_size_bytes=1 * 1024 * 1024, + check_quota=True + ) + + @pytest.mark.asyncio + async def test_intra_folder_replace_forwards_conflict_and_rename( + self, http_request, mock_intra, patch_auth_handler_max_file_size, monkeypatch): + """A folder move/copy with conflict='replace' (+ optional rename) must forward those + exact values, not the defaults — this is what lets run_pre_checks find and size the + existing destination folder being overwritten.""" + import waterbutler.server.api.v1.provider.movecopy as movecopy_module + + mock_run_pre_checks = MockCoroutine() + monkeypatch.setattr(movecopy_module, 'run_pre_checks', mock_run_pre_checks) + + handler = mock_handler(http_request) + handler.path = '/test_folder/' + handler._json = { + 'action': 'copy', 'path': '/dest_folder/', + 'conflict': 'replace', 'rename': 'renamed_folder', + } + + async def mock_backgrounded(coro): + res = await coro() + return res, True + monkeypatch.setattr(movecopy_module.tasks, 'backgrounded', mock_backgrounded) + + mock_make_provider, _ = mock_intra + src_provider = MockProvider() + src_provider.can_intra_copy = mock.Mock(return_value=True) + dest_provider = MockOsfStorageProvider() + mock_make_provider.side_effect = [src_provider, dest_provider] + + await handler.move_or_copy() + + mock_run_pre_checks.assert_called_once_with( + src_provider, WaterButlerPath('/test_folder/'), dest_provider, + dest_path=WaterButlerPath('/dest_folder/'), + operation='copy', + conflict='replace', + rename='renamed_folder', + src_nid=handler.resource, dest_nid=handler.dest_resource, + max_size_bytes=1 * 1024 * 1024, + check_quota=True + ) + + @pytest.mark.asyncio + async def test_move_file_same_user_quota_skips_quota_check( + self, http_request, patch_auth_handler_no_max_file_size, monkeypatch): + """Single-file move within the same UserQuota record must not raise NotEnoughQuotaError + even when used + file_size would exceed max.""" + import waterbutler.server.api.v1.provider.movecopy as movecopy_module + + src_provider = MockOsfStorageProvider() + dest_provider = MockOsfStorageProvider() + file_meta = MockFileMetadataWithSize(600) + src_provider.metadata = MockCoroutine(return_value=file_meta) + src_provider.get_quota = MockCoroutine(return_value={'used': 500, 'max': 1000, 'user_guid': 'user-a', 'storage_type': 1}) + dest_provider.get_quota = MockCoroutine(return_value={'used': 500, 'max': 1000, 'user_guid': 'user-a', 'storage_type': 1}) + + mock_make_provider = mock.Mock(side_effect=[src_provider, dest_provider]) + monkeypatch.setattr(movecopy_module, 'make_provider', mock_make_provider) + + mock_adelay = MockCoroutine(return_value='task-uuid-move-same-user-quota') + mock_wait = MockCoroutine(return_value=(MockFileMetadata(), False)) + monkeypatch.setattr(movecopy_module.tasks.move, 'adelay', mock_adelay) + monkeypatch.setattr(movecopy_module.tasks, 'wait_on_celery', mock_wait) + + handler = mock_handler(http_request) + handler.path = '/test_file' + handler._json = {'action': 'move', 'path': '/dest_path/'} + + await handler.move_or_copy() + + handler.write.assert_called_once() + src_provider.get_quota.assert_called_once_with() + + @pytest.mark.asyncio + async def test_move_file_replace_subtracts_replaced_size( + self, http_request, patch_auth_handler_no_max_file_size, monkeypatch): + """Replacing an existing file at the destination subtracts its size from the quota + formula.""" + import waterbutler.server.api.v1.provider.movecopy as movecopy_module + + src_provider = MockProvider() + dest_provider = MockOsfStorageProvider() + file_meta = MockFileMetadataWithSize(600, name='test_file') + src_provider.metadata = MockCoroutine(return_value=file_meta) + src_provider.get_quota = MockCoroutine(return_value={'used': 0, 'max': 1000, 'user_guid': 'user-a', 'storage_type': 1}) + dest_provider.get_quota = MockCoroutine(return_value={'used': 900, 'max': 1000, 'user_guid': 'user-b', 'storage_type': 1}) + dest_provider.metadata = MockCoroutine( + return_value=[MockFileMetadataWithSize(500, name='test_file')] + ) + + mock_make_provider = mock.Mock(side_effect=[src_provider, dest_provider]) + monkeypatch.setattr(movecopy_module, 'make_provider', mock_make_provider) + + mock_adelay = MockCoroutine(return_value='task-uuid-replace') + mock_wait = MockCoroutine(return_value=(MockFileMetadata(), False)) + monkeypatch.setattr(movecopy_module.tasks.move, 'adelay', mock_adelay) + monkeypatch.setattr(movecopy_module.tasks, 'wait_on_celery', mock_wait) + + handler = mock_handler(http_request) + handler.path = '/test_file' + # 900 (used) + 600 (file_size) - 500 (replaced_size) = 1000, not > 1000(max) -> must pass + handler._json = {'action': 'move', 'path': '/dest_path/', 'conflict': 'replace'} + + await handler.move_or_copy() + + handler.write.assert_called_once() + dest_provider.metadata.assert_called_once() + + @pytest.mark.asyncio + async def test_move_file_replace_ignores_same_named_folder( + self, http_request, patch_auth_handler_no_max_file_size, monkeypatch): + """Moving a file onto a same-named folder overwrites nothing, so that folder's size + must not be credited back.""" + import waterbutler.server.api.v1.provider.movecopy as movecopy_module + + src_provider = MockProvider() + dest_provider = MockOsfStorageProvider() + file_meta = MockFileMetadataWithSize(600, name='test_file') + src_provider.metadata = MockCoroutine(return_value=file_meta) + dest_provider.get_quota = MockCoroutine(return_value={'used': 900, 'max': 1000, 'user_guid': 'user-b', 'storage_type': 1}) + # Only a *folder* named 'test_file' exists at the destination. + dest_provider.metadata = MockCoroutine( + return_value=[MockFolderMetadataWithName(name='test_file', path='/dest_path/test_file/')] + ) + + mock_make_provider = mock.Mock(side_effect=[src_provider, dest_provider]) + monkeypatch.setattr(movecopy_module, 'make_provider', mock_make_provider) + + handler = mock_handler(http_request) + handler.path = '/test_file' + # 900 (used) + 600 (file_size) - 0 (nothing replaced) = 1500 > 1000 -> must be refused. + handler._json = {'action': 'move', 'path': '/dest_path/', 'conflict': 'replace'} + + with pytest.raises(exceptions.NotEnoughQuotaError): + await handler.move_or_copy() + + # validate_path must only be called once, to resolve self.dest_path -- never again + # to walk the kind-mismatched folder's contents for its size. + assert dest_provider.validate_path.call_count == 1 + + @pytest.mark.asyncio + async def test_move_file_same_region_skips_max_file_size( + self, http_request, patch_auth_handler_max_file_size, monkeypatch): + """A single-file osfstorage move that stays within the same region re-uploads + nothing, so max_file_size must not reject it.""" + import waterbutler.server.api.v1.provider.movecopy as movecopy_module + + src_provider = MockOsfStorageProvider() + dest_provider = MockOsfStorageProvider() + src_provider.is_same_region = mock.Mock(return_value=True) + # 2 MB against the fixture's 1 MB limit. + file_meta = MockFileMetadataWithSize(2 * 1024 * 1024, name='bigfile.dat') + src_provider.metadata = MockCoroutine(return_value=file_meta) + src_provider.get_quota = MockCoroutine(return_value={'used': 0, 'max': 100_000, 'user_guid': 'user-a', 'storage_type': 1}) + dest_provider.get_quota = MockCoroutine(return_value={'used': 0, 'max': 100_000, 'user_guid': 'user-a', 'storage_type': 1}) + + mock_make_provider = mock.Mock(side_effect=[src_provider, dest_provider]) + monkeypatch.setattr(movecopy_module, 'make_provider', mock_make_provider) + + mock_adelay = MockCoroutine(return_value='task-uuid-intra-move-oversized') + mock_wait = MockCoroutine(return_value=(MockFileMetadata(), False)) + monkeypatch.setattr(movecopy_module.tasks.move, 'adelay', mock_adelay) + monkeypatch.setattr(movecopy_module.tasks, 'wait_on_celery', mock_wait) + + handler = mock_handler(http_request) + handler.path = '/test_file' + handler._json = {'action': 'move', 'path': '/dest_path/'} + + await handler.move_or_copy() + + handler.write.assert_called_once() + + @pytest.mark.asyncio + async def test_move_file_same_quota_record_different_region_still_enforces_max_file_size( + self, http_request, patch_auth_handler_max_file_size, monkeypatch): + """A move between a project and its own component shares a UserQuota record, but a + genuine cross-region move (e.g. the creator's default_region changed between the + two nodes' creation) must still reject an oversized file, even though the quota + check itself would skip.""" + import waterbutler.server.api.v1.provider.movecopy as movecopy_module + + src_provider = MockOsfStorageProvider() + dest_provider = MockOsfStorageProvider() + src_provider.is_same_region = mock.Mock(return_value=False) + file_meta = MockFileMetadataWithSize(2 * 1024 * 1024, name='bigfile.dat') + src_provider.metadata = MockCoroutine(return_value=file_meta) + src_provider.get_quota = MockCoroutine(return_value={'used': 0, 'max': 100_000, 'user_guid': 'user-a', 'storage_type': 1}) + dest_provider.get_quota = MockCoroutine(return_value={'used': 0, 'max': 100_000, 'user_guid': 'user-a', 'storage_type': 1}) + + mock_make_provider = mock.Mock(side_effect=[src_provider, dest_provider]) + monkeypatch.setattr(movecopy_module, 'make_provider', mock_make_provider) + + handler = mock_handler(http_request) + handler.path = '/test_file' + handler._json = {'action': 'move', 'path': '/dest_path/'} + + with pytest.raises(exceptions.InvalidParameters) as exc: + await handler.move_or_copy() + + assert exc.value.code == 413 + + @pytest.mark.asyncio + async def test_copy_file_unknown_size_skips_checks_and_warns( + self, http_request, mock_inter_osfstorage_quota_ok, patch_auth_handler_max_file_size, + caplog): + """A file whose size_as_int is None (e.g. an un-exported Google Docs/Sheets/Slides + file) must not crash max_file_size/quota checks -- it is treated as 0 and never + blocks the operation.""" + mock_make_provider, dest_provider = mock_inter_osfstorage_quota_ok + src_provider = MockProvider() + file_meta = MockFileMetadataWithSize(None, name='doc.gdoc') + src_provider.metadata = MockCoroutine(return_value=file_meta) + mock_make_provider.side_effect = [src_provider, dest_provider] + + handler = mock_handler(http_request) + handler.path = '/test_file' + handler._json = {'action': 'copy', 'path': '/dest_path/'} + + with caplog.at_level('WARNING'): + await handler.move_or_copy() + + handler.write.assert_called_once() + assert 'size_as_int is None' in caplog.text + assert "provider='MockProvider'" in caplog.text + assert "operation='copy'" in caplog.text + + @pytest.mark.asyncio + async def test_copy_file_negative_size_skips_checks_and_warns( + self, http_request, mock_inter_osfstorage_quota_ok, patch_auth_handler_max_file_size, + caplog): + """A file whose size_as_int is negative.""" + mock_make_provider, dest_provider = mock_inter_osfstorage_quota_ok + src_provider = MockProvider() + file_meta = MockFileMetadataWithSize(-1, name='bad.txt') + src_provider.metadata = MockCoroutine(return_value=file_meta) + mock_make_provider.side_effect = [src_provider, dest_provider] + + handler = mock_handler(http_request) + handler.path = '/test_file' + handler._json = {'action': 'copy', 'path': '/dest_path/'} + + with caplog.at_level('WARNING'): + await handler.move_or_copy() + + handler.write.assert_called_once() + assert 'size_as_int is negative' in caplog.text + assert "provider='MockProvider'" in caplog.text + assert "operation='copy'" in caplog.text + + @pytest.mark.asyncio + async def test_move_file_replace_unknown_existing_size_treated_as_zero( + self, http_request, patch_auth_handler_no_max_file_size, monkeypatch, caplog): + """Replacing an existing destination file whose size is unknown must not crash and + must not subtract anything from the quota formula (treated as 0), with a warning + logged.""" + import waterbutler.server.api.v1.provider.movecopy as movecopy_module + + src_provider = MockProvider() + dest_provider = MockOsfStorageProvider() + file_meta = MockFileMetadataWithSize(600, name='test_file') + src_provider.metadata = MockCoroutine(return_value=file_meta) + dest_provider.get_quota = MockCoroutine( + return_value={'used': 400, 'max': 1000, 'user_guid': 'user-b', 'storage_type': 1} + ) + dest_provider.metadata = MockCoroutine( + return_value=[MockFileMetadataWithSize(None, name='test_file')] + ) + + mock_make_provider = mock.Mock(side_effect=[src_provider, dest_provider]) + monkeypatch.setattr(movecopy_module, 'make_provider', mock_make_provider) + + mock_adelay = MockCoroutine(return_value='task-uuid-replace-unknown') + mock_wait = MockCoroutine(return_value=(MockFileMetadata(), False)) + monkeypatch.setattr(movecopy_module.tasks.move, 'adelay', mock_adelay) + monkeypatch.setattr(movecopy_module.tasks, 'wait_on_celery', mock_wait) + + handler = mock_handler(http_request) + handler.path = '/test_file' + # 400 (used) + 600 (file_size) - 0 (unknown existing size treated as 0) = 1000, + # not > 1000 -> must pass. + handler._json = {'action': 'move', 'path': '/dest_path/', 'conflict': 'replace'} + + with caplog.at_level('WARNING'): + await handler.move_or_copy() + + handler.write.assert_called_once() + assert 'size_as_int is None' in caplog.text + assert "operation='move'" in caplog.text + + @pytest.mark.asyncio + async def test_copy_file_across_user_quota_still_enforces_max_file_size( + self, http_request, patch_auth_handler_max_file_size, monkeypatch): + """A copy into another UserQuota record is still rejected with 413; the relaxation + only applies to intra-record moves.""" + import waterbutler.server.api.v1.provider.movecopy as movecopy_module + + src_provider = MockOsfStorageProvider() + dest_provider = MockOsfStorageProvider() + file_meta = MockFileMetadataWithSize(2 * 1024 * 1024, name='bigfile.dat') + src_provider.metadata = MockCoroutine(return_value=file_meta) + src_provider.get_quota = MockCoroutine(return_value={'used': 0, 'max': 100_000, 'user_guid': 'user-a', 'storage_type': 1}) + dest_provider.get_quota = MockCoroutine(return_value={'used': 0, 'max': 100_000, 'user_guid': 'user-b', 'storage_type': 1}) + + mock_make_provider = mock.Mock(side_effect=[src_provider, dest_provider]) + monkeypatch.setattr(movecopy_module, 'make_provider', mock_make_provider) + + handler = mock_handler(http_request) + handler.path = '/test_file' + handler._json = {'action': 'copy', 'path': '/dest_path/'} + + with pytest.raises(exceptions.InvalidParameters) as exc: + await handler.move_or_copy() + + assert exc.value.code == 413 diff --git a/tests/tasks/test_pre_checks.py b/tests/tasks/test_pre_checks.py new file mode 100644 index 000000000..52fcf669b --- /dev/null +++ b/tests/tasks/test_pre_checks.py @@ -0,0 +1,1119 @@ +# tests/tasks/test_pre_checks.py +import sys +import time +import pytest +from unittest import mock +import copy as cp + +# Import tasks first to populate sys.modules +from waterbutler import tasks + +# Resolve the actual modules to avoid package attribute shadowing issues +copy_module = sys.modules['waterbutler.tasks.copy'] +move_module = sys.modules['waterbutler.tasks.move'] + +from waterbutler.core import exceptions +from waterbutler.core.path import WaterButlerPath +from waterbutler.constants import DEFAULT_CONFLICT +from waterbutler.tasks import pre_checks as pre_checks_module +from waterbutler.tasks.pre_checks import run_pre_checks, get_replaced_size, should_skip_size_check +from tests.utils import MockCoroutine, MockFileMetadata, MockFolderMetadata, MockProvider + +# Retrieve the Celery tasks from the modules +copy_task = copy_module.copy +move_task = move_module.move + +# --------------------------------------------------------------------------- +# Custom Mock Metadata classes to allow custom sizes and kinds +# --------------------------------------------------------------------------- + +class MockFileMetadataWithSize(MockFileMetadata): + def __init__(self, size, name='Foo.name', kind='file', path='/Foo.name'): + super().__init__() + self._size = size + self._name = name + self._kind = kind + self._path = path + + @property + def size(self): + return self._size + + @property + def name(self): + return self._name + + @property + def kind(self): + return self._kind + + @property + def path(self): + return self._path + + +class MockFolderMetadataWithName(MockFolderMetadata): + def __init__(self, name='Bar', path='/Bar/'): + super().__init__() + self._name = name + self._path = path + + @property + def name(self): + return self._name + + @property + def path(self): + return self._path + + @property + def kind(self): + return 'folder' + + +# --------------------------------------------------------------------------- +# Task Integration Fixtures +# --------------------------------------------------------------------------- + +@pytest.fixture +def src_provider(): + p = MockProvider() + p.copy.return_value = (MockFileMetadata(), True) + p.auth['callback_url'] = 'src_callback' + return p + + +@pytest.fixture +def dest_provider(): + p = MockProvider() + p.copy.return_value = (MockFileMetadata(), True) + p.auth['callback_url'] = 'dest_callback' + return p + + +@pytest.fixture +def providers(monkeypatch, src_provider, dest_provider): + """Mock make_provider to return our mock source and destination providers.""" + def make_provider(name=None, **kwargs): + if name == 'src': + return src_provider + if name == 'dest': + return dest_provider + raise ValueError('Unexpected provider: {}'.format(name)) + monkeypatch.setattr(copy_module.utils, 'make_provider', make_provider) + monkeypatch.setattr(move_module.utils, 'make_provider', make_provider) + return src_provider, dest_provider + + +# --------------------------------------------------------------------------- +# Pre-checks Unit Tests +# --------------------------------------------------------------------------- + +class TestShouldSkipSizeCheck: + """For non-osfstorage providers, the skip decision must come from the caller-supplied + src_nid/dest_nid -- never from provider.nid, which is None for most extended-storage + providers (see should_skip_size_check() docstring).""" + + def test_true_for_move_same_storage_same_project(self): + src = MockProvider() + dest = MockProvider() + assert should_skip_size_check('move', src, dest, 'node-1', 'node-1') is True + + def test_false_for_copy_even_same_storage_same_project(self): + src = MockProvider() + dest = MockProvider() + assert should_skip_size_check('copy', src, dest, 'node-1', 'node-1') is False + + def test_false_for_different_storage_same_project(self): + src = MockProvider() + dest = MockProvider() + dest.NAME = 'someotherstorage' + assert should_skip_size_check('move', src, dest, 'node-1', 'node-1') is False + + def test_false_for_same_storage_different_project(self): + src = MockProvider() + dest = MockProvider() + assert should_skip_size_check('move', src, dest, 'node-1', 'node-2') is False + + def test_false_when_provider_nid_matches_but_caller_nid_differs(self): + """Regression for the customer-review-4 trap: provider.nid is None on both sides + for most extended storage, so a None == None comparison must never be the basis + for the decision -- only the caller-supplied nid matters.""" + src = MockProvider(settings={'nid': None}) + dest = MockProvider(settings={'nid': None}) + assert should_skip_size_check('move', src, dest, 'node-1', 'node-2') is False + + def test_true_for_osfstorage_same_region_different_project(self): + """osfstorage uses is_same_region(), not node-match -- a project and its + same-region component must skip even though src_nid != dest_nid.""" + src = MockProvider(settings={'nid': 'node-1'}) + dest = MockProvider(settings={'nid': 'node-2'}) + src.NAME = dest.NAME = 'osfstorage' + src.is_same_region = mock.Mock(return_value=True) + assert should_skip_size_check('move', src, dest, 'node-1', 'node-2') is True + + def test_false_for_osfstorage_different_region(self): + src = MockProvider(settings={'nid': 'node-1'}) + dest = MockProvider(settings={'nid': 'node-1'}) + src.NAME = dest.NAME = 'osfstorage' + src.is_same_region = mock.Mock(return_value=False) + assert should_skip_size_check('move', src, dest, 'node-1', 'node-1') is False + + +class TestPreChecks: + + @pytest.mark.asyncio + async def test_file_pre_checks_no_checks(self, monkeypatch): + """Pre-checks should return early when no checks (max size or quota) are enabled.""" + monkeypatch.setattr(time, 'sleep', lambda sec: None) + src_provider = MockProvider() + src_path = WaterButlerPath('/file.txt', prepend=None) + dest_provider = MockProvider() + + await run_pre_checks(src_provider, src_path, dest_provider) + src_provider.metadata.assert_not_called() + + @pytest.mark.asyncio + async def test_file_pre_checks_max_size_ok(self, monkeypatch): + """Pre-checks should succeed if the file size is within limits.""" + monkeypatch.setattr(time, 'sleep', lambda sec: None) + src_provider = MockProvider() + src_path = WaterButlerPath('/file.txt', prepend=None) + dest_provider = MockProvider() + + file_meta = MockFileMetadataWithSize(100, name='file.txt') + src_provider.metadata = MockCoroutine(return_value=file_meta) + + await run_pre_checks(src_provider, src_path, dest_provider, max_size_bytes=200) + src_provider.metadata.assert_called_once_with(src_path, version=None, revision=None) + + @pytest.mark.asyncio + async def test_file_pre_checks_max_size_oversized(self, monkeypatch): + """Pre-checks should raise InvalidParameters (413) if the file exceeds max size.""" + monkeypatch.setattr(time, 'sleep', lambda sec: None) + src_provider = MockProvider() + src_path = WaterButlerPath('/file.txt', prepend=None) + dest_provider = MockProvider() + + file_meta = MockFileMetadataWithSize(300, name='bigfile.txt') + src_provider.metadata = MockCoroutine(return_value=file_meta) + + with pytest.raises(exceptions.InvalidParameters) as exc: + await run_pre_checks(src_provider, src_path, dest_provider, max_size_bytes=200) + + assert exc.value.code == 413 + assert exc.value.data['message'] == 'Move/Copy Failed due to oversized files.' + assert exc.value.data['oversized_files'] == [{'name': 'bigfile.txt', 'size': 300}] + + @pytest.mark.asyncio + async def test_file_pre_checks_quota_ok(self, monkeypatch): + """Pre-checks should succeed if the file fits in the destination quota.""" + monkeypatch.setattr(time, 'sleep', lambda sec: None) + src_provider = MockProvider() + src_path = WaterButlerPath('/file.txt', prepend=None) + dest_provider = MockProvider() + + file_meta = MockFileMetadataWithSize(100) + src_provider.metadata = MockCoroutine(return_value=file_meta) + dest_provider.get_quota = MockCoroutine(return_value={'used': 500, 'max': 1000}) + + await run_pre_checks(src_provider, src_path, dest_provider, check_quota=True) + dest_provider.get_quota.assert_called_once_with() + + @pytest.mark.asyncio + async def test_file_pre_checks_quota_exceeded(self, monkeypatch): + """Pre-checks should raise NotEnoughQuotaError if the file exceeds quota.""" + monkeypatch.setattr(time, 'sleep', lambda sec: None) + src_provider = MockProvider() + src_path = WaterButlerPath('/file.txt', prepend=None) + dest_provider = MockProvider() + + file_meta = MockFileMetadataWithSize(600) + src_provider.metadata = MockCoroutine(return_value=file_meta) + dest_provider.get_quota = MockCoroutine(return_value={'used': 500, 'max': 1000}) + + with pytest.raises(exceptions.NotEnoughQuotaError) as exc: + await run_pre_checks(src_provider, src_path, dest_provider, check_quota=True) + + assert exc.value.data['message_key'] == 'quota_exceeded' + + @pytest.mark.asyncio + async def test_fetch_all_pages_pagination(self, monkeypatch): + """Pre-checks should exhaust all pages when fetching paginated metadata.""" + monkeypatch.setattr(time, 'sleep', lambda sec: None) + src_provider = MockProvider() + src_path = WaterButlerPath('/folder/', prepend=None) + dest_provider = MockProvider() + + # Page 1 contains a file and a pagination token. Page 2 contains a file. + page1 = [MockFileMetadataWithSize(100, name='file1.txt'), 'page2_token'] + page2 = [MockFileMetadataWithSize(200, name='file2.txt')] + + src_provider.metadata = MockCoroutine(side_effect=[page1, page2]) + src_provider.handle_data = mock.Mock(return_value=([page1[0]], 'page2_token')) + + await run_pre_checks(src_provider, src_path, dest_provider, max_size_bytes=500) + + assert src_provider.metadata.call_count == 2 + src_provider.metadata.assert_has_calls([ + mock.call(src_path, version=None, revision=None, next_token=None), + mock.call(src_path, version=None, revision=None, next_token='page2_token') + ]) + src_provider.handle_data.assert_called_once_with(page1) + + @pytest.mark.asyncio + async def test_folder_pre_checks_max_size_ok(self, monkeypatch): + """Pre-checks should succeed if all files in the folder are within size limits.""" + monkeypatch.setattr(time, 'sleep', lambda sec: None) + src_provider = MockProvider() + src_path = WaterButlerPath('/folder/', prepend=None) + dest_provider = MockProvider() + + file1 = MockFileMetadataWithSize(50, name='file1.txt') + subfolder = MockFolderMetadataWithName(name='subfolder', path='/folder/subfolder/') + file2 = MockFileMetadataWithSize(80, name='file2.txt') + + src_provider.metadata = MockCoroutine(side_effect=[[file1, subfolder], [file2]]) + src_provider.validate_path = MockCoroutine(return_value=WaterButlerPath('/folder/subfolder/', prepend=None)) + + await run_pre_checks(src_provider, src_path, dest_provider, max_size_bytes=100) + + assert src_provider.metadata.call_count == 2 + src_provider.validate_path.assert_called_once_with('/folder/subfolder/') + + @pytest.mark.asyncio + async def test_folder_pre_checks_max_size_oversized(self, monkeypatch): + """Pre-checks should raise InvalidParameters listing all oversized files inside folder sorted properly.""" + monkeypatch.setattr(time, 'sleep', lambda sec: None) + src_provider = MockProvider() + src_path = WaterButlerPath('/folder/', prepend=None) + dest_provider = MockProvider() + + file_ok = MockFileMetadataWithSize(50, name='file_ok.txt') + bigfile1 = MockFileMetadataWithSize(150, name='bigfile1.txt') + subfolder = MockFolderMetadataWithName(name='subfolder', path='/folder/subfolder/') + bigfile2 = MockFileMetadataWithSize(180, name='bigfile2.txt') + + src_provider.metadata = MockCoroutine(side_effect=[ + [file_ok, bigfile1, subfolder], + [bigfile2] + ]) + src_provider.validate_path = MockCoroutine(return_value=WaterButlerPath('/folder/subfolder/', prepend=None)) + + with pytest.raises(exceptions.InvalidParameters) as exc: + await run_pre_checks(src_provider, src_path, dest_provider, max_size_bytes=100) + + assert exc.value.code == 413 + # subfolder is sorted first (kind=='folder'), then bigfile1.txt + assert exc.value.data['oversized_files'] == [ + {'name': 'bigfile2.txt', 'size': 180}, + {'name': 'bigfile1.txt', 'size': 150} + ] + + @pytest.mark.asyncio + async def test_folder_pre_checks_quota_ok(self, monkeypatch): + """Pre-checks should succeed if folder's recursive size fits within destination quota.""" + monkeypatch.setattr(time, 'sleep', lambda sec: None) + src_provider = MockProvider() + src_path = WaterButlerPath('/folder/', prepend=None) + dest_provider = MockProvider() + + file1 = MockFileMetadataWithSize(100) + subfolder = MockFolderMetadataWithName(name='subfolder', path='/folder/subfolder/') + file2 = MockFileMetadataWithSize(150) + + src_provider.metadata = MockCoroutine(side_effect=[[file1, subfolder], [file2]]) + src_provider.validate_path = MockCoroutine(return_value=WaterButlerPath('/folder/subfolder/', prepend=None)) + dest_provider.get_quota = MockCoroutine(return_value={'used': 500, 'max': 1000}) + + await run_pre_checks(src_provider, src_path, dest_provider, check_quota=True) + + @pytest.mark.asyncio + async def test_folder_pre_checks_quota_exceeded(self, monkeypatch): + """Pre-checks should raise NotEnoughQuotaError if folder's recursive size exceeds quota.""" + monkeypatch.setattr(time, 'sleep', lambda sec: None) + src_provider = MockProvider() + src_path = WaterButlerPath('/folder/', prepend=None) + dest_provider = MockProvider() + + file1 = MockFileMetadataWithSize(200) + subfolder = MockFolderMetadataWithName(name='subfolder', path='/folder/subfolder/') + file2 = MockFileMetadataWithSize(250) + + src_provider.metadata = MockCoroutine(side_effect=[[file1, subfolder], [file2]]) + src_provider.validate_path = MockCoroutine(return_value=WaterButlerPath('/folder/subfolder/', prepend=None)) + dest_provider.get_quota = MockCoroutine(return_value={'used': 600, 'max': 1000}) + + with pytest.raises(exceptions.NotEnoughQuotaError) as exc: + await run_pre_checks(src_provider, src_path, dest_provider, check_quota=True) + + assert exc.value.data['message_key'] == 'quota_exceeded' + + @pytest.mark.asyncio + async def test_move_same_user_quota_skips_quota_check_entirely(self, monkeypatch): + """Move sharing the same UserQuota record must skip the quota check, even if + used + file_size would otherwise exceed max.""" + monkeypatch.setattr(time, 'sleep', lambda sec: None) + src_provider = MockProvider() + src_path = WaterButlerPath('/file.txt', prepend=None) + dest_provider = MockProvider() + + file_meta = MockFileMetadataWithSize(600) + src_provider.NAME = 'osfstorage' + src_provider.metadata = MockCoroutine(return_value=file_meta) + src_provider.get_quota = MockCoroutine(return_value={'used': 500, 'max': 1000, 'user_guid': 'user-a', 'storage_type': 1}) + dest_provider.get_quota = MockCoroutine(return_value={'used': 500, 'max': 1000, 'user_guid': 'user-a', 'storage_type': 1}) + + await run_pre_checks( + src_provider, src_path, dest_provider, check_quota=True, operation='move' + ) + dest_provider.get_quota.assert_called_once_with() + src_provider.get_quota.assert_called_once_with() + + @pytest.mark.asyncio + async def test_move_from_non_osfstorage_src_skips_src_quota_fetch(self, monkeypatch): + """Moving from a non-osfstorage provider (e.g. Dropbox) into osfstorage must never call + src_provider.get_quota() — non-osfstorage providers have no `nid`, so that call would + build a bad URL and blow up. Only the destination quota should be checked.""" + monkeypatch.setattr(time, 'sleep', lambda sec: None) + src_provider = MockProvider() # NAME == 'MockProvider', not 'osfstorage' + src_path = WaterButlerPath('/file.txt', prepend=None) + dest_provider = MockProvider() + + file_meta = MockFileMetadataWithSize(100) + src_provider.metadata = MockCoroutine(return_value=file_meta) + src_provider.get_quota = MockCoroutine( + side_effect=AssertionError('src_provider.get_quota() must not be called for non-osfstorage src') + ) + dest_provider.get_quota = MockCoroutine(return_value={'used': 500, 'max': 1000}) + + await run_pre_checks( + src_provider, src_path, dest_provider, check_quota=True, operation='move' + ) + dest_provider.get_quota.assert_called_once_with() + src_provider.get_quota.assert_not_called() + + @pytest.mark.asyncio + async def test_move_different_user_quota_still_checks_quota(self, monkeypatch): + """Move across different UserQuota records must still apply used+size>max.""" + monkeypatch.setattr(time, 'sleep', lambda sec: None) + src_provider = MockProvider() + src_path = WaterButlerPath('/file.txt', prepend=None) + dest_provider = MockProvider() + + file_meta = MockFileMetadataWithSize(600) + src_provider.metadata = MockCoroutine(return_value=file_meta) + src_provider.get_quota = MockCoroutine(return_value={'used': 0, 'max': 1000, 'user_guid': 'user-a', 'storage_type': 1}) + dest_provider.get_quota = MockCoroutine(return_value={'used': 500, 'max': 1000, 'user_guid': 'user-b', 'storage_type': 1}) + + with pytest.raises(exceptions.NotEnoughQuotaError): + await run_pre_checks( + src_provider, src_path, dest_provider, check_quota=True, operation='move' + ) + + @pytest.mark.asyncio + async def test_copy_same_user_quota_still_checks_quota_and_does_not_fetch_src_quota(self, monkeypatch): + """Copy always creates new data, so it must NOT skip even when sharing the same UserQuota record, + and must not waste a call fetching the source's quota.""" + monkeypatch.setattr(time, 'sleep', lambda sec: None) + src_provider = MockProvider() + src_path = WaterButlerPath('/file.txt', prepend=None) + dest_provider = MockProvider() + + file_meta = MockFileMetadataWithSize(600) + src_provider.metadata = MockCoroutine(return_value=file_meta) + src_provider.get_quota = MockCoroutine(return_value={'used': 0, 'max': 1000, 'user_guid': 'user-a', 'storage_type': 1}) + dest_provider.get_quota = MockCoroutine(return_value={'used': 500, 'max': 1000, 'user_guid': 'user-a', 'storage_type': 1}) + + with pytest.raises(exceptions.NotEnoughQuotaError): + await run_pre_checks( + src_provider, src_path, dest_provider, check_quota=True, operation='copy' + ) + src_provider.get_quota.assert_not_called() + + @pytest.mark.asyncio + async def test_get_replaced_size_zero_when_conflict_not_replace(self, monkeypatch): + """conflict='keep' (or anything but 'replace') never subtracts anything, even if a + same-named item exists at the destination — nothing is actually being overwritten.""" + monkeypatch.setattr(time, 'sleep', lambda sec: None) + dest_provider = MockProvider() + dest_container_path = WaterButlerPath('/dest/', prepend=None) + dest_provider.metadata = MockCoroutine(return_value=[MockFileMetadataWithSize(999, name='Foo.txt')]) + + size = await get_replaced_size(dest_provider, dest_container_path, 'Foo.txt', 'keep', + 'file') + + assert size == 0 + dest_provider.metadata.assert_not_called() + + @pytest.mark.asyncio + async def test_get_replaced_size_zero_when_no_matching_child(self, monkeypatch): + """conflict='replace' but nothing at the destination shares the incoming name -> 0.""" + monkeypatch.setattr(time, 'sleep', lambda sec: None) + dest_provider = MockProvider() + dest_container_path = WaterButlerPath('/dest/', prepend=None) + dest_provider.metadata = MockCoroutine(return_value=[MockFileMetadataWithSize(999, name='Other.txt')]) + + size = await get_replaced_size(dest_provider, dest_container_path, 'Foo.txt', 'replace', + 'file') + + assert size == 0 + + @pytest.mark.asyncio + async def test_get_replaced_size_matches_file_by_name(self, monkeypatch): + """conflict='replace' with a matching file at the destination -> that file's size.""" + monkeypatch.setattr(time, 'sleep', lambda sec: None) + dest_provider = MockProvider() + dest_container_path = WaterButlerPath('/dest/', prepend=None) + dest_provider.metadata = MockCoroutine(return_value=[ + MockFileMetadataWithSize(999, name='Other.txt'), + MockFileMetadataWithSize(500, name='Foo.txt'), + ]) + + size = await get_replaced_size(dest_provider, dest_container_path, 'Foo.txt', 'replace', + 'file') + + assert size == 500 + + @pytest.mark.asyncio + async def test_get_replaced_size_sums_matching_folder_recursively(self, monkeypatch): + """conflict='replace' against a matching FOLDER sums its contents recursively.""" + monkeypatch.setattr(time, 'sleep', lambda sec: None) + dest_provider = MockProvider() + dest_container_path = WaterButlerPath('/dest/', prepend=None) + existing_folder = MockFolderMetadataWithName(name='Foo', path='/dest/Foo/') + existing_folder_path = WaterButlerPath('/dest/Foo/', prepend=None) + + dest_provider.metadata = MockCoroutine(side_effect=[ + [MockFileMetadataWithSize(999, name='Other.txt'), existing_folder], # listing dest_container_path + [MockFileMetadataWithSize(300, name='a.txt'), MockFileMetadataWithSize(200, name='b.txt')], + ]) + dest_provider.validate_path = MockCoroutine(return_value=existing_folder_path) + + size = await get_replaced_size(dest_provider, dest_container_path, 'Foo', 'replace', + 'folder') + + assert size == 500 + dest_provider.validate_path.assert_called_once_with('/dest/Foo/') + + @pytest.mark.asyncio + async def test_get_replaced_size_zero_when_only_kind_differs_file_over_folder(self, monkeypatch): + """A file replacing a same-named folder overwrites nothing, so the folder's size + must not be subtracted.""" + monkeypatch.setattr(time, 'sleep', lambda sec: None) + dest_provider = MockProvider() + dest_container_path = WaterButlerPath('/dest/', prepend=None) + existing_folder = MockFolderMetadataWithName(name='data', path='/dest/data/') + dest_provider.metadata = MockCoroutine(return_value=[existing_folder]) + dest_provider.validate_path = MockCoroutine( + side_effect=AssertionError('a kind mismatch must not be sized at all') + ) + + size = await get_replaced_size(dest_provider, dest_container_path, 'data', 'replace', + 'file') + + assert size == 0 + + @pytest.mark.asyncio + async def test_get_replaced_size_zero_when_only_kind_differs_folder_over_file(self, monkeypatch): + """A folder replacing a same-named file leaves the file in place, so nothing is freed.""" + monkeypatch.setattr(time, 'sleep', lambda sec: None) + dest_provider = MockProvider() + dest_container_path = WaterButlerPath('/dest/', prepend=None) + dest_provider.metadata = MockCoroutine( + return_value=[MockFileMetadataWithSize(999, name='data')] + ) + + size = await get_replaced_size(dest_provider, dest_container_path, 'data', 'replace', + 'folder') + + assert size == 0 + + @pytest.mark.asyncio + async def test_get_replaced_size_picks_matching_kind_among_duplicate_names(self, monkeypatch): + """With both a file and a folder named 'data' at the destination, only the one + matching the source's kind gets sized.""" + monkeypatch.setattr(time, 'sleep', lambda sec: None) + dest_provider = MockProvider() + dest_container_path = WaterButlerPath('/dest/', prepend=None) + existing_folder = MockFolderMetadataWithName(name='data', path='/dest/data/') + dest_provider.metadata = MockCoroutine(side_effect=[ + [MockFileMetadataWithSize(10, name='data'), existing_folder], # dest listing + [MockFileMetadataWithSize(50, name='inside.txt')], # folder's children + ]) + dest_provider.validate_path = MockCoroutine( + return_value=WaterButlerPath('/dest/data/', prepend=None) + ) + + assert await get_replaced_size(dest_provider, dest_container_path, 'data', 'replace', + 'file') == 10 + + dest_provider.metadata = MockCoroutine(side_effect=[ + [MockFileMetadataWithSize(10, name='data'), existing_folder], + [MockFileMetadataWithSize(50, name='inside.txt')], + ]) + assert await get_replaced_size(dest_provider, dest_container_path, 'data', 'replace', + 'folder') == 50 + + @pytest.mark.asyncio + async def test_folder_pre_check_quota_subtracts_replaced_folder_size(self, monkeypatch): + """A folder replace subtracts the existing destination folder's total size, not just + used + new_size.""" + monkeypatch.setattr(time, 'sleep', lambda sec: None) + src_provider = MockProvider() + src_path = WaterButlerPath('/src_folder/', prepend=None) + dest_provider = MockProvider() + dest_container_path = WaterButlerPath('/dest/', prepend=None) + existing_folder = MockFolderMetadataWithName(name='src_folder', path='/dest/src_folder/') + existing_folder_path = WaterButlerPath('/dest/src_folder/', prepend=None) + + src_provider.metadata = MockCoroutine(return_value=[MockFileMetadataWithSize(600, name='new.txt')]) + dest_provider.metadata = MockCoroutine(side_effect=[ + [existing_folder], # listing dest_container_path + [MockFileMetadataWithSize(500, name='old.txt')], # listing existing_folder's children + ]) + dest_provider.validate_path = MockCoroutine(return_value=existing_folder_path) + dest_provider.get_quota = MockCoroutine(return_value={'used': 900, 'max': 1000, 'user_guid': 'user-b', 'storage_type': 1}) + src_provider.get_quota = MockCoroutine(return_value={'used': 0, 'max': 1000, 'user_guid': 'user-a', 'storage_type': 1}) + + # 900 (used) + 600 (new folder) - 500 (existing folder being replaced) = 1000, not > 1000 -> must pass + await run_pre_checks( + src_provider, src_path, dest_provider, dest_path=dest_container_path, + check_quota=True, operation='copy', conflict='replace' + ) + + @pytest.mark.asyncio + async def test_folder_pre_check_quota_ignores_replace_when_no_dest_path(self, monkeypatch): + """Callers that don't pass dest_path keep replaced_size at 0, with no extra + metadata calls.""" + monkeypatch.setattr(time, 'sleep', lambda sec: None) + src_provider = MockProvider() + src_path = WaterButlerPath('/src_folder/', prepend=None) + dest_provider = MockProvider() + + src_provider.metadata = MockCoroutine(return_value=[MockFileMetadataWithSize(100, name='new.txt')]) + dest_provider.get_quota = MockCoroutine(return_value={'used': 500, 'max': 1000, 'user_guid': 'user-b', 'storage_type': 1}) + src_provider.get_quota = MockCoroutine(return_value={'used': 0, 'max': 1000, 'user_guid': 'user-a', 'storage_type': 1}) + + await run_pre_checks( + src_provider, src_path, dest_provider, check_quota=True, operation='copy' + ) + dest_provider.metadata.assert_not_called() + + @pytest.mark.asyncio + async def test_move_same_user_quota_does_not_walk_source_tree(self, monkeypatch): + """A move within one UserQuota record bails out of the quota check before recursing + into the source folder tree.""" + monkeypatch.setattr(time, 'sleep', lambda sec: None) + src_provider = MockProvider() + src_path = WaterButlerPath('/folder/', prepend=None) + dest_provider = MockProvider() + + subfolder = MockFolderMetadataWithName(name='subfolder', path='/folder/subfolder/') + src_provider.NAME = 'osfstorage' + # Top-level listing (always fetched) contains a subfolder; validate_path/further + # metadata calls are only reached by the recursive _get_total_size() walk. + src_provider.metadata = MockCoroutine(return_value=[subfolder]) + src_provider.validate_path = MockCoroutine( + side_effect=AssertionError('source tree must not be walked when quota check is skipped') + ) + src_provider.get_quota = MockCoroutine(return_value={'used': 500, 'max': 1000, 'user_guid': 'user-a', 'storage_type': 1}) + dest_provider.get_quota = MockCoroutine(return_value={'used': 500, 'max': 1000, 'user_guid': 'user-a', 'storage_type': 1}) + + await run_pre_checks( + src_provider, src_path, dest_provider, check_quota=True, operation='move' + ) + src_provider.validate_path.assert_not_called() + + @pytest.mark.asyncio + async def test_move_same_user_quota_does_not_look_up_replaced_item(self, monkeypatch): + """A same-UserQuota-record move also skips get_replaced_size(), since the quota + check itself is skipped.""" + monkeypatch.setattr(time, 'sleep', lambda sec: None) + src_provider = MockProvider() + src_path = WaterButlerPath('/file.txt', prepend=None) + dest_provider = MockProvider() + + file_meta = MockFileMetadataWithSize(600) + src_provider.NAME = 'osfstorage' + src_provider.metadata = MockCoroutine(return_value=file_meta) + src_provider.get_quota = MockCoroutine(return_value={'used': 500, 'max': 1000, 'user_guid': 'user-a', 'storage_type': 1}) + dest_provider.get_quota = MockCoroutine(return_value={'used': 500, 'max': 1000, 'user_guid': 'user-a', 'storage_type': 1}) + + mock_get_replaced_size = MockCoroutine() + monkeypatch.setattr(pre_checks_module, 'get_replaced_size', mock_get_replaced_size) + + await run_pre_checks( + src_provider, src_path, dest_provider, dest_path=WaterButlerPath('/dest/', prepend=None), + check_quota=True, operation='move', conflict='replace' + ) + mock_get_replaced_size.assert_not_called() + + @pytest.mark.asyncio + async def test_move_same_storage_same_region_skips_max_file_size_check(self, monkeypatch): + """A move that stays on osfstorage AND the same region re-uploads nothing, so + max_file_size must not apply -- regardless of node/project (customer review 4: + the differentiator for osfstorage is region, not node).""" + monkeypatch.setattr(time, 'sleep', lambda sec: None) + src_provider = MockProvider(settings={'nid': 'node-1'}) + dest_provider = MockProvider(settings={'nid': 'node-1'}) + src_provider.NAME = 'osfstorage' + dest_provider.NAME = 'osfstorage' + src_provider.is_same_region = mock.Mock(return_value=True) + src_provider.metadata = MockCoroutine( + return_value=[MockFileMetadataWithSize(6000, name='huge.bin')] + ) + src_provider.get_quota = MockCoroutine(return_value={'used': 500, 'max': 1000, 'user_guid': 'user-a', 'storage_type': 1}) + dest_provider.get_quota = MockCoroutine(return_value={'used': 500, 'max': 1000, 'user_guid': 'user-a', 'storage_type': 1}) + + await run_pre_checks( + src_provider, src_path=WaterButlerPath('/folder/', prepend=None), dest_provider=dest_provider, + max_size_bytes=100, check_quota=True, operation='move' + ) + src_provider.metadata.assert_not_called() + + @pytest.mark.asyncio + async def test_move_same_region_different_project_also_skips_max_file_size(self, monkeypatch): + """A move between a project and its same-region component must skip max_file_size + too, even with a different node/creator.""" + monkeypatch.setattr(time, 'sleep', lambda sec: None) + src_provider = MockProvider(settings={'nid': 'node-1'}) + dest_provider = MockProvider(settings={'nid': 'node-2'}) + src_provider.NAME = 'osfstorage' + dest_provider.NAME = 'osfstorage' + src_provider.is_same_region = mock.Mock(return_value=True) + src_provider.metadata = MockCoroutine( + return_value=[MockFileMetadataWithSize(6000, name='huge.bin')] + ) + src_provider.get_quota = MockCoroutine(return_value={'used': 500, 'max': 1000, 'user_guid': 'user-a', 'storage_type': 1}) + dest_provider.get_quota = MockCoroutine(return_value={'used': 500, 'max': 1000, 'user_guid': 'user-a', 'storage_type': 1}) + + await run_pre_checks( + src_provider, src_path=WaterButlerPath('/folder/', prepend=None), dest_provider=dest_provider, + max_size_bytes=100, check_quota=True, operation='move' + ) + src_provider.metadata.assert_not_called() + + @pytest.mark.asyncio + async def test_move_different_region_still_enforces_max_file_size(self, monkeypatch): + """A genuine cross-region osfstorage move (e.g. the creator's default_region + changed between the two nodes' creation) still enforces max_file_size, even + though both sides are osfstorage.""" + monkeypatch.setattr(time, 'sleep', lambda sec: None) + src_provider = MockProvider(settings={'nid': 'node-1'}) + dest_provider = MockProvider(settings={'nid': 'node-2'}) + src_provider.NAME = 'osfstorage' + dest_provider.NAME = 'osfstorage' + src_provider.is_same_region = mock.Mock(return_value=False) + src_provider.metadata = MockCoroutine( + return_value=[MockFileMetadataWithSize(6000, name='huge.bin')] + ) + src_provider.get_quota = MockCoroutine(return_value={'used': 500, 'max': 1000, 'user_guid': 'user-a', 'storage_type': 1}) + dest_provider.get_quota = MockCoroutine(return_value={'used': 500, 'max': 1000, 'user_guid': 'user-a', 'storage_type': 1}) + + with pytest.raises(exceptions.InvalidParameters) as exc: + await run_pre_checks( + src_provider, src_path=WaterButlerPath('/folder/', prepend=None), dest_provider=dest_provider, + max_size_bytes=100, check_quota=True, operation='move' + ) + assert exc.value.code == 413 + + @pytest.mark.asyncio + async def test_move_same_project_different_storage_still_enforces_max_file_size(self, monkeypatch): + """Two different storage types on the very same project never share "the same + storage", so max_file_size still applies regardless of the project match.""" + monkeypatch.setattr(time, 'sleep', lambda sec: None) + src_provider = MockProvider(settings={'nid': 'node-1'}) + dest_provider = MockProvider(settings={'nid': 'node-1'}) + src_provider.NAME = 'osfstorage' + dest_provider.NAME = 'someotherstorage' + src_provider.metadata = MockCoroutine( + return_value=[MockFileMetadataWithSize(6000, name='huge.bin')] + ) + + with pytest.raises(exceptions.InvalidParameters) as exc: + await run_pre_checks( + src_provider, src_path=WaterButlerPath('/folder/', prepend=None), dest_provider=dest_provider, + max_size_bytes=100, operation='move' + ) + assert exc.value.code == 413 + + @pytest.mark.asyncio + async def test_copy_across_user_quota_still_enforces_max_file_size(self, monkeypatch): + """A copy still enforces the size limit; the skip only applies to intra-record moves.""" + monkeypatch.setattr(time, 'sleep', lambda sec: None) + src_provider = MockProvider() + src_path = WaterButlerPath('/folder/', prepend=None) + dest_provider = MockProvider() + + src_provider.NAME = 'osfstorage' + src_provider.metadata = MockCoroutine( + return_value=[MockFileMetadataWithSize(6000, name='huge.bin')] + ) + src_provider.get_quota = MockCoroutine(return_value={'used': 0, 'max': 100_000, 'user_guid': 'user-a', 'storage_type': 1}) + dest_provider.get_quota = MockCoroutine(return_value={'used': 0, 'max': 100_000, 'user_guid': 'user-a', 'storage_type': 1}) + + with pytest.raises(exceptions.InvalidParameters) as exc: + await run_pre_checks( + src_provider, src_path, dest_provider, max_size_bytes=100, + check_quota=True, operation='copy' + ) + assert exc.value.code == 413 + + @pytest.mark.asyncio + async def test_move_to_other_storage_type_still_enforces_max_file_size(self, monkeypatch): + """A move into a different UserQuota record (other storage_type) still re-uploads, + so the size limit still applies.""" + monkeypatch.setattr(time, 'sleep', lambda sec: None) + src_provider = MockProvider() + src_path = WaterButlerPath('/folder/', prepend=None) + dest_provider = MockProvider() + + src_provider.NAME = 'osfstorage' + src_provider.metadata = MockCoroutine( + return_value=[MockFileMetadataWithSize(6000, name='huge.bin')] + ) + src_provider.get_quota = MockCoroutine(return_value={'used': 0, 'max': 100_000, 'user_guid': 'user-a', 'storage_type': 1}) + dest_provider.get_quota = MockCoroutine(return_value={'used': 0, 'max': 100_000, 'user_guid': 'user-a', 'storage_type': 2}) + + with pytest.raises(exceptions.InvalidParameters) as exc: + await run_pre_checks( + src_provider, src_path, dest_provider, max_size_bytes=100, + check_quota=True, operation='move' + ) + assert exc.value.code == 413 + + @pytest.mark.asyncio + async def test_move_non_osfstorage_different_project_still_enforces_max_file_size(self, monkeypatch): + """Destinations that have no quota to check (non-osfstorage) still enforce the + size limit when source and destination are different projects.""" + monkeypatch.setattr(time, 'sleep', lambda sec: None) + src_provider = MockProvider(settings={'nid': 'node-1'}) + dest_provider = MockProvider(settings={'nid': 'node-2'}) + src_provider.metadata = MockCoroutine( + return_value=[MockFileMetadataWithSize(6000, name='huge.bin')] + ) + dest_provider.get_quota = MockCoroutine( + side_effect=AssertionError('no quota lookup when check_quota is False') + ) + + with pytest.raises(exceptions.InvalidParameters) as exc: + await run_pre_checks( + src_provider, src_path=WaterButlerPath('/folder/', prepend=None), dest_provider=dest_provider, + max_size_bytes=100, operation='move', src_nid='node-1', dest_nid='node-2' + ) + assert exc.value.code == 413 + dest_provider.get_quota.assert_not_called() + + @pytest.mark.asyncio + async def test_move_non_osfstorage_different_project_enforces_even_when_provider_nid_matches(self, monkeypatch): + """Regression for `provider.nid`, the decision must use the caller-supplied src_nid/dest_nid.""" + monkeypatch.setattr(time, 'sleep', lambda sec: None) + src_provider = MockProvider(settings={'nid': None}) + dest_provider = MockProvider(settings={'nid': None}) + src_provider.metadata = MockCoroutine( + return_value=[MockFileMetadataWithSize(6000, name='huge.bin')] + ) + dest_provider.get_quota = MockCoroutine( + side_effect=AssertionError('no quota lookup when check_quota is False') + ) + + with pytest.raises(exceptions.InvalidParameters) as exc: + await run_pre_checks( + src_provider, src_path=WaterButlerPath('/folder/', prepend=None), dest_provider=dest_provider, + max_size_bytes=100, operation='move', src_nid='node-1', dest_nid='node-2' + ) + assert exc.value.code == 413 + + @pytest.mark.asyncio + async def test_move_non_osfstorage_same_project_skips_max_file_size(self, monkeypatch): + """A move between two extended-storage locations that is BOTH the same storage + type AND the same project must skip max_file_size too.""" + monkeypatch.setattr(time, 'sleep', lambda sec: None) + src_provider = MockProvider(settings={'nid': 'node-1'}) + dest_provider = MockProvider(settings={'nid': 'node-1'}) + src_provider.metadata = MockCoroutine( + return_value=[MockFileMetadataWithSize(6000, name='huge.bin')] + ) + dest_provider.get_quota = MockCoroutine( + side_effect=AssertionError('no quota lookup when check_quota is False') + ) + + await run_pre_checks( + src_provider, src_path=WaterButlerPath('/folder/', prepend=None), dest_provider=dest_provider, + max_size_bytes=100, operation='move', src_nid='node-1', dest_nid='node-1' + ) + src_provider.metadata.assert_not_called() + + @pytest.mark.asyncio + async def test_folder_quota_treats_unknown_size_as_zero_and_warns(self, monkeypatch, caplog): + """A file whose size_as_int is None (e.g. an un-exported Google Docs/Sheets/Slides + file) must not crash the quota calculation.""" + monkeypatch.setattr(time, 'sleep', lambda sec: None) + src_provider = MockProvider() + src_path = WaterButlerPath('/folder/', prepend=None) + dest_provider = MockProvider() + + known_file = MockFileMetadataWithSize(100, name='known.txt') + unknown_file = MockFileMetadataWithSize(None, name='doc.gdoc') + src_provider.metadata = MockCoroutine(return_value=[known_file, unknown_file]) + dest_provider.get_quota = MockCoroutine(return_value={'used': 0, 'max': 1000}) + + with caplog.at_level('WARNING'): + await run_pre_checks(src_provider, src_path, dest_provider, check_quota=True, + operation='copy') + + assert 'size_as_int is None' in caplog.text + assert "provider='MockProvider'" in caplog.text + assert "operation='copy'" in caplog.text + + @pytest.mark.asyncio + async def test_folder_quota_treats_negative_size_as_zero_and_warns(self, monkeypatch, caplog): + """A file whose size_as_int is negative (e.g. an osfstorage version whose size was + never set, default -1) must not be added to the running total as-is.""" + monkeypatch.setattr(time, 'sleep', lambda sec: None) + src_provider = MockProvider() + src_path = WaterButlerPath('/folder/', prepend=None) + dest_provider = MockProvider() + + known_file = MockFileMetadataWithSize(100, name='known.txt') + negative_file = MockFileMetadataWithSize(-1, name='bad.txt') + src_provider.metadata = MockCoroutine(return_value=[known_file, negative_file]) + dest_provider.get_quota = MockCoroutine(return_value={'used': 0, 'max': 1000}) + + with caplog.at_level('WARNING'): + await run_pre_checks(src_provider, src_path, dest_provider, check_quota=True, + operation='copy') + + assert 'size_as_int is negative' in caplog.text + assert "provider='MockProvider'" in caplog.text + assert "operation='copy'" in caplog.text + + @pytest.mark.asyncio + async def test_folder_max_size_skips_unknown_size_file_and_warns(self, monkeypatch, caplog): + """A file with unknown size has nothing to compare against max_file_size, so it must + never be flagged as oversized.""" + monkeypatch.setattr(time, 'sleep', lambda sec: None) + src_provider = MockProvider() + src_path = WaterButlerPath('/folder/', prepend=None) + dest_provider = MockProvider() + + unknown_file = MockFileMetadataWithSize(None, name='doc.gdoc') + src_provider.metadata = MockCoroutine(return_value=[unknown_file]) + + with caplog.at_level('WARNING'): + await run_pre_checks(src_provider, src_path, dest_provider, max_size_bytes=100, + operation='copy') + + assert 'size_as_int is None' in caplog.text + assert "operation='copy'" in caplog.text + + @pytest.mark.asyncio + async def test_folder_max_size_skips_negative_size_file_and_warns(self, monkeypatch, caplog): + """A file with a negative size has nothing meaningful to compare against + max_file_size, so it must never be flagged as oversized.""" + monkeypatch.setattr(time, 'sleep', lambda sec: None) + src_provider = MockProvider() + src_path = WaterButlerPath('/folder/', prepend=None) + dest_provider = MockProvider() + + negative_file = MockFileMetadataWithSize(-1, name='bad.txt') + src_provider.metadata = MockCoroutine(return_value=[negative_file]) + + with caplog.at_level('WARNING'): + await run_pre_checks(src_provider, src_path, dest_provider, max_size_bytes=100, + operation='copy') + + assert 'size_as_int is negative' in caplog.text + assert "operation='copy'" in caplog.text + + @pytest.mark.asyncio + async def test_get_replaced_size_treats_unknown_size_as_zero_and_warns(self, monkeypatch, caplog): + """An existing destination file with unknown size (being overwritten) must be + treated as 0, not crash get_replaced_size.""" + monkeypatch.setattr(time, 'sleep', lambda sec: None) + dest_provider = MockProvider() + dest_container_path = WaterButlerPath('/dest/', prepend=None) + dest_provider.metadata = MockCoroutine( + return_value=[MockFileMetadataWithSize(None, name='Foo.txt')] + ) + + with caplog.at_level('WARNING'): + size = await get_replaced_size(dest_provider, dest_container_path, 'Foo.txt', + 'replace', 'file', operation='move') + + assert size == 0 + assert 'size_as_int is None' in caplog.text + assert "operation='move'" in caplog.text + + @pytest.mark.asyncio + async def test_get_replaced_size_treats_negative_size_as_zero_and_warns(self, monkeypatch, caplog): + """An existing destination file with a negative size (being overwritten) must be + treated as 0, not passed through as replaced_size.""" + monkeypatch.setattr(time, 'sleep', lambda sec: None) + dest_provider = MockProvider() + dest_container_path = WaterButlerPath('/dest/', prepend=None) + dest_provider.metadata = MockCoroutine( + return_value=[MockFileMetadataWithSize(-1, name='Foo.txt')] + ) + + with caplog.at_level('WARNING'): + size = await get_replaced_size(dest_provider, dest_container_path, 'Foo.txt', + 'replace', 'file', operation='move') + + assert size == 0 + assert 'size_as_int is negative' in caplog.text + assert "operation='move'" in caplog.text + + @pytest.mark.asyncio + async def test_folder_pre_check_matches_replaced_item_by_kind(self, monkeypatch): + """A folder move onto a same-named file frees nothing, so the quota formula must + not subtract that file's size.""" + monkeypatch.setattr(time, 'sleep', lambda sec: None) + src_provider = MockProvider() + src_path = WaterButlerPath('/data/', prepend=None) + dest_provider = MockProvider() + dest_container_path = WaterButlerPath('/dest/', prepend=None) + + src_provider.metadata = MockCoroutine(return_value=[MockFileMetadataWithSize(600, name='new.txt')]) + # A *file* named 'data' at the destination: same name, different kind. + dest_provider.metadata = MockCoroutine( + return_value=[MockFileMetadataWithSize(500, name='data')] + ) + dest_provider.get_quota = MockCoroutine(return_value={'used': 900, 'max': 1000, 'user_guid': 'user-b', 'storage_type': 1}) + src_provider.get_quota = MockCoroutine(return_value={'used': 0, 'max': 1000, 'user_guid': 'user-a', 'storage_type': 1}) + + # 900 + 600 - 0 (nothing replaced) = 1500 > 1000 -> must be refused. + with pytest.raises(exceptions.NotEnoughQuotaError): + await run_pre_checks( + src_provider, src_path, dest_provider, dest_path=dest_container_path, + check_quota=True, operation='copy', conflict='replace' + ) + + +# --------------------------------------------------------------------------- +# Celery Task Integration Tests +# --------------------------------------------------------------------------- + +class TestPreChecksTaskIntegration: + + def test_copy_task_calls_pre_checks(self, monkeypatch, providers, bundles, callback): + """Copy task should execute pre-checks before triggering copy.""" + src, dest = providers + src_bundle, dest_bundle = bundles + + mock_run_pre_checks = MockCoroutine() + monkeypatch.setattr(copy_module, 'run_pre_checks', mock_run_pre_checks) + + copy_task( + cp.deepcopy(src_bundle), + cp.deepcopy(dest_bundle), + max_size_bytes=1000, + check_quota=True + ) + + mock_run_pre_checks.assert_called_once_with( + src, src_bundle['path'], dest, + dest_path=dest_bundle['path'], + max_size_bytes=1000, + check_quota=True, + operation='copy', + conflict=DEFAULT_CONFLICT, + rename=None, + src_nid=src_bundle['nid'], dest_nid=dest_bundle['nid'] + ) + assert src.copy.called + + def test_move_task_calls_pre_checks(self, monkeypatch, providers, bundles, callback): + """Move task should execute pre-checks before triggering move.""" + src, dest = providers + src_bundle, dest_bundle = bundles + + mock_run_pre_checks = MockCoroutine() + monkeypatch.setattr(move_module, 'run_pre_checks', mock_run_pre_checks) + + src.move.return_value = (MockFileMetadata(), True) + + move_task( + cp.deepcopy(src_bundle), + cp.deepcopy(dest_bundle), + max_size_bytes=1000, + check_quota=True + ) + + mock_run_pre_checks.assert_called_once_with( + src, src_bundle['path'], dest, + dest_path=dest_bundle['path'], + max_size_bytes=1000, + check_quota=True, + operation='move', + conflict=DEFAULT_CONFLICT, + rename=None, + src_nid=src_bundle['nid'], dest_nid=dest_bundle['nid'] + ) + assert src.move.called + + def test_copy_task_forwards_conflict_and_rename_to_pre_checks(self, monkeypatch, providers, bundles, callback): + """conflict/rename passed to the celery task (e.g. from an explicit replace request) + must reach run_pre_checks unchanged, not just the defaults.""" + src, dest = providers + src_bundle, dest_bundle = bundles + + mock_run_pre_checks = MockCoroutine() + monkeypatch.setattr(copy_module, 'run_pre_checks', mock_run_pre_checks) + + copy_task( + cp.deepcopy(src_bundle), + cp.deepcopy(dest_bundle), + max_size_bytes=1000, + check_quota=True, + conflict='replace', + rename='renamed.txt' + ) + + mock_run_pre_checks.assert_called_once_with( + src, src_bundle['path'], dest, + dest_path=dest_bundle['path'], + max_size_bytes=1000, + check_quota=True, + operation='copy', + conflict='replace', + rename='renamed.txt', + src_nid=src_bundle['nid'], dest_nid=dest_bundle['nid'] + ) + + def test_copy_task_pre_checks_failure_aborts_copy(self, monkeypatch, providers, bundles, callback): + """Copy task should abort and raise if pre-checks raise InvalidParameters.""" + src, dest = providers + src_bundle, dest_bundle = bundles + + mock_run_pre_checks = MockCoroutine(side_effect=exceptions.InvalidParameters('Oversized files', code=413)) + monkeypatch.setattr(copy_module, 'run_pre_checks', mock_run_pre_checks) + + with pytest.raises(exceptions.InvalidParameters): + copy_task( + cp.deepcopy(src_bundle), + cp.deepcopy(dest_bundle), + max_size_bytes=1000, + check_quota=True + ) + + assert not src.copy.called + + def test_move_task_pre_checks_failure_aborts_move(self, monkeypatch, providers, bundles, callback): + """Move task should abort and raise if pre-checks raise NotEnoughQuotaError.""" + src, dest = providers + src_bundle, dest_bundle = bundles + + mock_run_pre_checks = MockCoroutine(side_effect=exceptions.NotEnoughQuotaError('Quota exceeded')) + monkeypatch.setattr(move_module, 'run_pre_checks', mock_run_pre_checks) + + with pytest.raises(exceptions.NotEnoughQuotaError): + move_task( + cp.deepcopy(src_bundle), + cp.deepcopy(dest_bundle), + max_size_bytes=1000, + check_quota=True + ) + + assert not src.move.called diff --git a/waterbutler/providers/osfstorage/provider.py b/waterbutler/providers/osfstorage/provider.py index 8606a202f..ad64def6c 100644 --- a/waterbutler/providers/osfstorage/provider.py +++ b/waterbutler/providers/osfstorage/provider.py @@ -478,8 +478,30 @@ async def _do_intra_move_or_copy(self, action: str, dest_provider, src_path, des """ created = True + replaced_size = 0 if dest_path.identifier: created = False + try: + dest_meta = await dest_provider.metadata(dest_path) + if hasattr(dest_meta, 'size_as_int'): + size_as_int = dest_meta.size_as_int + if size_as_int is None: + logger.warning( + 'dest_meta.size_as_int is None for %s (provider=%s, action=%s); ' + 'treating replaced size as 0 for quota calculation', + dest_path, dest_provider.NAME, action + ) + elif size_as_int < 0: + logger.warning( + 'dest_meta.size_as_int is negative (%s) for %s (provider=%s, action=%s); ' + 'treating replaced size as 0 for quota calculation', + size_as_int, dest_path, dest_provider.NAME, action + ) + else: + replaced_size = size_as_int + except Exception as e: + logger.error('Failed to fetch dest_meta for replaced_size calculation: %s', e) + raise exceptions.ProviderError({'message': 'Failed to fetch dest_meta for replaced_size calculation.'}, code=500) await dest_provider.delete(dest_path) resp = await self.make_signed_request( @@ -492,7 +514,8 @@ async def _do_intra_move_or_copy(self, action: str, dest_provider, src_path, des 'name': dest_path.name, 'node': dest_provider.nid, 'parent': dest_path.parent.identifier - } + }, + 'replaced_size': replaced_size, }), headers={'Content-Type': 'application/json'}, expects=(200, 201) diff --git a/waterbutler/server/api/v1/provider/movecopy.py b/waterbutler/server/api/v1/provider/movecopy.py index 8cfd7a638..478fc017d 100644 --- a/waterbutler/server/api/v1/provider/movecopy.py +++ b/waterbutler/server/api/v1/provider/movecopy.py @@ -1,4 +1,5 @@ import json +import logging from http import HTTPStatus from waterbutler import tasks @@ -12,8 +13,13 @@ from waterbutler.constants import DEFAULT_CONFLICT from waterbutler.auth.osf.handler import EXPORT_DATA_FAKE_NODE_ID from waterbutler.tasks.settings import SYNCHRONOUS_TIMEOUT +from waterbutler.tasks.pre_checks import ( + run_pre_checks, get_replaced_size, resolve_quota_context, check_quota_limit, + should_skip_size_check, +) auth_handler = AuthHandler(settings.AUTH_HANDLERS) +logger = logging.getLogger(__name__) class MoveCopyMixin: @@ -101,12 +107,13 @@ async def move_or_copy(self): self.auth['settings'] ) self.path = await self.provider.validate_v1_path(self.path, **self.arguments) - + check_kwargs = {} if auth_action == 'rename': # 'rename' implies the file/folder does not change location self.dest_auth = self.auth self.dest_provider = self.provider self.dest_path = self.path.parent self.dest_resource = self.resource + conflict = self.json.get('conflict', DEFAULT_CONFLICT) else: path = self.json.get('path', None) if path is None: @@ -145,6 +152,90 @@ async def move_or_copy(self): ) self.dest_path = await self.dest_provider.validate_path(**self.json) + conflict = self.json.get('conflict', DEFAULT_CONFLICT) + + # Check if the file/folder is oversized + max_size_mb = self.dest_auth['settings'].get('max_file_size') + max_size_bytes = (int(max_size_mb) * 1024 * 1024) if max_size_mb else None + + if not self.path.is_dir: + # Single-file path: read metadata once and check inline. + # No recursion needed — the item is guaranteed to be a file. + file_meta = await self.provider.metadata( + self.path, version=None, revision=None + ) + # size_as_int is None when the provider can't report a size (e.g. a Google + # Docs/Sheets/Slides file that hasn't been exported yet) -- int(file_meta.size) + # would raise TypeError in that case instead of failing the check gracefully. + # A negative size_as_int (e.g. an osfstorage version whose size was never set, + # default -1) is equally unusable and gets the same treatment. + file_size = file_meta.size_as_int + if file_size is None: + logger.warning( + 'size_as_int is None for {!r} (provider={!r}, operation={!r}); ' + 'treating size as 0 for max_file_size/quota calculation'.format( + file_meta.materialized_path, self.provider.NAME, provider_action + ) + ) + elif file_size < 0: + logger.warning( + 'size_as_int is negative ({!r}) for {!r} (provider={!r}, operation={!r}); ' + 'treating size as 0 for max_file_size/quota calculation'.format( + file_size, file_meta.materialized_path, self.provider.NAME, + provider_action + ) + ) + + # max_file_size and quota are skipped on two different, independent + # conditions -- see should_skip_size_check() and resolve_quota_context(). + # An unknown or negative file_size also skips the max_file_size check, + # since there's nothing meaningful to compare against. + known_file_size = file_size is not None and file_size >= 0 + run_size_check = ( + max_size_bytes is not None and known_file_size and + not should_skip_size_check( + provider_action, self.provider, self.dest_provider, + self.resource, self.dest_resource + ) + ) + + # Check max_file_size first -- quota is fetched only after this check + # passes, so an oversized file never triggers a creator_quota request. + if run_size_check and file_size > max_size_bytes: + raise exceptions.InvalidParameters({ + 'message': 'Move/Copy Failed due to oversized files.', + 'oversized_files': [{'name': file_meta.name, 'size': file_size}], + 'max_size': max_size_bytes, + }, code=413) + + # Check quota (osfstorage only) + if self.dest_provider.NAME == 'osfstorage': + skip_quota, dest_quota = await resolve_quota_context( + provider_action, self.provider, self.dest_provider + ) + if not skip_quota: + resolved_name = self.json.get('rename') or self.path.name + replaced_size = await get_replaced_size( + self.dest_provider, self.dest_path, resolved_name, conflict, 'file', + operation=provider_action, + ) + # An unknown or negative file_size can't be validated against quota + # either -- treat it as 0, same as the max_file_size check above. + # (Not `file_size or 0`: a real negative int is truthy in Python and + # would slide through unchanged instead of being normalized to 0.) + check_quota_limit( + dest_quota, file_size if known_file_size else 0, replaced_size + ) + check_kwargs = { + 'max_size_bytes': None, + 'check_quota': False, + } + else: + check_kwargs = { + 'max_size_bytes': max_size_bytes, + 'check_quota': (self.dest_provider.NAME == 'osfstorage'), + } + if not getattr(self.provider, 'can_intra_' + provider_action)(self.dest_provider, self.path): # this weird signature syntax courtesy of py3.4 not liking trailing commas on kwargs conflict = self.json.get('conflict', DEFAULT_CONFLICT) @@ -158,6 +249,7 @@ async def move_or_copy(self): request=remote_logging._serialize_request(self.request), *self.build_args(), **task_kwargs, + **check_kwargs, ) synchronous = self.json.get('synchronous', 'false') synchronous = True if isinstance(synchronous, bool) and synchronous is True else False @@ -168,16 +260,26 @@ async def move_or_copy(self): # Use default timeout value for asynchronous processes metadata, created = await tasks.wait_on_celery(result) else: - metadata, created = ( - await tasks.backgrounded( - getattr(self.provider, provider_action), + async def _intra_task(): + if self.path.is_dir: + await run_pre_checks( + self.provider, self.path, self.dest_provider, + dest_path=self.dest_path, + operation=provider_action, + conflict=conflict, + rename=self.json.get('rename'), + src_nid=self.resource, dest_nid=self.dest_resource, + **check_kwargs + ) + return await getattr(self.provider, provider_action)( self.dest_provider, self.path, self.dest_path, rename=self.json.get('rename'), - conflict=self.json.get('conflict', DEFAULT_CONFLICT), + conflict=conflict, ) - ) + + metadata, created = await tasks.backgrounded(_intra_task) self.dest_meta = metadata diff --git a/waterbutler/tasks/copy.py b/waterbutler/tasks/copy.py index 878f14aff..841b80ffc 100644 --- a/waterbutler/tasks/copy.py +++ b/waterbutler/tasks/copy.py @@ -5,12 +5,15 @@ from waterbutler.core.path import WaterButlerPath from waterbutler.core import utils, remote_logging from waterbutler.core.log_payload import LogPayload +from waterbutler.constants import DEFAULT_CONFLICT +from waterbutler.tasks.pre_checks import run_pre_checks logger = logging.getLogger(__name__) @core.celery_task -async def copy(src_bundle, dest_bundle, request=None, start_time=None, **kwargs): +async def copy(src_bundle, dest_bundle, request=None, start_time=None, + max_size_bytes=None, check_quota=False, **kwargs): request = request or {} start_time = start_time or time.time() @@ -26,6 +29,14 @@ async def copy(src_bundle, dest_bundle, request=None, start_time=None, **kwargs) metadata, errors = None, [] try: + # Run pre-checks before attempting the copy to avoid partial copies and ensure we can report all errors at once + await run_pre_checks(src_provider, src_path, dest_provider, + dest_path=dest_path, + max_size_bytes=max_size_bytes, check_quota=check_quota, + operation='copy', + conflict=kwargs.get('conflict', DEFAULT_CONFLICT), + rename=kwargs.get('rename'), + src_nid=src_bundle['nid'], dest_nid=dest_bundle['nid']) metadata, created = await src_provider.copy(dest_provider, src_path, dest_path, **kwargs) except Exception as e: logger.error('Copy failed with error {!r}'.format(e)) diff --git a/waterbutler/tasks/move.py b/waterbutler/tasks/move.py index d25d96ab2..2a4d923dc 100644 --- a/waterbutler/tasks/move.py +++ b/waterbutler/tasks/move.py @@ -5,12 +5,15 @@ from waterbutler.core.path import WaterButlerPath from waterbutler.core import utils, remote_logging from waterbutler.core.log_payload import LogPayload +from waterbutler.constants import DEFAULT_CONFLICT +from waterbutler.tasks.pre_checks import run_pre_checks logger = logging.getLogger(__name__) @core.celery_task -async def move(src_bundle, dest_bundle, request=None, start_time=None, **kwargs): +async def move(src_bundle, dest_bundle, request=None, start_time=None, + max_size_bytes=None, check_quota=False, **kwargs): request = request or {} start_time = start_time or time.time() @@ -26,6 +29,14 @@ async def move(src_bundle, dest_bundle, request=None, start_time=None, **kwargs) metadata, errors = None, [] try: + # Run pre-checks before attempting the move to avoid partial moves and ensure we can report all errors at once + await run_pre_checks(src_provider, src_path, dest_provider, + dest_path=dest_path, + max_size_bytes=max_size_bytes, check_quota=check_quota, + operation='move', + conflict=kwargs.get('conflict', DEFAULT_CONFLICT), + rename=kwargs.get('rename'), + src_nid=src_bundle['nid'], dest_nid=dest_bundle['nid']) metadata, created = await src_provider.move(dest_provider, src_path, dest_path, **kwargs) except Exception as e: logger.error('Move failed with error {!r}'.format(e)) diff --git a/waterbutler/tasks/pre_checks.py b/waterbutler/tasks/pre_checks.py new file mode 100644 index 000000000..18f141525 --- /dev/null +++ b/waterbutler/tasks/pre_checks.py @@ -0,0 +1,236 @@ +import logging + +from waterbutler.core import exceptions + +logger = logging.getLogger(__name__) + + +async def _fetch_all_pages(provider, path): + """Fetch all children with pagination support.""" + all_data = [] + next_token = None + while True: + data = await provider.metadata(path, version=None, revision=None, next_token=next_token) + if data and isinstance(data[-1], str): + data, next_token = provider.handle_data(data) + else: + next_token = None + all_data.extend(data) + if not next_token: + break + return all_data + + +async def _get_total_size(provider, data, operation=None): + """Recursively calculate total size of all files.""" + size = 0 + for item in data: + if item.kind == 'file': + item_size = item.size_as_int + if item_size is None: + logger.warning( + 'size_as_int is None for {!r} (provider={!r}, operation={!r}); ' + 'treating size as 0 for quota calculation'.format( + item.materialized_path, provider.NAME, operation + ) + ) + item_size = 0 + elif item_size < 0: + logger.warning( + 'size_as_int is negative ({!r}) for {!r} (provider={!r}, operation={!r}); ' + 'treating size as 0 for quota calculation'.format( + item_size, item.materialized_path, provider.NAME, operation + ) + ) + item_size = 0 + size += item_size + else: + child_path = await provider.validate_path(item.path) + children = await _fetch_all_pages(provider, child_path) + size += await _get_total_size(provider, children, operation=operation) + return size + + +async def _get_oversized_files(provider, data, max_size_bytes, operation=None): + """Recursively find files exceeding max_size_bytes.""" + oversized = [] + for item in sorted(data, key=lambda i: (0 if i.kind == 'folder' else 1, i.name.lower())): + if item.kind == 'file': + item_size = item.size_as_int + if item_size is None: + logger.warning( + 'size_as_int is None for {!r} (provider={!r}, operation={!r}); ' + 'skipping max_file_size check for this file'.format( + item.materialized_path, provider.NAME, operation + ) + ) + elif item_size < 0: + logger.warning( + 'size_as_int is negative ({!r}) for {!r} (provider={!r}, operation={!r}); ' + 'skipping max_file_size check for this file'.format( + item_size, item.materialized_path, provider.NAME, operation + ) + ) + elif item_size > max_size_bytes: + oversized.append({'name': item.name, 'size': item_size}) + else: + child_path = await provider.validate_path(item.path) + children = await _fetch_all_pages(provider, child_path) + oversized.extend( + await _get_oversized_files(provider, children, max_size_bytes, operation=operation) + ) + return oversized + + +async def get_replaced_size(dest_provider, dest_container_path, resolved_name, conflict, + src_kind, operation=None): + """Size of the existing file/folder being overwritten on replace, else 0. + + ``src_kind`` is the kind ('file' or 'folder') of the item being moved/copied. Only a + destination child of that same kind is actually overwritten: osfstorage allows a file + and a folder to share one name (``can_duplicate_names()`` is True), so matching on the + name alone would subtract the size of an item that survives the operation and let the + user push ``used`` past ``max``. + """ + if conflict != 'replace' or dest_container_path is None: + return 0 + + children = await _fetch_all_pages(dest_provider, dest_container_path) + existing = next((child for child in children + if child.name == resolved_name and child.kind == src_kind), None) + if existing is None: + return 0 + + if existing.kind == 'file': + item_size = existing.size_as_int + if item_size is None: + logger.warning( + 'size_as_int is None for {!r} (provider={!r}, operation={!r}); ' + 'treating replaced size as 0 for quota calculation'.format( + existing.materialized_path, dest_provider.NAME, operation + ) + ) + return 0 + if item_size < 0: + logger.warning( + 'size_as_int is negative ({!r}) for {!r} (provider={!r}, operation={!r}); ' + 'treating replaced size as 0 for quota calculation'.format( + item_size, existing.materialized_path, dest_provider.NAME, operation + ) + ) + return 0 + return item_size + + existing_path = await dest_provider.validate_path(existing.path) + existing_children = await _fetch_all_pages(dest_provider, existing_path) + return await _get_total_size(dest_provider, existing_children, operation=operation) + + +async def resolve_quota_context(operation, src_provider, dest_provider): + """Fetch the destination quota and decide whether the check can be skipped outright. + + Returns ``(skip, dest_quota)``. When ``skip`` is True the operation stays inside one + UserQuota record, so ``used`` cannot grow and no size needs to be computed at all -- + callers should bail out *before* walking the source tree or looking up the item being + replaced, since neither result would change the outcome. + + ``dest_quota`` is returned so callers can hand it straight to check_quota_limit() + instead of re-fetching it; this keeps the number of creator_quota requests identical + to the previous single-function implementation. + """ + dest_quota = await dest_provider.get_quota() + + if operation == 'move' and src_provider.NAME == 'osfstorage': + src_quota = await src_provider.get_quota() + src_user_guid = src_quota.get('user_guid') + if (src_user_guid is not None and + src_user_guid == dest_quota.get('user_guid') and + src_quota.get('storage_type') == dest_quota.get('storage_type')): + return True, dest_quota + + return False, dest_quota + + +def check_quota_limit(dest_quota, file_size, replaced_size=0): + """Raise NotEnoughQuotaError when the operation would push `used` past `max`.""" + if dest_quota['used'] + file_size - replaced_size > dest_quota['max']: + raise exceptions.NotEnoughQuotaError({ + 'message_key': 'quota_exceeded', + 'message': 'You do not have enough available quota.', + }) + + +def should_skip_size_check(operation, src_provider, dest_provider, src_nid, dest_nid): + """True when a move stays within the same storage and location (size can't change). + + For osfstorage, "same location" means same region (``is_same_region()``), since a + same-region move never changes which bucket holds the data. Other providers fall back + to node-match via the caller-supplied ``src_nid``/``dest_nid`` -- not ``provider.nid``, + which is usually ``None`` on both sides and would wrongly match every cross-project move. + + Independent from resolve_quota_context()'s skip: the two can diverge (e.g. different + creator keeps this skip but not quota's; cross-region keeps quota's skip but not this). + """ + if operation != 'move' or src_provider.NAME != dest_provider.NAME: + return False + if src_provider.NAME == 'osfstorage': + return src_provider.is_same_region(dest_provider) + return src_nid == dest_nid + + +async def run_pre_checks(src_provider, src_path, dest_provider, dest_path=None, + max_size_bytes=None, check_quota=False, operation=None, + conflict='replace', rename=None, src_nid=None, dest_nid=None): + """Run max_file_size and quota pre-checks inside the Celery task. + + The two checks have independent skip conditions: see should_skip_size_check() and + resolve_quota_context(). The quota check (and its creator_quota fetch) is deferred + until *after* the max_file_size check has run and passed, so an oversized file is + rejected with 413 without ever calling creator_quota. + """ + run_size_check = max_size_bytes is not None and not should_skip_size_check( + operation, src_provider, dest_provider, src_nid, dest_nid) + + if not run_size_check and not check_quota: + return + + data = None + + # Check 1: max file size. Only fetches source data when the check actually applies, + # so a same-storage/same-project move that also skips quota never walks the tree. + if run_size_check: + if src_path.is_dir: + data = await _fetch_all_pages(src_provider, src_path) + else: + data = [await src_provider.metadata(src_path, version=None, revision=None)] + + oversized = await _get_oversized_files(src_provider, data, max_size_bytes, + operation=operation) + if oversized: + raise exceptions.InvalidParameters({ + 'message': 'Move/Copy Failed due to oversized files.', + 'oversized_files': oversized, + 'max_size': max_size_bytes, + }, code=413) + + if not check_quota: + return + + # Check 2: quota -- resolved only now that check 1 has passed (or didn't apply). + skip, dest_quota = await resolve_quota_context(operation, src_provider, dest_provider) + if skip: + return + + if data is None: + if src_path.is_dir: + data = await _fetch_all_pages(src_provider, src_path) + else: + data = [await src_provider.metadata(src_path, version=None, revision=None)] + + file_size = await _get_total_size(src_provider, data, operation=operation) + resolved_name = rename or src_path.name + src_kind = 'folder' if src_path.is_dir else 'file' + replaced_size = await get_replaced_size(dest_provider, dest_path, + resolved_name, conflict, src_kind, + operation=operation) + check_quota_limit(dest_quota, file_size, replaced_size)