feat(s3): S3プロバイダにSigV4署名を追加

requirements.txt

@@ -1,7 +1,13 @@
 aiocontextvars==0.2.2  # recommended for sentry-sdk
 aiohttp==3.6.2
 git+https://github.com/felliott/boto.git@feature/gen-url-query-params-6#egg=boto
-boto3==1.16.63
+boto3==1.16.52
+aiobotocore==1.2.2
+# aiobotocore 1.2.2 requires botocore<1.19.53,>=1.19.52, but neither it nor boto3 1.16.52
+# pins the patch level tightly enough to keep pip inside that window: resolving this file
+# without the line below installs botocore 1.19.63 and `pip check` reports the conflict.
+# aiobotocore reaches into botocore internals, so the window is not advisory.
+botocore==1.19.52
 moto==1.3.16
 aws-sam-translator==1.42.0
 celery==3.1.17

tests/core/test_metadata.py

@@ -1,6 +1,9 @@
 import hashlib
 
+import pytest
+
 from tests import utils
+from waterbutler.core import metadata
 
 
 class TestBaseMetadata:
@@ -114,3 +117,28 @@ def test_file_revision_json_api_serialize(self):
             'modified_utc': 'never',
             'versionIdentifier': 'versions',
         }
+
+
+class TestNoPayloadDrivenConstruction:
+    """CL M-1 / 決定-21: core carries no way to build a metadata class named by a payload.
+
+    The ``dehydrate``/``rehydrate`` pair came from COS's Celery serialization work (ENG-7534)
+    and has nothing to do with SigV4.  ``rehydrate`` read a dotted name out of the payload and
+    turned it into a class -- ``importlib.import_module(...)`` then ``getattr`` -- so anything
+    that ever reached it with attacker-shaped input would be choosing which class WaterButler
+    instantiates and what it is called with.  Nothing in GRDM calls either method, so the
+    gadget sat there earning nothing.
+
+    ``core`` is inherited by every provider, so this is also the one place where the S3 branch
+    was widening its blast radius beyond S3.  These assertions are what keeps a later merge
+    from restoring the pair without the decision being revisited (台帳 U-11).
+    """
+
+    @pytest.mark.parametrize('name', ['dehydrate', '_dehydrate', 'rehydrate', '_rehydrate'])
+    def test_base_metadata_has_no_hydration_hooks(self, name):
+        assert not hasattr(metadata.BaseMetadata, name)
+
+    def test_core_metadata_does_not_import_importlib(self):
+        """The import is the gadget's only ingredient; its absence is what makes the removal
+        real rather than a rename."""
+        assert not hasattr(metadata, 'importlib')

tests/providers/s3/fixtures.py

@@ -30,7 +30,8 @@ def credentials():
 @pytest.fixture
 def settings():
     return {
-        'bucket': 'that kerning',
+        'id': 'that-kerning:/my-subfolder/',
+        'bucket': 'that-kerning',
         'encrypt_uploads': False
     }
 
@@ -42,49 +43,49 @@ def file_content():
 
 @pytest.fixture
 def folder_metadata():
-    with open(os.path.join(os.path.dirname(__file__), 'fixtures/folder_metadata.xml'), 'r') as fp:
+    with open(os.path.join(os.path.dirname(__file__), 'fixtures/folder_metadata.xml')) as fp:
         return fp.read()
 
 
 @pytest.fixture
 def folder_single_item_metadata():
     with open(os.path.join(os.path.dirname(__file__),
-                           'fixtures/folder_single_item_metadata.xml'), 'r') as fp:
+                           'fixtures/folder_single_item_metadata.xml')) as fp:
         return fp.read()
 
 
 @pytest.fixture
 def folder_item_metadata():
     with open(os.path.join(os.path.dirname(__file__),
-                           'fixtures/folder_item_metadata.xml'), 'r') as fp:
+                           'fixtures/folder_item_metadata.xml')) as fp:
         return fp.read()
 
 
 @pytest.fixture
 def folder_and_contents():
     with open(os.path.join(os.path.dirname(__file__),
-                           'fixtures/folder_and_contents.xml'), 'r') as fp:
+                           'fixtures/folder_and_contents.xml')) as fp:
         return fp.read()
 
 
 @pytest.fixture
 def version_metadata():
     with open(os.path.join(os.path.dirname(__file__),
-                           'fixtures/version_metadata.xml'), 'r') as fp:
+                           'fixtures/version_metadata.xml')) as fp:
         return fp.read()
 
 
 @pytest.fixture
 def single_version_metadata():
     with open(os.path.join(os.path.dirname(__file__),
-                           'fixtures/single_version_metadata.xml'), 'r') as fp:
+                           'fixtures/single_version_metadata.xml')) as fp:
         return fp.read()
 
 
 @pytest.fixture
 def folder_empty_metadata():
     with open(os.path.join(os.path.dirname(__file__),
-                           'fixtures/folder_empty_metadata.xml'), 'r') as fp:
+                           'fixtures/folder_empty_metadata.xml')) as fp:
         return fp.read()
 
 
@@ -94,7 +95,7 @@ def file_header_metadata():
         'Content-Length': '9001',
         'Last-Modified': 'SomeTime',
         'Content-Type': 'binary/octet-stream',
-        'Etag': '"fba9dede5f27731c9771645a39863328"',
+        'ETag': '"fba9dede5f27731c9771645a39863328"',
         'x-amz-server-side-encryption': 'AES256'
     }
 
@@ -108,7 +109,7 @@ def file_metadata_headers_object(file_header_metadata):
 def file_metadata_object():
     content = OrderedDict(Key='my-image.jpg',
                           LastModified='2009-10-12T17:50:30.000Z',
-                          ETag="fba9dede5f27731c9771645a39863328",
+                          ETag='"fba9dede5f27731c9771645a39863328"',
                           Size='434234',
                           StorageClass='STANDARD')
 
@@ -154,47 +155,47 @@ def revision_metadata_object():
 @pytest.fixture
 def create_session_resp():
     file_path = 'fixtures/chunked_uploads/create_session_resp.xml'
-    with open(os.path.join(os.path.dirname(__file__), file_path), 'r') as fp:
+    with open(os.path.join(os.path.dirname(__file__), file_path)) as fp:
         return fp.read()
 
 
 @pytest.fixture
 def generic_http_404_resp():
     file_path = 'fixtures/chunked_uploads/generic_http_404_resp.xml'
-    with open(os.path.join(os.path.dirname(__file__), file_path), 'r') as fp:
+    with open(os.path.join(os.path.dirname(__file__), file_path)) as fp:
         return fp.read()
 
 
 @pytest.fixture
 def generic_http_403_resp():
     file_path = 'fixtures/chunked_uploads/generic_http_403_resp.xml'
-    with open(os.path.join(os.path.dirname(__file__), file_path), 'r') as fp:
+    with open(os.path.join(os.path.dirname(__file__), file_path)) as fp:
         return fp.read()
 
 
 @pytest.fixture
 def list_parts_resp_empty():
     file_path = 'fixtures/chunked_uploads/list_parts_resp_empty.xml'
-    with open(os.path.join(os.path.dirname(__file__), file_path), 'r') as fp:
+    with open(os.path.join(os.path.dirname(__file__), file_path)) as fp:
         return fp.read()
 
 
 @pytest.fixture
 def list_parts_resp_not_empty():
     file_path = 'fixtures/chunked_uploads/list_parts_resp_not_empty.xml'
-    with open(os.path.join(os.path.dirname(__file__), file_path), 'r') as fp:
+    with open(os.path.join(os.path.dirname(__file__), file_path)) as fp:
         return fp.read()
 
 
 @pytest.fixture
 def complete_upload_resp():
     file_path = 'fixtures/chunked_uploads/complete_upload_resp.xml'
-    with open(os.path.join(os.path.dirname(__file__), file_path), 'r') as fp:
+    with open(os.path.join(os.path.dirname(__file__), file_path)) as fp:
         return fp.read()
 
 
 @pytest.fixture
 def upload_parts_headers_list():
     file_path = 'fixtures/chunked_uploads/upload_parts_headers_list.json'
-    with open(os.path.join(os.path.dirname(__file__), file_path), 'r') as fp:
+    with open(os.path.join(os.path.dirname(__file__), file_path)) as fp:
         return fp.read()

tests/providers/s3/test_metadata.py

@@ -1,5 +1,12 @@
+from collections import OrderedDict
+
 import pytest
 
+from waterbutler.providers.s3.metadata import (
+    S3Metadata,
+    S3FileMetadata,
+    S3FileMetadataHeaders,
+)
 from tests.providers.s3.fixtures import (
     file_metadata_headers_object,
     file_header_metadata,
@@ -76,6 +83,38 @@ def test_file_metadata(self, file_metadata_object):
         assert file_metadata_object._json_api_links('guid0') == links
 
 
+class TestEtagConsistency:
+    """HEAD and ListObjects must report the same ETag for one object."""
+
+    def _listing(self, quoted_etag):
+        return S3FileMetadata(OrderedDict(
+            Key='my-image.jpg',
+            LastModified='2009-10-12T17:50:30.000Z',
+            ETag=quoted_etag,
+            Size='434234',
+            StorageClass='STANDARD',
+        ))
+
+    @pytest.mark.parametrize('quoted_etag', ['"abc"', '"abc-4"'])
+    def test_headers_and_listing_agree(self, quoted_etag):
+        headers = S3FileMetadataHeaders('my-image.jpg', {
+            'Content-Length': '434234',
+            'Last-Modified': '2009-10-12T17:50:30.000Z',
+            'Content-Type': 'binary/octet-stream',
+            'ETag': quoted_etag,
+        })
+        listing = self._listing(quoted_etag)
+
+        assert headers.etag == listing.etag
+        assert headers.extra['md5'] == listing.extra['md5']
+        assert headers.serialized()['etag'] == listing.serialized()['etag']
+
+    def test_boto_spelling_of_the_header_agrees_too(self):
+        headers = S3FileMetadataHeaders('my-image.jpg', {'Etag': '"abc"'})
+
+        assert headers.etag == self._listing('"abc"').etag
+
+
 class TestFolderMetadata:
 
     def test_folder_metadata(self, folder_metadata_object):
@@ -136,3 +175,17 @@ def test_revisions_metadata_not_lastest(self, revision_metadata_object):
 
         assert revision_metadata_object.version == '3/L4kqtJl40Nr8X8gdRQBpUMLUo'
 
+
+class TestNoHydrationHooks:
+    """The S3 half of the removed Celery serialization hooks is gone too.
+
+    ``S3FileMetadataHeaders`` overrode ``_dehydrate``/``_rehydrate`` to carry ``_path`` through
+    the payload.  With the base pair removed those overrides call a ``super()`` that no longer
+    exists, so leaving them behind would be a method that raises the moment anything reaches
+    it.  See ``tests.core.test_metadata.TestNoPayloadDrivenConstruction`` for why the pair went.
+    """
+
+    @pytest.mark.parametrize('name', ['dehydrate', '_dehydrate', 'rehydrate', '_rehydrate'])
+    def test_s3_metadata_has_no_hydration_hooks(self, name):
+        assert not hasattr(S3Metadata, name)
+        assert not hasattr(S3FileMetadataHeaders, name)