mock date in s3 tests when using botocore

scrapy/core/downloader/handlers/s3.py

@@ -68,10 +68,10 @@ class S3DownloadHandler(object):
             except Exception as ex:
                 raise NotConfigured(str(ex))
         else:
-            SignerCls = botocore.auth.AUTH_TYPE_MAPS['s3']
-            # TODO - anon
-            self._signer = SignerCls(botocore.credentials.Credentials(
-                aws_access_key_id, aws_secret_access_key))
+            if not self.anon:
+                SignerCls = botocore.auth.AUTH_TYPE_MAPS['s3']
+                self._signer = SignerCls(botocore.credentials.Credentials(
+                    aws_access_key_id, aws_secret_access_key))
 
         self._download_http = httpdownloadhandler(settings).download_request
 

tests/test_downloader_handlers.py

 import os
 import six
+import contextlib
+try:
+    from unittest import mock
+except ImportError:
+    import mock
 
 from twisted.trial import unittest
 from twisted.protocols.policies import WrappingFactory
@@ -433,16 +438,16 @@ class HttpDownloadHandlerMock(object):
 
 
 class BaseS3TestCase(unittest.TestCase):
-    if six.PY3:
-        try:
-            import botocore
-        except ImportError:
+    try:
+        import botocore
+    except ImportError:
+        if six.PY2:
+            try:
+                import boto
+            except ImportError:
+                skip = 'missing botocore or boto library'
+        else:
             skip = 'missing botocore library'
-    else:
-        try:
-            import boto
-        except ImportError:
-            skip = 'missing boto library'
 
 
 class S3AnonTestCase(BaseS3TestCase):
@@ -464,7 +469,7 @@ class S3AnonTestCase(BaseS3TestCase):
             httpreq.url, 'http://aws-publicdatasets.s3.amazonaws.com/')
 
 
-class S3TestCase(unittest.TestCase):
+class S3TestCase(BaseS3TestCase):
     download_handler_cls = S3DownloadHandler
 
     # test use same example keys than amazon developer guide
@@ -481,63 +486,89 @@ class S3TestCase(unittest.TestCase):
         self.download_request = s3reqh.download_request
         self.spider = Spider('foo')
 
+    @contextlib.contextmanager
+    def _mocked_date(self, date):
+        try:
+            import botocore.auth
+        except ImportError:
+            yield
+        else:
+            # We need to mock botocore.auth.formatdate, because otherwise
+            # botocore overrides Date header with current date and time
+            # and Authorization header is different each time
+            with mock.patch('botocore.auth.formatdate') as mock_formatdate:
+                mock_formatdate.return_value = date
+                yield
+
     def test_request_signing1(self):
         # gets an object from the johnsmith bucket.
-        req = Request('s3://johnsmith/photos/puppy.jpg',
-                headers={'Date': 'Tue, 27 Mar 2007 19:36:42 +0000'})
-        httpreq = self.download_request(req, self.spider)
+        date ='Tue, 27 Mar 2007 19:36:42 +0000'
+        req = Request('s3://johnsmith/photos/puppy.jpg', headers={'Date': date})
+        with self._mocked_date(date):
+            httpreq = self.download_request(req, self.spider)
         self.assertEqual(httpreq.headers['Authorization'], \
                 b'AWS 0PN5J17HBGZHT7JJ3X82:xXjDGYUmKxnwqr5KXNPGldn5LbA=')
 
     def test_request_signing2(self):
         # puts an object into the johnsmith bucket.
+        date = 'Tue, 27 Mar 2007 21:15:45 +0000'
         req = Request('s3://johnsmith/photos/puppy.jpg', method='PUT', headers={
             'Content-Type': 'image/jpeg',
-            'Date': 'Tue, 27 Mar 2007 21:15:45 +0000',
+            'Date': date,
             'Content-Length': '94328',
             })
-        httpreq = self.download_request(req, self.spider)
+        with self._mocked_date(date):
+            httpreq = self.download_request(req, self.spider)
         self.assertEqual(httpreq.headers['Authorization'], \
                 b'AWS 0PN5J17HBGZHT7JJ3X82:hcicpDDvL9SsO6AkvxqmIWkmOuQ=')
 
     def test_request_signing3(self):
         # lists the content of the johnsmith bucket.
+        date = 'Tue, 27 Mar 2007 19:42:41 +0000'
         req = Request('s3://johnsmith/?prefix=photos&max-keys=50&marker=puppy', \
                 method='GET', headers={
                     'User-Agent': 'Mozilla/5.0',
-                    'Date': 'Tue, 27 Mar 2007 19:42:41 +0000',
+                    'Date': date,
                     })
-        httpreq = self.download_request(req, self.spider)
+        with self._mocked_date(date):
+            httpreq = self.download_request(req, self.spider)
         self.assertEqual(httpreq.headers['Authorization'], \
                 b'AWS 0PN5J17HBGZHT7JJ3X82:jsRt/rhG+Vtp88HrYL706QhE4w4=')
 
     def test_request_signing4(self):
         # fetches the access control policy sub-resource for the 'johnsmith' bucket.
-        req = Request('s3://johnsmith/?acl', \
-                method='GET', headers={'Date': 'Tue, 27 Mar 2007 19:44:46 +0000'})
-        httpreq = self.download_request(req, self.spider)
+        date = 'Tue, 27 Mar 2007 19:44:46 +0000'
+        req = Request('s3://johnsmith/?acl',
+            method='GET', headers={'Date': date})
+        with self._mocked_date(date):
+            httpreq = self.download_request(req, self.spider)
         self.assertEqual(httpreq.headers['Authorization'], \
                 b'AWS 0PN5J17HBGZHT7JJ3X82:thdUi9VAkzhkniLj96JIrOPGi0g=')
 
     def test_request_signing5(self):
         # deletes an object from the 'johnsmith' bucket using the
         # path-style and Date alternative.
+        date = 'Tue, 27 Mar 2007 21:20:27 +0000'
         req = Request('s3://johnsmith/photos/puppy.jpg', \
                 method='DELETE', headers={
-                    'Date': 'Tue, 27 Mar 2007 21:20:27 +0000',
+                    'Date': date,
                     'x-amz-date': 'Tue, 27 Mar 2007 21:20:26 +0000',
                     })
-        httpreq = self.download_request(req, self.spider)
-        self.assertEqual(httpreq.headers['Authorization'], \
-                b'AWS 0PN5J17HBGZHT7JJ3X82:k3nL7gH3+PadhTEVn5Ip83xlYzk=')
+        with self._mocked_date(date):
+            httpreq = self.download_request(req, self.spider)
+        # botocore does not override Date with x-amz-date
+        self.assertIn(httpreq.headers['Authorization'], [
+                b'AWS 0PN5J17HBGZHT7JJ3X82:k3nL7gH3+PadhTEVn5Ip83xlYzk=',
+                b'AWS 0PN5J17HBGZHT7JJ3X82:otYM2krxnuHhAofO4oqIV7wcfdU='])
 
     def test_request_signing6(self):
         # uploads an object to a CNAME style virtual hosted bucket with metadata.
+        date = 'Tue, 27 Mar 2007 21:06:08 +0000'
         req = Request('s3://static.johnsmith.net:8080/db-backup.dat.gz', \
                 method='PUT', headers={
                     'User-Agent': 'curl/7.15.5',
                     'Host': 'static.johnsmith.net:8080',
-                    'Date': 'Tue, 27 Mar 2007 21:06:08 +0000',
+                    'Date': date,
                     'x-amz-acl': 'public-read',
                     'content-type': 'application/x-download',
                     'Content-MD5': '4gJE4saaMU4BqNR0kLY+lw==',
@@ -548,20 +579,22 @@ class S3TestCase(unittest.TestCase):
                     'Content-Encoding': 'gzip',
                     'Content-Length': '5913339',
                     })
-        httpreq = self.download_request(req, self.spider)
+        with self._mocked_date(date):
+            httpreq = self.download_request(req, self.spider)
         self.assertEqual(httpreq.headers['Authorization'], \
                 b'AWS 0PN5J17HBGZHT7JJ3X82:C0FlOtU8Ylb9KDTpZqYkZPX91iI=')
 
     def test_request_signing7(self):
         # ensure that spaces are quoted properly before signing
+        date = 'Tue, 27 Mar 2007 19:42:41 +0000'
         req = Request(
             ("s3://johnsmith/photos/my puppy.jpg"
              "?response-content-disposition=my puppy.jpg"),
             method='GET',
-            headers={
-                'Date': 'Tue, 27 Mar 2007 19:42:41 +0000',
-            })
-        httpreq = self.download_request(req, self.spider)
+            headers={'Date': date},
+            )
+        with self._mocked_date(date):
+            httpreq = self.download_request(req, self.spider)
         self.assertEqual(
             httpreq.headers['Authorization'],
             b'AWS 0PN5J17HBGZHT7JJ3X82:+CfvG8EZ3YccOrRVMXNaK2eKZmM=')