S3FeedStorageTest: add botocore support, and organize boto/botocore checks

scrapy/core/downloader/handlers/s3.py

-import six
 from six.moves.urllib.parse import unquote
 
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.httpobj import urlparse_cached
-from scrapy.utils.python import to_unicode
+from scrapy.utils.boto import is_botocore
 from .http import HTTPDownloadHandler
 
 
-def get_s3_connection():
-    try:
-        from boto.s3.connection import S3Connection
-    except ImportError:
-        return None
+def _get_boto_connection():
+    from boto.s3.connection import S3Connection
 
     class _v19_S3Connection(S3Connection):
         """A dummy S3Connection wrapper that doesn't do any synchronous download"""
@@ -53,21 +49,9 @@ class S3DownloadHandler(object):
         self.anon = kw.get('anon')
 
         self._signer = None
-        try:
+        if is_botocore():
             import botocore.auth
             import botocore.credentials
-        except ImportError:
-            if six.PY3:
-                raise NotConfigured("missing botocore library")
-            _S3Connection = get_s3_connection()
-            if _S3Connection is None:
-                raise NotConfigured("missing botocore or boto library")
-            try:
-                self.conn = _S3Connection(
-                    aws_access_key_id, aws_secret_access_key, **kw)
-            except Exception as ex:
-                raise NotConfigured(str(ex))
-        else:
             kw.pop('anon', None)
             if kw:
                 raise TypeError('Unexpected keyword arguments: %s' % kw)
@@ -75,6 +59,13 @@ class S3DownloadHandler(object):
                 SignerCls = botocore.auth.AUTH_TYPE_MAPS['s3']
                 self._signer = SignerCls(botocore.credentials.Credentials(
                     aws_access_key_id, aws_secret_access_key))
+        else:
+            _S3Connection = _get_boto_connection()
+            try:
+                self.conn = _S3Connection(
+                    aws_access_key_id, aws_secret_access_key, **kw)
+            except Exception as ex:
+                raise NotConfigured(str(ex))
 
         self._download_http = httpdownloadhandler(settings).download_request
 

scrapy/extensions/feedexport.py

@@ -24,6 +24,7 @@ from scrapy.exceptions import NotConfigured
 from scrapy.utils.misc import load_object
 from scrapy.utils.log import failure_to_exc_info
 from scrapy.utils.python import without_none_values
+from scrapy.utils.boto import is_botocore
 
 logger = logging.getLogger(__name__)
 
@@ -90,24 +91,33 @@ class S3FeedStorage(BlockingFeedStorage):
 
     def __init__(self, uri):
         from scrapy.conf import settings
-        try:
-            import boto
-        except ImportError:
-            raise NotConfigured
-        self.connect_s3 = boto.connect_s3
         u = urlparse(uri)
         self.bucketname = u.hostname
         self.access_key = u.username or settings['AWS_ACCESS_KEY_ID']
         self.secret_key = u.password or settings['AWS_SECRET_ACCESS_KEY']
-        self.keyname = u.path
+        self.is_botocore = is_botocore()
+        self.keyname = u.path[1:]  # remove first "/"
+        if self.is_botocore:
+            import botocore.session
+            session = botocore.session.get_session()
+            self.s3_client = session.create_client(
+                's3', aws_access_key_id=self.access_key,
+                aws_secret_access_key=self.secret_key)
+        else:
+            import boto
+            self.connect_s3 = boto.connect_s3
 
     def _store_in_thread(self, file):
         file.seek(0)
-        conn = self.connect_s3(self.access_key, self.secret_key)
-        bucket = conn.get_bucket(self.bucketname, validate=False)
-        key = bucket.new_key(self.keyname)
-        key.set_contents_from_file(file)
-        key.close()
+        if self.is_botocore:
+            self.s3_client.put_object(
+                Bucket=self.bucketname, Key=self.keyname, Body=file)
+        else:
+            conn = self.connect_s3(self.access_key, self.secret_key)
+            bucket = conn.get_bucket(self.bucketname, validate=False)
+            key = bucket.new_key(self.keyname)
+            key.set_contents_from_file(file)
+            key.close()
 
 
 class FTPFeedStorage(BlockingFeedStorage):

scrapy/utils/boto.py

+"""Boto/botocore helpers"""
+
+import six
+
+from scrapy.exceptions import NotConfigured
+
+
+def is_botocore():
+    try:
+        import botocore
+        return True
+    except ImportError:
+        if six.PY2:
+            try:
+                import boto
+                return False
+            except ImportError:
+                raise NotConfigured('missing botocore or boto library')
+        else:
+            raise NotConfigured('missing botocore library')

scrapy/utils/test.py

@@ -5,9 +5,11 @@ This module contains some assorted functions used in tests
 import os
 
 from importlib import import_module
-import six
 from twisted.trial.unittest import SkipTest
 
+from scrapy.exceptions import NotConfigured
+from scrapy.utils.boto import is_botocore
+
 
 def assert_aws_environ():
     """Asserts the current environment is suitable for running AWS testsi.
@@ -19,15 +21,9 @@ def assert_aws_environ():
 
 def skip_if_no_boto():
     try:
-        import botocore
-    except ImportError:
-        if six.PY2:
-            try:
-                import boto
-            except ImportError:
-                raise SkipTest('missing botocore or boto library')
-        else:
-            raise SkipTest('missing botocore library')
+        is_botocore()
+    except NotConfigured as e:
+        raise SkipTest(e.message)
 
 def get_crawler(spidercls=None, settings_dict=None):
     """Return an unconfigured Crawler object. If settings_dict is given, it

tests/test_feedexport.py

@@ -22,6 +22,7 @@ from scrapy.extensions.feedexport import (
 )
 from scrapy.utils.test import assert_aws_environ
 from scrapy.utils.python import to_native_str
+from scrapy.utils.boto import is_botocore
 
 
 class FileFeedStorageTest(unittest.TestCase):
@@ -95,17 +96,30 @@ class S3FeedStorageTest(unittest.TestCase):
         uri = os.environ.get('FEEDTEST_S3_URI')
         if not uri:
             raise unittest.SkipTest("No S3 URI available for testing")
-        from boto import connect_s3
         storage = S3FeedStorage(uri)
         verifyObject(IFeedStorage, storage)
         file = storage.open(scrapy.Spider("default"))
         file.write("content")
         yield storage.store(file)
         u = urlparse(uri)
-        bucket = connect_s3().get_bucket(u.hostname, validate=False)
-        key = bucket.get_key(u.path)
-        self.assertEqual(key.get_contents_as_string(), "content")
-        bucket.delete_key(u.path)
+        content = self._get_content_and_delete(u.hostname, u.path[1:])
+        self.assertEqual(content, "content")
+
+    def _get_content_and_delete(self, bucket, path):
+        if is_botocore():
+            import botocore.session
+            session = botocore.session.get_session()
+            client = session.create_client('s3')
+            key = client.get_object(Bucket=bucket, Key=path)
+            content = key['Body'].read()
+            client.delete_object(Bucket=bucket, Key=path)
+        else:
+            from boto import connect_s3
+            bucket = connect_s3().get_bucket(bucket, validate=False)
+            key = bucket.get_key(path)
+            content = key.get_contents_as_string()
+            bucket.delete_key(path)
+        return content
 
 
 class StdoutFeedStorageTest(unittest.TestCase):