Support libcurl stroage backend(read-only)

  1. url: <protocol>://[<username>[:<password>]@]<host>/<path>
     protocol: 'http', 'https', 'ftp', or 'ftps'
  2. -blockdev driver=https, url=https://a.b.c.d/image,
     username=xx, password-secret=xx, sslverify=off,readahead=xx,
     timeout=xxx
Signed-off-by: NZhenchao Liu <zhencliu@redhat.com>

shared/cfg/guest-hw.cfg

@@ -272,6 +272,34 @@ variants image_backend:
         # On an unexpected disconnect, the nbd client tries to connect again
         # until succeeding or encountering a serious error, optional
         #nbd_reconnect_delay =
+    - libcurl:
+        storage_type = curl
+        enable_curl = yes
+        force_create_image = no
+        remove_image = no
+        image_readonly = yes
+        skip_hash = yes
+        # Address of the remote server, required
+        #curl_server =
+        # 'http', 'https', 'ftp', or 'ftps', required
+        #curl_protocol =
+        # Image path on the remote server, required
+        #curl_path =
+        # Whether to verify the remote server's certificate when connecting over SSL.
+        # It can have the value 'on'(default) or 'off', https/ftps only, optional
+        #curl_sslverify =
+        # Cookie data provided in a secure way, http/https only, optional
+        #curl_cookie_secret =
+        # Username for authentication to the remote server, optional
+        #curl_username =
+        # Password for authentication to the remote server, optional
+        #curl_password =
+        # The amount of data to read ahead, which may optionally have the suffix
+        # 'T', 'G', 'M', 'K', 'k' or 'b'. If it does not have a suffix, it is in bytes.
+        # The value must be a multiple of 512 bytes. It defaults to 256k, optional
+        #curl_readahead =
+        # The timeout in seconds of the CURL connection, it defaults to 5s, optional
+        #curl_timeout =
     - gluster_direct:
         # Access gluster storage directly by URI
         storage_type = glusterfs-direct

virttest/curl.py

+from virttest import error_context
+
+from avocado.utils import process
+
+
+@error_context.context_aware
+def get_image_filename(curl_protocol, curl_server, curl_path,
+                       curl_user=None, curl_passwd=None):
+    """
+    Form the url: <protocol>://[<username>[:<password>]@]<host>/<path>
+
+    :param curl_protocol: one of 'http', 'https', 'ftp', 'ftps'
+    :param curl_server: Address of the remote server
+    :param curl_path: Path on the remote server, including any query string
+    :param curl_user: Username for authentication to the remote server
+    :param curl_passwd: Password for authentication to the remote server
+    :return: Remote image filename in url format
+    """
+    url = "{protocol}://{auth}{host}/{path}"
+
+    protocols = ('http', 'https', 'ftp', 'ftps')
+    if curl_protocol not in protocols:
+        raise ValueError('curl_protocol should be in %s.' % str(protocols))
+
+    auth = ''
+    if curl_user:
+        auth = '%s:%s@' % (curl_user,
+                           curl_passwd) if curl_passwd else '%s@' % curl_user
+
+    return url.format(protocol=curl_protocol, auth=auth,
+                      host=curl_server, path=curl_path)
+
+
+@error_context.context_aware
+def file_exists(params, filename):
+    """
+    Check whether the image on curl storage exists.
+
+    :param params: A dict containing image parameters.
+    :param filename: The image filename
+    :return: True if the image exists, else False
+    """
+    curl_cmd = 'curl -I -L -k {tmo} {filename}'
+    t = '-m %s' % params['curl_timeout'] if params.get('curl_timeout') else ''
+    o = process.run(curl_cmd.format(tmo=t, filename=filename),
+                    ignore_status=True, verbose=True).stdout_text.strip()
+    return 'Content-Length:' in o

virttest/qemu_devices/qcontainer.py

@@ -1660,6 +1660,16 @@ class DevContainer(object):
                         else:
                             # -drive: u/p included in the filename
                             pass
+                    elif access.storage_type == 'curl':
+                        if access.cookie:
+                            secrets.append((access.cookie, 'cookie'))
+
+                        if Flags.BLOCKDEV in self.caps:
+                            # -blockdev requires password-secret while
+                            # -drive includes u/p in the filename
+                            if access.data:
+                                secrets.append((access, 'password'))
+
             return secrets
 
         def define_hbas(qtype, atype, bus, unit, port, qbus, pci_bus, iothread,
@@ -1737,8 +1747,7 @@ class DevContainer(object):
             if image_encryption.key_secret:
                 secret_obj = devices[-1]
 
-        access_secret_obj = None
-        access_secret, secret_type = None, None
+        secret_info = []
         image_secrets = _get_access_secrets(image_access)
         for sec, sectype in image_secrets:
             # create and add all secret objects: -object secret
@@ -1748,14 +1757,13 @@ class DevContainer(object):
 
             if sectype == 'password':
                 devices[-1].set_param("file", sec.filename)
-            elif sectype == 'key':
+            elif sectype == 'key' or sectype == 'cookie':
                 devices[-1].set_param("data", sec.data)
 
             if sec.image == name:
                 # only the top image should be associated
                 # with its secure object
-                access_secret_obj = devices[-1]
-                access_secret, secret_type = sec, sectype
+                secret_info.append((devices[-1], sectype))
 
         tls_creds = None
         tls_creds_obj = None
@@ -1778,6 +1786,9 @@ class DevContainer(object):
         gluster_logfile = None
         gluster_peers = {}
         reconnect_delay = None
+        curl_sslverify = None
+        curl_readahead = None
+        curl_timeout = None
         access = image_access.image_auth if image_access else None
         if access is not None:
             if access.storage_type == 'iscsi-direct':
@@ -1801,6 +1812,10 @@ class DevContainer(object):
                                      for k, v in six.iteritems(server)})
             elif access.storage_type == 'nbd':
                 reconnect_delay = access.reconnect_delay
+            elif access.storage_type == 'curl':
+                curl_sslverify = access.sslverify
+                curl_timeout = access.timeout
+                curl_readahead = access.readahead
 
         use_device = self.has_option("device")
         if fmt == "scsi":   # fmt=scsi force the old version of devices
@@ -1949,6 +1964,14 @@ class DevContainer(object):
                 protocol_cls = qdevices.QBlockdevProtocolNVMe
             elif filename.startswith('ssh:'):
                 protocol_cls = qdevices.QBlockdevProtocolSSH
+            elif filename.startswith('https:'):
+                protocol_cls = qdevices.QBlockdevProtocolHTTPS
+            elif filename.startswith('http:'):
+                protocol_cls = qdevices.QBlockdevProtocolHTTP
+            elif filename.startswith('ftps:'):
+                protocol_cls = qdevices.QBlockdevProtocolFTPS
+            elif filename.startswith('ftp:'):
+                protocol_cls = qdevices.QBlockdevProtocolFTP
             elif fmt in ('scsi-generic', 'scsi-block'):
                 protocol_cls = qdevices.QBlockdevProtocolHostDevice
             elif blkdebug is not None:
@@ -2058,13 +2081,16 @@ class DevContainer(object):
                 for key, value in six.iteritems(file_opts):
                     devices[-2].set_param(key, value)
 
-            if access_secret is not None:
+            for access_secret_obj, secret_type in secret_info:
                 if secret_type == 'password':
                     devices[-2].set_param('password-secret',
                                           access_secret_obj.get_qid())
                 elif secret_type == 'key':
                     devices[-2].set_param('key-secret',
                                           access_secret_obj.get_qid())
+                elif secret_type == 'cookie':
+                    devices[-2].set_param('cookie-secret',
+                                          access_secret_obj.get_qid())
 
             if tls_creds is not None:
                 devices[-2].set_param('tls-creds', tls_creds_obj.get_qid())
@@ -2076,6 +2102,12 @@ class DevContainer(object):
                 devices[-2].set_param('debug', int(gluster_debug))
             if gluster_logfile:
                 devices[-2].set_param('logfile', gluster_logfile)
+            if curl_sslverify:
+                devices[-2].set_param('sslverify', curl_sslverify)
+            if curl_readahead:
+                devices[-2].set_param('readahead', curl_readahead)
+            if curl_timeout:
+                devices[-2].set_param('timeout', curl_timeout)
             for key, value in six.iteritems(gluster_peers):
                 devices[-2].set_param(key, value)
 
@@ -2097,13 +2129,16 @@ class DevContainer(object):
             else:
                 devices[-1].set_param('file', filename)
 
-            if access_secret is not None:
+            for access_secret_obj, secret_type in secret_info:
                 if secret_type == 'password':
                     devices[-1].set_param('file.password-secret',
                                           access_secret_obj.get_qid())
                 elif secret_type == 'key':
                     devices[-1].set_param('file.key-secret',
                                           access_secret_obj.get_qid())
+                elif secret_type == 'cookie':
+                    devices[-1].set_param('file.cookie-secret',
+                                          access_secret_obj.get_qid())
 
             if tls_creds is not None:
                 devices[-1].set_param('file.tls-creds',
@@ -2117,6 +2152,12 @@ class DevContainer(object):
                 devices[-1].set_param('file.debug', int(gluster_debug))
             if gluster_logfile:
                 devices[-1].set_param('file.logfile', gluster_logfile)
+            if curl_sslverify:
+                devices[-1].set_param('file.sslverify', curl_sslverify)
+            if curl_readahead:
+                devices[-1].set_param('file.readahead', curl_readahead)
+            if curl_timeout:
+                devices[-1].set_param('file.timeout', curl_timeout)
 
         if drv_extra_params:
             drv_extra_params = (_.split('=', 1) for _ in
@@ -2560,6 +2601,10 @@ class DevContainer(object):
                 scsi_hba = "virtio-scsi-ccw"
         shared_dir = os.path.join(data_dir.get_data_dir(), "shared")
         cache_mode = image_params.get('image_aio') == 'native' and 'none' or ''
+
+        # iso image can be stored in a network storage, e.g. http server
+        image_access = storage.ImageAccessInfo.access_info_define_by_params(
+            name, image_params)
         return self.images_define_by_variables(name,
                                                storage.get_image_filename(
                                                    image_params,
@@ -2615,7 +2660,8 @@ class DevContainer(object):
                                                None,
                                                image_params.get(
                                                    "bus_extra_params"),
-                                               image_params.get("force_drive_format"))
+                                               image_params.get("force_drive_format"),
+                                               None, image_access)
 
     def pcic_by_params(self, name, params, parent_bus=None):
         """

virttest/qemu_devices/qdevices.py

@@ -926,6 +926,26 @@ class QBlockdevProtocolSSH(QBlockdevProtocol):
     TYPE = 'ssh'
 
 
+class QBlockdevProtocolHTTP(QBlockdevProtocol):
+    """ New a protocol http blockdev node. """
+    TYPE = 'http'
+
+
+class QBlockdevProtocolHTTPS(QBlockdevProtocol):
+    """ New a protocol https blockdev node. """
+    TYPE = 'https'
+
+
+class QBlockdevProtocolFTP(QBlockdevProtocol):
+    """ New a protocol ftp blockdev node. """
+    TYPE = 'ftp'
+
+
+class QBlockdevProtocolFTPS(QBlockdevProtocol):
+    """ New a protocol ftps blockdev node. """
+    TYPE = 'ftps'
+
+
 class QDevice(QCustomDevice):
 
     """

virttest/qemu_storage.py

@@ -160,6 +160,26 @@ def filename_to_file_opts(filename):
                             'host-key-check.type': m['type'],
                             'host-key-check.hash': m['hash']
                         })
+    elif re.match(r'(http|https|ftp|ftps)://', filename):
+        filename_pattern = re.compile(
+            r'(?P<protocol>.+?)://((?P<user>.+?)(:(?P<password>.+?))?@)?'
+            r'(?P<server>.+?)/(?P<path>.+)')
+        matches = filename_pattern.match(filename)
+        if matches:
+            matches = matches.groupdict()
+            if all((matches['protocol'], matches['server'], matches['path'])):
+                # required libcurl options, note server can be hostname:port
+                file_opts = {
+                    'driver': matches['protocol'],
+                    'url': '{protocol}://{server}/{path}'.format(
+                        protocol=matches['protocol'],
+                        server=matches['server'],
+                        path=matches['path']
+                    )
+                }
+
+                if matches['user'] is not None:
+                    file_opts['username'] = matches['user']
     # FIXME: Judge the host device by the string starts with "/dev/".
     elif filename.startswith('/dev/'):
         file_opts = {'driver': 'host_device', 'filename': filename}
@@ -229,6 +249,19 @@ def _get_image_meta(image, params, root_dir):
                 meta['file']['tls-creds'] = auth_info.aid
             if auth_info.reconnect_delay:
                 meta['file']['reconnect-delay'] = auth_info.reconnect_delay
+        elif auth_info.storage_type == 'curl':
+            mapping = {
+                'password-secret': (auth_info.data, auth_info.aid),
+                'sslverify': (auth_info.sslverify, auth_info.sslverify),
+                'cookie-secret': (auth_info.cookie,
+                                  auth_info.cookie.aid
+                                  if auth_info.cookie else ''),
+                'readahead': (auth_info.readahead, auth_info.readahead),
+                'timeout': (auth_info.timeout, auth_info.timeout)
+            }
+            meta['file'].update({
+                k: v[1] for k, v in six.iteritems(mapping) if v[0]
+            })
 
     return meta
 
@@ -296,6 +329,12 @@ def get_image_repr(image, params, root_dir, representation=None):
             elif auth_info.storage_type == 'nbd':
                 # tls-creds, reconnect_delay represent in json
                 access_needed = True
+            elif auth_info.storage_type == 'curl':
+                # u/p can be included in url, while the others should be
+                # represented in json
+                if any((auth_info.sslverify, auth_info.cookie,
+                        auth_info.readahead, auth_info.timeout)):
+                    access_needed = True
 
         func = mapping["json"] if image_secret or access_needed else mapping["filename"]
     return func(image, params, root_dir)
@@ -452,12 +491,13 @@ class QemuImg(storage.QemuImg):
                 ("data_file=%s" % self.data_file.image_filename,
                  "data_file_raw=%s" % params.get("image_data_file_raw", "off")))
 
-        access_secret, secret_type = self._image_access_secret
-        if access_secret is not None:
+        for access_secret, secret_type in self._image_access_secret:
             if secret_type == 'password':
                 options.append("password-secret=%s" % access_secret.aid)
             elif secret_type == 'key':
                 options.append("key-secret=%s" % access_secret.aid)
+            elif secret_type == 'cookie':
+                options.append("cookie-secret=%s" % access_secret.aid)
 
         image_extra_params = params.get("image_extra_params")
         if image_extra_params:
@@ -521,21 +561,29 @@ class QemuImg(storage.QemuImg):
     def _image_access_secret(self):
         """
         Get the access secret object and its type of the image itself,
-        the type can be 'key' or 'password'
+        the type can be 'key' or 'password' or 'cookie'
 
-        :return: a tuple, (StorageAuth object, secret type)
+        :return: a list of tuple(StorageAuth object, secret type) or []
+        :note: an image can have more than one secret objects, e.g.
+               access secret object and cookie secret object for libcurl
         """
-        sec, sec_type = None, None
+        secrets = []
         auth = self.image_access.image_auth if self.image_access else None
 
         if auth is not None:
             if auth.storage_type == 'ceph':
-                # Only ceph image access requires secret object by
+                # ceph image access requires secret object by
                 # qemu-img and only 'password-secret' is supported
                 if auth.data:
-                    sec, sec_type = auth, 'password'
+                    secrets.append((auth, 'password'))
+            elif auth.storage_type == 'curl':
+                # a libcurl image can have more than one secret object
+                if auth.data:
+                    secrets.append((auth, 'password'))
+                if auth.cookie:
+                    secrets.append((auth.cookie, 'cookie'))
 
-        return sec, sec_type
+        return secrets
 
     @property
     def _backing_access_secrets(self):
@@ -550,10 +598,15 @@ class QemuImg(storage.QemuImg):
 
         for auth in info:
             if auth.storage_type == 'ceph':
-                # Only ceph image access requires secret object by
+                # ceph image access requires secret object by
                 # qemu-img and only 'password-secret' is supported
                 if auth.data:
                     secrets.append((auth, 'password'))
+            elif auth.storage_type == 'curl':
+                if auth.data:
+                    secrets.append((auth, 'password'))
+                if auth.cookie:
+                    secrets.append((auth.cookie, 'cookie'))
 
         return secrets
 
@@ -585,17 +638,17 @@ class QemuImg(storage.QemuImg):
     @property
     def _image_access_secret_object(self):
         """Get the secret object str of the image itself."""
-        secret_obj_str = ''
-        access_secret, secret_type = self._image_access_secret
+        secrets = []
 
-        if access_secret is not None:
+        for access_secret, secret_type in self._image_access_secret:
+            secret_obj_str = ''
             if secret_type == 'password':
-                secret = '--object secret,id={s.aid},format={s.data_format},file={s.filename}'
-            elif secret_type == 'key':
-                secret = '--object secret,id={s.aid},format={s.data_format},data={s.data}'
-            secret_obj_str = secret.format(s=access_secret)
+                secret_obj_str = '--object secret,id={s.aid},format={s.data_format},file={s.filename}'
+            elif secret_type == 'key' or secret_type == 'cookie':
+                secret_obj_str = '--object secret,id={s.aid},format={s.data_format},data={s.data}'
+            secrets.append(secret_obj_str.format(s=access_secret))
 
-        return secret_obj_str
+        return secrets
 
     @property
     def _backing_access_secret_objects(self):
@@ -606,7 +659,7 @@ class QemuImg(storage.QemuImg):
             secret_obj_str = ''
             if secret_type == 'password':
                 secret_obj_str = "--object secret,id={s.aid},format={s.data_format},file={s.filename}"
-            elif secret_type == 'key':
+            elif secret_type == 'key' or secret_type == 'cookie':
                 secret_obj_str = "--object secret,id={s.aid},format={s.data_format},data={s.data}"
             secrets.append(secret_obj_str.format(s=access_secret))
 
@@ -683,7 +736,7 @@ class QemuImg(storage.QemuImg):
 
             # secret object of the image itself
             if self._image_access_secret_object:
-                secret_objects.append(self._image_access_secret_object)
+                secret_objects.extend(self._image_access_secret_object)
 
             image_secret_objects = self._secret_objects
             if image_secret_objects:
@@ -818,12 +871,12 @@ class QemuImg(storage.QemuImg):
 
         # secret object of the source image itself
         if self._image_access_secret_object:
-            secret_objects.append(self._image_access_secret_object)
+            secret_objects.extend(self._image_access_secret_object)
 
         # target image access secret object
         # target image to be converted never has backing images
         if convert_image._image_access_secret_object:
-            secret_objects.append(convert_image._image_access_secret_object)
+            secret_objects.extend(convert_image._image_access_secret_object)
 
         # target image secret(luks)
         if convert_image.encryption_config.key_secret:
@@ -1062,7 +1115,7 @@ class QemuImg(storage.QemuImg):
 
         if self._image_access_secret_object:
             # secret object of the image itself
-            cmd += " %s" % self._image_access_secret_object
+            cmd += " %s" % " ".join(self._image_access_secret_object)
 
         if self._image_access_tls_creds_object:
             # tls creds object of the image itself
@@ -1202,7 +1255,7 @@ class QemuImg(storage.QemuImg):
 
         # source image access secret object
         if self._image_access_secret_object:
-            secret_objects.append(self._image_access_secret_object)
+            secret_objects.extend(self._image_access_secret_object)
 
         # target image's backing access secret objects
         if target_image._backing_access_secret_objects:
@@ -1210,7 +1263,7 @@ class QemuImg(storage.QemuImg):
 
         # target image access secret object
         if target_image._image_access_secret_object:
-            secret_objects.append(target_image._image_access_secret_object)
+            secret_objects.extend(target_image._image_access_secret_object)
 
         # if compared images are in the same snapshot chain,
         # needs to remove duplicated secrets
@@ -1288,7 +1341,7 @@ class QemuImg(storage.QemuImg):
 
         # access secret object of the image itself
         if self._image_access_secret_object:
-            secret_objects.append(self._image_access_secret_object)
+            secret_objects.extend(self._image_access_secret_object)
 
         # image(e.g. luks image) secret objects
         image_secret_objects = self._secret_objects
@@ -1550,7 +1603,7 @@ class QemuImg(storage.QemuImg):
 
         # access secret object of the image itself
         if self._image_access_secret_object:
-            secret_objects.append(self._image_access_secret_object)
+            secret_objects.extend(self._image_access_secret_object)
 
         if secret_objects:
             cmd_dict["secret_object"] = " ".join(secret_objects)

virttest/qemu_vm.py

@@ -2678,11 +2678,13 @@ class VM(virt_vm.BaseVM):
                 continue
             if cdrom_params.get("enable_ssh") == "yes":
                 continue
-            iso = cdrom_params.get("cdrom")
+
+            iso = storage.get_iso_filename(cdrom_params,
+                                           data_dir.get_data_dir())
             if iso:
-                iso = utils_misc.get_path(data_dir.get_data_dir(), iso)
-                if not os.path.exists(iso):
+                if not storage.file_exists(cdrom_params, iso):
                     raise virt_vm.VMImageMissingError(iso)
+
                 compare = False
                 if cdrom_params.get("skip_hash", "no") == "yes":
                     logging.debug("Skipping hash comparison")

virttest/storage.py

@@ -20,6 +20,7 @@ from avocado.utils import process
 
 from virttest import storage_ssh
 from virttest import nbd
+from virttest import curl
 from virttest import iscsi
 from virttest import utils_misc
 from virttest import utils_numeric
@@ -93,6 +94,9 @@ def file_exists(params, filename_path):
     if params.get('enable_ssh') == 'yes':
         return storage_ssh.file_exists(params, filename_path)
 
+    if params.get('enable_curl') == 'yes':
+        return curl.file_exists(params, filename_path)
+
     return os.path.exists(filename_path)
 
 
@@ -164,6 +168,7 @@ def get_image_filename(params, root_dir, basename=False):
     :raise VMDeviceError: When no matching disk found (in indirect method).
     """
     enable_ssh = params.get("enable_ssh") == "yes"
+    enable_curl = params.get("enable_curl") == "yes"
     enable_nbd = params.get("enable_nbd") == "yes"
     enable_gluster = params.get("enable_gluster") == "yes"
     enable_ceph = params.get("enable_ceph") == "yes"
@@ -173,6 +178,18 @@ def get_image_filename(params, root_dir, basename=False):
     storage_type = params.get("storage_type")
     if image_name:
         image_format = params.get("image_format", "qcow2")
+        if enable_curl:
+            # required libcurl params
+            curl_protocol = params['curl_protocol']
+            curl_server = params['curl_server']
+            curl_path = params['curl_path']
+
+            # optional libcurl params
+            curl_user = params.get('curl_username')
+            curl_passwd = params.get('curl_password')
+
+            return curl.get_image_filename(curl_protocol, curl_server,
+                                           curl_path, curl_user, curl_passwd)
         if enable_nbd:
             return nbd.get_image_filename(params)
         if enable_iscsi:
@@ -297,6 +314,36 @@ def get_image_filename_filesytem(params, root_dir, basename=False):
     return image_filename
 
 
+def get_iso_filename(cdrom_params, root_dir, basename=False):
+    """
+    Generate an iso image path from params and root_dir.
+
+    :param cdrom_params: Dictionary containing the test parameters.
+    :param root_dir: Base directory for relative iso image.
+    :param basename: True to use only basename of iso image.
+    :return: iso filename
+    """
+    enable_nvme = cdrom_params.get("enable_nvme") == "yes"
+    enable_nbd = cdrom_params.get("enable_nbd") == "yes"
+    enable_gluster = cdrom_params.get("enable_gluster") == "yes"
+    enable_ceph = cdrom_params.get("enable_ceph") == "yes"
+    enable_iscsi = cdrom_params.get("enable_iscsi") == "yes"
+    enable_curl = cdrom_params.get("enable_curl") == "yes"
+    enable_ssh = cdrom_params.get("enable_ssh") == "yes"
+
+    if enable_nvme:
+        return None
+    elif any((enable_nbd, enable_gluster, enable_ceph,
+              enable_iscsi, enable_curl, enable_ssh)):
+        return get_image_filename(cdrom_params, None, basename)
+    else:
+        iso = cdrom_params.get("cdrom")
+        if iso:
+            iso = os.path.basename(iso) if basename else utils_misc.get_path(
+                                                               root_dir, iso)
+        return iso
+
+
 secret_dir = os.path.join(data_dir.get_data_dir(), "images/secrets")
 
 
@@ -340,12 +387,38 @@ class ImageSecret(object):
             fd.write(self.data)
 
 
+class Cookie(object):
+    """
+    Cookie data stored in secret object
+    """
+
+    def __init__(self, image, cookie_data, cookie_data_format):
+        """
+        :param image: image tag name
+        :param cookie_data: cookie data string
+        :param cookie_data_format: raw or base64
+        """
+        self.image = image
+        self.data = cookie_data
+        self.data_format = cookie_data_format
+        self.aid = '%s_cookie_secret' % self.image
+        self.filename = os.path.join(secret_dir, "%s.secret" % self.aid)
+        self.save_to_file()
+
+    def save_to_file(self):
+        """Save secret data to file."""
+        _make_secret_dir()
+        with open(self.filename, "w") as fd:
+            fd.write(self.data)
+
+
 class StorageAuth(object):
     """
     Image storage authentication class.
     iscsi auth: initiator + password
     ceph auth: ceph key
     nbd auth: tls creds
+    libcurl auth: password sslverify timeout readahead cookie
     """
 
     def __init__(self, image, data, data_format, storage_type, **info):
@@ -353,11 +426,12 @@ class StorageAuth(object):
         :param image: image tag name
         :param data: sensitive data like password
         :param data_format: raw or base64
-        :param storage_type: ceph, glusterfs-direct or iscsi-direct
+        :param storage_type: ceph, glusterfs-direct, iscsi-direct, nbd, curl
         :param info: other access information, such as:
                      iscsi-direct: initiator
                      gluster-direct: debug, logfile, peers
                      nbd: tls creds path for client access
+                     libcurl: password, sslverify, timeout, readahead, cookie
         """
         self.image = image
         self.aid = '%s_access' % self.image
@@ -383,8 +457,15 @@ class StorageAuth(object):
             # e.g. tls-creds-psk
             self.tls_creds = info['tls_creds']
             self.reconnect_delay = info['reconnect_delay']
+        elif self.storage_type == 'curl':
+            self._password = data
+            self.cookie = info['cookie']
+            self.sslverify = info['sslverify']
+            self.readahead = info['readahead']
+            self.timeout = info['timeout']
 
         if self.data is not None:
+            self.filename = os.path.join(secret_dir, "%s.secret" % self.aid)
             self.save_to_file()
 
     @property
@@ -393,6 +474,8 @@ class StorageAuth(object):
             return self._chap_passwd
         elif self.storage_type == 'ceph':
             return self._ceph_key
+        elif self.storage_type == 'curl':
+            return self._password
         else:
             return None
 
@@ -414,6 +497,7 @@ class StorageAuth(object):
         enable_iscsi = params.get("enable_iscsi") == "yes"
         enable_gluster = params.get("enable_gluster") == "yes"
         enable_nbd = params.get("enable_nbd") == "yes"
+        enable_curl = params.get("enable_curl") == "yes"
 
         if enable_iscsi:
             if storage_type == 'iscsi-direct':
@@ -444,6 +528,33 @@ class StorageAuth(object):
                 image, None, None, storage_type,
                 tls_creds=tls_creds, reconnect_delay=reconnect_delay
             ) if tls_creds or reconnect_delay else None
+        elif enable_curl:
+            # cookie data in a secure way, only for http/https
+            cookie = Cookie(
+                image, params['curl_cookie_secret'],
+                params.get('curl_cookie_secret_format', 'raw'),
+            ) if params.get('curl_cookie_secret') else None
+
+            # sslverify, only for https/ftps
+            sslverify = params.get('curl_sslverify')
+
+            # size of the read-ahead cache
+            readahead = int(float(
+                utils_numeric.normalize_data_size(params['curl_readahead'],
+                                                  order_magnitude="B")
+            )) if params.get('curl_readahead') else None
+
+            # timeout for connections in seconds
+            timeout = params.get('curl_timeout')
+
+            # password
+            data = params.get('curl_password')
+            data_format = params.get('curl_password_format', 'raw')
+
+            if any((data, cookie, sslverify, readahead, timeout)):
+                auth = cls(image, data, data_format, storage_type,
+                           cookie=cookie, sslverify=sslverify,
+                           readahead=readahead, timeout=timeout)
 
         return auth
 
@@ -655,8 +766,10 @@ class QemuImg(object):
         self.check_output = params.get("check_output") == "yes"
         self.image_blkdebug_filename = get_image_blkdebug_filename(params,
                                                                    root_dir)
-        self.remote_keywords = params.get("remote_image",
-                                          "gluster iscsi rbd nbd nvme").split()
+        self.remote_keywords = params.get(
+            "remote_image",
+            "gluster iscsi rbd nbd nvme http https ftp ftps"
+        ).split()
         self.encryption_config = ImageEncryption.encryption_define_by_params(
             tag, params)
         image_chain = params.get("image_chain")