diff -Nru cinder-17.0.1/api-ref/source/v3/attachments.inc cinder-17.4.0/api-ref/source/v3/attachments.inc --- cinder-17.0.1/api-ref/source/v3/attachments.inc 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/api-ref/source/v3/attachments.inc 2022-04-20 17:38:33.000000000 +0000 @@ -253,7 +253,7 @@ - project_id: project_id_path - attachment: attachment - - instance_uuid: instance_uuid_req + - instance_uuid: instance_uuid - connector: connector - volume_uuid: volume_id_attachment - mode: attach_mode diff -Nru cinder-17.0.1/api-ref/source/v3/parameters.yaml cinder-17.4.0/api-ref/source/v3/parameters.yaml --- cinder-17.0.1/api-ref/source/v3/parameters.yaml 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/api-ref/source/v3/parameters.yaml 2022-04-20 17:38:33.000000000 +0000 @@ -863,6 +863,13 @@ in: body required: false type: string +container_format_upload: + description: | + Container format for the new image. Default is bare. (Note: Volumes + of an encrypted volume type must use a bare container format.) + in: body + required: false + type: string control_location: description: | Notional service where encryption is performed. Valid values are @@ -1110,6 +1117,13 @@ in: body required: false type: string +disk_format_upload: + description: | + Disk format for the new image. Default is raw. (Note: volumes of an + encrypted volume type can only be uploaded in raw format.) + in: body + required: false + type: string display_name: description: | The name of volume backend capabilities. diff -Nru cinder-17.0.1/api-ref/source/v3/volumes-v3-volumes-actions.inc cinder-17.4.0/api-ref/source/v3/volumes-v3-volumes-actions.inc --- cinder-17.0.1/api-ref/source/v3/volumes-v3-volumes-actions.inc 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/api-ref/source/v3/volumes-v3-volumes-actions.inc 2022-04-20 17:38:33.000000000 +0000 @@ -699,8 +699,8 @@ - os-volume_upload_image: os-volume_upload_image - image_name: image_name - force: force_upload_vol - - disk_format: disk_format - - container_format: container_format + - disk_format: disk_format_upload + - container_format: container_format_upload - visibility: visibility_min - protected: protected diff -Nru cinder-17.0.1/bindep.txt cinder-17.4.0/bindep.txt --- cinder-17.0.1/bindep.txt 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/bindep.txt 2022-04-20 17:38:33.000000000 +0000 @@ -44,3 +44,5 @@ qemu-img [platform:redhat] qemu-tools [platform:suse] qemu-utils [platform:dpkg] +libcgroup-tools [platform:rpm] +cgroup-tools [platform:dpkg] diff -Nru cinder-17.0.1/cinder/api/contrib/volume_actions.py cinder-17.4.0/cinder/api/contrib/volume_actions.py --- cinder-17.0.1/cinder/api/contrib/volume_actions.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/api/contrib/volume_actions.py 2022-04-20 17:38:33.000000000 +0000 @@ -219,6 +219,14 @@ "name": params["image_name"]} if volume.encryption_key_id: + # encrypted volumes cannot be converted on upload + if (image_metadata['disk_format'] != 'raw' + or image_metadata['container_format'] != 'bare'): + msg = _("An encrypted volume uploaded as an image must use " + "'raw' disk_format and 'bare' container_format, " + "which are the defaults for these options.") + raise webob.exc.HTTPBadRequest(explanation=msg) + # Clone volume encryption key: the current key cannot # be reused because it will be deleted when the volume is # deleted. diff -Nru cinder-17.0.1/cinder/api/schemas/attachments.py cinder-17.4.0/cinder/api/schemas/attachments.py --- cinder-17.0.1/cinder/api/schemas/attachments.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/api/schemas/attachments.py 2022-04-20 17:38:33.000000000 +0000 @@ -17,6 +17,7 @@ Schema for V3 Attachments API. """ +import copy from cinder.api.validation import parameter_types @@ -32,7 +33,7 @@ 'connector': {'type': ['object', 'null']}, 'volume_uuid': parameter_types.uuid, }, - 'required': ['instance_uuid', 'volume_uuid'], + 'required': ['volume_uuid'], 'additionalProperties': False, }, }, @@ -56,3 +57,7 @@ 'required': ['attachment'], 'additionalProperties': False, } + +create_v354 = copy.deepcopy(create) +create_v354['properties']['attachment']['properties']['mode'] = ( + {'type': 'string', 'enum': ['rw', 'ro']}) diff -Nru cinder-17.0.1/cinder/api/schemas/volume_manage.py cinder-17.4.0/cinder/api/schemas/volume_manage.py --- cinder-17.0.1/cinder/api/schemas/volume_manage.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/api/schemas/volume_manage.py 2022-04-20 17:38:33.000000000 +0000 @@ -33,7 +33,7 @@ "bootable": parameter_types.boolean, "volume_type": parameter_types.name_allow_zero_min_length, "name": parameter_types.name_allow_zero_min_length, - "host": parameter_types.hostname, + "host": parameter_types.cinder_host, "ref": {'type': ['object', 'string']}, "metadata": parameter_types.metadata_allows_null, }, diff -Nru cinder-17.0.1/cinder/api/v3/attachments.py cinder-17.4.0/cinder/api/v3/attachments.py --- cinder-17.0.1/cinder/api/v3/attachments.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/api/v3/attachments.py 2022-04-20 17:38:33.000000000 +0000 @@ -101,7 +101,9 @@ @wsgi.Controller.api_version(mv.NEW_ATTACH) @wsgi.response(http_client.OK) - @validation.schema(attachment.create) + @validation.schema(attachment.create, mv.NEW_ATTACH, + mv.get_prior_version(mv.ATTACHMENT_CREATE_MODE_ARG)) + @validation.schema(attachment.create_v354, mv.ATTACHMENT_CREATE_MODE_ARG) def create(self, req, body): """Create an attachment. @@ -124,6 +126,9 @@ referenced below is the UUID of the Instance, for non-nova consumers this can be a server UUID or some other arbitrary unique identifier. + Starting from microversion 3.54, we can pass the attach mode as + argument in the request body. + Expected format of the input parameter 'body': .. code-block:: json @@ -132,8 +137,9 @@ "attachment": { "volume_uuid": "volume-uuid", - "instance_uuid": "nova-server-uuid", - "connector": "null|" + "instance_uuid": "null|nova-server-uuid", + "connector": "null|", + "mode": "null|rw|ro" } } @@ -161,7 +167,7 @@ returns: A summary view of the attachment object """ context = req.environ['cinder.context'] - instance_uuid = body['attachment']['instance_uuid'] + instance_uuid = body['attachment'].get('instance_uuid') volume_uuid = body['attachment']['volume_uuid'] volume_ref = objects.Volume.get_by_id( context, diff -Nru cinder-17.0.1/cinder/api/validation/parameter_types.py cinder-17.4.0/cinder/api/validation/parameter_types.py --- cinder-17.0.1/cinder/api/validation/parameter_types.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/api/validation/parameter_types.py 2022-04-20 17:38:33.000000000 +0000 @@ -232,6 +232,18 @@ 'pattern': '^[a-zA-Z0-9-._#@:/+]*$' } +cinder_host = { + # A string that represents a cinder host. + # Examples: + # hostname + # hostname.domain + # hostname.domain@backend + # hostname.domain@backend#pool + # hostname.domain@backend#[dead:beef::cafe]:/complex_ipv6_pool_w_share + 'type': ['string', 'null'], 'minLength': 1, 'maxLength': 255, + 'pattern': r'^[a-zA-Z0-9-._#@:/+\[\]]*$' # hostname plus brackets +} + resource_type = {'type': ['string', 'null'], 'minLength': 0, 'maxLength': 40} diff -Nru cinder-17.0.1/cinder/backup/drivers/ceph.py cinder-17.4.0/cinder/backup/drivers/ceph.py --- cinder-17.0.1/cinder/backup/drivers/ceph.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/backup/drivers/ceph.py 2022-04-20 17:38:33.000000000 +0000 @@ -244,6 +244,11 @@ """Determine if journaling is supported by our version of librbd.""" return hasattr(self.rbd, 'RBD_FEATURE_JOURNALING') + @property + def _supports_fast_diff(self): + """Determine if fast-diff is supported by our version of librbd.""" + return hasattr(self.rbd, 'RBD_FEATURE_FAST_DIFF') + def _get_rbd_support(self): """Determine RBD features supported by our version of librbd.""" old_format = True @@ -255,24 +260,22 @@ old_format = False features |= self.rbd.RBD_FEATURE_STRIPINGV2 - # journaling requires exclusive_lock; check both together if CONF.backup_ceph_image_journals: - if self._supports_exclusive_lock and self._supports_journaling: - old_format = False - features |= (self.rbd.RBD_FEATURE_EXCLUSIVE_LOCK | - self.rbd.RBD_FEATURE_JOURNALING) - else: - # FIXME (tasker): when the backup manager supports loading the - # driver during its initialization, this exception should be - # moved to the driver's initialization so that it can stop - # the service from starting when the underyling RBD does not - # support the requested features. - LOG.error("RBD journaling not supported - unable to " - "support per image mirroring in backup pool") - raise exception.BackupInvalidCephArgs( - _("Image Journaling set but RBD backend does " - "not support journaling") - ) + LOG.debug("RBD journaling supported by backend and requested " + "via config. Enabling it together with " + "exclusive-lock") + old_format = False + features |= (self.rbd.RBD_FEATURE_EXCLUSIVE_LOCK | + self.rbd.RBD_FEATURE_JOURNALING) + + # NOTE(christian_rohmann): Check for fast-diff support and enable it + if self._supports_fast_diff: + LOG.debug("RBD also supports fast-diff, enabling it " + "together with exclusive-lock and object-map") + old_format = False + features |= (self.rbd.RBD_FEATURE_EXCLUSIVE_LOCK | + self.rbd.RBD_FEATURE_OBJECT_MAP | + self.rbd.RBD_FEATURE_FAST_DIFF) return (old_format, features) @@ -294,6 +297,16 @@ with rbd_driver.RADOSClient(self, self._ceph_backup_pool): pass + # NOTE(christian_rohmann): Check features required for journaling + if CONF.backup_ceph_image_journals: + if not self._supports_exclusive_lock and self._supports_journaling: + LOG.error("RBD journaling not supported - unable to " + "support per image mirroring in backup pool") + raise exception.BackupInvalidCephArgs( + _("Image Journaling set but RBD backend does " + "not support journaling") + ) + def _connect_to_rados(self, pool=None): """Establish connection to the backup Ceph cluster.""" client = eventlet.tpool.Proxy(self.rados.Rados( @@ -806,7 +819,10 @@ image. """ volume_id = backup.volume_id - backup_name = self._get_backup_base_name(volume_id, backup=backup) + if backup.snapshot_id: + backup_name = self._get_backup_base_name(volume_id) + else: + backup_name = self._get_backup_base_name(volume_id, backup=backup) with eventlet.tpool.Proxy(rbd_driver.RADOSClient(self, backup.container)) as client: diff -Nru cinder-17.0.1/cinder/backup/manager.py cinder-17.4.0/cinder/backup/manager.py --- cinder-17.0.1/cinder/backup/manager.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/backup/manager.py 2022-04-20 17:38:33.000000000 +0000 @@ -1070,7 +1070,7 @@ if not is_snapshot: rpcapi.terminate_connection(ctxt, device, properties, force=force) - rpcapi.remove_export(ctxt, device) + rpcapi.remove_export(ctxt, device, sync=True) else: rpcapi.terminate_connection_snapshot(ctxt, device, properties, force=force) diff -Nru cinder-17.0.1/cinder/brick/local_dev/lvm.py cinder-17.4.0/cinder/brick/local_dev/lvm.py --- cinder-17.0.1/cinder/brick/local_dev/lvm.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/brick/local_dev/lvm.py 2022-04-20 17:38:33.000000000 +0000 @@ -149,6 +149,25 @@ def _create_vg(self, pv_list): cinder.privsep.lvm.create_volume(self.vg_name, pv_list) + @utils.retry(retry=utils.retry_if_exit_code, retry_param=139, interval=0.5, + backoff_rate=0.5) + def _run_lvm_command(self, + cmd_arg_list: list, + root_helper: str = None, + run_as_root: bool = True) -> tuple: + """Run LVM commands with a retry on code 139 to work around LVM bugs. + + Refer to LP bug 1901783, LP bug 1932188. + """ + if not root_helper: + root_helper = self._root_helper + + (out, err) = self._execute(*cmd_arg_list, + root_helper=root_helper, + run_as_root=run_as_root) + + return (out, err) + def _get_thin_pool_free_space(self, vg_name, thin_pool_name): """Returns available thin pool free space. @@ -168,9 +187,7 @@ free_space = 0.0 try: - (out, err) = self._execute(*cmd, - root_helper=self._root_helper, - run_as_root=True) + (out, err) = self._run_lvm_command(cmd) if out is not None: out = out.strip() data = out.split(':') @@ -278,6 +295,8 @@ return LVM._supports_pvs_ignoreskippedcluster @staticmethod + @utils.retry(retry=utils.retry_if_exit_code, retry_param=139, interval=0.5, + backoff_rate=0.5) # Bug#1901783 def get_lv_info(root_helper, vg_name=None, lv_name=None): """Retrieve info about LVs (all, in a VG, or a single LV). @@ -526,9 +545,7 @@ 'size': size_str, 'free': self.vg_free_space}) - self._execute(*cmd, - root_helper=self._root_helper, - run_as_root=True) + self._run_lvm_command(cmd) self.vg_thin_pool = name return size_str @@ -562,9 +579,7 @@ cmd.extend(['-R', str(rsize)]) try: - self._execute(*cmd, - root_helper=self._root_helper, - run_as_root=True) + self._run_lvm_command(cmd) except putils.ProcessExecutionError as err: LOG.exception('Error creating Volume') LOG.error('Cmd :%s', err.cmd) @@ -595,9 +610,7 @@ cmd.extend(['-L', '%sg' % (size)]) try: - self._execute(*cmd, - root_helper=self._root_helper, - run_as_root=True) + self._run_lvm_command(cmd) except putils.ProcessExecutionError as err: LOG.exception('Error creating snapshot') LOG.error('Cmd :%s', err.cmd) @@ -615,9 +628,7 @@ def _lv_is_active(self, name): cmd = LVM.LVM_CMD_PREFIX + ['lvdisplay', '--noheading', '-C', '-o', 'Attr', '%s/%s' % (self.vg_name, name)] - out, _err = self._execute(*cmd, - root_helper=self._root_helper, - run_as_root=True) + out, _err = self._run_lvm_command(cmd) if out: out = out.strip() if (out[4] == 'a'): @@ -644,7 +655,7 @@ # order to prevent a race condition. self._wait_for_volume_deactivation(name) - @utils.retry(exceptions=exception.VolumeNotDeactivated, retries=5, + @utils.retry(retry_param=exception.VolumeNotDeactivated, retries=5, backoff_rate=2) def _wait_for_volume_deactivation(self, name): LOG.debug("Checking to see if volume %s has been deactivated.", @@ -761,10 +772,9 @@ def lv_has_snapshot(self, name): cmd = LVM.LVM_CMD_PREFIX + ['lvdisplay', '--noheading', '-C', '-o', - 'Attr', '%s/%s' % (self.vg_name, name)] - out, _err = self._execute(*cmd, - root_helper=self._root_helper, - run_as_root=True) + 'Attr', '--readonly', + '%s/%s' % (self.vg_name, name)] + out, _err = self._run_lvm_command(cmd) if out: out = out.strip() if (out[0] == 'o') or (out[0] == 'O'): @@ -775,9 +785,7 @@ """Return True if LV is a snapshot, False otherwise.""" cmd = LVM.LVM_CMD_PREFIX + ['lvdisplay', '--noheading', '-C', '-o', 'Attr', '%s/%s' % (self.vg_name, name)] - out, _err = self._execute(*cmd, - root_helper=self._root_helper, - run_as_root=True) + out, _err = self._run_lvm_command(cmd) out = out.strip() if out: if (out[0] == 's'): @@ -788,9 +796,7 @@ """Return True if LV is currently open, False otherwise.""" cmd = LVM.LVM_CMD_PREFIX + ['lvdisplay', '--noheading', '-C', '-o', 'Attr', '%s/%s' % (self.vg_name, name)] - out, _err = self._execute(*cmd, - root_helper=self._root_helper, - run_as_root=True) + out, _err = self._run_lvm_command(cmd) out = out.strip() if out: if (out[5] == 'o'): @@ -801,9 +807,7 @@ """Return the origin of an LV that is a snapshot, None otherwise.""" cmd = LVM.LVM_CMD_PREFIX + ['lvdisplay', '--noheading', '-C', '-o', 'Origin', '%s/%s' % (self.vg_name, name)] - out, _err = self._execute(*cmd, - root_helper=self._root_helper, - run_as_root=True) + out, _err = self._run_lvm_command(cmd) out = out.strip() if out: return out @@ -821,8 +825,7 @@ try: cmd = LVM.LVM_CMD_PREFIX + ['lvextend', '-L', new_size, '%s/%s' % (self.vg_name, lv_name)] - self._execute(*cmd, root_helper=self._root_helper, - run_as_root=True) + self._run_lvm_command(cmd) except putils.ProcessExecutionError as err: LOG.exception('Error extending Volume') LOG.error('Cmd :%s', err.cmd) diff -Nru cinder-17.0.1/cinder/common/config.py cinder-17.4.0/cinder/common/config.py --- cinder-17.0.1/cinder/common/config.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/common/config.py 2022-04-20 17:38:33.000000000 +0000 @@ -84,12 +84,12 @@ cfg.StrOpt('scheduler_manager', default='cinder.scheduler.manager.SchedulerManager', help='Full class name for the Manager for scheduler'), - cfg.HostAddressOpt('host', - sample_default='localhost', - default=socket.gethostname(), - help='Name of this node. This can be an opaque ' - 'identifier. It is not necessarily a host name, ' - 'FQDN, or IP address.'), + cfg.StrOpt('host', + sample_default='localhost', + default=socket.gethostname(), + help='Name of this node. This can be an opaque ' + 'identifier. It is not necessarily a host name, ' + 'FQDN, or IP address.'), # NOTE(vish): default to nova for compatibility with nova installs cfg.StrOpt('storage_availability_zone', default='nova', diff -Nru cinder-17.0.1/cinder/compute/nova.py cinder-17.4.0/cinder/compute/nova.py --- cinder-17.0.1/cinder/compute/nova.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/compute/nova.py 2022-04-20 17:38:33.000000000 +0000 @@ -173,13 +173,6 @@ return False return True - def has_extension(self, context, extension, timeout=None): - try: - nova_exts = novaclient(context).list_extensions.show_all() - except request_exceptions.Timeout: - raise exception.APITimeout(service='Nova') - return extension in [e.name for e in nova_exts] - def update_server_volume(self, context, server_id, src_volid, new_volume_id): nova = novaclient(context, privileged_user=True) diff -Nru cinder-17.0.1/cinder/db/sqlalchemy/api.py cinder-17.4.0/cinder/db/sqlalchemy/api.py --- cinder-17.0.1/cinder/db/sqlalchemy/api.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/db/sqlalchemy/api.py 2022-04-20 17:38:33.000000000 +0000 @@ -65,6 +65,9 @@ CONF = cfg.CONF LOG = logging.getLogger(__name__) +# Map with cases where attach status differs from volume status +ATTACH_STATUS_MAP = {'attached': 'in-use', 'detached': 'available'} + options.set_defaults(CONF, connection='sqlite:///$state_path/cinder.sqlite') @@ -1704,75 +1707,83 @@ partial_rename, filters) +def _get_statuses_from_attachments(context, session, volume_id): + """Get volume status and attach_status based on existing attachments.""" + # NOTE: Current implementation ignores attachments on error attaching, + # since they will not have been used by any consumer because os-brick's + # connect_volume has not been called yet. This leads to cases where a + # volume will be in 'available' state yet have attachments. + + # If we sort status of attachments alphabetically, ignoring errors, the + # first element will be the attachment status for the volume: + # attached > attaching > detaching > reserved + attach_status = session.query(models.VolumeAttachment.attach_status).\ + filter_by(deleted=False).\ + filter_by(volume_id=volume_id).\ + filter(~models.VolumeAttachment.attach_status.startswith('error_')).\ + order_by(models.VolumeAttachment.attach_status.asc()).\ + limit(1).\ + scalar() + + # No volume attachment records means the volume is detached. + attach_status = attach_status or 'detached' + + # Check cases where volume status is different from attach status, and + # default to the same value if it's not one of those cases. + status = ATTACH_STATUS_MAP.get(attach_status, attach_status) + + return (status, attach_status) + + @require_admin_context def volume_detached(context, volume_id, attachment_id): - """This updates a volume attachment and marks it as detached. + """Delete an attachment and update the volume accordingly. + + After marking the attachment as detached the method will decide the status + and attach_status values for the volume based on the current status and the + remaining attachments and their status. - This method also ensures that the volume entry is correctly - marked as either still attached/in-use or detached/available - if this was the last detachment made. + Volume status may be changed to: in-use, attaching, detaching, reserved, or + available. + Volume attach_status will be changed to one of: attached, attaching, + detaching, reserved, or detached. """ # NOTE(jdg): This is a funky band-aid for the earlier attempts at # multiattach, it's a bummer because these things aren't really being used # but at the same time we don't want to break them until we work out the # new proposal for multi-attach - remain_attachment = True session = get_session() with session.begin(): + # Only load basic volume info necessary to check various status and use + # the volume row as a lock with the for_update. + volume = _volume_get(context, volume_id, session=session, + joined_load=False, for_update=True) + try: attachment = _attachment_get(context, attachment_id, session=session) + attachment_updates = attachment.delete(session) except exception.VolumeAttachmentNotFound: attachment_updates = None - attachment = None - if attachment: - now = timeutils.utcnow() - attachment_updates = { - 'attach_status': fields.VolumeAttachStatus.DETACHED, - 'detach_time': now, - 'deleted': True, - 'deleted_at': now, - 'updated_at': - literal_column('updated_at'), - } - attachment.update(attachment_updates) - attachment.save(session=session) - del attachment_updates['updated_at'] + status, attach_status = _get_statuses_from_attachments(context, + session, + volume_id) + + volume_updates = {'updated_at': volume.updated_at, + 'attach_status': attach_status} + + # Hide volume status update to available on volume migration or upload, + # as status is updated later on those flows. + if ((attach_status != 'detached') + or (not volume.migration_status and volume.status != 'uploading') + or volume.migration_status in ('success', 'error')): + volume_updates['status'] = status - attachment_list = None - volume_ref = _volume_get(context, volume_id, - session=session) - volume_updates = {'updated_at': literal_column('updated_at')} - if not volume_ref.volume_attachment: - # NOTE(jdg): We kept the old arg style allowing session exclusively - # for this one call - attachment_list = volume_attachment_get_all_by_volume_id( - context, volume_id, session=session) - remain_attachment = False - if attachment_list and len(attachment_list) > 0: - remain_attachment = True - - if not remain_attachment: - # Hide status update from user if we're performing volume migration - # or uploading it to image - if ((not volume_ref.migration_status and - not (volume_ref.status == 'uploading')) or - volume_ref.migration_status in ('success', 'error')): - volume_updates['status'] = 'available' - - volume_updates['attach_status'] = ( - fields.VolumeAttachStatus.DETACHED) - else: - # Volume is still attached - volume_updates['status'] = 'in-use' - volume_updates['attach_status'] = ( - fields.VolumeAttachStatus.ATTACHED) - - volume_ref.update(volume_updates) - volume_ref.save(session=session) + volume.update(volume_updates) + volume.save(session=session) del volume_updates['updated_at'] return (volume_updates, attachment_updates) @@ -1856,11 +1867,14 @@ @require_context -def _volume_get(context, volume_id, session=None, joined_load=True): +def _volume_get(context, volume_id, session=None, joined_load=True, + for_update=False): result = _volume_get_query(context, session=session, project_only=True, joined_load=joined_load) if joined_load: result = result.options(joinedload('volume_type.extra_specs')) + if for_update: + result = result.with_for_update() result = result.filter_by(id=volume_id).first() if not result: diff -Nru cinder-17.0.1/cinder/db/sqlalchemy/models.py cinder-17.4.0/cinder/db/sqlalchemy/models.py --- cinder-17.0.1/cinder/db/sqlalchemy/models.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/db/sqlalchemy/models.py 2022-04-20 17:38:33.000000000 +0000 @@ -55,8 +55,10 @@ def delete(self, session): """Delete this object.""" updated_values = self.delete_values() + updated_values['updated_at'] = self.updated_at self.update(updated_values) self.save(session=session) + del updated_values['updated_at'] return updated_values @@ -376,6 +378,14 @@ # Stores a serialized json dict of host connector information from brick. connector = Column(Text) + @staticmethod + def delete_values(): + now = timeutils.utcnow() + return {'deleted': True, + 'deleted_at': now, + 'attach_status': 'detached', + 'detach_time': now} + class VolumeType(BASE, CinderBase): """Represent possible volume_types of volumes offered.""" diff -Nru cinder-17.0.1/cinder/image/image_utils.py cinder-17.4.0/cinder/image/image_utils.py --- cinder-17.0.1/cinder/image/image_utils.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/image/image_utils.py 2022-04-20 17:38:33.000000000 +0000 @@ -645,7 +645,7 @@ # large and cause disk full errors which would confuse users. # Unfortunately it seems that you can't pipe to 'qemu-img convert' because # it seeks. Maybe we can think of something for a future version. - with temporary_file() as tmp: + with temporary_file(prefix='image_download_%s_' % image_id) as tmp: has_meta = False if not image_meta else True try: format_raw = True if image_meta['disk_format'] == 'raw' else False @@ -763,7 +763,7 @@ base_image_ref=base_image_ref) return - with temporary_file() as tmp: + with temporary_file(prefix='vol_upload_') as tmp: LOG.debug("%s was %s, converting to %s", image_id, volume_format, image_meta['disk_format']) @@ -997,7 +997,8 @@ @contextlib.contextmanager def fetch(cls, image_service, context, image_id, suffix=''): tmp_images = cls.for_image_service(image_service).temporary_images - with temporary_file(suffix=suffix) as tmp: + with temporary_file(prefix='image_fetch_%s_' % image_id, + suffix=suffix) as tmp: fetch_verify_image(context, image_service, image_id, tmp) user = context.user_id if not tmp_images.get(user): diff -Nru cinder-17.0.1/cinder/objects/volume.py cinder-17.4.0/cinder/objects/volume.py --- cinder-17.0.1/cinder/objects/volume.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/objects/volume.py 2022-04-20 17:38:33.000000000 +0000 @@ -138,7 +138,8 @@ @classmethod def _get_expected_attrs(cls, context, *args, **kwargs): - expected_attrs = ['metadata', 'volume_type', 'volume_type.extra_specs'] + expected_attrs = ['metadata', 'volume_type', 'volume_type.extra_specs', + 'volume_attachment'] if context.is_admin: expected_attrs.append('admin_metadata') @@ -180,6 +181,13 @@ md = {d['key']: d['value'] for d in value} self.admin_metadata = md + def admin_metadata_update(self, metadata, delete, add=True, update=True): + new_metadata = db.volume_admin_metadata_update(self._context, self.id, + metadata, delete, add, + update) + self.admin_metadata = new_metadata + self._reset_metadata_tracking(fields=('admin_metadata',)) + @property def volume_glance_metadata(self): md = [MetadataObject(k, v) for k, v in self.glance_metadata.items()] @@ -344,7 +352,8 @@ volume_types.get_default_volume_type()['id']) db_volume = db.volume_create(self._context, updates) - self._from_db_object(self._context, self, db_volume) + expected_attrs = self._get_expected_attrs(self._context) + self._from_db_object(self._context, self, db_volume, expected_attrs) def save(self): updates = self.cinder_obj_get_changes() @@ -482,7 +491,8 @@ # end of migration because we want to keep the original volume id # in the DB but now pointing to the migrated volume. skip = ({'id', 'provider_location', 'glance_metadata', - 'volume_type'} | set(self.obj_extra_fields)) + 'volume_type', 'volume_attachment'} + | set(self.obj_extra_fields)) for key in set(dest_volume.fields.keys()) - skip: # Only swap attributes that are already set. We do not want to # unexpectedly trigger a lazy-load. diff -Nru cinder-17.0.1/cinder/policies/group_snapshot_actions.py cinder-17.4.0/cinder/policies/group_snapshot_actions.py --- cinder-17.0.1/cinder/policies/group_snapshot_actions.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/policies/group_snapshot_actions.py 2022-04-20 17:38:33.000000000 +0000 @@ -24,7 +24,7 @@ group_snapshot_actions_policies = [ policy.DocumentedRuleDefault( name=RESET_STATUS, - check_str=base.RULE_ADMIN_OR_OWNER, + check_str=base.RULE_ADMIN_API, description="Reset status of group snapshot.", operations=[ { diff -Nru cinder-17.0.1/cinder/scheduler/filters/instance_locality_filter.py cinder-17.4.0/cinder/scheduler/filters/instance_locality_filter.py --- cinder-17.0.1/cinder/scheduler/filters/instance_locality_filter.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/scheduler/filters/instance_locality_filter.py 2022-04-20 17:38:33.000000000 +0000 @@ -56,19 +56,6 @@ self._cache = {} super(InstanceLocalityFilter, self).__init__() - def _nova_has_extended_server_attributes(self, context): - """Check Extended Server Attributes presence - - Find out whether the Extended Server Attributes extension is activated - in Nova or not. Cache the result to query Nova only once. - """ - - if not hasattr(self, '_nova_ext_srv_attr'): - self._nova_ext_srv_attr = nova.API().has_extension( - context, 'ExtendedServerAttributes', timeout=REQUESTS_TIMEOUT) - - return self._nova_ext_srv_attr - def backend_passes(self, backend_state, filter_properties): context = filter_properties['context'] backend = volume_utils.extract_host(backend_state.backend_id, 'host') @@ -95,13 +82,6 @@ if instance_uuid in self._cache: return self._cache[instance_uuid] == backend - if not self._nova_has_extended_server_attributes(context): - LOG.warning('Hint "%s" dropped because ' - 'ExtendedServerAttributes not active in Nova.', - HINT_KEYWORD) - raise exception.CinderException(_('Hint "%s" not supported.') % - HINT_KEYWORD) - server = nova.API().get_server(context, instance_uuid, privileged_user=True, timeout=REQUESTS_TIMEOUT) diff -Nru cinder-17.0.1/cinder/tests/unit/api/contrib/test_volume_actions.py cinder-17.4.0/cinder/tests/unit/api/contrib/test_volume_actions.py --- cinder-17.0.1/cinder/tests/unit/api/contrib/test_volume_actions.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/tests/unit/api/contrib/test_volume_actions.py 2022-04-20 17:38:33.000000000 +0000 @@ -1003,6 +1003,38 @@ id, body=body) + @mock.patch.object(volume_api.API, 'get', fake_volume_get_obj) + def test_copy_volume_to_image_bad_disk_format_for_encrypted_vol(self): + id = ENCRYPTED_VOLUME_ID + vol = {"container_format": 'bare', + "disk_format": 'qcow2', + "image_name": 'image_name', + "force": True} + body = {"os-volume_upload_image": vol} + req = fakes.HTTPRequest.blank('/v3/%s/volumes/%s/action' + % (fake.PROJECT_ID, id)) + self.assertRaises(webob.exc.HTTPBadRequest, + self.controller._volume_upload_image, + req, + id, + body=body) + + @mock.patch.object(volume_api.API, 'get', fake_volume_get_obj) + def test_copy_volume_to_image_bad_container_format_for_encrypted_vol(self): + id = ENCRYPTED_VOLUME_ID + vol = {"container_format": 'ovf', + "disk_format": 'raw', + "image_name": 'image_name', + "force": True} + body = {"os-volume_upload_image": vol} + req = fakes.HTTPRequest.blank('/v3/%s/volumes/%s/action' + % (fake.PROJECT_ID, id)) + self.assertRaises(webob.exc.HTTPBadRequest, + self.controller._volume_upload_image, + req, + id, + body=body) + @mock.patch.object(volume_api.API, "copy_volume_to_image") def test_copy_volume_to_image_disk_format_ploop(self, mock_copy_to_image): diff -Nru cinder-17.0.1/cinder/tests/unit/api/contrib/test_volume_manage.py cinder-17.4.0/cinder/tests/unit/api/contrib/test_volume_manage.py --- cinder-17.0.1/cinder/tests/unit/api/contrib/test_volume_manage.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/tests/unit/api/contrib/test_volume_manage.py 2022-04-20 17:38:33.000000000 +0000 @@ -205,7 +205,8 @@ @ddt.data({'host': 'host_ok'}, {'host': 'user@host#backend:/vol_path'}, - {'host': 'host@backend#parts+of+pool'}) + {'host': 'host@backend#parts+of+pool'}, + {'host': 'host@backend#[dead:beef::cafe]:/vol01'}) @ddt.unpack @mock.patch('cinder.volume.api.API.manage_existing', wraps=api_manage) def test_manage_volume_ok(self, mock_api_manage, host): diff -Nru cinder-17.0.1/cinder/tests/unit/api/v3/test_attachments.py cinder-17.4.0/cinder/tests/unit/api/v3/test_attachments.py --- cinder-17.0.1/cinder/tests/unit/api/v3/test_attachments.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/tests/unit/api/v3/test_attachments.py 2022-04-20 17:38:33.000000000 +0000 @@ -203,13 +203,13 @@ self.controller.delete(req, attachment.id) volume2 = objects.Volume.get_by_id(self.ctxt, volume1.id) - if status == 'reserved': - self.assertEqual('detached', volume2.attach_status) - self.assertRaises( - exception.VolumeAttachmentNotFound, - objects.VolumeAttachment.get_by_id, self.ctxt, attachment.id) - else: - self.assertEqual('attached', volume2.attach_status) + # Volume and attachment status is changed on the API service + self.assertEqual('detached', volume2.attach_status) + self.assertEqual('available', volume2.status) + self.assertRaises( + exception.VolumeAttachmentNotFound, + objects.VolumeAttachment.get_by_id, self.ctxt, attachment.id) + if status != 'reserved': mock_delete.assert_called_once_with(req.environ['cinder.context'], attachment.id, mock.ANY) diff -Nru cinder-17.0.1/cinder/tests/unit/attachments/test_attachments_manager.py cinder-17.4.0/cinder/tests/unit/attachments/test_attachments_manager.py --- cinder-17.0.1/cinder/tests/unit/attachments/test_attachments_manager.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/tests/unit/attachments/test_attachments_manager.py 2022-04-20 17:38:33.000000000 +0000 @@ -17,7 +17,6 @@ from cinder import context from cinder import db -from cinder import exception from cinder.objects import fields from cinder.objects import volume_attachment from cinder.tests.unit.api.v2 import fakes as v2_fakes @@ -136,13 +135,49 @@ attachment_ref = db.volume_attachment_get( self.context, attachment_ref['id']) + + vref.refresh() + expected_status = (vref.status, vref.attach_status, + attachment_ref.attach_status) + self.manager.attachment_delete(self.context, attachment_ref['id'], vref) - self.assertRaises(exception.VolumeAttachmentNotFound, - db.volume_attachment_get, - self.context, - attachment_ref.id) + # Manager doesn't change the resource status. It is changed on the API + attachment_ref = db.volume_attachment_get(self.context, + attachment_ref.id) + vref.refresh() + self.assertEqual( + expected_status, + (vref.status, vref.attach_status, attachment_ref.attach_status)) + + def test_attachment_delete_remove_export_fail(self): + """attachment_delete removes attachment on remove_export failure.""" + self.mock_object(self.manager.driver, 'remove_export', + side_effect=Exception) + # Report that the connection is not shared + self.mock_object(self.manager, '_connection_terminate', + return_value=False) + + vref = tests_utils.create_volume(self.context, status='in-use', + attach_status='attached') + values = {'volume_id': vref.id, 'volume_host': vref.host, + 'attach_status': 'reserved', 'instance_uuid': fake.UUID1} + attach = db.volume_attach(self.context, values) + # Confirm the volume OVO has the attachment before the deletion + vref.refresh() + expected_vol_status = (vref.status, vref.attach_status) + self.assertEqual(1, len(vref.volume_attachment)) + + self.manager.attachment_delete(self.context, attach.id, vref) + + # Manager doesn't change the resource status. It is changed on the API + attachment = db.volume_attachment_get(self.context, attach.id) + self.assertEqual(attach.attach_status, attachment.attach_status) + + vref = db.volume_get(self.context, vref.id) + self.assertEqual(expected_vol_status, + (vref.status, vref.attach_status)) def test_attachment_delete_multiple_attachments(self): volume_params = {'status': 'available'} @@ -163,6 +198,12 @@ mock_elevated, mock_db_detached, mock_db_meta_delete): mock_elevated.return_value = self.context mock_con_term.return_value = False + mock_db_detached.return_value = ( + {'status': 'available', + 'attach_status': fields.VolumeAttachStatus.DETACHED}, + {'attach_status': fields.VolumeAttachStatus.DETACHED, + 'deleted': True} + ) # test single attachment. This should call # detach and remove_export diff -Nru cinder-17.0.1/cinder/tests/unit/backup/drivers/test_backup_ceph.py cinder-17.4.0/cinder/tests/unit/backup/drivers/test_backup_ceph.py --- cinder-17.0.1/cinder/tests/unit/backup/drivers/test_backup_ceph.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/tests/unit/backup/drivers/test_backup_ceph.py 2022-04-20 17:38:33.000000000 +0000 @@ -237,11 +237,15 @@ del self.service.rbd.RBD_FEATURE_STRIPINGV2 del self.service.rbd.RBD_FEATURE_EXCLUSIVE_LOCK del self.service.rbd.RBD_FEATURE_JOURNALING + del self.service.rbd.RBD_FEATURE_OBJECT_MAP + del self.service.rbd.RBD_FEATURE_FAST_DIFF self.assertFalse(hasattr(self.service.rbd, 'RBD_FEATURE_LAYERING')) self.assertFalse(hasattr(self.service.rbd, 'RBD_FEATURE_STRIPINGV2')) self.assertFalse(hasattr(self.service.rbd, 'RBD_FEATURE_EXCLUSIVE_LOCK')) self.assertFalse(hasattr(self.service.rbd, 'RBD_FEATURE_JOURNALING')) + self.assertFalse(hasattr(self.service.rbd, 'RBD_FEATURE_OBJECT_MAP')) + self.assertFalse(hasattr(self.service.rbd, 'RBD_FEATURE_FAST_DIFF')) oldformat, features = self.service._get_rbd_support() self.assertTrue(oldformat) @@ -281,6 +285,17 @@ self.assertFalse(oldformat) self.assertEqual(1 | 2 | 4 | 64, features) + # + # test that FAST_DIFF is enabled if supported by RBD + # this also enables OBJECT_MAP as required by Ceph + # + self.service.rbd.RBD_FEATURE_OBJECT_MAP = 8 + self.service.rbd.RBD_FEATURE_FAST_DIFF = 16 + + oldformat, features = self.service._get_rbd_support() + self.assertFalse(oldformat) + self.assertEqual(1 | 2 | 4 | 8 | 16 | 64, features) + @common_mocks def test_get_backup_snap_name(self): snap_name = 'backup.%s.snap.3824923.1412' % (fake.VOLUME3_ID) diff -Nru cinder-17.0.1/cinder/tests/unit/brick/test_brick_lvm.py cinder-17.4.0/cinder/tests/unit/brick/test_brick_lvm.py --- cinder-17.0.1/cinder/tests/unit/brick/test_brick_lvm.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/tests/unit/brick/test_brick_lvm.py 2022-04-20 17:38:33.000000000 +0000 @@ -16,6 +16,7 @@ from unittest import mock import ddt +from os_brick import executor as os_brick_executor from oslo_concurrency import processutils from cinder.brick.local_dev import lvm as brick @@ -231,6 +232,49 @@ 'sudo', vg_name='fake-vg') ) + @mock.patch('tenacity.nap.sleep', mock.Mock()) + @mock.patch.object(brick.putils, 'execute') + def test_get_lv_info_retry(self, exec_mock): + exec_mock.side_effect = ( + processutils.ProcessExecutionError('', '', exit_code=139), + ('vg name size', ''), + ) + + self.assertEqual( + [{'name': 'name', 'vg': 'vg', 'size': 'size'}], + self.vg.get_lv_info('sudo', vg_name='vg', lv_name='name') + ) + + self.assertEqual(2, exec_mock.call_count) + args = ['env', 'LC_ALL=C', 'lvs', '--noheadings', '--unit=g', '-o', + 'vg_name,name,size', '--nosuffix', 'vg/name'] + if self.configuration.lvm_suppress_fd_warnings: + args.insert(2, 'LVM_SUPPRESS_FD_WARNINGS=1') + lvs_call = mock.call(*args, root_helper='sudo', run_as_root=True) + exec_mock.assert_has_calls([lvs_call, lvs_call]) + + @mock.patch('tenacity.nap.sleep', mock.Mock()) + @mock.patch.object(os_brick_executor.Executor, '_execute') + def test_get_thin_pool_free_space_retry(self, exec_mock): + exec_mock.side_effect = ( + processutils.ProcessExecutionError('', '', exit_code=139), + ('15.84:50', ''), + ) + + self.assertEqual( + 7.92, + self.vg._get_thin_pool_free_space('vg', 'thinpool') + ) + + self.assertEqual(2, exec_mock.call_count) + args = ['env', 'LC_ALL=C', 'lvs', '--noheadings', '--unit=g', '-o', + 'size,data_percent', '--separator', ':', '--nosuffix', + '/dev/vg/thinpool'] + if self.configuration.lvm_suppress_fd_warnings: + args.insert(2, 'LVM_SUPPRESS_FD_WARNINGS=1') + lvs_call = mock.call(*args, root_helper='sudo', run_as_root=True) + exec_mock.assert_has_calls([lvs_call, lvs_call]) + def test_get_all_physical_volumes(self): # Filtered VG version pvs = self.vg.get_all_physical_volumes('sudo', 'fake-vg') @@ -353,7 +397,7 @@ pool_name = vg_name + "-pool" self.vg.create_thin_pool(pool_name, "1G") - with mock.patch.object(self.vg, '_execute'): + with mock.patch.object(self.vg, '_execute', return_value=(0, 0)): self.vg.create_volume("test", "1G", lv_type='thin') if self.configuration.lvm_suppress_fd_warnings is False: self.vg._execute.assert_called_once_with( @@ -412,7 +456,7 @@ @ddt.data(True, False) def test_lv_extend(self, has_snapshot): - with mock.patch.object(self.vg, '_execute'): + with mock.patch.object(self.vg, '_execute', return_value=('', '')): with mock.patch.object(self.vg, 'lv_has_snapshot'): self.vg.deactivate_lv = mock.MagicMock() self.vg.activate_lv = mock.MagicMock() @@ -429,7 +473,7 @@ self.vg.deactivate_lv.assert_not_called() def test_lv_deactivate(self): - with mock.patch.object(self.vg, '_execute'): + with mock.patch.object(self.vg, '_execute', return_value=(0, 0)): is_active_mock = mock.Mock() is_active_mock.return_value = False self.vg._lv_is_active = is_active_mock @@ -438,7 +482,7 @@ @mock.patch('time.sleep') def test_lv_deactivate_timeout(self, _mock_sleep): - with mock.patch.object(self.vg, '_execute'): + with mock.patch.object(self.vg, '_execute', return_value=(0, 0)): is_active_mock = mock.Mock() is_active_mock.return_value = True self.vg._lv_is_active = is_active_mock diff -Nru cinder-17.0.1/cinder/tests/unit/fake_volume.py cinder-17.4.0/cinder/tests/unit/fake_volume.py --- cinder-17.0.1/cinder/tests/unit/fake_volume.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/tests/unit/fake_volume.py 2022-04-20 17:38:33.000000000 +0000 @@ -107,8 +107,11 @@ if updates.get('encryption_key_id'): assert is_uuid_like(updates['encryption_key_id']) + updates['volume_attachment'] = updates.get('volume_attachment') or [] + expected_attrs = updates.pop('expected_attrs', - ['metadata', 'admin_metadata']) + ['metadata', 'admin_metadata', + 'volume_attachment']) vol = objects.Volume._from_db_object(context, objects.Volume(), fake_db_volume(**updates), expected_attrs=expected_attrs) diff -Nru cinder-17.0.1/cinder/tests/unit/objects/test_volume.py cinder-17.4.0/cinder/tests/unit/objects/test_volume.py --- cinder-17.0.1/cinder/tests/unit/objects/test_volume.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/tests/unit/objects/test_volume.py 2022-04-20 17:38:33.000000000 +0000 @@ -430,7 +430,8 @@ # finish_volume_migration ignore_keys = ('id', 'provider_location', '_name_id', 'migration_status', 'display_description', 'status', - 'volume_glance_metadata', 'volume_type') + 'volume_glance_metadata', 'volume_type', + 'volume_attachment') dest_vol_dict = {k: updated_dest_volume[k] for k in updated_dest_volume.keys() if k not in ignore_keys} @@ -478,7 +479,8 @@ self, volume_attachment_get, volume_detached, metadata_delete): - va_objs = [objects.VolumeAttachment(context=self.context, id=i) + va_objs = [objects.VolumeAttachment(context=self.context, id=i, + volume_id=fake.VOLUME_ID) for i in [fake.OBJECT_ID, fake.OBJECT2_ID, fake.OBJECT3_ID]] # As changes are not saved, we need reset it here. Later changes # will be checked. @@ -491,6 +493,7 @@ admin_context = context.get_admin_context() volume = fake_volume.fake_volume_obj( admin_context, + id=fake.VOLUME_ID, volume_attachment=va_list, volume_admin_metadata=[{'key': 'attached_mode', 'value': 'rw'}]) @@ -522,7 +525,7 @@ admin_context, volume_admin_metadata=[{'key': 'attached_mode', 'value': 'rw'}]) - self.assertFalse(volume.obj_attr_is_set('volume_attachment')) + self.assertEqual([], volume.volume_attachment.objects) volume_detached.return_value = ({'status': 'in-use'}, None) with mock.patch.object(admin_context, 'elevated') as mock_elevated: mock_elevated.return_value = admin_context diff -Nru cinder-17.0.1/cinder/tests/unit/scheduler/test_host_filters.py cinder-17.4.0/cinder/tests/unit/scheduler/test_host_filters.py --- cinder-17.0.1/cinder/tests/unit/scheduler/test_host_filters.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/tests/unit/scheduler/test_host_filters.py 2022-04-20 17:38:33.000000000 +0000 @@ -1068,20 +1068,6 @@ filt_cls.backend_passes, host, filter_properties) @mock.patch('cinder.compute.nova.novaclient') - def test_nova_no_extended_server_attributes(self, _mock_novaclient): - _mock_novaclient.return_value = fakes.FakeNovaClient( - ext_srv_attr=False) - filt_cls = self.class_map['InstanceLocalityFilter']() - host = fakes.FakeBackendState('host1', {}) - uuid = nova.novaclient().servers.create('host1') - - filter_properties = {'context': self.context, - 'scheduler_hints': {'local_to_instance': uuid}, - 'request_spec': {'volume_id': fake.VOLUME_ID}} - self.assertRaises(exception.CinderException, - filt_cls.backend_passes, host, filter_properties) - - @mock.patch('cinder.compute.nova.novaclient') def test_nova_down_does_not_alter_other_filters(self, _mock_novaclient): # Simulate Nova API is not available _mock_novaclient.side_effect = Exception @@ -1096,8 +1082,8 @@ @mock.patch('cinder.compute.nova.novaclient') def test_nova_timeout(self, mock_novaclient): # Simulate a HTTP timeout - mock_show_all = mock_novaclient.return_value.list_extensions.show_all - mock_show_all.side_effect = request_exceptions.Timeout + mock_get = mock_novaclient.return_value.servers.get + mock_get.side_effect = request_exceptions.Timeout filt_cls = self.class_map['InstanceLocalityFilter']() host = fakes.FakeBackendState('host1', {}) diff -Nru cinder-17.0.1/cinder/tests/unit/test_db_api.py cinder-17.4.0/cinder/tests/unit/test_db_api.py --- cinder-17.0.1/cinder/tests/unit/test_db_api.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/tests/unit/test_db_api.py 2022-04-20 17:38:33.000000000 +0000 @@ -571,11 +571,13 @@ 'instance_uuid': instance_uuid, 'attach_status': fields.VolumeAttachStatus.ATTACHING, } attachment = db.volume_attach(self.ctxt, values) + db.volume_attached(self.ctxt, attachment.id, instance_uuid, + None, '/tmp') values2 = {'volume_id': volume.id, 'instance_uuid': fake.OBJECT_ID, 'attach_status': fields.VolumeAttachStatus.ATTACHING, } - db.volume_attach(self.ctxt, values2) - db.volume_attached(self.ctxt, attachment.id, + attachment2 = db.volume_attach(self.ctxt, values2) + db.volume_attached(self.ctxt, attachment2.id, instance_uuid, None, '/tmp') volume_updates, attachment_updates = ( diff -Nru cinder-17.0.1/cinder/tests/unit/test_image_utils.py cinder-17.4.0/cinder/tests/unit/test_image_utils.py --- cinder-17.0.1/cinder/tests/unit/test_image_utils.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/tests/unit/test_image_utils.py 2022-04-20 17:38:33.000000000 +0000 @@ -1142,7 +1142,8 @@ volume_format, blocksize) self.assertIsNone(output) - mock_temp.assert_called_once_with() + mock_temp.assert_called_once_with(prefix='image_download_%s_' % + image_id) mock_info.assert_has_calls([ mock.call(tmp, force_share=False, run_as_root=True), mock.call(tmp, run_as_root=True)]) @@ -1194,7 +1195,8 @@ run_as_root=run_as_root) self.assertIsNone(output) - mock_temp.assert_called_once_with() + mock_temp.assert_called_once_with(prefix='image_download_%s_' % + image_id) mock_info.assert_has_calls([ mock.call(tmp, force_share=False, run_as_root=run_as_root), mock.call(tmp, run_as_root=run_as_root)]) @@ -1250,7 +1252,8 @@ run_as_root=run_as_root) self.assertIsNone(output) - mock_temp.assert_called_once_with() + mock_temp.assert_called_once_with(prefix='image_download_%s_' % + image_id) mock_info.assert_has_calls([ mock.call(tmp, force_share=False, run_as_root=run_as_root), mock.call(tmp, run_as_root=run_as_root)]) @@ -1302,7 +1305,8 @@ run_as_root=run_as_root) self.assertIsNone(output) - mock_temp.assert_called_once_with() + mock_temp.assert_called_once_with(prefix='image_download_%s_' % + image_id) mock_info.assert_has_calls([ mock.call(tmp, force_share=False, run_as_root=run_as_root), mock.call(tmp, run_as_root=run_as_root)]) @@ -1406,7 +1410,8 @@ self.assertIsNone(output) image_service.show.assert_called_once_with(ctxt, image_id) - mock_temp.assert_called_once_with() + mock_temp.assert_called_once_with(prefix='image_download_%s_' % + image_id) mock_info.assert_called_once_with(tmp, force_share=False, run_as_root=run_as_root) @@ -1453,7 +1458,8 @@ run_as_root=run_as_root) image_service.show.assert_called_once_with(ctxt, image_id) - mock_temp.assert_called_once_with() + mock_temp.assert_called_once_with(prefix='image_download_%s_' % + image_id) mock_info.assert_called_once_with(tmp, force_share=False, run_as_root=run_as_root) @@ -1498,7 +1504,8 @@ run_as_root=run_as_root) image_service.show.assert_called_once_with(ctxt, image_id) - mock_temp.assert_called_once_with() + mock_temp.assert_called_once_with(prefix='image_download_%s_' % + image_id) mock_info.assert_called_once_with(tmp, force_share=False, run_as_root=run_as_root) @@ -1549,7 +1556,8 @@ run_as_root=run_as_root) image_service.show.assert_called_once_with(ctxt, image_id) - mock_temp.assert_called_once_with() + mock_temp.assert_called_once_with(prefix='image_download_%s_' % + image_id) mock_info.assert_has_calls([ mock.call(tmp, force_share=False, run_as_root=run_as_root), mock.call(tmp, run_as_root=run_as_root)]) @@ -1597,7 +1605,8 @@ run_as_root=run_as_root) image_service.show.assert_called_once_with(ctxt, image_id) - mock_temp.assert_called_once_with() + mock_temp.assert_called_once_with(prefix='image_download_%s_' % + image_id) mock_info.assert_has_calls([ mock.call(tmp, force_share=False, run_as_root=run_as_root), mock.call(tmp, run_as_root=run_as_root)]) @@ -1645,7 +1654,8 @@ run_as_root=run_as_root) image_service.show.assert_called_once_with(ctxt, image_id) - mock_temp.assert_called_once_with() + mock_temp.assert_called_once_with(prefix='image_download_%s_' % + image_id) mock_info.assert_has_calls([ mock.call(tmp, force_share=False, run_as_root=run_as_root), mock.call(tmp, run_as_root=run_as_root)]) @@ -1694,7 +1704,8 @@ run_as_root=run_as_root) self.assertIsNone(output) - mock_temp.assert_called_once_with() + mock_temp.assert_called_once_with(prefix='image_download_%s_' % + image_id) mock_info.assert_has_calls([ mock.call(tmp, force_share=False, run_as_root=run_as_root), mock.call(tmp, run_as_root=run_as_root)]) @@ -1852,7 +1863,8 @@ volume_format, blocksize) self.assertIsNone(output) - mock_temp.assert_called_once_with() + mock_temp.assert_called_once_with(prefix='image_download_%s_' % + image_id) mock_info.assert_has_calls([ mock.call(tmp, force_share=False, run_as_root=True), mock.call(tmp, run_as_root=True)]) diff -Nru cinder-17.0.1/cinder/tests/unit/test_utils.py cinder-17.4.0/cinder/tests/unit/test_utils.py --- cinder-17.0.1/cinder/tests/unit/test_utils.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/tests/unit/test_utils.py 2022-04-20 17:38:33.000000000 +0000 @@ -1057,6 +1057,33 @@ self.assertRaises(WrongException, raise_unexpected_error) self.assertFalse(mock_sleep.called) + @mock.patch('tenacity.nap.sleep') + def test_retry_exit_code(self, sleep_mock): + + exit_code = 5 + exception = utils.processutils.ProcessExecutionError + + @utils.retry(retry=utils.retry_if_exit_code, retry_param=exit_code) + def raise_retriable_exit_code(): + raise exception(exit_code=exit_code) + + self.assertRaises(exception, raise_retriable_exit_code) + self.assertEqual(2, sleep_mock.call_count) + sleep_mock.assert_has_calls([mock.call(1), mock.call(2)]) + + @mock.patch('tenacity.nap.sleep') + def test_retry_exit_code_non_retriable(self, sleep_mock): + + exit_code = 5 + exception = utils.processutils.ProcessExecutionError + + @utils.retry(retry=utils.retry_if_exit_code, retry_param=exit_code) + def raise_non_retriable_exit_code(): + raise exception(exit_code=exit_code + 1) + + self.assertRaises(exception, raise_non_retriable_exit_code) + sleep_mock.assert_not_called() + @ddt.ddt class LogTracingTestCase(test.TestCase): diff -Nru cinder-17.0.1/cinder/tests/unit/utils.py cinder-17.4.0/cinder/tests/unit/utils.py --- cinder-17.0.1/cinder/tests/unit/utils.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/tests/unit/utils.py 2022-04-20 17:38:33.000000000 +0000 @@ -108,6 +108,12 @@ def attach_volume(ctxt, volume_id, instance_uuid, attached_host, mountpoint, mode='rw'): + if isinstance(volume_id, objects.Volume): + volume_ovo = volume_id + volume_id = volume_ovo.id + else: + volume_ovo = None + now = timeutils.utcnow() values = {} values['volume_id'] = volume_id @@ -119,6 +125,13 @@ volume, updated_values = db.volume_attached( ctxt, attachment['id'], instance_uuid, attached_host, mountpoint, mode) + + if volume_ovo: + cls = objects.Volume + expected_attrs = cls._get_expected_attrs(ctxt) + volume = cls._from_db_object(ctxt, cls(ctxt), volume, + expected_attrs=expected_attrs) + return volume diff -Nru cinder-17.0.1/cinder/tests/unit/volume/drivers/dell_emc/powerflex/test_migrate_volume.py cinder-17.4.0/cinder/tests/unit/volume/drivers/dell_emc/powerflex/test_migrate_volume.py --- cinder-17.0.1/cinder/tests/unit/volume/drivers/dell_emc/powerflex/test_migrate_volume.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/tests/unit/volume/drivers/dell_emc/powerflex/test_migrate_volume.py 2022-04-20 17:38:33.000000000 +0000 @@ -29,15 +29,15 @@ MIGRATE_VOLUME_PARAMS_CASES = ( # Cases for testing _get_volume_params function. # +----------------------------------------------------+------------------+ - # |Volume type|Real provisioning|Conversion|Compression|Pool support thick| + # |Volume Type|Real provisioning|Conversion|Compression|Pool support thick| # +-----------+-----------------+----------+-----------+-----+------------+ - ('thin', 'ThinProvisioned', 'NoConversion', 'None', False), - ('thin', 'ThickProvisioned', 'ThickToThin', 'None', True), - ('thick', 'ThinProvisioned', 'NoConversion', 'None', False), - ('thick', 'ThinProvisioned', 'ThinToThick', 'None', True), - ('compressed', 'ThinProvisioned', 'NoConversion', 'Normal', False), - ('compressed', 'ThickProvisioned', 'ThickToThin', 'Normal', False), - ('compressed', 'ThickProvisioned', 'ThickToThin', 'None', False) + ('ThinProvisioned', 'ThinProvisioned', 'NoConversion', 'None', False), + ('ThinProvisioned', 'ThickProvisioned', 'ThickToThin', 'None', True), + ('ThickProvisioned', 'ThinProvisioned', 'NoConversion', 'None', False), + ('ThickProvisioned', 'ThinProvisioned', 'ThinToThick', 'None', True), + ('ThinProvisioned', 'ThinProvisioned', 'NoConversion', 'Normal', False), + ('ThinProvisioned', 'ThickProvisioned', 'ThickToThin', 'Normal', False), + ('ThinProvisioned', 'ThickProvisioned', 'ThickToThin', 'None', False) ) diff -Nru cinder-17.0.1/cinder/tests/unit/volume/drivers/dell_emc/powermax/powermax_data.py cinder-17.4.0/cinder/tests/unit/volume/drivers/dell_emc/powermax/powermax_data.py --- cinder-17.0.1/cinder/tests/unit/volume/drivers/dell_emc/powermax/powermax_data.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/tests/unit/volume/drivers/dell_emc/powermax/powermax_data.py 2022-04-20 17:38:33.000000000 +0000 @@ -1467,6 +1467,7 @@ snap_device_label = ('%(dev)s:%(label)s' % {'dev': device_id, 'label': managed_snap_id}) + priv_snap_response = { 'deviceName': snap_device_label, 'snapshotLnks': [], 'snapshotSrcs': [ @@ -1478,6 +1479,9 @@ 'snapshotName': test_snapshot_snap_name, 'state': 'Established'}]} + priv_snap_response_no_label = deepcopy(priv_snap_response) + priv_snap_response_no_label.update({'deviceName': device_id}) + volume_metadata = { 'DeviceID': device_id, 'ArrayID': array, 'ArrayModel': array_model} diff -Nru cinder-17.0.1/cinder/tests/unit/volume/drivers/dell_emc/powermax/test_powermax_common.py cinder-17.4.0/cinder/tests/unit/volume/drivers/dell_emc/powermax/test_powermax_common.py --- cinder-17.0.1/cinder/tests/unit/volume/drivers/dell_emc/powermax/test_powermax_common.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/tests/unit/volume/drivers/dell_emc/powermax/test_powermax_common.py 2022-04-20 17:38:33.000000000 +0000 @@ -3803,6 +3803,22 @@ array, device_id, snap_name) self.assertEqual(ref_metadata, act_metadata) + @mock.patch.object( + rest.PowerMaxRest, 'get_volume_snap_info', + return_value=(tpd.PowerMaxData.priv_snap_response_no_label)) + def test_get_snapshot_metadata_no_label(self, mck_snap): + array = self.data.array + device_id = self.data.device_id + snap_name = self.data.test_snapshot_snap_name + ref_metadata = {'SnapshotLabel': snap_name, + 'SourceDeviceID': device_id, + 'SnapIdList': six.text_type(self.data.snap_id), + 'is_snap_id': True} + + act_metadata = self.common.get_snapshot_metadata( + array, device_id, snap_name) + self.assertEqual(ref_metadata, act_metadata) + def test_update_metadata(self): model_update = {'provider_location': six.text_type( self.data.provider_location)} @@ -3970,6 +3986,26 @@ self.data.test_volume, None) self.assertEqual(self.data.rep_extra_specs_metro, extra_specs) + @mock.patch.object(utils.PowerMaxUtils, 'get_rdf_management_group_name') + def test_retype_volume_promotion_get_extra_specs_mgmt_group(self, mck_get): + array = self.data.array + srp = self.data.srp + device_id = self.data.device_id + volume = self.data.test_volume + volume_name = self.data.volume_id + extra_specs = deepcopy(self.data.rep_extra_specs) + target_slo = self.data.slo_silver + target_workload = self.data.workload + target_extra_specs = deepcopy(self.data.extra_specs) + target_extra_specs[utils.DISABLECOMPRESSION] = False + extra_specs[utils.REP_CONFIG] = self.data.rep_config_async + self.common.promotion = True + self.common._retype_volume( + array, srp, device_id, volume, volume_name, extra_specs, + target_slo, target_workload, target_extra_specs) + self.common.promotion = False + mck_get.assert_called_once_with(extra_specs[utils.REP_CONFIG]) + @mock.patch.object(rest.PowerMaxRest, 'is_volume_in_storagegroup', return_value=True) @mock.patch.object(masking.PowerMaxMasking, diff -Nru cinder-17.0.1/cinder/tests/unit/volume/drivers/dell_emc/powermax/test_powermax_replication.py cinder-17.4.0/cinder/tests/unit/volume/drivers/dell_emc/powermax/test_powermax_replication.py --- cinder-17.0.1/cinder/tests/unit/volume/drivers/dell_emc/powermax/test_powermax_replication.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/tests/unit/volume/drivers/dell_emc/powermax/test_powermax_replication.py 2022-04-20 17:38:33.000000000 +0000 @@ -494,6 +494,20 @@ extra_specs1, rep_config) self.assertEqual(ref_specs2, rep_extra_specs2) + @mock.patch.object(common.PowerMaxCommon, 'get_rdf_details', + return_value=(1, True)) + @mock.patch.object(rest.PowerMaxRest, + 'get_array_model_info', + return_value=('VMAX250F', False)) + def test_get_replication_extra_specs_get_rdf_group_promotion( + self, mock_model, mck_rdf): + self.common.promotion = True + remote_array = self.data.remote_array + rep_config = self.data.rep_config_sync + extra_specs1 = deepcopy(self.extra_specs) + self.common._get_replication_extra_specs(extra_specs1, rep_config) + mck_rdf.assert_called_with(remote_array, rep_config) + @mock.patch.object(rest.PowerMaxRest, 'get_array_model_info', return_value=('PowerMax 2000', True)) @@ -1186,15 +1200,16 @@ return_value=(True, tpd.PowerMaxData.defaultstoragegroup_name)) def test_migrate_volume_success_rep_promotion( self, mck_retype, mck_get, mck_break, mck_valid): - array_id = self.data.array + array_id = self.data.remote_array volume = self.data.test_rep_volume device_id = self.data.device_id - srp = self.data.srp + srp = 'SRP_2' target_slo = self.data.slo_silver target_workload = self.data.workload volume_name = volume.name new_type = {'extra_specs': {}} extra_specs = self.data.rep_extra_specs_rep_config + updated_host = 'HostX@Backend#Diamond+DSS+SRP_2+000197800124' self.common.promotion = True target_extra_specs = { utils.SRP: srp, utils.ARRAY: array_id, utils.SLO: target_slo, @@ -1205,6 +1220,7 @@ success, model_update = self.common._migrate_volume( array_id, volume, device_id, srp, target_slo, target_workload, volume_name, new_type, extra_specs) + self.assertEqual(model_update['host'], updated_host) mck_break.assert_called_once_with( array_id, device_id, volume_name, extra_specs) mck_retype.assert_called_once_with( @@ -1214,6 +1230,10 @@ self.common.promotion = False @mock.patch.object( + common.PowerMaxCommon, 'update_metadata', + return_value={'metadata': { + 'Configuration': 'RDF2+TDEV', 'ReplicationEnabled': 'True'}}) + @mock.patch.object( common.PowerMaxCommon, '_rdf_vols_partitioned', return_value=True) @mock.patch.object( @@ -1227,7 +1247,8 @@ common.PowerMaxCommon, '_retype_volume', return_value=(True, tpd.PowerMaxData.defaultstoragegroup_name)) def test_migrate_volume_success_rep_partitioned( - self, mck_retype, mck_get, mck_break, mck_valid, mck_partitioned): + self, mck_retype, mck_get, mck_break, mck_valid, mck_partitioned, + mck_update): array_id = self.data.array volume = self.data.test_rep_volume device_id = self.data.device_id @@ -1253,6 +1274,10 @@ target_slo, target_workload, target_extra_specs) self.assertTrue(success) self.common.promotion = False + config_metadata = model_update['metadata']['Configuration'] + rep_metadata = model_update['metadata']['ReplicationEnabled'] + self.assertEqual('TDEV', config_metadata) + self.assertEqual('False', rep_metadata) @mock.patch.object(masking.PowerMaxMasking, 'add_volume_to_storage_group') @mock.patch.object(provision.PowerMaxProvision, 'get_or_create_group') diff -Nru cinder-17.0.1/cinder/tests/unit/volume/drivers/dell_emc/powermax/test_powermax_rest.py cinder-17.4.0/cinder/tests/unit/volume/drivers/dell_emc/powermax/test_powermax_rest.py --- cinder-17.0.1/cinder/tests/unit/volume/drivers/dell_emc/powermax/test_powermax_rest.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/tests/unit/volume/drivers/dell_emc/powermax/test_powermax_rest.py 2022-04-20 17:38:33.000000000 +0000 @@ -628,6 +628,22 @@ self.assertTrue(is_child1) self.assertFalse(is_child2) + def test_is_child_sg_in_parent_sg_case_not_matching(self): + lower_case_host = 'OS-hostx-SRP_1-DiamondDSS-os-fibre-PG' + + is_child1 = self.rest.is_child_sg_in_parent_sg( + self.data.array, lower_case_host, + self.data.parent_sg_f) + self.assertTrue(is_child1) + + def test_is_child_sg_in_parent_sg_spelling_mistake(self): + lower_case_host = 'OS-hosty-SRP_1-DiamondDSS-os-fiber-PG' + + is_child1 = self.rest.is_child_sg_in_parent_sg( + self.data.array, lower_case_host, + self.data.parent_sg_f) + self.assertFalse(is_child1) + def test_add_child_sg_to_parent_sg(self): payload = {'editStorageGroupActionParam': { 'expandStorageGroupParam': { diff -Nru cinder-17.0.1/cinder/tests/unit/volume/drivers/dell_emc/powerstore/test_base.py cinder-17.4.0/cinder/tests/unit/volume/drivers/dell_emc/powerstore/test_base.py --- cinder-17.0.1/cinder/tests/unit/volume/drivers/dell_emc/powerstore/test_base.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/tests/unit/volume/drivers/dell_emc/powerstore/test_base.py 2022-04-20 17:38:33.000000000 +0000 @@ -21,8 +21,10 @@ class TestBase(powerstore.TestPowerStoreDriver): @mock.patch("cinder.volume.drivers.dell_emc.powerstore.client." + "PowerStoreClient.get_chap_config") + @mock.patch("cinder.volume.drivers.dell_emc.powerstore.client." "PowerStoreClient.get_appliance_id_by_name") - def test_configuration(self, mock_appliance): + def test_configuration(self, mock_appliance, mock_chap): mock_appliance.return_value = "A1" self.driver.check_for_setup_error() @@ -51,10 +53,15 @@ self.assertIn("Failed to query PowerStore appliances.", error.msg) @mock.patch("cinder.volume.drivers.dell_emc.powerstore.client." + "PowerStoreClient.get_chap_config") + @mock.patch("cinder.volume.drivers.dell_emc.powerstore.client." "PowerStoreClient.get_appliance_id_by_name") @mock.patch("cinder.volume.drivers.dell_emc.powerstore.client." "PowerStoreClient.get_appliance_metrics") - def test_update_volume_stats(self, mock_metrics, mock_appliance): + def test_update_volume_stats(self, + mock_metrics, + mock_appliance, + mock_chap): mock_appliance.return_value = "A1" mock_metrics.return_value = { "physical_total": 2147483648, @@ -64,11 +71,14 @@ self.driver._update_volume_stats() @mock.patch("cinder.volume.drivers.dell_emc.powerstore.client." + "PowerStoreClient.get_chap_config") + @mock.patch("cinder.volume.drivers.dell_emc.powerstore.client." "PowerStoreClient.get_appliance_id_by_name") @mock.patch("requests.request") def test_update_volume_stats_bad_status(self, mock_metrics, - mock_appliance): + mock_appliance, + mock_chap): mock_appliance.return_value = "A1" mock_metrics.return_value = powerstore.MockResponse(rc=400) self.driver.check_for_setup_error() diff -Nru cinder-17.0.1/cinder/tests/unit/volume/drivers/dell_emc/powerstore/test_snapshot_create_delete_revert.py cinder-17.4.0/cinder/tests/unit/volume/drivers/dell_emc/powerstore/test_snapshot_create_delete_revert.py --- cinder-17.0.1/cinder/tests/unit/volume/drivers/dell_emc/powerstore/test_snapshot_create_delete_revert.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/tests/unit/volume/drivers/dell_emc/powerstore/test_snapshot_create_delete_revert.py 2022-04-20 17:38:33.000000000 +0000 @@ -23,8 +23,10 @@ class TestSnapshotCreateDelete(powerstore.TestPowerStoreDriver): @mock.patch("cinder.volume.drivers.dell_emc.powerstore.client." + "PowerStoreClient.get_chap_config") + @mock.patch("cinder.volume.drivers.dell_emc.powerstore.client." "PowerStoreClient.get_appliance_id_by_name") - def setUp(self, mock_appliance): + def setUp(self, mock_appliance, mock_chap): super(TestSnapshotCreateDelete, self).setUp() mock_appliance.return_value = "A1" self.driver.check_for_setup_error() diff -Nru cinder-17.0.1/cinder/tests/unit/volume/drivers/dell_emc/powerstore/test_volume_attach_detach.py cinder-17.4.0/cinder/tests/unit/volume/drivers/dell_emc/powerstore/test_volume_attach_detach.py --- cinder-17.0.1/cinder/tests/unit/volume/drivers/dell_emc/powerstore/test_volume_attach_detach.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/tests/unit/volume/drivers/dell_emc/powerstore/test_volume_attach_detach.py 2022-04-20 17:38:33.000000000 +0000 @@ -25,10 +25,13 @@ class TestVolumeAttachDetach(powerstore.TestPowerStoreDriver): @mock.patch("cinder.volume.drivers.dell_emc.powerstore.client." + "PowerStoreClient.get_chap_config") + @mock.patch("cinder.volume.drivers.dell_emc.powerstore.client." "PowerStoreClient.get_appliance_id_by_name") - def setUp(self, mock_appliance): + def setUp(self, mock_appliance, mock_chap): super(TestVolumeAttachDetach, self).setUp() mock_appliance.return_value = "A1" + mock_chap.return_value = {"mode": "Single"} self.iscsi_driver.check_for_setup_error() self.fc_driver.check_for_setup_error() self.volume = fake_volume.fake_volume_obj( @@ -50,7 +53,7 @@ attached_host=self.volume.host ) ] - self.fake_iscsi_targets_response = [ + fake_iscsi_targets_response = [ { "address": "1.2.3.4", "ip_port": { @@ -66,7 +69,7 @@ }, }, ] - self.fake_fc_wwns_response = [ + fake_fc_wwns_response = [ { "wwn": "58:cc:f0:98:49:21:07:02" }, @@ -79,18 +82,52 @@ "wwpns": ["58:cc:f0:98:49:21:07:02", "58:cc:f0:98:49:23:07:02"], "initiator": "fake_initiator", } + self.iscsi_targets_mock = self.mock_object( + self.iscsi_driver.adapter.client, + "get_ip_pool_address", + return_value=fake_iscsi_targets_response + ) + self.fc_wwns_mock = self.mock_object( + self.fc_driver.adapter.client, + "get_fc_port", + return_value=fake_fc_wwns_response + ) - @mock.patch("cinder.volume.drivers.dell_emc.powerstore.client." - "PowerStoreClient.get_fc_port") - def test_get_fc_targets(self, mock_get_ip_pool): - mock_get_ip_pool.return_value = self.fake_fc_wwns_response + def test_initialize_connection_chap_enabled(self): + self.iscsi_driver.adapter.use_chap_auth = True + with mock.patch.object(self.iscsi_driver.adapter, + "_create_host_and_attach", + return_value=( + utils.get_chap_credentials(), + 1 + )): + connection_properties = self.iscsi_driver.initialize_connection( + self.volume, + self.fake_connector + ) + self.assertIn("auth_username", connection_properties["data"]) + self.assertIn("auth_password", connection_properties["data"]) + + def test_initialize_connection_chap_disabled(self): + self.iscsi_driver.adapter.use_chap_auth = False + with mock.patch.object(self.iscsi_driver.adapter, + "_create_host_and_attach", + return_value=( + utils.get_chap_credentials(), + 1 + )): + connection_properties = self.iscsi_driver.initialize_connection( + self.volume, + self.fake_connector + ) + self.assertNotIn("auth_username", connection_properties["data"]) + self.assertNotIn("auth_password", connection_properties["data"]) + + def test_get_fc_targets(self): wwns = self.fc_driver.adapter._get_fc_targets("A1") self.assertEqual(2, len(wwns)) - @mock.patch("cinder.volume.drivers.dell_emc.powerstore.client." - "PowerStoreClient.get_fc_port") - def test_get_fc_targets_filtered(self, mock_get_ip_pool): - mock_get_ip_pool.return_value = self.fake_fc_wwns_response + def test_get_fc_targets_filtered(self): self.fc_driver.adapter.allowed_ports = ["58:cc:f0:98:49:23:07:02"] wwns = self.fc_driver.adapter._get_fc_targets("A1") self.assertEqual(1, len(wwns)) @@ -98,10 +135,7 @@ utils.fc_wwn_to_string("58:cc:f0:98:49:21:07:02") in wwns ) - @mock.patch("cinder.volume.drivers.dell_emc.powerstore.client." - "PowerStoreClient.get_fc_port") - def test_get_fc_targets_filtered_no_matched_ports(self, mock_get_ip_pool): - mock_get_ip_pool.return_value = self.fake_fc_wwns_response + def test_get_fc_targets_filtered_no_matched_ports(self): self.fc_driver.adapter.allowed_ports = ["fc_wwn_1", "fc_wwn_2"] error = self.assertRaises(exception.VolumeBackendAPIException, self.fc_driver.adapter._get_fc_targets, @@ -109,18 +143,12 @@ self.assertIn("There are no accessible Fibre Channel targets on the " "system.", error.msg) - @mock.patch("cinder.volume.drivers.dell_emc.powerstore.client." - "PowerStoreClient.get_ip_pool_address") - def test_get_iscsi_targets(self, mock_get_ip_pool): - mock_get_ip_pool.return_value = self.fake_iscsi_targets_response + def test_get_iscsi_targets(self): iqns, portals = self.iscsi_driver.adapter._get_iscsi_targets("A1") self.assertTrue(len(iqns) == len(portals)) self.assertEqual(2, len(portals)) - @mock.patch("cinder.volume.drivers.dell_emc.powerstore.client." - "PowerStoreClient.get_ip_pool_address") - def test_get_iscsi_targets_filtered(self, mock_get_ip_pool): - mock_get_ip_pool.return_value = self.fake_iscsi_targets_response + def test_get_iscsi_targets_filtered(self): self.iscsi_driver.adapter.allowed_ports = ["1.2.3.4"] iqns, portals = self.iscsi_driver.adapter._get_iscsi_targets("A1") self.assertTrue(len(iqns) == len(portals)) @@ -129,11 +157,7 @@ "iqn.2020-07.com.dell:dellemc-powerstore-test-iqn-2" in iqns ) - @mock.patch("cinder.volume.drivers.dell_emc.powerstore.client." - "PowerStoreClient.get_ip_pool_address") - def test_get_iscsi_targets_filtered_no_matched_ports(self, - mock_get_ip_pool): - mock_get_ip_pool.return_value = self.fake_iscsi_targets_response + def test_get_iscsi_targets_filtered_no_matched_ports(self): self.iscsi_driver.adapter.allowed_ports = ["1.1.1.1", "2.2.2.2"] error = self.assertRaises(exception.VolumeBackendAPIException, self.iscsi_driver.adapter._get_iscsi_targets, diff -Nru cinder-17.0.1/cinder/tests/unit/volume/drivers/dell_emc/powerstore/test_volume_create_delete_extend.py cinder-17.4.0/cinder/tests/unit/volume/drivers/dell_emc/powerstore/test_volume_create_delete_extend.py --- cinder-17.0.1/cinder/tests/unit/volume/drivers/dell_emc/powerstore/test_volume_create_delete_extend.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/tests/unit/volume/drivers/dell_emc/powerstore/test_volume_create_delete_extend.py 2022-04-20 17:38:33.000000000 +0000 @@ -23,8 +23,10 @@ class TestVolumeCreateDeleteExtend(powerstore.TestPowerStoreDriver): @mock.patch("cinder.volume.drivers.dell_emc.powerstore.client." + "PowerStoreClient.get_chap_config") + @mock.patch("cinder.volume.drivers.dell_emc.powerstore.client." "PowerStoreClient.get_appliance_id_by_name") - def setUp(self, mock_appliance): + def setUp(self, mock_appliance, mock_chap): super(TestVolumeCreateDeleteExtend, self).setUp() mock_appliance.return_value = "A1" self.driver.check_for_setup_error() diff -Nru cinder-17.0.1/cinder/tests/unit/volume/drivers/dell_emc/powerstore/test_volume_create_from_source.py cinder-17.4.0/cinder/tests/unit/volume/drivers/dell_emc/powerstore/test_volume_create_from_source.py --- cinder-17.0.1/cinder/tests/unit/volume/drivers/dell_emc/powerstore/test_volume_create_from_source.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/tests/unit/volume/drivers/dell_emc/powerstore/test_volume_create_from_source.py 2022-04-20 17:38:33.000000000 +0000 @@ -23,8 +23,10 @@ class TestVolumeCreateFromSource(powerstore.TestPowerStoreDriver): @mock.patch("cinder.volume.drivers.dell_emc.powerstore.client." + "PowerStoreClient.get_chap_config") + @mock.patch("cinder.volume.drivers.dell_emc.powerstore.client." "PowerStoreClient.get_appliance_id_by_name") - def setUp(self, mock_appliance): + def setUp(self, mock_appliance, mock_chap): super(TestVolumeCreateFromSource, self).setUp() mock_appliance.return_value = "A1" self.driver.check_for_setup_error() diff -Nru cinder-17.0.1/cinder/tests/unit/volume/drivers/dell_emc/test_xtremio.py cinder-17.4.0/cinder/tests/unit/volume/drivers/dell_emc/test_xtremio.py --- cinder-17.0.1/cinder/tests/unit/volume/drivers/dell_emc/test_xtremio.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/tests/unit/volume/drivers/dell_emc/test_xtremio.py 2022-04-20 17:38:33.000000000 +0000 @@ -62,6 +62,14 @@ "name": "10.205.68.5/16", "index": 1, }, + '10.205.68.6/16': + {"port-address": + "iqn.2008-05.com.xtremio:002e67939c34", + "ip-port": 3260, + "ip-addr": "10.205.68.6/16", + "name": "10.205.68.6/16", + "index": 1, + }, }, 'targets': {'X1-SC2-target1': {'index': 1, "name": "X1-SC2-fc1", "port-address": @@ -347,7 +355,8 @@ driver_ssl_cert_path='/test/path/root_ca.crt', xtremio_array_busy_retry_count=5, xtremio_array_busy_retry_interval=5, - xtremio_clean_unused_ig=False) + xtremio_clean_unused_ig=False, + xtremio_ports=[]) def safe_get(key): return getattr(self.config, key) @@ -650,10 +659,26 @@ portals = xms_data['iscsi-portals'].copy() xms_data['iscsi-portals'].clear() lunmap = {'lun': 4} - self.assertRaises(exception.VolumeDriverException, + self.assertRaises(exception.VolumeBackendAPIException, self.driver._get_iscsi_properties, lunmap) xms_data['iscsi-portals'] = portals + def test_no_allowed_portals(self, req): + req.side_effect = xms_request + lunmap = {'lun': 4} + self.driver.allowed_ports = ['1.2.3.4'] + self.assertRaises(exception.VolumeBackendAPIException, + self.driver._get_iscsi_properties, lunmap) + + def test_filtered_portals(self, req): + req.side_effect = xms_request + lunmap = {'lun': 4} + self.driver.allowed_ports = ['10.205.68.6'] + connection_properties = self.driver._get_iscsi_properties(lunmap) + self.assertEqual(1, len(connection_properties['target_portals'])) + self.assertIn('10.205.68.6:3260', + connection_properties['target_portals']) + def test_initialize_connection(self, req): req.side_effect = xms_request self.driver.create_volume(self.data.test_volume) @@ -1365,6 +1390,27 @@ configuration=self.config) # ##### Connection FC##### + def test_no_targets_configured(self, req): + req.side_effect = xms_request + targets = xms_data['targets'].copy() + xms_data['targets'].clear() + self.assertRaises(exception.VolumeBackendAPIException, + self.driver.get_targets) + xms_data['targets'] = targets + + def test_no_allowed_targets(self, req): + req.side_effect = xms_request + self.driver.allowed_ports = ['58:cc:f0:98:49:22:07:02'] + self.assertRaises(exception.VolumeBackendAPIException, + self.driver.get_targets) + + def test_filtered_targets(self, req): + req.side_effect = xms_request + self.driver.allowed_ports = ['21:00:00:24:ff:57:b2:36'] + targets = self.driver.get_targets() + self.assertEqual(1, len(targets)) + self.assertIn('21000024ff57b236', targets) + def test_initialize_connection(self, req): req.side_effect = xms_request diff -Nru cinder-17.0.1/cinder/tests/unit/volume/drivers/hpe/test_hpe3par.py cinder-17.4.0/cinder/tests/unit/volume/drivers/hpe/test_hpe3par.py --- cinder-17.0.1/cinder/tests/unit/volume/drivers/hpe/test_hpe3par.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/tests/unit/volume/drivers/hpe/test_hpe3par.py 2022-04-20 17:38:33.000000000 +0000 @@ -763,14 +763,15 @@ spec=True, ) def setup_mock_client(self, _m_client, driver, conf=None, m_conf=None, - is_primera=False): + is_primera=False, + wsapi_version=wsapi_version_latest): _m_client = _m_client.return_value # Configure the base constants, defaults etc... _m_client.configure_mock(**self.mock_client_conf) - _m_client.getWsApiVersion.return_value = self.wsapi_version_latest + _m_client.getWsApiVersion.return_value = wsapi_version _m_client.is_primera_array.return_value = is_primera @@ -8901,10 +8902,49 @@ return mock_client - def test_iscsi_primera(self): + def test_iscsi_primera_old(self): + # primera 4.0.xx.yyy + wsapi_version_primera_old = {'major': 1, + 'build': 40000128, + 'minor': 8, + 'revision': 1} + self.assertRaises(NotImplementedError, self.setup_mock_client, driver=hpedriver.HPE3PARISCSIDriver, - is_primera=True) + is_primera=True, + wsapi_version=wsapi_version_primera_old) + + def test_iscsi_primera_new(self, config=None, mock_conf=None): + # primera 4.2.xx.yyy + wsapi_version_primera_new = {'major': 1, + 'build': 40202010, + 'minor': 8, + 'revision': 1} + + self.ctxt = context.get_admin_context() + + mock_client = self.setup_mock_client( + conf=config, + m_conf=mock_conf, + driver=hpedriver.HPE3PARISCSIDriver, + is_primera=True, + wsapi_version=wsapi_version_primera_new) + + expected_get_cpgs = [ + mock.call.getCPG(HPE3PAR_CPG), + mock.call.getCPG(HPE3PAR_CPG2)] + expected_get_ports = [mock.call.getPorts()] + expected_primera = [ + mock.call.is_primera_array(), + mock.call.getWsApiVersion()] + mock_client.assert_has_calls( + self.standard_login + + expected_get_cpgs + + self.standard_logout + + expected_primera + + self.standard_login + + expected_get_ports + + self.standard_logout) @ddt.data('volume', 'volume_name_id') def test_initialize_connection(self, volume_attr): diff -Nru cinder-17.0.1/cinder/tests/unit/volume/drivers/ibm/test_storwize_svc.py cinder-17.4.0/cinder/tests/unit/volume/drivers/ibm/test_storwize_svc.py --- cinder-17.0.1/cinder/tests/unit/volume/drivers/ibm/test_storwize_svc.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/tests/unit/volume/drivers/ibm/test_storwize_svc.py 2022-04-20 17:38:33.000000000 +0000 @@ -25,6 +25,7 @@ import ddt from oslo_concurrency import processutils from oslo_config import cfg +from oslo_service import loopingcall from oslo_utils import importutils from oslo_utils import units import paramiko @@ -5537,7 +5538,7 @@ self._set_flag('reserved_percentage', 25) self._set_flag('storwize_svc_multihostmap_enabled', True) self._set_flag('storwize_svc_vol_rsize', rsize) - stats = self.driver.get_volume_stats() + stats = self.driver.get_volume_stats(True) for each_pool in stats['pools']: self.assertIn(each_pool['pool_name'], self._def_flags['storwize_svc_volpool_name']) @@ -8269,6 +8270,7 @@ list(resp.select('port_id', 'port_status'))) +@ddt.ddt class StorwizeHelpersTestCase(test.TestCase): def setUp(self): super(StorwizeHelpersTestCase, self).setUp() @@ -8494,7 +8496,239 @@ 'status': 'copying', 'target_vdisk_name': 'testvol'} self.storwize_svc_common.pretreatment_before_revert(vol) - stopfcmap.assert_called_once_with('4', split=True) + stopfcmap.assert_called_once_with('4') + + @ddt.data({'copy_rate': '50', 'progress': '3', 'status': 'copying'}, + {'copy_rate': '50', 'progress': '100', 'status': 'copying'}, + {'copy_rate': '0', 'progress': '0', 'status': 'copying'}, + {'copy_rate': '50', 'progress': '0', 'status': 'copying'}, + {'copy_rate': '0', 'progress': '0', 'status': 'idle_or_copied'}) + @mock.patch.object(storwize_svc_common.StorwizeSSH, 'chfcmap') + @mock.patch.object(storwize_svc_common.StorwizeSSH, 'stopfcmap') + @mock.patch.object(storwize_svc_common.StorwizeSSH, 'rmfcmap') + @mock.patch.object(storwize_svc_common.StorwizeHelpers, + '_get_flashcopy_mapping_attributes') + @mock.patch.object(storwize_svc_common.StorwizeHelpers, + '_get_vdisk_fc_mappings') + def test_check_vdisk_fc_mappings(self, + fc_data, + get_vdisk_fc_mappings, + get_fc_mapping_attributes, + rmfcmap, stopfcmap, chfcmap): + vol = 'testvol' + get_vdisk_fc_mappings.return_value = ['4'] + get_fc_mapping_attributes.return_value = { + 'copy_rate': fc_data['copy_rate'], + 'progress': fc_data['progress'], + 'status': fc_data['status'], + 'target_vdisk_name': 'tar-testvol', + 'rc_controlled': 'no', + 'source_vdisk_name': 'testvol'} + + if(fc_data['copy_rate'] != '0' and fc_data['progress'] == '100' + and fc_data['status'] == 'copying'): + (self.assertRaises(loopingcall.LoopingCallDone, + self.storwize_svc_common._check_vdisk_fc_mappings, vol, True, + False)) + stopfcmap.assert_called_with('4') + self.assertEqual(1, stopfcmap.call_count) + else: + self.storwize_svc_common._check_vdisk_fc_mappings(vol, True, + False) + stopfcmap.assert_not_called() + self.assertEqual(0, stopfcmap.call_count) + + get_vdisk_fc_mappings.assert_called() + get_fc_mapping_attributes.assert_called_with('4') + rmfcmap.assert_not_called() + self.assertEqual(1, get_fc_mapping_attributes.call_count) + self.assertEqual(0, rmfcmap.call_count) + + if(fc_data['copy_rate'] == '0' and fc_data['progress'] == '0' + and fc_data['status'] in ['copying', 'idle_or_copied']): + chfcmap.assert_called_with('4', copyrate='50', autodel='on') + self.assertEqual(1, chfcmap.call_count) + else: + chfcmap.assert_not_called() + self.assertEqual(0, chfcmap.call_count) + + @mock.patch.object(storwize_svc_common.StorwizeSSH, 'chfcmap') + @mock.patch.object(storwize_svc_common.StorwizeSSH, 'stopfcmap') + @mock.patch.object(storwize_svc_common.StorwizeSSH, 'rmfcmap') + @mock.patch.object(storwize_svc_common.StorwizeHelpers, + '_get_flashcopy_mapping_attributes') + @mock.patch.object(storwize_svc_common.StorwizeHelpers, + '_get_vdisk_fc_mappings') + def test_check_vdisk_fc_mappings_tarisvol(self, + get_vdisk_fc_mappings, + get_fc_mapping_attributes, + rmfcmap, stopfcmap, chfcmap): + vol = 'tar-testvol' + get_vdisk_fc_mappings.return_value = ['4'] + get_fc_mapping_attributes.return_value = { + 'copy_rate': '0', + 'progress': '0', + 'status': 'idle_or_copied', + 'target_vdisk_name': 'tar-testvol', + 'rc_controlled': 'no', + 'source_vdisk_name': 'testvol'} + + self.assertRaises(loopingcall.LoopingCallDone, + self.storwize_svc_common._check_vdisk_fc_mappings, + vol, True, False) + + get_vdisk_fc_mappings.assert_called() + get_fc_mapping_attributes.assert_called_with('4') + stopfcmap.assert_not_called() + rmfcmap.assert_called_with('4') + chfcmap.assert_not_called() + self.assertEqual(1, get_fc_mapping_attributes.call_count) + self.assertEqual(0, stopfcmap.call_count) + self.assertEqual(1, rmfcmap.call_count) + self.assertEqual(0, chfcmap.call_count) + + @ddt.data(([{'cp_rate': '0', 'prgs': '0', 'status': 'idle_or_copied', + 'trg_vdisk': 'testvol', 'src_vdisk': 'tar_testvol'}, + {'cp_rate': '50', 'prgs': '100', 'status': 'copying', + 'trg_vdisk': 'tar_testvol', 'src_vdisk': 'testvol'}, + {'cp_rate': '50', 'prgs': '3', 'status': 'copying', + 'trg_vdisk': 'tar_testvol', 'src_vdisk': 'testvol'}], 1), + ([{'cp_rate': '50', 'prgs': '100', 'status': 'idle_or_copied', + 'trg_vdisk': 'testvol', 'src_vdisk': 'tar_testvol'}, + {'cp_rate': '50', 'prgs': '100', 'status': 'copying', + 'trg_vdisk': 'tar_testvol', 'src_vdisk': 'testvol'}, + {'cp_rate': '50', 'prgs': '100', 'status': 'copying', + 'trg_vdisk': 'testvol', 'src_vdisk': 'tar_testvol'}], 1), + ([{'cp_rate': '50', 'prgs': '100', 'status': 'idle_or_copied', + 'trg_vdisk': 'testvol', 'src_vdisk': 'tar_testvol'}, + {'cp_rate': '50', 'prgs': '100', 'status': 'copying', + 'trg_vdisk': 'tar_testvol', 'src_vdisk': 'testvol'}, + {'cp_rate': '50', 'prgs': '100', 'status': 'copying', + 'trg_vdisk': 'tar_testvol_1', 'src_vdisk': 'testvol'}], 2), + ([{'cp_rate': '0', 'prgs': '0', 'status': 'copying', + 'trg_vdisk': 'testvol', 'src_vdisk': 'snap_testvol'}, + {'cp_rate': '50', 'prgs': '0', 'status': 'copying', + 'trg_vdisk': 'tar_testvol', 'src_vdisk': 'testvol'}, + {'cp_rate': '50', 'prgs': '0', 'status': 'copying', + 'trg_vdisk': 'tar_testvol_1', 'src_vdisk': 'testvol'}], 0)) + @mock.patch.object(storwize_svc_common.StorwizeSSH, 'chfcmap') + @mock.patch.object(storwize_svc_common.StorwizeSSH, 'stopfcmap') + @mock.patch.object(storwize_svc_common.StorwizeSSH, 'rmfcmap') + @mock.patch.object(storwize_svc_common.StorwizeHelpers, + '_get_flashcopy_mapping_attributes') + @mock.patch.object(storwize_svc_common.StorwizeHelpers, + '_get_vdisk_fc_mappings') + @ddt.unpack + def test_check_vdisk_fc_mappings_mul_fcs(self, + fc_data, stopfc_count, + get_vdisk_fc_mappings, + get_fc_mapping_attributes, + rmfcmap, stopfcmap, chfcmap): + vol = 'testvol' + get_vdisk_fc_mappings.return_value = ['4', '5', '7'] + get_fc_mapping_attributes.side_effect = [ + { + 'copy_rate': fc_data[0]['cp_rate'], + 'progress': fc_data[0]['prgs'], + 'status': fc_data[0]['status'], + 'target_vdisk_name': fc_data[0]['trg_vdisk'], + 'rc_controlled': 'no', + 'source_vdisk_name': fc_data[0]['src_vdisk']}, + { + 'copy_rate': fc_data[1]['cp_rate'], + 'progress': fc_data[1]['prgs'], + 'status': fc_data[1]['status'], + 'target_vdisk_name': fc_data[1]['trg_vdisk'], + 'rc_controlled': 'no', + 'source_vdisk_name': fc_data[1]['src_vdisk']}, + { + 'copy_rate': fc_data[2]['cp_rate'], + 'progress': fc_data[2]['prgs'], + 'status': fc_data[2]['status'], + 'target_vdisk_name': fc_data[2]['trg_vdisk'], + 'rc_controlled': 'no', + 'source_vdisk_name': fc_data[2]['src_vdisk']}] + + self.storwize_svc_common._check_vdisk_fc_mappings(vol, True, True) + get_vdisk_fc_mappings.assert_called() + get_fc_mapping_attributes.assert_called() + rmfcmap.assert_not_called() + chfcmap.assert_not_called() + self.assertEqual(3, get_fc_mapping_attributes.call_count) + self.assertEqual(stopfc_count, stopfcmap.call_count) + self.assertEqual(0, rmfcmap.call_count) + self.assertEqual(0, chfcmap.call_count) + + @ddt.data(([{'cp_rate': '0', 'prgs': '0', 'status': 'idle_or_copied', + 'trg_vdisk': 'vdisk', 'src_vdisk': 'Hyp_vol'}, + {'cp_rate': '50', 'prgs': '0', 'status': 'idle_or_copied', + 'trg_vdisk': 'Hyp_vol', 'src_vdisk': 'vdisk'}, + {'cp_rate': '50', 'prgs': '3', 'status': 'copying', + 'trg_vdisk': 'Snap_vol', 'src_vdisk': 'Hyp_vol'}, + {'cp_rate': '50', 'prgs': '0', 'status': 'copying', + 'trg_vdisk': 'Snap_vol_1', 'src_vdisk': 'Hyp_vol'}], 0), + ([{'cp_rate': '0', 'prgs': '0', 'status': 'idle_or_copied', + 'trg_vdisk': 'vdisk', 'src_vdisk': 'Hyp_vol'}, + {'cp_rate': '50', 'prgs': '0', 'status': 'idle_or_copied', + 'trg_vdisk': 'Hyp_vol', 'src_vdisk': 'vdisk'}, + {'cp_rate': '50', 'prgs': '100', 'status': 'copying', + 'trg_vdisk': 'Snap_vol', 'src_vdisk': 'Hyp_vol'}, + {'cp_rate': '50', 'prgs': '0', 'status': 'copying', + 'trg_vdisk': 'Snap_vol_1', 'src_vdisk': 'Hyp_vol'}], 1)) + @mock.patch.object(storwize_svc_common.StorwizeSSH, 'chfcmap') + @mock.patch.object(storwize_svc_common.StorwizeSSH, 'stopfcmap') + @mock.patch.object(storwize_svc_common.StorwizeSSH, 'rmfcmap') + @mock.patch.object(storwize_svc_common.StorwizeHelpers, + '_get_flashcopy_mapping_attributes') + @mock.patch.object(storwize_svc_common.StorwizeHelpers, + '_get_vdisk_fc_mappings') + @ddt.unpack + def test_check_vdisk_fc_mappings_rc_cont(self, + fc_data, stopfc_count, + get_vdisk_fc_mappings, + get_fc_mapping_attributes, + rmfcmap, stopfcmap, chfcmap): + vol = 'Hyp_vol' + get_vdisk_fc_mappings.return_value = ['4', '5', '7', '9'] + get_fc_mapping_attributes.side_effect = [ + { + 'copy_rate': fc_data[0]['cp_rate'], + 'progress': fc_data[0]['prgs'], + 'status': fc_data[0]['status'], + 'target_vdisk_name': fc_data[0]['trg_vdisk'], + 'rc_controlled': 'yes', + 'source_vdisk_name': fc_data[0]['src_vdisk']}, + { + 'copy_rate': fc_data[1]['cp_rate'], + 'progress': fc_data[1]['prgs'], + 'status': fc_data[1]['status'], + 'target_vdisk_name': fc_data[1]['trg_vdisk'], + 'rc_controlled': 'yes', + 'source_vdisk_name': fc_data[1]['src_vdisk']}, + { + 'copy_rate': fc_data[2]['cp_rate'], + 'progress': fc_data[2]['prgs'], + 'status': fc_data[2]['status'], + 'target_vdisk_name': fc_data[2]['trg_vdisk'], + 'rc_controlled': 'no', + 'source_vdisk_name': fc_data[2]['src_vdisk']}, + { + 'copy_rate': fc_data[3]['cp_rate'], + 'progress': fc_data[3]['prgs'], + 'status': fc_data[3]['status'], + 'target_vdisk_name': fc_data[3]['trg_vdisk'], + 'rc_controlled': 'no', + 'source_vdisk_name': fc_data[3]['src_vdisk']}] + + self.storwize_svc_common._check_vdisk_fc_mappings(vol, True, True) + get_vdisk_fc_mappings.assert_called() + get_fc_mapping_attributes.assert_called() + rmfcmap.assert_not_called() + chfcmap.assert_not_called() + self.assertEqual(4, get_fc_mapping_attributes.call_count) + self.assertEqual(stopfc_count, stopfcmap.call_count) + self.assertEqual(0, rmfcmap.call_count) + self.assertEqual(0, chfcmap.call_count) def test_storwize_check_flashcopy_rate_invalid1(self): with mock.patch.object(storwize_svc_common.StorwizeHelpers, @@ -8522,6 +8756,35 @@ self.storwize_svc_common.check_flashcopy_rate, flashcopy_rate) + @ddt.data(({'mirror_pool': 'openstack2', + 'volume_topology': None, + 'peer_pool': None}, True, 1), + ({'mirror_pool': 'openstack2', + 'volume_topology': None, + 'peer_pool': None}, False, 2), + ({'mirror_pool': None, + 'volume_topology': 'hyperswap', + 'peer_pool': 'openstack1'}, True, 1), + ({'mirror_pool': None, + 'volume_topology': 'hyperswap', + 'peer_pool': 'openstack1'}, False, 2)) + @mock.patch.object(storwize_svc_common.StorwizeHelpers, + 'is_data_reduction_pool') + @ddt.unpack + def test_is_volume_type_dr_pools_dr_pool(self, opts, is_drp, call_count, + is_data_reduction_pool): + is_data_reduction_pool.return_value = is_drp + pool = 'openstack' + rep_type = None + rep_target_pool = None + + isdrpool = (self.storwize_svc_common. + is_volume_type_dr_pools(pool, opts, rep_type, + rep_target_pool)) + self.assertEqual(is_drp, isdrpool) + is_data_reduction_pool.assert_called() + self.assertEqual(call_count, is_data_reduction_pool.call_count) + @ddt.ddt class StorwizeSSHTestCase(test.TestCase): diff -Nru cinder-17.0.1/cinder/tests/unit/volume/drivers/netapp/dataontap/client/test_client_cmode.py cinder-17.4.0/cinder/tests/unit/volume/drivers/netapp/dataontap/client/test_client_cmode.py --- cinder-17.0.1/cinder/tests/unit/volume/drivers/netapp/dataontap/client/test_client_cmode.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/tests/unit/volume/drivers/netapp/dataontap/client/test_client_cmode.py 2022-04-20 17:38:33.000000000 +0000 @@ -736,6 +736,42 @@ netapp_api.NaElement.create_node_with_children( 'clone-create', **clone_create_args), True) + @ddt.data(0, 1) + def test_clone_lun_is_sub_clone(self, block_count): + + self.client.clone_lun( + 'volume', 'fakeLUN', 'newFakeLUN', block_count=block_count) + + clone_create_args = { + 'volume': 'volume', + 'source-path': 'fakeLUN', + 'destination-path': 'newFakeLUN', + } + + is_sub_clone = block_count > 0 + if not is_sub_clone: + clone_create_args['space-reserve'] = 'true' + + # build the expected request + expected_clone_create_request = \ + netapp_api.NaElement.create_node_with_children( + 'clone-create', **clone_create_args) + + # add expected fields in the request if it's a sub-clone + if is_sub_clone: + block_ranges = netapp_api.NaElement("block-ranges") + block_range = \ + netapp_api.NaElement.create_node_with_children( + 'block-range', + **{'source-block-number': '0', + 'destination-block-number': '0', + 'block-count': '1'}) + block_ranges.add_child_elem(block_range) + expected_clone_create_request.add_child_elem(block_ranges) + + self.connection.invoke_successfully.assert_called_once_with( + expected_clone_create_request, True) + def test_clone_lun_multiple_zapi_calls(self): """Test for when lun clone requires more than one zapi call.""" diff -Nru cinder-17.0.1/cinder/tests/unit/volume/drivers/netapp/dataontap/test_nfs_base.py cinder-17.4.0/cinder/tests/unit/volume/drivers/netapp/dataontap/test_nfs_base.py --- cinder-17.0.1/cinder/tests/unit/volume/drivers/netapp/dataontap/test_nfs_base.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/tests/unit/volume/drivers/netapp/dataontap/test_nfs_base.py 2022-04-20 17:38:33.000000000 +0000 @@ -940,3 +940,9 @@ self.assertFalse(cloned) mock_call__is_share_clone_compatible.assert_not_called() mock_call__do_clone_rel_img_cache.assert_not_called() + + def test_update_migrated_volume(self): + self.assertRaises(NotImplementedError, + self.driver.update_migrated_volume, self.ctxt, + fake.test_volume, mock.sentinel.new_volume, + mock.sentinel.original_status) diff -Nru cinder-17.0.1/cinder/tests/unit/volume/drivers/netapp/fakes.py cinder-17.4.0/cinder/tests/unit/volume/drivers/netapp/fakes.py --- cinder-17.0.1/cinder/tests/unit/volume/drivers/netapp/fakes.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/tests/unit/volume/drivers/netapp/fakes.py 2022-04-20 17:38:33.000000000 +0000 @@ -16,6 +16,7 @@ # under the License. +from cinder.tests.unit import fake_volume from cinder.volume import configuration as conf import cinder.volume.drivers.netapp.options as na_opts @@ -71,13 +72,12 @@ VOLUME_ID = 'fake_volume_id' VOLUME_TYPE_ID = 'fake_volume_type_id' -VOLUME = { - 'name': VOLUME_NAME, - 'size': 42, - 'id': VOLUME_ID, - 'host': 'fake_host@fake_backend#fake_pool', - 'volume_type_id': VOLUME_TYPE_ID, -} +VOLUME = fake_volume.fake_volume_obj(None, + name=VOLUME_NAME, + size=42, + id=VOLUME_ID, + host='fake_host@fake_backend#fake_pool', + volume_type_id=VOLUME_TYPE_ID) SNAPSHOT_NAME = 'fake_snapshot_name' SNAPSHOT_ID = 'fake_snapshot_id' diff -Nru cinder-17.0.1/cinder/tests/unit/volume/drivers/netapp/test_utils.py cinder-17.4.0/cinder/tests/unit/volume/drivers/netapp/test_utils.py --- cinder-17.0.1/cinder/tests/unit/volume/drivers/netapp/test_utils.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/tests/unit/volume/drivers/netapp/test_utils.py 2022-04-20 17:38:33.000000000 +0000 @@ -352,13 +352,24 @@ self.assertEqual(expected, result) def test_get_qos_policy_group_name_no_id(self): - volume = copy.deepcopy(fake.VOLUME) - del(volume['id']) - - result = na_utils.get_qos_policy_group_name(volume) + delattr(fake.VOLUME, '_obj_id') + try: + result = na_utils.get_qos_policy_group_name(fake.VOLUME) + finally: + fake.VOLUME._obj_id = fake.VOLUME_ID self.assertIsNone(result) + def test_get_qos_policy_group_name_migrated_volume(self): + fake.VOLUME._name_id = 'asdf' + try: + expected = 'openstack-' + fake.VOLUME.name_id + result = na_utils.get_qos_policy_group_name(fake.VOLUME) + finally: + fake.VOLUME._name_id = None + + self.assertEqual(expected, result) + def test_get_qos_policy_group_name_from_info(self): expected = 'openstack-%s' % fake.VOLUME_ID result = na_utils.get_qos_policy_group_name_from_info( diff -Nru cinder-17.0.1/cinder/tests/unit/volume/drivers/solidfire/test_solidfire.py cinder-17.4.0/cinder/tests/unit/volume/drivers/solidfire/test_solidfire.py --- cinder-17.0.1/cinder/tests/unit/volume/drivers/solidfire/test_solidfire.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/tests/unit/volume/drivers/solidfire/test_solidfire.py 2022-04-20 17:38:33.000000000 +0000 @@ -370,11 +370,14 @@ 'attributes': {'uuid': f_uuid[1]}, 'qos': None, 'iqn': test_name}]}} - if params and params['startVolumeID']: + if params and params.get('startVolumeID', None): volumes = result['result']['volumes'] - selected_volumes = [v for v in volumes if v.get('volumeID') - != params['startVolumeID']] + selected_volumes = [v for v in volumes if v.get('volumeID') != + params['startVolumeID']] result['result']['volumes'] = selected_volumes + else: + result = {'result': {'volumes': []}} + return result elif method == 'DeleteSnapshot': return {'result': {}} diff -Nru cinder-17.0.1/cinder/tests/unit/volume/drivers/test_nfs.py cinder-17.4.0/cinder/tests/unit/volume/drivers/test_nfs.py --- cinder-17.0.1/cinder/tests/unit/volume/drivers/test_nfs.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/tests/unit/volume/drivers/test_nfs.py 2022-04-20 17:38:33.000000000 +0000 @@ -1354,12 +1354,13 @@ run_as_root=True) mock_permission.assert_called_once_with(dest_vol_path) - @ddt.data([NFS_CONFIG1, QEMU_IMG_INFO_OUT3], - [NFS_CONFIG2, QEMU_IMG_INFO_OUT4], - [NFS_CONFIG3, QEMU_IMG_INFO_OUT3], - [NFS_CONFIG4, QEMU_IMG_INFO_OUT4]) + @ddt.data([NFS_CONFIG1, QEMU_IMG_INFO_OUT3, 'available'], + [NFS_CONFIG2, QEMU_IMG_INFO_OUT4, 'backing-up'], + [NFS_CONFIG3, QEMU_IMG_INFO_OUT3, 'available'], + [NFS_CONFIG4, QEMU_IMG_INFO_OUT4, 'backing-up']) @ddt.unpack - def test_create_volume_from_snapshot(self, nfs_conf, qemu_img_info): + def test_create_volume_from_snapshot(self, nfs_conf, qemu_img_info, + snap_status): self._set_driver(extra_confs=nfs_conf) drv = self._driver @@ -1376,7 +1377,7 @@ # Fake snapshot based in the previous created volume snap_file = src_volume.name + '.' + fake_snap.id fake_snap.volume = src_volume - fake_snap.status = 'available' + fake_snap.status = snap_status fake_snap.size = 10 # New fake volume where the snap will be copied @@ -1419,7 +1420,9 @@ mock_ensure.assert_called_once() mock_find_share.assert_called_once_with(new_volume) - def test_create_volume_from_snapshot_status_not_available(self): + @ddt.data('error', 'creating', 'deleting', 'deleted', 'updating', + 'error_deleting', 'unmanaging', 'restoring') + def test_create_volume_from_snapshot_invalid_status(self, snap_status): """Expect an error when the snapshot's status is not 'available'.""" self._set_driver() drv = self._driver @@ -1428,6 +1431,7 @@ fake_snap = fake_snapshot.fake_snapshot_obj(self.context) fake_snap.volume = src_volume + fake_snap.status = snap_status new_volume = self._simple_volume() new_volume['size'] = fake_snap['volume_size'] @@ -1504,6 +1508,8 @@ mock.patch.object(drv, '_read_info_file', return_value={}), \ mock.patch.object(drv, '_do_create_snapshot') \ as mock_do_create_snapshot, \ + mock.patch.object(drv, '_check_snapshot_support') \ + as mock_check_support, \ mock.patch.object(drv, '_write_info_file') \ as mock_write_info_file, \ mock.patch.object(drv, 'get_active_image_from_info', @@ -1512,6 +1518,7 @@ return_value=snap_path): self._driver.create_snapshot(fake_snap) + mock_check_support.assert_called_once() mock_do_create_snapshot.assert_called_with(fake_snap, volume['name'], snap_path) mock_write_info_file.assert_called_with( diff -Nru cinder-17.0.1/cinder/tests/unit/volume/drivers/test_pure.py cinder-17.4.0/cinder/tests/unit/volume/drivers/test_pure.py --- cinder-17.0.1/cinder/tests/unit/volume/drivers/test_pure.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/tests/unit/volume/drivers/test_pure.py 2022-04-20 17:38:33.000000000 +0000 @@ -52,6 +52,7 @@ ISCSI_DRIVER_OBJ = DRIVER_PATH + ".PureISCSIDriver" FC_DRIVER_OBJ = DRIVER_PATH + ".PureFCDriver" ARRAY_OBJ = DRIVER_PATH + ".FlashArray" +UNMANAGED_SUFFIX = "-unmanaged" GET_ARRAY_PRIMARY = {"version": "99.9.9", "revision": "201411230504+8a400f7", @@ -495,6 +496,7 @@ 'source_reference': {'name': MANAGEABLE_PURE_SNAPS[2]['source']}, } ] +MAX_SNAP_LENGTH = 96 class FakePureStorageHTTPError(Exception): @@ -602,6 +604,7 @@ vol.volume_type = voltype vol.volume_type_id = voltype.id + vol.volume_attachment = None return vol, vol_name @@ -1635,34 +1638,6 @@ self.assertEqual((None, None), result) mock_create_cg.assert_called_with(mock_context, mock_group) self.assertTrue(self.array.create_pgroup_snapshot.called) - self.assertEqual(num_volumes, self.array.copy_volume.call_count) - self.assertEqual(num_volumes, self.array.set_pgroup.call_count) - self.assertTrue(self.array.destroy_pgroup.called) - - @mock.patch(BASE_DRIVER_OBJ + ".create_consistencygroup") - def test_create_consistencygroup_from_cg_with_error(self, mock_create_cg): - num_volumes = 5 - mock_context = mock.MagicMock() - mock_group = mock.MagicMock() - mock_source_cg = mock.MagicMock() - mock_volumes = [mock.MagicMock() for i in range(num_volumes)] - mock_source_vols = [mock.MagicMock() for i in range(num_volumes)] - - self.array.copy_volume.side_effect = FakePureStorageHTTPError() - - self.assertRaises( - FakePureStorageHTTPError, - self.driver.create_consistencygroup_from_src, - mock_context, - mock_group, - mock_volumes, - source_cg=mock_source_cg, - source_vols=mock_source_vols - ) - mock_create_cg.assert_called_with(mock_context, mock_group) - self.assertTrue(self.array.create_pgroup_snapshot.called) - # Make sure that the temp snapshot is cleaned up even when copying - # the volume fails! self.assertTrue(self.array.destroy_pgroup.called) @mock.patch(BASE_DRIVER_OBJ + ".delete_volume", autospec=True) @@ -2042,7 +2017,7 @@ def test_unmanage(self): vol, vol_name = self.new_fake_vol() - unmanaged_vol_name = vol_name + "-unmanaged" + unmanaged_vol_name = vol_name + UNMANAGED_SUFFIX self.driver.unmanage(vol) @@ -2057,7 +2032,7 @@ def test_unmanage_with_deleted_volume(self): vol, vol_name = self.new_fake_vol() - unmanaged_vol_name = vol_name + "-unmanaged" + unmanaged_vol_name = vol_name + UNMANAGED_SUFFIX self.array.rename_volume.side_effect = \ self.purestorage_module.PureHTTPError( text="Volume does not exist.", @@ -2206,12 +2181,23 @@ self.driver.manage_existing_snapshot_get_size, snap, {'name': PURE_SNAPSHOT['name']}) - def test_unmanage_snapshot(self): - snap, snap_name = self.new_fake_snap() - unmanaged_snap_name = snap_name + "-unmanaged" + @ddt.data( + # 96 chars, will exceed allowable length + 'volume-1e5177e7-95e5-4a0f-b170-e45f4b469f6a-cinder.' + 'snapshot-253b2878-ec60-4793-ad19-e65496ec7aab', + # short_name that will require no adjustment + 'volume-1e5177e7-cinder.snapshot-e65496ec7aab') + @mock.patch(BASE_DRIVER_OBJ + "._get_snap_name") + def test_unmanage_snapshot(self, fake_name, mock_get_snap_name): + snap, _ = self.new_fake_snap() + mock_get_snap_name.return_value = fake_name self.driver.unmanage_snapshot(snap) - self.array.rename_volume.assert_called_with(snap_name, - unmanaged_snap_name) + self.array.rename_volume.assert_called_once() + old_name = self.array.rename_volume.call_args[0][0] + new_name = self.array.rename_volume.call_args[0][1] + self.assertEqual(fake_name, old_name) + self.assertLessEqual(len(new_name), MAX_SNAP_LENGTH) + self.assertTrue(new_name.endswith(UNMANAGED_SUFFIX)) def test_unmanage_snapshot_error_propagates(self): snap, _ = self.new_fake_snap() @@ -2221,7 +2207,11 @@ def test_unmanage_snapshot_with_deleted_snapshot(self): snap, snap_name = self.new_fake_snap() - unmanaged_snap_name = snap_name + "-unmanaged" + if len(snap_name + UNMANAGED_SUFFIX) > MAX_SNAP_LENGTH: + unmanaged_snap_name = snap_name[:-len(UNMANAGED_SUFFIX)] + \ + UNMANAGED_SUFFIX + else: + unmanaged_snap_name = snap_name self.array.rename_volume.side_effect = \ self.purestorage_module.PureHTTPError( text="Snapshot does not exist.", @@ -3727,9 +3717,9 @@ self.driver.terminate_connection(vol, FC_CONNECTOR) mock_disconnect.assert_has_calls([ mock.call(mock_secondary, vol, FC_CONNECTOR, - remove_remote_hosts=False), + is_multiattach=False, remove_remote_hosts=False), mock.call(self.array, vol, FC_CONNECTOR, - remove_remote_hosts=False) + is_multiattach=False, remove_remote_hosts=False) ]) diff -Nru cinder-17.0.1/cinder/tests/unit/volume/drivers/test_rbd.py cinder-17.4.0/cinder/tests/unit/volume/drivers/test_rbd.py --- cinder-17.0.1/cinder/tests/unit/volume/drivers/test_rbd.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/tests/unit/volume/drivers/test_rbd.py 2022-04-20 17:38:33.000000000 +0000 @@ -881,6 +881,52 @@ @common_mocks @mock.patch('cinder.objects.Volume.get_by_id') + @mock.patch.object(driver.RBDDriver, '_resize', mock.Mock()) + def test_log_create_vol_from_snap_w_v2_clone_api(self, volume_get_by_id): + volume_get_by_id.return_value = self.volume_a + + self.mock_proxy().__enter__().volume.op_features.return_value = 1 + self.mock_rbd.RBD_OPERATION_FEATURE_CLONE_PARENT = 1 + + self.cfg.rbd_flatten_volume_from_snapshot = False + + with mock.patch.object(driver, 'LOG') as mock_log: + with mock.patch.object(self.driver.rbd.Image(), 'stripe_unit') as \ + mock_rbd_image_stripe_unit: + mock_rbd_image_stripe_unit.return_value = 4194304 + self.driver.create_volume_from_snapshot(self.volume_a, + self.snapshot) + + mock_log.info.assert_called_once() + self.assertTrue(self.driver._clone_v2_api_checked) + + @common_mocks + @mock.patch('cinder.objects.Volume.get_by_id') + @mock.patch.object(driver.RBDDriver, '_resize', mock.Mock()) + def test_log_create_vol_from_snap_without_v2_clone_api(self, + volume_get_by_id): + volume_get_by_id.return_value = self.volume_a + + self.mock_proxy().__enter__().volume.op_features.return_value = 0 + self.mock_rbd.RBD_OPERATION_FEATURE_CLONE_PARENT = 1 + + self.cfg.rbd_flatten_volume_from_snapshot = False + + with mock.patch.object(driver, 'LOG') as mock_log: + with mock.patch.object(self.driver.rbd.Image(), 'stripe_unit') as \ + mock_rbd_image_stripe_unit: + mock_rbd_image_stripe_unit.return_value = 4194304 + self.driver.create_volume_from_snapshot(self.volume_a, + self.snapshot) + + self.assertTrue(any(m for m in mock_log.warning.call_args_list + if 'Not using v2 clone API' in m[0][0])) + + mock_log.warning.assert_called_once() + self.assertTrue(self.driver._clone_v2_api_checked) + + @common_mocks + @mock.patch('cinder.objects.Volume.get_by_id') def test_delete_snapshot(self, volume_get_by_id): volume_get_by_id.return_value = self.volume_a proxy = self.mock_proxy.return_value @@ -1313,17 +1359,29 @@ self.driver._is_cloneable(location, {'disk_format': f})) self.assertTrue(mock_get_fsid.called) - def _copy_image(self): + def _copy_image(self, volume_busy=False): with mock.patch.object(tempfile, 'NamedTemporaryFile'): with mock.patch.object(os.path, 'exists') as mock_exists: mock_exists.return_value = True with mock.patch.object(image_utils, 'fetch_to_raw'): - with mock.patch.object(self.driver, 'delete_volume'): + with mock.patch.object(self.driver, 'delete_volume') \ + as mock_dv: with mock.patch.object(self.driver, '_resize'): mock_image_service = mock.MagicMock() args = [None, self.volume_a, mock_image_service, None] - self.driver.copy_image_to_volume(*args) + if volume_busy: + mock_dv.side_effect = ( + exception.VolumeIsBusy("doh")) + self.assertRaises( + exception.VolumeIsBusy, + self.driver.copy_image_to_volume, + *args) + self.assertEqual( + self.cfg.rados_connection_retries, + mock_dv.call_count) + else: + self.driver.copy_image_to_volume(*args) @mock.patch('cinder.volume.drivers.rbd.fileutils.delete_if_exists') @mock.patch('cinder.volume.volume_utils.check_encryption_provider', @@ -1373,6 +1431,11 @@ self.cfg.image_conversion_dir = '/var/run/cinder/tmp' self._copy_image_encrypted() + @common_mocks + def test_copy_image_busy_volume(self): + self.cfg.image_conversion_dir = '/var/run/cinder/tmp' + self._copy_image(volume_busy=True) + @ddt.data(True, False) @common_mocks @mock.patch('cinder.volume.drivers.rbd.RBDDriver._get_usage_info') @@ -1527,9 +1590,9 @@ return_value=dynamic_total): result = self.driver._get_pool_stats() client.cluster.mon_command.assert_has_calls([ - mock.call('{"prefix":"df", "format":"json"}', ''), + mock.call('{"prefix":"df", "format":"json"}', b''), mock.call('{"prefix":"osd pool get-quota", "pool": "rbd",' - ' "format":"json"}', ''), + ' "format":"json"}', b''), ]) self.assertEqual((free_capacity, total_capacity), result) @@ -1550,9 +1613,9 @@ ] result = self.driver._get_pool_stats() client.cluster.mon_command.assert_has_calls([ - mock.call('{"prefix":"df", "format":"json"}', ''), + mock.call('{"prefix":"df", "format":"json"}', b''), mock.call('{"prefix":"osd pool get-quota", "pool": "rbd",' - ' "format":"json"}', ''), + ' "format":"json"}', b''), ]) free_capacity = 1.56 total_capacity = 3.0 @@ -1640,7 +1703,8 @@ # OpenStack usage doesn't have the rbd_keyring_conf Oslo Config option cfg_file = '/etc/ceph/ceph.client.admin.keyring' self.driver.configuration.rbd_keyring_conf = cfg_file - self.driver._set_keyring_attributes() + with mock.patch('os.path.isfile', return_value=False): + self.driver._set_keyring_attributes() self.assertEqual(cfg_file, self.driver.keyring_file) self.assertIsNone(self.driver.keyring_data) @@ -1685,13 +1749,13 @@ self.assertEqual(cfg_file, self.driver.keyring_file) self.assertIsNone(self.driver.keyring_data) - @ddt.data({'rbd_chunk_size': 1, 'order': 20}, - {'rbd_chunk_size': 8, 'order': 23}, - {'rbd_chunk_size': 32, 'order': 25}) + @ddt.data({'rbd_chunk_size': 1}, + {'rbd_chunk_size': 8}, + {'rbd_chunk_size': 32}) @ddt.unpack @common_mocks @mock.patch.object(driver.RBDDriver, '_enable_replication') - def test_clone(self, mock_enable_repl, rbd_chunk_size, order): + def test_clone(self, mock_enable_repl, rbd_chunk_size): self.cfg.rbd_store_chunk_size = rbd_chunk_size src_pool = u'images' src_image = u'image-name' @@ -1709,14 +1773,20 @@ # capture both rados client used to perform the clone client.__enter__.side_effect = mock__enter__(client) - res = self.driver._clone(self.volume_a, src_pool, src_image, src_snap) + with mock.patch.object(self.driver.rbd.Image(), 'stripe_unit') as \ + mock_rbd_image_stripe_unit: + mock_rbd_image_stripe_unit.return_value = 4194304 + res = self.driver._clone(self.volume_a, src_pool, src_image, + src_snap) self.assertEqual({}, res) args = [client_stack[0].ioctx, str(src_image), str(src_snap), client_stack[1].ioctx, str(self.volume_a.name)] + stripe_unit = max(4194304, rbd_chunk_size * 1048576) + expected_order = int(math.log(stripe_unit, 2)) kwargs = {'features': client.features, - 'order': order} + 'order': expected_order} self.mock_rbd.RBD.return_value.clone.assert_called_once_with( *args, **kwargs) self.assertEqual(2, client.__enter__.call_count) @@ -1725,8 +1795,9 @@ @common_mocks @mock.patch.object(driver.RBDDriver, '_enable_replication') def test_clone_replicated(self, mock_enable_repl): - rbd_chunk_size = 1 order = 20 + rbd_chunk_size = 1 + stripe_unit = 1048576 self.volume_a.volume_type = fake_volume.fake_volume_type_obj( self.context, id=fake.VOLUME_TYPE_ID, @@ -1755,7 +1826,11 @@ # capture both rados client used to perform the clone client.__enter__.side_effect = mock__enter__(client) - res = self.driver._clone(self.volume_a, src_pool, src_image, src_snap) + with mock.patch.object(self.driver.rbd.Image(), 'stripe_unit') as \ + mock_rbd_image_stripe_unit: + mock_rbd_image_stripe_unit.return_value = stripe_unit + res = self.driver._clone(self.volume_a, src_pool, src_image, + src_snap) self.assertEqual(expected_update, res) mock_enable_repl.assert_called_once_with(self.volume_a) diff -Nru cinder-17.0.1/cinder/tests/unit/volume/drivers/test_remotefs.py cinder-17.4.0/cinder/tests/unit/volume/drivers/test_remotefs.py --- cinder-17.0.1/cinder/tests/unit/volume/drivers/test_remotefs.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/tests/unit/volume/drivers/test_remotefs.py 2022-04-20 17:38:33.000000000 +0000 @@ -74,7 +74,7 @@ self._fake_snap_c = fake_snapshot.fake_snapshot_obj(self.context) self._fake_snap_c.volume = self.volume_c self.volume_c_path = os.path.join(self._FAKE_MNT_POINT, - self._fake_snap_c.name) + self.volume_c.name) self._fake_snap_c_path = (self.volume_c_path + '.' + self._fake_snap_c.id) @@ -356,6 +356,18 @@ snapshot.volume.status = 'backing-up' snapshot.volume.attach_status = 'attached' expected_method_called = '_create_snapshot_online' + conn_info = ('{"driver_volume_type": "nfs",' + '"export": "localhost:/srv/nfs1",' + '"name": "old_name"}') + attachment = fake_volume.volume_attachment_ovo( + self.context, connection_info=conn_info) + snapshot.volume.volume_attachment.objects.append(attachment) + mock_save = self.mock_object(attachment, 'save') + + # After the snapshot the connection info should change the name of + # the file + expected = copy.deepcopy(attachment.connection_info) + expected['name'] = snapshot.volume.name + '.' + snapshot.id else: expected_method_called = '_do_create_snapshot' @@ -372,6 +384,11 @@ mock.sentinel.fake_info_path, expected_snapshot_info) + if volume_in_use: + mock_save.assert_called_once() + changed_fields = attachment.cinder_obj_get_changes() + self.assertEqual(expected, changed_fields['connection_info']) + @ddt.data({'encryption': True}, {'encryption': False}) def test_create_snapshot_volume_available(self, encryption): self._test_create_snapshot(encryption=encryption) diff -Nru cinder-17.0.1/cinder/tests/unit/volume/flows/test_create_volume_flow.py cinder-17.4.0/cinder/tests/unit/volume/flows/test_create_volume_flow.py --- cinder-17.0.1/cinder/tests/unit/volume/flows/test_create_volume_flow.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/tests/unit/volume/flows/test_create_volume_flow.py 2022-04-20 17:38:33.000000000 +0000 @@ -205,7 +205,7 @@ snapshot_obj = fake_snapshot.fake_snapshot_obj(self.ctxt) snapshot_get_by_id.return_value = snapshot_obj volume_get_by_id.return_value = volume_obj - volume_create.return_value = {'id': '123456'} + volume_create.return_value = {'id': '123456', 'volume_attachment': []} task = create_volume.EntryCreateTask() @@ -254,7 +254,7 @@ volume_db = {'bootable': bootable} volume_obj = fake_volume.fake_volume_obj(self.ctxt, **volume_db) volume_get_by_id.return_value = volume_obj - volume_create.return_value = {'id': '123456'} + volume_create.return_value = {'id': '123456', 'volume_attachment': []} task = create_volume.EntryCreateTask() result = task.execute(self.ctxt, @@ -1255,12 +1255,68 @@ exception=err) +@ddt.ddt class CreateVolumeFlowManagerGlanceCinderBackendCase(test.TestCase): def setUp(self): super(CreateVolumeFlowManagerGlanceCinderBackendCase, self).setUp() self.ctxt = context.get_admin_context() + # data for test__extract_cinder_ids + # legacy glance cinder URI: cinder:// + # new-style glance cinder URI: cinder:/// + LEGACY_VOL2 = 'cinder://%s' % fakes.VOLUME2_ID + NEW_VOL3 = 'cinder://glance-store-name/%s' % fakes.VOLUME3_ID + # these *may* be illegal names in glance, but check anyway + NEW_VOL4 = 'cinder://glance/store/name/%s' % fakes.VOLUME4_ID + NEW_VOL5 = 'cinder://glance:store:name/%s' % fakes.VOLUME5_ID + NEW_VOL6 = 'cinder://glance:store,name/%s' % fakes.VOLUME6_ID + NOT_CINDER1 = 'rbd://%s' % fakes.UUID1 + NOT_CINDER2 = 'http://%s' % fakes.UUID2 + NOGOOD3 = 'cinder://glance:store,name/%s/garbage' % fakes.UUID3 + NOGOOD4 = 'cinder://glance:store,name/%s-garbage' % fakes.UUID4 + NOGOOD5 = fakes.UUID5 + NOGOOD6 = 'cinder://store-name/12345678' + NOGOOD7 = 'cinder://' + NOGOOD8 = 'some-random-crap' + NOGOOD9 = None + + TEST_CASE_DATA = ( + # the format of these is: (input, expected output) + ([LEGACY_VOL2], [fakes.VOLUME2_ID]), + ([NEW_VOL3], [fakes.VOLUME3_ID]), + ([NEW_VOL4], [fakes.VOLUME4_ID]), + ([NEW_VOL5], [fakes.VOLUME5_ID]), + ([NEW_VOL6], [fakes.VOLUME6_ID]), + ([], []), + ([''], []), + ([NOT_CINDER1], []), + ([NOT_CINDER2], []), + ([NOGOOD3], []), + ([NOGOOD4], []), + ([NOGOOD5], []), + ([NOGOOD6], []), + ([NOGOOD7], []), + ([NOGOOD8], []), + ([NOGOOD9], []), + ([NOT_CINDER1, NOGOOD4], []), + # mix of URIs should only get the cinder IDs + ([LEGACY_VOL2, NOT_CINDER1, NEW_VOL3, NOT_CINDER2], + [fakes.VOLUME2_ID, fakes.VOLUME3_ID]), + # a bad cinder URI early in the list shouldn't prevent us from + # processing a good one later in the list + ([NOGOOD6, NEW_VOL3, NOGOOD7, LEGACY_VOL2], + [fakes.VOLUME3_ID, fakes.VOLUME2_ID]), + ) + + @ddt.data(*TEST_CASE_DATA) + @ddt.unpack + def test__extract_cinder_ids(self, url_list, id_list): + """Test utility function that gets IDs from Glance location URIs""" + klass = create_volume_manager.CreateVolumeFromSpecTask + actual = klass._extract_cinder_ids(url_list) + self.assertEqual(id_list, actual) + @mock.patch('cinder.volume.flows.manager.create_volume.' 'CreateVolumeFromSpecTask.' '_cleanup_cg_in_volume') @@ -1322,6 +1378,10 @@ self.assertFalse(fake_driver.create_cloned_volume.called) mock_cleanup_cg.assert_called_once_with(volume) + LEGACY_URI = 'cinder://%s' % fakes.VOLUME_ID + MULTISTORE_URI = 'cinder://fake-store/%s' % fakes.VOLUME_ID + + @ddt.data(LEGACY_URI, MULTISTORE_URI) @mock.patch('cinder.volume.flows.manager.create_volume.' 'CreateVolumeFromSpecTask.' '_cleanup_cg_in_volume') @@ -1330,7 +1390,8 @@ 'CreateVolumeFromSpecTask.' '_handle_bootable_volume_glance_meta') @mock.patch('cinder.image.image_utils.qemu_img_info') - def test_create_from_image_volume_ignore_size(self, mock_qemu_info, + def test_create_from_image_volume_ignore_size(self, location_uri, + mock_qemu_info, handle_bootable, mock_fetch_img, mock_cleanup_cg, @@ -1357,7 +1418,7 @@ # will fail because of free space being too low. image_info.virtual_size = '1073741824000000000000' mock_qemu_info.return_value = image_info - url = 'cinder://%s' % image_volume['id'] + url = location_uri image_location = None if location: image_location = (url, [{'url': url, 'metadata': {}}]) diff -Nru cinder-17.0.1/cinder/tests/unit/volume/test_volume_migration.py cinder-17.4.0/cinder/tests/unit/volume/test_volume_migration.py --- cinder-17.0.1/cinder/tests/unit/volume/test_volume_migration.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/tests/unit/volume/test_volume_migration.py 2022-04-20 17:38:33.000000000 +0000 @@ -233,12 +233,11 @@ volume_get.return_value = fake_new_volume volume = tests_utils.create_volume(self.context, size=1, host=CONF.host) - volume_attach = tests_utils.attach_volume( - self.context, volume['id'], fake_uuid, attached_host, '/dev/vda') - self.assertIsNotNone(volume_attach['volume_attachment'][0]['id']) - self.assertEqual( - fake_uuid, volume_attach['volume_attachment'][0]['instance_uuid']) - self.assertEqual('in-use', volume_attach['status']) + volume = tests_utils.attach_volume( + self.context, volume, fake_uuid, attached_host, '/dev/vda') + self.assertIsNotNone(volume.volume_attachment[0].id) + self.assertEqual(fake_uuid, volume.volume_attachment[0].instance_uuid) + self.assertEqual('in-use', volume.status) self.volume._migrate_volume_generic(self.context, volume, host_obj, None) self.assertFalse(migrate_volume_completion.called) @@ -603,11 +602,12 @@ previous_status=previous_status) attachment = None if status == 'in-use': - vol = tests_utils.attach_volume(self.context, old_volume.id, - instance_uuid, attached_host, - '/dev/vda') - self.assertEqual('in-use', vol['status']) - attachment = vol['volume_attachment'][0] + old_volume = tests_utils.attach_volume(self.context, old_volume, + instance_uuid, + attached_host, + '/dev/vda') + self.assertEqual('in-use', old_volume.status) + attachment = old_volume.volume_attachment[0] target_status = 'target:%s' % old_volume.id new_host = CONF.host + 'new' new_volume = tests_utils.create_volume(self.context, size=0, diff -Nru cinder-17.0.1/cinder/tests/unit/volume/test_volume.py cinder-17.4.0/cinder/tests/unit/volume/test_volume.py --- cinder-17.0.1/cinder/tests/unit/volume/test_volume.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/tests/unit/volume/test_volume.py 2022-04-20 17:38:33.000000000 +0000 @@ -256,7 +256,7 @@ volume_id = volume['id'] self.assertIsNone(volume['encryption_key_id']) - mock_notify.assert_not_called() + self.assertRaises(exception.DriverNotInitialized, self.volume.create_volume, self.context, volume) @@ -339,7 +339,6 @@ **self.volume_params) self.assertIsNone(volume['encryption_key_id']) - mock_notify.assert_not_called() self.assertRaises(exception.DriverNotInitialized, self.volume.delete_volume, self.context, volume) @@ -359,8 +358,6 @@ **self.volume_params) volume_id = volume['id'] - mock_notify.assert_not_called() - self.assertIsNone(volume['encryption_key_id']) self.volume.create_volume(self.context, volume) diff -Nru cinder-17.0.1/cinder/utils.py cinder-17.4.0/cinder/utils.py --- cinder-17.0.1/cinder/utils.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/utils.py 2022-04-20 17:38:33.000000000 +0000 @@ -655,8 +655,19 @@ return self._compare(other, lambda s, o: s != o) -def retry(exceptions, interval=1, retries=3, backoff_rate=2, - wait_random=False): +class retry_if_exit_code(tenacity.retry_if_exception): + """Retry on ProcessExecutionError specific exit codes.""" + def __init__(self, codes): + self.codes = (codes,) if isinstance(codes, int) else codes + super(retry_if_exit_code, self).__init__(self._check_exit_code) + + def _check_exit_code(self, exc): + return (exc and isinstance(exc, processutils.ProcessExecutionError) and + exc.exit_code in self.codes) + + +def retry(retry_param, interval=1, retries=3, backoff_rate=2, + wait_random=False, retry=tenacity.retry_if_exception_type): if retries < 1: raise ValueError('Retries must be greater than or ' @@ -678,7 +689,7 @@ after=tenacity.after_log(LOG, logging.DEBUG), stop=tenacity.stop_after_attempt(retries), reraise=True, - retry=tenacity.retry_if_exception_type(exceptions), + retry=retry(retry_param), wait=wait) return r.call(f, *args, **kwargs) diff -Nru cinder-17.0.1/cinder/volume/api.py cinder-17.4.0/cinder/volume/api.py --- cinder-17.0.1/cinder/volume/api.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/volume/api.py 2022-04-20 17:38:33.000000000 +0000 @@ -52,7 +52,6 @@ from cinder.policies import snapshots as snapshot_policy from cinder.policies import volume_actions as vol_action_policy from cinder.policies import volume_metadata as vol_meta_policy -from cinder.policies import volume_transfer as vol_transfer_policy from cinder.policies import volumes as vol_policy from cinder import quota from cinder import quota_utils @@ -354,10 +353,6 @@ if flow_engine.storage.fetch('refresh_az'): self.list_availability_zones(enable_cache=True, refresh_cache=True) - # Refresh the object here, otherwise things ain't right - vref = objects.Volume.get_by_id( - context, vref['id']) - vref.save() LOG.info("Create volume request issued successfully.", resource=vref) return vref @@ -833,8 +828,6 @@ def accept_transfer(self, context, volume, new_user, new_project, no_snapshots=False): - context.authorize(vol_transfer_policy.ACCEPT_POLICY, - target_obj=volume) if volume['status'] == 'maintenance': LOG.info('Unable to accept transfer for volume, ' 'because it is in maintenance.', resource=volume) @@ -1196,15 +1189,17 @@ context.authorize(vol_meta_policy.UPDATE_ADMIN_METADATA_POLICY, target_obj=volume) utils.check_metadata_properties(metadata) - db_meta = self.db.volume_admin_metadata_update(context, volume.id, - metadata, delete, add, - update) + # Policy could allow non admin users to update admin metadata, but + # underlying DB methods require admin privileges, so we elevate the + # context. + with volume.obj_as_admin(): + volume.admin_metadata_update(metadata, delete, add, update) # TODO(jdg): Implement an RPC call for drivers that may use this info LOG.info("Update volume admin metadata completed successfully.", resource=volume) - return db_meta + return volume.admin_metadata def get_snapshot_metadata(self, context, snapshot): """Get all metadata associated with a snapshot.""" @@ -2267,52 +2262,28 @@ ctxt.authorize(attachment_policy.DELETE_POLICY, target_obj=attachment) volume = objects.Volume.get_by_id(ctxt, attachment.volume_id) + if attachment.attach_status == fields.VolumeAttachStatus.RESERVED: - self.db.volume_detached(ctxt.elevated(), attachment.volume_id, - attachment.get('id')) - self.db.volume_admin_metadata_delete(ctxt.elevated(), - attachment.volume_id, - 'attached_mode') - volume_utils.notify_about_volume_usage(ctxt, volume, "detach.end") + volume_utils.notify_about_volume_usage(ctxt, + volume, "detach.start") else: - self.volume_rpcapi.attachment_delete(ctxt, - attachment.id, - volume) - status_updates = {'status': 'available', - 'attach_status': 'detached'} - remaining_attachments = AO_LIST.get_all_by_volume_id(ctxt, volume.id) - LOG.debug("Remaining volume attachments: %s", remaining_attachments, - resource=volume) - - # NOTE(jdg) Try and figure out the > state we have left and set that - # attached > attaching > > detaching > reserved - pending_status_list = [] - for attachment in remaining_attachments: - pending_status_list.append(attachment.attach_status) - LOG.debug("Adding status of: %s to pending status list " - "for volume.", attachment.attach_status, - resource=volume) - - LOG.debug("Pending status list for volume during " - "attachment-delete: %s", - pending_status_list, resource=volume) - if 'attached' in pending_status_list: - status_updates['status'] = 'in-use' - status_updates['attach_status'] = 'attached' - elif 'attaching' in pending_status_list: - status_updates['status'] = 'attaching' - status_updates['attach_status'] = 'attaching' - elif 'detaching' in pending_status_list: - status_updates['status'] = 'detaching' - status_updates['attach_status'] = 'detaching' - elif 'reserved' in pending_status_list: - status_updates['status'] = 'reserved' - status_updates['attach_status'] = 'reserved' - - volume.status = status_updates['status'] - volume.attach_status = status_updates['attach_status'] - volume.save() - return remaining_attachments + # Generate the detach.start notification on the volume service to + # include the host doing the operation. + self.volume_rpcapi.attachment_delete(ctxt, attachment.id, volume) + + # Trigger attachments lazy load (missing since volume was loaded in the + # attachment without joined tables). With them loaded the finish_detach + # call removes the detached one from the list, and the notification and + # return have the right attachment list. + volume.volume_attachment + volume.finish_detach(attachment.id) + + # Do end notification on the API so it comes after finish_detach. + # Doing it on the volume service leads to race condition from bug + # #1937084, and doing the notification there with the finish here leads + # to bug #1916980. + volume_utils.notify_about_volume_usage(ctxt, volume, "detach.end") + return volume.volume_attachment class HostAPI(base.Base): diff -Nru cinder-17.0.1/cinder/volume/drivers/dell_emc/powerflex/driver.py cinder-17.4.0/cinder/volume/drivers/dell_emc/powerflex/driver.py --- cinder-17.0.1/cinder/volume/drivers/dell_emc/powerflex/driver.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/volume/drivers/dell_emc/powerflex/driver.py 2022-04-20 17:38:33.000000000 +0000 @@ -92,9 +92,11 @@ 3.5.3 - Add revert volume to snapshot support 3.5.4 - Fix for Bug #1823200. See OSSN-0086 for details. 3.5.5 - Rebrand VxFlex OS to PowerFlex. + 3.5.6 - Fix for Bug #1897598 when volume can be migrated without + conversion of its type. """ - VERSION = "3.5.5" + VERSION = "3.5.6" # ThirdPartySystems wiki CI_WIKI_NAME = "DellEMC_PowerFlex_CI" @@ -1396,13 +1398,13 @@ ) if ( real_provisioning == "ThickProvisioned" and - (provisioning in ["thin", "compressed"] or + (provisioning == "ThinProvisioned" or not pool_supports_thick_vols) ): params["volTypeConversion"] = "ThickToThin" elif ( real_provisioning == "ThinProvisioned" and - provisioning == "thick" and + provisioning == "ThickProvisioned" and pool_supports_thick_vols ): params["volTypeConversion"] = "ThinToThick" diff -Nru cinder-17.0.1/cinder/volume/drivers/dell_emc/powermax/common.py cinder-17.4.0/cinder/volume/drivers/dell_emc/powermax/common.py --- cinder-17.0.1/cinder/volume/drivers/dell_emc/powermax/common.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/volume/drivers/dell_emc/powermax/common.py 2022-04-20 17:38:33.000000000 +0000 @@ -3940,6 +3940,7 @@ resume_target_sg, resume_original_sg = False, False resume_original_sg_dict = dict() orig_mgmt_sg_name = '' + is_partitioned = False target_extra_specs = dict(new_type['extra_specs']) target_extra_specs.update({ @@ -4086,9 +4087,17 @@ previous_host = volume.get('host') host_details = previous_host.split('+') array_index = len(host_details) - 1 + srp_index = len(host_details) - 2 host_details[array_index] = array + host_details[srp_index] = srp updated_host = '+'.join(host_details) model_update['host'] = updated_host + if is_partitioned: + # Must set these here as offline R1 promotion does + # not perform rdf cleanup. + model_update[ + 'metadata']['ReplicationEnabled'] = 'False' + model_update['metadata']['Configuration'] = 'TDEV' target_backend_id = None if is_rep_enabled: @@ -4221,7 +4230,12 @@ parent_sg = None if self.utils.is_replication_enabled(target_extra_specs): is_re, rep_mode = True, target_extra_specs['rep_mode'] - if self.utils.is_replication_enabled(extra_specs): + mgmt_sg_name = self.utils.get_rdf_management_group_name( + target_extra_specs[utils.REP_CONFIG]) + if self.promotion and self.utils.is_replication_enabled(extra_specs): + # Need to check this when performing promotion while R1 is offline + # as RDF cleanup is not performed. Target is not RDF enabled + # in that scenario. mgmt_sg_name = self.utils.get_rdf_management_group_name( extra_specs[utils.REP_CONFIG]) @@ -5386,9 +5400,11 @@ rep_extra_specs[utils.PORTGROUPNAME] = rep_config['portgroup'] # Get the RDF Group label & number + array = (rep_config[utils.ARRAY] if self.promotion else + extra_specs[utils.ARRAY]) rep_extra_specs['rdf_group_label'] = rep_config['rdf_group_label'] rdf_group_no, __ = self.get_rdf_details( - extra_specs['array'], rep_config) + array, rep_config) rep_extra_specs['rdf_group_no'] = rdf_group_no # Get the SRDF wait/retries settings rep_extra_specs['sync_retries'] = rep_config['sync_retries'] @@ -6831,14 +6847,17 @@ snap_id_list.append(snapshot_src.get( 'snap_id') if self.rest.is_snap_id else snapshot_src.get( 'generation')) - device_label = device_name.split(':')[1] + try: + device_label = device_name.split(':')[1] if device_name else None + except IndexError: + device_label = None metadata = {'SnapshotLabel': snap_name, 'SourceDeviceID': device_id, - 'SourceDeviceLabel': device_label, 'SnapIdList': ', '.join( six.text_type(v) for v in snap_id_list), 'is_snap_id': self.rest.is_snap_id} - + if device_label: + metadata['SourceDeviceLabel'] = device_label return metadata def _check_and_add_tags_to_storage_array( diff -Nru cinder-17.0.1/cinder/volume/drivers/dell_emc/powermax/fc.py cinder-17.4.0/cinder/volume/drivers/dell_emc/powermax/fc.py --- cinder-17.0.1/cinder/volume/drivers/dell_emc/powermax/fc.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/volume/drivers/dell_emc/powermax/fc.py 2022-04-20 17:38:33.000000000 +0000 @@ -130,9 +130,11 @@ - Use of snap id instead of generation (bp powermax-snapset-ids) - Support for Failover Abilities (bp/powermax-failover-abilities) 4.3.1 - Fix non-temporary snapshot delete (#1887962) + 4.3.2 - Fix for create snapshot (#1939139) + 4.3.3 - Allow for case mismatch in SGs (bug #1929429) """ - VERSION = "4.3.1" + VERSION = "4.3.3" # ThirdPartySystems wiki CI_WIKI_NAME = "EMC_VMAX_CI" diff -Nru cinder-17.0.1/cinder/volume/drivers/dell_emc/powermax/iscsi.py cinder-17.4.0/cinder/volume/drivers/dell_emc/powermax/iscsi.py --- cinder-17.0.1/cinder/volume/drivers/dell_emc/powermax/iscsi.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/volume/drivers/dell_emc/powermax/iscsi.py 2022-04-20 17:38:33.000000000 +0000 @@ -136,9 +136,11 @@ - Use of snap id instead of generation (bp powermax-snapset-ids) - Support for Failover Abilities (bp/powermax-failover-abilities) 4.3.1 - Fix non-temporary snapshot delete (#1887962) + 4.3.2 - Fix for create snapshot (#1939139) + 4.3.3 - Allow for case mismatch in SGs (bug #1929429) """ - VERSION = "4.3.1" + VERSION = "4.3.3" # ThirdPartySystems wiki CI_WIKI_NAME = "EMC_VMAX_CI" diff -Nru cinder-17.0.1/cinder/volume/drivers/dell_emc/powermax/rest.py cinder-17.4.0/cinder/volume/drivers/dell_emc/powermax/rest.py --- cinder-17.0.1/cinder/volume/drivers/dell_emc/powermax/rest.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/volume/drivers/dell_emc/powermax/rest.py 2022-04-20 17:38:33.000000000 +0000 @@ -925,7 +925,8 @@ parent_sg = self.get_storage_group(array, parent_name) if parent_sg and parent_sg.get('child_storage_group'): child_sg_list = parent_sg['child_storage_group'] - if child_name in child_sg_list: + if child_name.lower() in ( + child.lower() for child in child_sg_list): return True return False diff -Nru cinder-17.0.1/cinder/volume/drivers/dell_emc/powerstore/adapter.py cinder-17.4.0/cinder/volume/drivers/dell_emc/powerstore/adapter.py --- cinder-17.0.1/cinder/volume/drivers/dell_emc/powerstore/adapter.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/volume/drivers/dell_emc/powerstore/adapter.py 2022-04-20 17:38:33.000000000 +0000 @@ -16,6 +16,7 @@ """Adapter for Dell EMC PowerStore Cinder driver.""" from oslo_log import log as logging +from oslo_utils import strutils from cinder import coordination from cinder import exception @@ -30,6 +31,7 @@ LOG = logging.getLogger(__name__) PROTOCOL_FC = "FC" PROTOCOL_ISCSI = "iSCSI" +CHAP_MODE_SINGLE = "Single" class CommonAdapter(object): @@ -41,6 +43,7 @@ self.configuration = configuration self.storage_protocol = None self.allowed_ports = None + self.use_chap_auth = None @staticmethod def initiators(connector): @@ -83,13 +86,20 @@ self.appliances_to_ids_map[appliance_name] = ( self.client.get_appliance_id_by_name(appliance_name) ) + self.use_chap_auth = False + if self.storage_protocol == PROTOCOL_ISCSI: + chap_config = self.client.get_chap_config() + if chap_config.get("mode") == CHAP_MODE_SINGLE: + self.use_chap_auth = True LOG.debug("Successfully initialized PowerStore %(protocol)s adapter. " "PowerStore appliances: %(appliances)s. " - "Allowed ports: %(allowed_ports)s.", + "Allowed ports: %(allowed_ports)s. " + "Use CHAP authentication: %(use_chap_auth)s.", { "protocol": self.storage_protocol, "appliances": self.appliances, "allowed_ports": self.allowed_ports, + "use_chap_auth": self.use_chap_auth, }) def create_volume(self, volume): @@ -314,7 +324,9 @@ { "volume_name": volume.name, "volume_id": volume.id, - "connection_properties": connection_properties, + "connection_properties": strutils.mask_password( + connection_properties + ), }) return connection_properties @@ -429,13 +441,17 @@ """Create PowerStore host if it does not exist. :param connector: connection properties - :return: PowerStore host object + :return: PowerStore host object, iSCSI CHAP credentials """ initiators = self.initiators(connector) host = self._filter_hosts_by_initiators(initiators) + if self.use_chap_auth: + chap_credentials = utils.get_chap_credentials() + else: + chap_credentials = {} if host: - self._modify_host_initiators(host, initiators) + self._modify_host_initiators(host, chap_credentials, initiators) else: host_name = utils.powerstore_host_name( connector, @@ -451,6 +467,7 @@ { "port_name": initiator, "port_type": self.storage_protocol, + **chap_credentials, } for initiator in initiators ] host = self.client.create_host(host_name, ports) @@ -463,12 +480,13 @@ "initiators": initiators, "host_provider_id": host["id"], }) - return host + return host, chap_credentials - def _modify_host_initiators(self, host, initiators): + def _modify_host_initiators(self, host, chap_credentials, initiators): """Update PowerStore host initiators if needed. :param host: PowerStore host object + :param chap_credentials: iSCSI CHAP credentials :param initiators: list of initiators :return: None """ @@ -476,17 +494,22 @@ initiators_added = [ initiator["port_name"] for initiator in host["host_initiators"] ] + initiators_to_add = [] + initiators_to_modify = [] initiators_to_remove = [ initiator for initiator in initiators_added if initiator not in initiators ] - initiators_to_add = [ - { + for initiator in initiators: + initiator_add_modify = { "port_name": initiator, - "port_type": self.storage_protocol, - } for initiator in initiators - if initiator not in initiators_added - ] + **chap_credentials, + } + if initiator not in initiators_added: + initiator_add_modify["port_type"] = self.storage_protocol + initiators_to_add.append(initiator_add_modify) + elif self.use_chap_auth: + initiators_to_modify.append(initiator_add_modify) if initiators_to_remove: LOG.debug("Remove initiators from PowerStore host %(host_name)s. " "Initiators: %(initiators_to_remove)s. " @@ -514,7 +537,9 @@ "%(host_provider_id)s.", { "host_name": host["name"], - "initiators_to_add": initiators_to_add, + "initiators_to_add": strutils.mask_password( + initiators_to_add + ), "host_provider_id": host["id"], }) self.client.modify_host_initiators( @@ -526,7 +551,34 @@ "PowerStore host id: %(host_provider_id)s.", { "host_name": host["name"], - "initiators_to_add": initiators_to_add, + "initiators_to_add": strutils.mask_password( + initiators_to_add + ), + "host_provider_id": host["id"], + }) + if initiators_to_modify: + LOG.debug("Modify initiators of PowerStore host %(host_name)s. " + "Initiators: %(initiators_to_modify)s. " + "PowerStore host id: %(host_provider_id)s.", + { + "host_name": host["name"], + "initiators_to_modify": strutils.mask_password( + initiators_to_modify + ), + "host_provider_id": host["id"], + }) + self.client.modify_host_initiators( + host["id"], + modify_initiators=initiators_to_modify + ) + LOG.debug("Successfully modified initiators of PowerStore host " + "%(host_name)s. Initiators: %(initiators_to_modify)s. " + "PowerStore host id: %(host_provider_id)s.", + { + "host_name": host["name"], + "initiators_to_modify": strutils.mask_password( + initiators_to_modify + ), "host_provider_id": host["id"], }) @@ -572,11 +624,11 @@ :param connector: connection properties :param volume: OpenStack volume object - :return: attached volume logical number + :return: iSCSI CHAP credentials, volume logical number """ - host = self._create_host_if_not_exist(connector) - return self._attach_volume_to_host(host, volume) + host, chap_credentials = self._create_host_if_not_exist(connector) + return chap_credentials, self._attach_volume_to_host(host, volume) def _connect_volume(self, volume, connector): """Attach PowerStore volume and return it's connection properties. @@ -588,12 +640,21 @@ appliance_name = volume_utils.extract_host(volume.host, "pool") appliance_id = self.appliances_to_ids_map[appliance_name] - volume_lun = self._create_host_and_attach( + chap_credentials, volume_lun = self._create_host_and_attach( connector, volume ) - return self._get_connection_properties(appliance_id, - volume_lun) + connection_properties = self._get_connection_properties(appliance_id, + volume_lun) + if self.use_chap_auth: + connection_properties["data"]["auth_method"] = "CHAP" + connection_properties["data"]["auth_username"] = ( + chap_credentials.get("chap_single_username") + ) + connection_properties["data"]["auth_password"] = ( + chap_credentials.get("chap_single_password") + ) + return connection_properties def _detach_volume_from_hosts(self, volume, hosts_to_detach=None): """Detach volume from PowerStore hosts. @@ -731,7 +792,7 @@ return { "driver_volume_type": self.driver_volume_type, "data": { - "target_discovered": True, + "target_discovered": False, "target_lun": volume_lun, "target_wwn": target_wwns, } @@ -782,7 +843,10 @@ return { "driver_volume_type": self.driver_volume_type, "data": { - "target_discovered": True, + "target_discovered": False, + "target_portal": portals[0], + "target_iqn": iqns[0], + "target_lun": volume_lun, "target_portals": portals, "target_iqns": iqns, "target_luns": [volume_lun] * len(portals), diff -Nru cinder-17.0.1/cinder/volume/drivers/dell_emc/powerstore/client.py cinder-17.4.0/cinder/volume/drivers/dell_emc/powerstore/client.py --- cinder-17.0.1/cinder/volume/drivers/dell_emc/powerstore/client.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/volume/drivers/dell_emc/powerstore/client.py 2022-04-20 17:38:33.000000000 +0000 @@ -19,6 +19,7 @@ import json from oslo_log import log as logging +from oslo_utils import strutils import requests from cinder import exception @@ -83,7 +84,12 @@ "verify_cert": self._verify_cert, }) - def _send_request(self, method, url, payload=None, params=None): + def _send_request(self, + method, + url, + payload=None, + params=None, + log_response_data=True): if not payload: payload = {} if not params: @@ -106,11 +112,12 @@ "REST Request: %s %s with body %s", r.request.method, r.request.url, - r.request.body) - LOG.log(log_level, - "REST Response: %s with data %s", - r.status_code, - r.text) + strutils.mask_password(r.request.body)) + if log_response_data or log_level == logging.ERROR: + msg = "REST Response: %s with data %s" % (r.status_code, r.text) + else: + msg = "REST Response: %s" % r.status_code + LOG.log(log_level, msg) try: response = r.json() @@ -123,6 +130,19 @@ _send_patch_request = functools.partialmethod(_send_request, "PATCH") _send_delete_request = functools.partialmethod(_send_request, "DELETE") + def get_chap_config(self): + r, response = self._send_get_request( + "/chap_config/0", + params={ + "select": "mode" + } + ) + if r.status_code not in self.ok_codes: + msg = _("Failed to query PowerStore CHAP configuration.") + LOG.error(msg) + raise exception.VolumeBackendAPIException(data=msg) + return response + def get_appliance_id_by_name(self, appliance_name): r, response = self._send_get_request( "/appliance", @@ -148,7 +168,8 @@ payload={ "entity": "space_metrics_by_appliance", "entity_id": appliance_id, - } + }, + log_response_data=False ) if r.status_code not in self.ok_codes: msg = (_("Failed to query metrics for " @@ -363,7 +384,7 @@ "/ip_pool_address", params={ "appliance_id": "eq.%s" % appliance_id, - "purposes": "eq.{Storage_Iscsi_Target}", + "purposes": "cs.{Storage_Iscsi_Target}", "select": "address,ip_port(target_iqn)" } diff -Nru cinder-17.0.1/cinder/volume/drivers/dell_emc/powerstore/driver.py cinder-17.4.0/cinder/volume/drivers/dell_emc/powerstore/driver.py --- cinder-17.0.1/cinder/volume/drivers/dell_emc/powerstore/driver.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/volume/drivers/dell_emc/powerstore/driver.py 2022-04-20 17:38:33.000000000 +0000 @@ -37,9 +37,13 @@ Version history: 1.0.0 - Initial version + 1.0.1 - Add CHAP support + 1.0.2 - Fix iSCSI targets not being returned from the REST API call if + targets are used for multiple purposes + (iSCSI target, Replication target, etc.) """ - VERSION = "1.0.0" + VERSION = "1.0.2" VENDOR = "Dell EMC" # ThirdPartySystems wiki page diff -Nru cinder-17.0.1/cinder/volume/drivers/dell_emc/powerstore/utils.py cinder-17.4.0/cinder/volume/drivers/dell_emc/powerstore/utils.py --- cinder-17.0.1/cinder/volume/drivers/dell_emc/powerstore/utils.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/volume/drivers/dell_emc/powerstore/utils.py 2022-04-20 17:38:33.000000000 +0000 @@ -23,9 +23,12 @@ from cinder import exception from cinder.i18n import _ from cinder.objects import fields +from cinder.volume import volume_utils LOG = logging.getLogger(__name__) +CHAP_DEFAULT_USERNAME = "PowerStore_iSCSI_CHAP_Username" +CHAP_DEFAULT_SECRET_LENGTH = 60 def bytes_to_gib(size_in_bytes): @@ -134,3 +137,17 @@ attachment.attached_host == host_name) ] return len(attachments) > 1 + + +def get_chap_credentials(): + """Generate CHAP credentials. + + :return: CHAP username and secret + """ + + return { + "chap_single_username": CHAP_DEFAULT_USERNAME, + "chap_single_password": volume_utils.generate_password( + CHAP_DEFAULT_SECRET_LENGTH + ) + } diff -Nru cinder-17.0.1/cinder/volume/drivers/dell_emc/sc/storagecenter_api.py cinder-17.4.0/cinder/volume/drivers/dell_emc/sc/storagecenter_api.py --- cinder-17.0.1/cinder/volume/drivers/dell_emc/sc/storagecenter_api.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/volume/drivers/dell_emc/sc/storagecenter_api.py 2022-04-20 17:38:33.000000000 +0000 @@ -232,8 +232,8 @@ raise exception.VolumeBackendAPIException(message=msg) return rest_response - @utils.retry(exceptions=(requests.ConnectionError, - DellDriverRetryableException)) + @utils.retry(retry_param=(requests.ConnectionError, + DellDriverRetryableException)) def get(self, url): LOG.debug('get: %(url)s', {'url': url}) rest_response = self.session.get(self.__formatUrl(url), @@ -247,7 +247,7 @@ raise DellDriverRetryableException() return rest_response - @utils.retry(exceptions=(requests.ConnectionError,)) + @utils.retry(retry_param=(requests.ConnectionError,)) def post(self, url, payload, async_call=False): LOG.debug('post: %(url)s data: %(payload)s', {'url': url, @@ -261,7 +261,7 @@ self.asynctimeout if async_call else self.synctimeout)), async_call) - @utils.retry(exceptions=(requests.ConnectionError,)) + @utils.retry(retry_param=(requests.ConnectionError,)) def put(self, url, payload, async_call=False): LOG.debug('put: %(url)s data: %(payload)s', {'url': url, @@ -275,7 +275,7 @@ self.asynctimeout if async_call else self.synctimeout)), async_call) - @utils.retry(exceptions=(requests.ConnectionError,)) + @utils.retry(retry_param=(requests.ConnectionError,)) def delete(self, url, payload=None, async_call=False): LOG.debug('delete: %(url)s data: %(payload)s', {'url': url, 'payload': payload}) diff -Nru cinder-17.0.1/cinder/volume/drivers/dell_emc/vnx/client.py cinder-17.4.0/cinder/volume/drivers/dell_emc/vnx/client.py --- cinder-17.0.1/cinder/volume/drivers/dell_emc/vnx/client.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/volume/drivers/dell_emc/vnx/client.py 2022-04-20 17:38:33.000000000 +0000 @@ -200,7 +200,7 @@ def modify_lun(self): pass - @cinder_utils.retry(exceptions=const.VNXTargetNotReadyError, + @cinder_utils.retry(retry_param=const.VNXTargetNotReadyError, interval=15, retries=5, backoff_rate=1) def migrate_lun(self, src_id, dst_id, diff -Nru cinder-17.0.1/cinder/volume/drivers/dell_emc/xtremio.py cinder-17.4.0/cinder/volume/drivers/dell_emc/xtremio.py --- cinder-17.0.1/cinder/volume/drivers/dell_emc/xtremio.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/volume/drivers/dell_emc/xtremio.py 2022-04-20 17:38:33.000000000 +0000 @@ -31,6 +31,7 @@ 1.0.9 - performance improvements, support force detach, support for X2 1.0.10 - option to clean unused IGs 1.0.11 - add support for multiattach + 1.0.12 - add support for ports filtering """ import json @@ -82,7 +83,13 @@ 'the IG be, we default to False (not deleting IGs ' 'without connected volumes); setting this parameter ' 'to True will remove any IG after terminating its ' - 'connection to the last volume.')] + 'connection to the last volume.'), + cfg.ListOpt('xtremio_ports', + default=[], + help='Allowed ports. Comma separated list of XtremIO ' + 'iSCSI IPs or FC WWNs (ex. 58:cc:f0:98:49:22:07:02) ' + 'to be used. If option is not set all ports are allowed.') +] CONF.register_opts(XTREMIO_OPTS, group=configuration.SHARED_CONF_GROUP) @@ -420,7 +427,7 @@ class XtremIOVolumeDriver(san.SanDriver): """Executes commands relating to Volumes.""" - VERSION = '1.0.11' + VERSION = '1.0.12' # ThirdPartySystems wiki CI_WIKI_NAME = "EMC_XIO_CI" @@ -444,6 +451,10 @@ self.clean_ig = (self.configuration.safe_get('xtremio_clean_unused_ig') or False) self._stats = {} + self.allowed_ports = [ + port.strip().lower() for port in + self.configuration.safe_get('xtremio_ports') + ] self.client = XtremIOClient3(self.configuration, self.cluster_id) @classmethod @@ -1054,6 +1065,19 @@ raise (exception.VolumeBackendAPIException (data=_("Failed to create IG, %s") % name)) + def _port_is_allowed(self, port): + """Check if port is in allowed ports list. + + If allowed ports are empty then all ports are allowed. + + :param port: iSCSI IP/FC WWN to check + :return: is port allowed + """ + + if not self.allowed_ports: + return True + return port.lower() in self.allowed_ports + @interface.volumedriver class XtremIOISCSIDriver(XtremIOVolumeDriver, driver.ISCSIDriver): @@ -1171,26 +1195,33 @@ multiple values. The main portal information is also returned in :target_iqn, :target_portal, :target_lun for backward compatibility. """ - portals = self.client.get_iscsi_portals() - if not portals: - msg = _("XtremIO not configured correctly, no iscsi portals found") - LOG.error(msg) - raise exception.VolumeDriverException(message=msg) - portal = RANDOM.choice(portals) + iscsi_portals = self.client.get_iscsi_portals() + allowed_portals = [] + for iscsi_portal in iscsi_portals: + iscsi_portal['ip-addr'] = iscsi_portal['ip-addr'].split('/')[0] + if self._port_is_allowed(iscsi_portal['ip-addr']): + allowed_portals.append(iscsi_portal) + if not allowed_portals: + msg = _("There are no accessible iSCSI targets on the " + "system.") + raise exception.VolumeBackendAPIException(data=msg) + portal = RANDOM.choice(allowed_portals) portal_addr = ('%(ip)s:%(port)d' % - {'ip': portal['ip-addr'].split('/')[0], + {'ip': portal['ip-addr'], 'port': portal['ip-port']}) - tg_portals = ['%(ip)s:%(port)d' % {'ip': p['ip-addr'].split('/')[0], + tg_portals = ['%(ip)s:%(port)d' % {'ip': p['ip-addr'], 'port': p['ip-port']} - for p in portals] + for p in allowed_portals] properties = {'target_discovered': False, 'target_iqn': portal['port-address'], 'target_lun': lunmap['lun'], 'target_portal': portal_addr, - 'target_iqns': [p['port-address'] for p in portals], + 'target_iqns': [ + p['port-address'] for p in allowed_portals + ], 'target_portals': tg_portals, - 'target_luns': [lunmap['lun']] * len(portals)} + 'target_luns': [lunmap['lun']] * len(allowed_portals)} return properties def _get_initiator_names(self, connector): @@ -1213,8 +1244,17 @@ if not self._targets: try: targets = self.client.get_fc_up_ports() - self._targets = [target['port-address'].replace(':', '') - for target in targets] + allowed_targets = [] + for target in targets: + if self._port_is_allowed(target['port-address']): + allowed_targets.append( + target['port-address'].replace(':', '') + ) + if not allowed_targets: + msg = _("There are no accessible Fibre Channel targets " + "on the system.") + raise exception.VolumeBackendAPIException(data=msg) + self._targets = allowed_targets except exception.NotFound: raise (exception.VolumeBackendAPIException (data=_("Failed to get targets"))) diff -Nru cinder-17.0.1/cinder/volume/drivers/hpe/hpe_3par_iscsi.py cinder-17.4.0/cinder/volume/drivers/hpe/hpe_3par_iscsi.py --- cinder-17.0.1/cinder/volume/drivers/hpe/hpe_3par_iscsi.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/volume/drivers/hpe/hpe_3par_iscsi.py 2022-04-20 17:38:33.000000000 +0000 @@ -128,10 +128,11 @@ failover. bug #1773069 4.0.4 - Added Peer Persistence feature 4.0.5 - Added Primera array check. bug #1849525 + 4.0.6 - Allow iSCSI support for Primera 4.2 onwards """ - VERSION = "4.0.5" + VERSION = "4.0.6" # The name of the CI wiki page. CI_WIKI_NAME = "HPE_Storage_CI" @@ -144,9 +145,17 @@ client_obj = common.client is_primera = client_obj.is_primera_array() if is_primera: - LOG.error("For Primera, only FC is supported. " - "iSCSI cannot be used") - raise NotImplementedError() + api_version = client_obj.getWsApiVersion() + array_version = api_version['build'] + LOG.debug("array version: %(version)s", + {'version': array_version}) + if array_version < 40200000: + err_msg = (_('The iSCSI driver is not supported for ' + 'Primera %(version)s. It is supported ' + 'for Primera 4.2 or higher versions.') + % {'version': array_version}) + LOG.error(err_msg) + raise NotImplementedError() self.iscsi_ips = {} common.client_login() diff -Nru cinder-17.0.1/cinder/volume/drivers/ibm/storwize_svc/storwize_svc_common.py cinder-17.4.0/cinder/volume/drivers/ibm/storwize_svc/storwize_svc_common.py --- cinder-17.0.1/cinder/volume/drivers/ibm/storwize_svc/storwize_svc_common.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/volume/drivers/ibm/storwize_svc/storwize_svc_common.py 2022-04-20 17:38:33.000000000 +0000 @@ -756,6 +756,7 @@ self.ssh = StorwizeSSH(run_ssh) self.check_fcmapping_interval = 3 self.code_level = None + self.stats = {} @staticmethod def handle_keyerror(cmd, out): @@ -825,6 +826,12 @@ def is_data_reduction_pool(self, pool_name): """Check if pool is data reduction pool.""" + # Check pool is data reduction pool or not from pool information + # saved in stats. + for pool in self.stats.get('pools', []): + if pool['pool_name'] == pool_name: + return pool['data_reduction'] + pool_data = self.get_pool_attrs(pool_name) if (pool_data and 'data_reduction' in pool_data and pool_data['data_reduction'] == 'yes'): @@ -2069,10 +2076,10 @@ return None return resp[0] - def _check_vdisk_fc_mappings(self, name, - allow_snaps=True, allow_fctgt=False): + @cinder_utils.trace + def _check_delete_vdisk_fc_mappings(self, name, allow_snaps=True, + allow_fctgt=False): """FlashCopy mapping check helper.""" - LOG.debug('Loopcall: _check_vdisk_fc_mappings(), vdisk %s.', name) mapping_ids = self._get_vdisk_fc_mappings(name) wait_for_copy = False for map_id in mapping_ids: @@ -2085,10 +2092,23 @@ target = attrs['target_vdisk_name'] copy_rate = attrs['copy_rate'] status = attrs['status'] + progress = attrs['progress'] + LOG.debug('Loopcall: source: %s, target: %s, copy_rate: %s, ' + 'status: %s, progress: %s, mapid: %s', source, target, + copy_rate, status, progress, map_id) if allow_fctgt and target == name and status == 'copying': - self.ssh.stopfcmap(map_id) - attrs = self._get_flashcopy_mapping_attributes(map_id) + try: + self.ssh.stopfcmap(map_id) + except exception.VolumeBackendAPIException as ex: + LOG.warning(ex) + wait_for_copy = True + try: + attrs = self._get_flashcopy_mapping_attributes(map_id) + except exception.VolumeBackendAPIException as ex: + LOG.warning(ex) + wait_for_copy = True + continue if attrs: status = attrs['status'] else: @@ -2110,28 +2130,86 @@ {'name': name, 'src': source, 'tgt': target}) LOG.error(msg) raise exception.VolumeDriverException(message=msg) - if status in ['copying', 'prepared']: - self.ssh.stopfcmap(map_id) - # Need to wait for the fcmap to change to - # stopped state before remove fcmap - wait_for_copy = True - elif status in ['stopping', 'preparing']: + try: + if status in ['copying', 'prepared']: + self.ssh.stopfcmap(map_id) + # Need to wait for the fcmap to change to + # stopped state before remove fcmap + wait_for_copy = True + elif status in ['stopping', 'preparing']: + wait_for_copy = True + else: + self.ssh.rmfcmap(map_id) + except exception.VolumeBackendAPIException as ex: + LOG.warning(ex) wait_for_copy = True - else: - self.ssh.rmfcmap(map_id) # Case 4: Copy in progress - wait and will autodelete else: - if status == 'prepared': - self.ssh.stopfcmap(map_id) - self.ssh.rmfcmap(map_id) - elif status in ['idle_or_copied', 'stopped']: - # Prepare failed or stopped - self.ssh.rmfcmap(map_id) - else: + try: + if status == 'prepared': + self.ssh.stopfcmap(map_id) + self.ssh.rmfcmap(map_id) + elif status in ['idle_or_copied', 'stopped']: + # Prepare failed or stopped + self.ssh.rmfcmap(map_id) + elif (status in ['copying', 'prepared'] and + progress == '100'): + self.ssh.stopfcmap(map_id) + else: + wait_for_copy = True + except exception.VolumeBackendAPIException as ex: + LOG.warning(ex) wait_for_copy = True + if not wait_for_copy or not len(mapping_ids): raise loopingcall.LoopingCallDone(retvalue=True) + @cinder_utils.trace + def _check_vdisk_fc_mappings(self, name, allow_snaps=True, + allow_fctgt=False): + """FlashCopy mapping check helper.""" + # if this is a remove disk we need to be down to one fc clone + mapping_ids = self._get_vdisk_fc_mappings(name) + Rc_mapping_ids = [] + if len(mapping_ids) > 1 and allow_fctgt: + LOG.debug('Loopcall: vdisk %s has ' + 'more than one fc map. Waiting.', name) + for map_id in mapping_ids: + attrs = self._get_flashcopy_mapping_attributes(map_id) + if not attrs: + continue + if 'yes' == attrs.get('rc_controlled', None): + Rc_mapping_ids.append(map_id) + continue + + source = attrs['source_vdisk_name'] + target = attrs['target_vdisk_name'] + copy_rate = attrs['copy_rate'] + status = attrs['status'] + progress = attrs['progress'] + LOG.debug('Loopcall: source: %s, target: %s, copy_rate: %s, ' + 'status: %s, progress: %s, mapid: %s', + source, target, copy_rate, status, progress, map_id) + + if copy_rate != '0' and source == name: + try: + if status in ['copying'] and progress == '100': + self.ssh.stopfcmap(map_id) + elif status == 'idle_or_copied' and progress == '100': + # wait for auto-delete of fcmap. + continue + elif status in ['idle_or_copied', 'stopped']: + # Prepare failed or stopped + self.ssh.rmfcmap(map_id) + # handle VolumeBackendAPIException to let it go through + # next attempts in case of any cli exception. + except exception.VolumeBackendAPIException as ex: + LOG.warning(ex) + if len(mapping_ids) - len(Rc_mapping_ids) > 1: + return + return self._check_delete_vdisk_fc_mappings( + name, allow_snaps=allow_snaps, allow_fctgt=allow_fctgt) + def ensure_vdisk_no_fc_mappings(self, name, allow_snaps=True, allow_fctgt=False): """Ensure vdisk has no flashcopy mappings.""" @@ -2584,7 +2662,7 @@ elif copy_rate != '0' and progress == '100': LOG.debug('Split completed clone map_id=%(map_id)s fcmap', {'map_id': map_id}) - self.ssh.stopfcmap(map_id, split=True) + self.ssh.stopfcmap(map_id) class CLIResponse(object): @@ -2778,6 +2856,9 @@ # This is used to save the available pools in failed-over status self._secondary_pools = None + # This dictionary is used to save pools information. + self._stats = {} + # Storwize has the limitation that can not burst more than 3 new ssh # connections within 1 second. So slow down the initialization. time.sleep(1) @@ -2795,6 +2876,12 @@ # Get list of all volumes self._get_all_volumes() + # Update the pool stats + self._update_volume_stats() + + # Save the pool stats information in helpers class. + self._master_backend_helpers.stats = self._stats + # Build the list of in-progress vdisk copy operations if ctxt is None: admin_context = context.get_admin_context() @@ -3583,6 +3670,8 @@ self._state = self._master_state self._update_volume_stats() + self._master_backend_helpers.stats = self._stats + return storwize_const.FAILBACK_VALUE, volumes_update, groups_update def _failback_replica_volumes(self, ctxt, rep_volumes): @@ -3832,6 +3921,8 @@ self._state = self._aux_state self._update_volume_stats() + self._aux_backend_helpers.stats = self._stats + return self._active_backend_id, volumes_update, groups_update def _failover_replica_volumes(self, ctxt, rep_volumes): @@ -5632,6 +5723,7 @@ """Build pool status""" QoS_support = True pool_stats = {} + is_dr_pool = False pool_data = self._helpers.get_pool_attrs(pool) if pool_data: easy_tier = pool_data['easy_tier'] in ['on', 'auto'] @@ -5655,6 +5747,14 @@ storwize_svc_multihostmap_enabled) backend_state = ('up' if pool_data['status'] == 'online' else 'down') + + # Get the data_reduction information for pool and set + # is_dr_pool flag. + if pool_data.get('data_reduction') == 'Yes': + is_dr_pool = True + elif pool_data.get('data_reduction') == 'No': + is_dr_pool = False + pool_stats = { 'pool_name': pool_data['name'], 'total_capacity_gb': total_capacity_gb, @@ -5674,6 +5774,7 @@ 'max_over_subscription_ratio': over_sub_ratio, 'consistent_group_snapshot_enabled': True, 'backend_state': backend_state, + 'data_reduction': is_dr_pool, } if self._replica_enabled: pool_stats.update({ @@ -5695,6 +5796,7 @@ 'thick_provisioning_support': False, 'max_over_subscription_ratio': 0, 'reserved_percentage': 0, + 'data_reduction': is_dr_pool, 'backend_state': 'down'} return pool_stats diff -Nru cinder-17.0.1/cinder/volume/drivers/inspur/as13000/as13000_driver.py cinder-17.4.0/cinder/volume/drivers/inspur/as13000/as13000_driver.py --- cinder-17.0.1/cinder/volume/drivers/inspur/as13000/as13000_driver.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/volume/drivers/inspur/as13000/as13000_driver.py 2022-04-20 17:38:33.000000000 +0000 @@ -704,7 +704,7 @@ request_type=request_type) @utils.trace - @utils.retry(exceptions=exception.VolumeDriverException, + @utils.retry(retry_param=exception.VolumeDriverException, interval=1, retries=3) def _add_lun_to_target(self, target_name, volume): diff -Nru cinder-17.0.1/cinder/volume/drivers/netapp/dataontap/client/client_cmode.py cinder-17.4.0/cinder/volume/drivers/netapp/dataontap/client/client_cmode.py --- cinder-17.0.1/cinder/volume/drivers/netapp/dataontap/client/client_cmode.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/volume/drivers/netapp/dataontap/client/client_cmode.py 2022-04-20 17:38:33.000000000 +0000 @@ -468,12 +468,14 @@ else: block_count = zbc + is_sub_clone = block_count > 0 zapi_args = { 'volume': volume, 'source-path': name, 'destination-path': new_name, - 'space-reserve': space_reserved, } + if not is_sub_clone: + zapi_args['space-reserve'] = space_reserved if source_snapshot: zapi_args['snapshot-name'] = source_snapshot if is_snapshot and self.features.BACKUP_CLONE_PARAM: @@ -488,7 +490,7 @@ else: clone_create.add_new_child('qos-policy-group-name', qos_policy_group_name) - if block_count > 0: + if is_sub_clone: block_ranges = netapp_api.NaElement("block-ranges") segments = int(math.ceil(block_count / float(bc_limit))) bc = block_count diff -Nru cinder-17.0.1/cinder/volume/drivers/netapp/dataontap/nfs_base.py cinder-17.4.0/cinder/volume/drivers/netapp/dataontap/nfs_base.py --- cinder-17.0.1/cinder/volume/drivers/netapp/dataontap/nfs_base.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/volume/drivers/netapp/dataontap/nfs_base.py 2022-04-20 17:38:33.000000000 +0000 @@ -1068,3 +1068,10 @@ vol_path = os.path.join(volume['provider_location'], vol_str) LOG.info('Cinder NFS volume with current path "%(cr)s" is ' 'no longer being managed.', {'cr': vol_path}) + + def update_migrated_volume(self, ctxt, volume, new_volume, + original_volume_status): + # Implemented to prevent NFSDriver's implementation renaming the file + # and breaking volume's backend QoS. + msg = _("The method update_migrated_volume is not implemented.") + raise NotImplementedError(msg) diff -Nru cinder-17.0.1/cinder/volume/drivers/netapp/utils.py cinder-17.4.0/cinder/volume/drivers/netapp/utils.py --- cinder-17.0.1/cinder/volume/drivers/netapp/utils.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/volume/drivers/netapp/utils.py 2022-04-20 17:38:33.000000000 +0000 @@ -266,7 +266,7 @@ def get_qos_policy_group_name(volume): """Return the name of backend QOS policy group based on its volume id.""" if 'id' in volume: - return OPENSTACK_PREFIX + volume['id'] + return OPENSTACK_PREFIX + volume.name_id return None @@ -313,7 +313,7 @@ info = dict(legacy=None, spec=None) try: volume_type = get_volume_type_from_volume(volume) - except KeyError: + except (KeyError, exception.NotFound): LOG.exception('Cannot get QoS spec for volume %s.', volume['id']) return info if volume_type is None: diff -Nru cinder-17.0.1/cinder/volume/drivers/nfs.py cinder-17.4.0/cinder/volume/drivers/nfs.py --- cinder-17.0.1/cinder/volume/drivers/nfs.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/volume/drivers/nfs.py 2022-04-20 17:38:33.000000000 +0000 @@ -585,7 +585,6 @@ def delete_snapshot(self, snapshot): """Apply locking to the delete snapshot operation.""" - self._check_snapshot_support() return self._delete_snapshot(snapshot) def _copy_volume_from_snapshot(self, snapshot, volume, volume_size, diff -Nru cinder-17.0.1/cinder/volume/drivers/pure.py cinder-17.4.0/cinder/volume/drivers/pure.py --- cinder-17.0.1/cinder/volume/drivers/pure.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/volume/drivers/pure.py 2022-04-20 17:38:33.000000000 +0000 @@ -59,8 +59,9 @@ "this calculated value will override the " "max_over_subscription_ratio config option."), cfg.StrOpt("pure_host_personality", + default=None, choices=['aix', 'esxi', 'hitachi-vsp', 'hpux', - 'oracle-vm-server', 'solaris', 'vms'], + 'oracle-vm-server', 'solaris', 'vms', None], help="Determines how the Purity system tunes the protocol used " "between the array and the initiator."), # These are used as default settings. In future these can be overridden @@ -124,6 +125,8 @@ EXTRA_SPECS_REPL_ENABLED = "replication_enabled" EXTRA_SPECS_REPL_TYPE = "replication_type" +MAX_VOL_LENGTH = 63 +MAX_SNAP_LENGTH = 96 UNMANAGED_SUFFIX = '-unmanaged' SYNC_REPLICATION_REQUIRED_API_VERSIONS = ['1.13', '1.14'] ASYNC_REPLICATION_REQUIRED_API_VERSIONS = [ @@ -186,6 +189,8 @@ SUPPORTED_REST_API_VERSIONS = ['1.2', '1.3', '1.4', '1.5', '1.13', '1.14'] + SUPPORTS_ACTIVE_ACTIVE = True + # ThirdPartySystems wiki page CI_WIKI_NAME = "Pure_Storage_CI" @@ -608,8 +613,23 @@ """ raise NotImplementedError + def _is_multiattach_to_host(self, volume_attachment, host_name): + # When multiattach is enabled a volume could be attached to multiple + # instances which are hosted on the same Nova compute. + # Because Purity cannot recognize the volume is attached more than + # one instance we should keep the volume attached to the Nova compute + # until the volume is detached from the last instance + if not volume_attachment: + return False + + attachment = [a for a in volume_attachment + if a.attach_status == "attached" and + a.attached_host == host_name] + return len(attachment) > 1 + @pure_driver_debug_trace - def _disconnect(self, array, volume, connector, remove_remote_hosts=False): + def _disconnect(self, array, volume, connector, remove_remote_hosts=False, + is_multiattach=False): """Disconnect the volume from the host described by the connector. If no connector is specified it will remove *all* attachments for @@ -640,11 +660,16 @@ remote=remove_remote_hosts) if hosts: any_in_use = False + host_in_use = False for host in hosts: host_name = host["name"] - host_in_use = self._disconnect_host(array, - host_name, - vol_name) + if not is_multiattach: + host_in_use = self._disconnect_host(array, + host_name, + vol_name) + else: + LOG.warning("Unable to disconnect host from volume. " + "Volume is multi-attached.") any_in_use = any_in_use or host_in_use return any_in_use else: @@ -657,20 +682,27 @@ def terminate_connection(self, volume, connector, **kwargs): """Terminate connection.""" vol_name = self._get_vol_name(volume) + # None `connector` indicates force detach, then delete all even + # if the volume is multi-attached. + multiattach = (connector is not None and + self._is_multiattach_to_host(volume.volume_attachment, + connector["host"])) if self._is_vol_in_pod(vol_name): # Try to disconnect from each host, they may not be online though # so if they fail don't cause a problem. for array in self._uniform_active_cluster_target_arrays: try: self._disconnect(array, volume, connector, - remove_remote_hosts=False) + remove_remote_hosts=False, + is_multiattach=multiattach) except purestorage.PureError as err: # Swallow any exception, just warn and continue LOG.warning("Disconnect on secondary array failed with" " message: %(msg)s", {"msg": err.text}) # Now disconnect from the current array self._disconnect(self._get_current_array(), volume, - connector, remove_remote_hosts=False) + connector, remove_remote_hosts=False, + is_multiattach=multiattach) @pure_driver_debug_trace def _disconnect_host(self, array, host_name, vol_name): @@ -894,6 +926,7 @@ 'source_group': source_group.id}) current_array = self._get_current_array() current_array.create_pgroup_snapshot(pgroup_name, suffix=tmp_suffix) + volumes, _ = self.update_provider_info(volumes, None) try: for source_vol, cloned_vol in zip(source_vols, volumes): source_snap_name = self._get_pgroup_vol_snap_name( @@ -1268,7 +1301,11 @@ """ vol_name = self._get_vol_name(volume) - unmanaged_vol_name = vol_name + UNMANAGED_SUFFIX + if len(vol_name + UNMANAGED_SUFFIX) > MAX_VOL_LENGTH: + unmanaged_vol_name = vol_name[:-len(UNMANAGED_SUFFIX)] + \ + UNMANAGED_SUFFIX + else: + unmanaged_vol_name = vol_name + UNMANAGED_SUFFIX LOG.info("Renaming existing volume %(ref_name)s to %(new_name)s", {"ref_name": vol_name, "new_name": unmanaged_vol_name}) self._rename_volume_object(vol_name, unmanaged_vol_name) @@ -1325,7 +1362,11 @@ """ self._verify_manage_snap_api_requirements() snap_name = self._get_snap_name(snapshot) - unmanaged_snap_name = snap_name + UNMANAGED_SUFFIX + if len(snap_name + UNMANAGED_SUFFIX) > MAX_SNAP_LENGTH: + unmanaged_snap_name = snap_name[:-len(UNMANAGED_SUFFIX)] + \ + UNMANAGED_SUFFIX + else: + unmanaged_snap_name = snap_name + UNMANAGED_SUFFIX LOG.info("Renaming existing snapshot %(ref_name)s to " "%(new_name)s", {"ref_name": snap_name, "new_name": unmanaged_snap_name}) @@ -1554,13 +1595,14 @@ base_name = volume.name # Some OpenStack deployments, eg PowerVC, create a volume.name that - # when appended with out '-cinder' string will exceed the maximum + # when appended with our '-cinder' string will exceed the maximum # volume name length for Pure, so here we left truncate the true volume # name before the opennstack volume_name_template affected it and # then put back the template format if len(base_name) > 56: - actual_name = base_name[7:] - base_name = "volume-" + actual_name[-52:] + actual_name = base_name[(len(CONF.volume_name_template) - 2):] + base_name = CONF.volume_name_template % \ + actual_name[-(56 - len(CONF.volume_name_template)):] repl_type = self._get_replication_type_from_vol_type( volume.volume_type) @@ -2726,6 +2768,11 @@ def terminate_connection(self, volume, connector, **kwargs): """Terminate connection.""" vol_name = self._get_vol_name(volume) + # None `connector` indicates force detach, then delete all even + # if the volume is multi-attached. + multiattach = (connector is not None and + self._is_multiattach_to_host(volume.volume_attachment, + connector["host"])) unused_wwns = [] if self._is_vol_in_pod(vol_name): @@ -2734,7 +2781,8 @@ for array in self._uniform_active_cluster_target_arrays: try: no_more_connections = self._disconnect( - array, volume, connector, remove_remote_hosts=False) + array, volume, connector, remove_remote_hosts=False, + is_multiattach=multiattach) if no_more_connections: unused_wwns += self._get_array_wwns(array) except purestorage.PureError as err: @@ -2747,7 +2795,8 @@ current_array = self._get_current_array() no_more_connections = self._disconnect(current_array, volume, connector, - remove_remote_hosts=False) + remove_remote_hosts=False, + is_multiattach=multiattach) if no_more_connections: unused_wwns += self._get_array_wwns(current_array) diff -Nru cinder-17.0.1/cinder/volume/drivers/rbd.py cinder-17.4.0/cinder/volume/drivers/rbd.py --- cinder-17.0.1/cinder/volume/drivers/rbd.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/volume/drivers/rbd.py 2022-04-20 17:38:33.000000000 +0000 @@ -102,13 +102,16 @@ 'dynamic value (used + current free) and to False to ' 'report a static value (quota max bytes if defined and ' 'global size of cluster if not).'), - cfg.BoolOpt('rbd_exclusive_cinder_pool', default=False, - help="Set to True if the pool is used exclusively by Cinder. " + cfg.BoolOpt('rbd_exclusive_cinder_pool', default=True, + help="Set to False if the pool is shared with other usages. " "On exclusive use driver won't query images' provisioned " "size as they will match the value calculated by the " "Cinder core code for allocated_capacity_gb. This " "reduces the load on the Ceph cluster as well as on the " - "volume service."), + "volume service. On non exclusive use driver will query " + "the Ceph cluster for per image used disk, this is an " + "intensive operation having an independent request for " + "each image."), cfg.BoolOpt('enable_deferred_deletion', default=False, help='Enable deferred deletion. Upon deletion, volumes are ' 'tagged for deletion but will only be removed ' @@ -248,6 +251,7 @@ self._is_replication_enabled = False self._replication_targets = [] self._target_names = [] + self._clone_v2_api_checked = False if self.rbd is not None: self.RBD_FEATURE_LAYERING = self.rbd.RBD_FEATURE_LAYERING @@ -294,6 +298,22 @@ 'max_over_subscription_ratio', 'volume_dd_blocksize') return RBD_OPTS + additional_opts + def _show_msg_check_clone_v2_api(self, volume_name): + if not self._clone_v2_api_checked: + self._clone_v2_api_checked = True + with RBDVolumeProxy(self, volume_name, read_only=True) as volume: + try: + if (volume.volume.op_features() & + self.rbd.RBD_OPERATION_FEATURE_CLONE_PARENT): + LOG.info('Using v2 Clone API') + return + except AttributeError: + pass + LOG.warning('Not using v2 clone API, please upgrade to' + ' mimic+ and set the OSD minimum client' + ' compat version to mimic for better' + ' performance, fewer deletion issues') + def _get_target_config(self, target_id): """Get a replication target from known replication targets.""" for target in self._replication_targets: @@ -558,14 +578,14 @@ with RADOSClient(self) as client: ret, df_outbuf, __ = client.cluster.mon_command( - '{"prefix":"df", "format":"json"}', '') + '{"prefix":"df", "format":"json"}', b'') if ret: LOG.warning('Unable to get rados pool stats.') return 'unknown', 'unknown' ret, quota_outbuf, __ = client.cluster.mon_command( '{"prefix":"osd pool get-quota", "pool": "%s",' - ' "format":"json"}' % pool_name, '') + ' "format":"json"}' % pool_name, b'') if ret: LOG.warning('Unable to get rados pool quotas.') return 'unknown', 'unknown' @@ -648,7 +668,9 @@ def _get_clone_depth(self, client, volume_name, depth=0): """Returns the number of ancestral clones of the given volume.""" - parent_volume = self.rbd.Image(client.ioctx, volume_name) + parent_volume = self.rbd.Image(client.ioctx, + volume_name, + read_only=True) try: _pool, parent, _snap = self._get_clone_info(parent_volume, volume_name) @@ -984,16 +1006,35 @@ with RBDVolumeProxy(self, volume_name, pool) as vol: vol.flatten() + def _get_stripe_unit(self, ioctx, volume_name): + """Return the correct stripe unit for a cloned volume. + + A cloned volume must be created with a stripe unit at least as large + as the source volume. We compute the desired stripe width from + rbd_store_chunk_size and compare that to the incoming source volume's + stripe width, selecting the larger to avoid error. + """ + default_stripe_unit = \ + self.configuration.rbd_store_chunk_size * units.Mi + + image = self.rbd.Image(ioctx, volume_name, read_only=True) + try: + image_stripe_unit = image.stripe_unit() + finally: + image.close() + + return max(image_stripe_unit, default_stripe_unit) + def _clone(self, volume, src_pool, src_image, src_snap): LOG.debug('cloning %(pool)s/%(img)s@%(snap)s to %(dst)s', dict(pool=src_pool, img=src_image, snap=src_snap, dst=volume.name)) - chunk_size = self.configuration.rbd_store_chunk_size * units.Mi - order = int(math.log(chunk_size, 2)) vol_name = utils.convert_str(volume.name) with RADOSClient(self, src_pool) as src_client: + stripe_unit = self._get_stripe_unit(src_client.ioctx, src_image) + order = int(math.log(stripe_unit, 2)) with RADOSClient(self) as dest_client: self.RBDProxy().clone(src_client.ioctx, utils.convert_str(src_image), @@ -1002,7 +1043,6 @@ vol_name, features=src_client.features, order=order) - try: volume_update = self._setup_volume(volume) except Exception: @@ -1028,6 +1068,8 @@ self._flatten(self.configuration.rbd_pool, volume.name) if int(volume.size): self._resize(volume) + + self._show_msg_check_clone_v2_api(snapshot.volume_name) return volume_update def _delete_backup_snaps(self, rbd_image): @@ -1604,7 +1646,13 @@ if encrypted: self._encrypt_image(context, volume, tmp_dir, tmp.name) - self.delete_volume(volume) + @utils.retry(exception.VolumeIsBusy, + self.configuration.rados_connection_interval, + self.configuration.rados_connection_retries) + def _delete_volume(volume): + self.delete_volume(volume) + + _delete_volume(volume) chunk_size = self.configuration.rbd_store_chunk_size * units.Mi order = int(math.log(chunk_size, 2)) @@ -1698,7 +1746,9 @@ with RADOSClient(self) as client: # Raise an exception if we didn't find a suitable rbd image. try: - rbd_image = self.rbd.Image(client.ioctx, rbd_name) + rbd_image = self.rbd.Image(client.ioctx, + rbd_name, + read_only=True) except self.rbd.ImageNotFound: kwargs = {'existing_ref': rbd_name, 'reason': 'Specified rbd image does not exist.'} @@ -1925,7 +1975,8 @@ # Raise an exception if we didn't find a suitable rbd image. try: rbd_snapshot = self.rbd.Image(client.ioctx, volume_name, - snapshot=snapshot_name) + snapshot=snapshot_name, + read_only=True) except self.rbd.ImageNotFound: kwargs = {'existing_ref': snapshot_name, 'reason': 'Specified snapshot does not exist.'} diff -Nru cinder-17.0.1/cinder/volume/drivers/remotefs.py cinder-17.4.0/cinder/volume/drivers/remotefs.py --- cinder-17.0.1/cinder/volume/drivers/remotefs.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/volume/drivers/remotefs.py 2022-04-20 17:38:33.000000000 +0000 @@ -1377,11 +1377,11 @@ LOG.debug('Creating volume %(vol)s from snapshot %(snap)s', {'vol': volume.id, 'snap': snapshot.id}) - if snapshot.status != 'available': - msg = _('Snapshot status must be "available" to clone. ' - 'But is: %(status)s') % {'status': snapshot.status} - - raise exception.InvalidSnapshot(msg) + status = snapshot.status + acceptable_states = ['available', 'backing-up'] + self._validate_state(status, acceptable_states, + obj_description='snapshot', + invalid_exc=exception.InvalidSnapshot) self._ensure_shares_mounted() @@ -1637,18 +1637,25 @@ backing_filename = self.get_active_image_from_info( snapshot.volume) new_snap_path = self._get_new_snap_path(snapshot) + active = os.path.basename(new_snap_path) if self._is_volume_attached(snapshot.volume): self._create_snapshot_online(snapshot, backing_filename, new_snap_path) + # Update reference in the only attachment (no multi-attach support) + attachment = snapshot.volume.volume_attachment[0] + attachment.connection_info['name'] = active + # Let OVO know it has been updated + attachment.connection_info = attachment.connection_info + attachment.save() else: self._do_create_snapshot(snapshot, backing_filename, new_snap_path) - snap_info['active'] = os.path.basename(new_snap_path) - snap_info[snapshot.id] = os.path.basename(new_snap_path) + snap_info['active'] = active + snap_info[snapshot.id] = active self._write_info_file(info_path, snap_info) def _create_snapshot_online(self, snapshot, backing_filename, diff -Nru cinder-17.0.1/cinder/volume/drivers/solidfire.py cinder-17.4.0/cinder/volume/drivers/solidfire.py --- cinder-17.0.1/cinder/volume/drivers/solidfire.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/volume/drivers/solidfire.py 2022-04-20 17:38:33.000000000 +0000 @@ -101,17 +101,26 @@ min=30, help='Sets time in seconds to wait for a migrating volume to ' 'complete pairing and sync.'), + cfg.IntOpt('sf_api_request_timeout', default=30, min=30, help='Sets time in seconds to wait for an api request to ' 'complete.'), + cfg.IntOpt('sf_volume_clone_timeout', default=600, min=60, help='Sets time in seconds to wait for a clone of a volume or ' 'snapshot to complete.' - )] + ), + + cfg.IntOpt('sf_volume_create_timeout', + default=60, + min=30, + help='Sets time in seconds to wait for a create volume ' + 'operation to complete.')] + CONF = cfg.CONF CONF.register_opts(sf_opts, group=configuration.SHARED_CONF_GROUP) @@ -123,10 +132,6 @@ xNotInVolumeAccessGroup = 'xNotInVolumeAccessGroup' -class DuplicateSfVolumeNames(exception.Duplicate): - message = _("Detected more than one volume with name %(vol_name)s") - - class SolidFireAPIException(exception.VolumeBackendAPIException): message = _("Bad response from SolidFire API") @@ -161,6 +166,11 @@ message = _("Data sync volumes timed out") +class SolidFireDuplicateVolumeNames(SolidFireDriverException): + message = _("Volume name [%(vol_name)s] already exists " + "in the SolidFire backend.") + + def retry(exc_tuple, tries=5, delay=1, backoff=2): def retry_dec(f): @six.wraps(f) @@ -262,9 +272,13 @@ - Enable Active/Active support flag - Implement Active/Active replication support 2.2.0 - Add storage assisted volume migration support + 2.2.1 - Fix bug #1891914 fix error on cluster workload rebalancing + by adding xNotPrimary to the retryable exception list + 2.2.2 - Fix bug #1896112 SolidFire Driver creates duplicate volume + when API response is lost """ - VERSION = '2.2.0' + VERSION = '2.2.2' SUPPORTS_ACTIVE_ACTIVE = True @@ -302,7 +316,8 @@ 'xMaxSnapshotsPerNodeExceeded', 'xMaxClonesPerNodeExceeded', 'xSliceNotRegistered', - 'xNotReadyForIO'] + 'xNotReadyForIO', + 'xNotPrimary'] def __init__(self, *args, **kwargs): super(SolidFireDriver, self).__init__(*args, **kwargs) @@ -1000,10 +1015,59 @@ params['attributes'] = attributes return self._issue_api_request('ModifyVolume', params) + def _list_volumes_by_name(self, sf_volume_name): + params = {'volumeName': sf_volume_name} + return self._issue_api_request( + 'ListVolumes', params, version='8.0')['result']['volumes'] + + def _wait_volume_is_active(self, sf_volume_name): + + def _wait(): + volumes = self._list_volumes_by_name(sf_volume_name) + if volumes: + LOG.debug("Found Volume [%s] in SolidFire backend. " + "Current status is [%s].", + sf_volume_name, volumes[0]['status']) + if volumes[0]['status'] == 'active': + raise loopingcall.LoopingCallDone(volumes[0]) + + try: + timer = loopingcall.FixedIntervalWithTimeoutLoopingCall( + _wait) + sf_volume = (timer.start( + interval=1, + timeout=self.configuration.sf_volume_create_timeout).wait()) + + return sf_volume + except loopingcall.LoopingCallTimeOut: + msg = ("Timeout while waiting volume [%s] " + "to be in active state." % sf_volume_name) + LOG.error(msg) + raise SolidFireAPIException(msg) + def _do_volume_create(self, sf_account, params, endpoint=None): - params['accountID'] = sf_account['accountID'] - sf_volid = self._issue_api_request( - 'CreateVolume', params, endpoint=endpoint)['result']['volumeID'] + + sf_volume_name = params['name'] + volumes_found = self._list_volumes_by_name(sf_volume_name) + if volumes_found: + raise SolidFireDuplicateVolumeNames(vol_name=sf_volume_name) + + sf_volid = None + try: + params['accountID'] = sf_account['accountID'] + response = self._issue_api_request( + 'CreateVolume', params, endpoint=endpoint) + sf_volid = response['result']['volumeID'] + + except requests.exceptions.ReadTimeout: + LOG.debug("Read Timeout exception caught while creating " + "volume [%s].", sf_volume_name) + # Check if volume was created for the given name, + # in case the backend has processed the request but failed + # to deliver the response before api request timeout. + volume_created = self._wait_volume_is_active(sf_volume_name) + sf_volid = volume_created['volumeID'] + return self._get_model_info(sf_account, sf_volid, endpoint=endpoint) def _do_snapshot_create(self, params): @@ -1156,7 +1220,7 @@ LOG.error("Found %(count)s volumes mapped to id: %(uuid)s.", {'count': found_count, 'uuid': uuid}) - raise DuplicateSfVolumeNames(vol_name=uuid) + raise SolidFireDuplicateVolumeNames(vol_name=uuid) return sf_volref diff -Nru cinder-17.0.1/cinder/volume/flows/manager/create_volume.py cinder-17.4.0/cinder/volume/flows/manager/create_volume.py --- cinder-17.0.1/cinder/volume/flows/manager/create_volume.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/volume/flows/manager/create_volume.py 2022-04-20 17:38:33.000000000 +0000 @@ -20,7 +20,9 @@ from oslo_log import log as logging from oslo_utils import excutils from oslo_utils import fileutils +from oslo_utils import netutils from oslo_utils import timeutils +from oslo_utils import uuidutils import six import taskflow.engines from taskflow.patterns import linear_flow @@ -656,6 +658,34 @@ self.db.volume_glance_metadata_bulk_create(context, volume_id, volume_metadata) + @staticmethod + def _extract_cinder_ids(urls): + """Process a list of location URIs from glance + + :param urls: list of glance location URIs + :return: list of IDs extracted from the 'cinder://' URIs + + """ + ids = [] + for url in urls: + # The url can also be None and a TypeError is raised + # TypeError: a bytes-like object is required, not 'str' + if not url: + continue + parts = netutils.urlsplit(url) + if parts.scheme == 'cinder': + if parts.path: + vol_id = parts.path.split('/')[-1] + else: + vol_id = parts.netloc + if uuidutils.is_uuid_like(vol_id): + ids.append(vol_id) + else: + LOG.debug("Ignoring malformed image location uri " + "'%(url)s'", {'url': url}) + + return ids + def _clone_image_volume(self, context, volume, image_location, image_meta): """Create a volume efficiently from an existing image. @@ -675,9 +705,9 @@ image_volume = None direct_url, locations = image_location - urls = set([direct_url] + [loc.get('url') for loc in locations or []]) - image_volume_ids = [url[9:] for url in urls - if url and url.startswith('cinder://')] + urls = list(set([direct_url] + + [loc.get('url') for loc in locations or []])) + image_volume_ids = self._extract_cinder_ids(urls) image_volumes = self.db.volume_get_all_by_host( context, volume['host'], filters={'id': image_volume_ids}) diff -Nru cinder-17.0.1/cinder/volume/manager.py cinder-17.4.0/cinder/volume/manager.py --- cinder-17.0.1/cinder/volume/manager.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/volume/manager.py 2022-04-20 17:38:33.000000000 +0000 @@ -45,6 +45,7 @@ from oslo_service import periodic_task from oslo_utils import excutils from oslo_utils import importutils +from oslo_utils import strutils from oslo_utils import timeutils from oslo_utils import units from oslo_utils import uuidutils @@ -2575,7 +2576,20 @@ resource={'type': 'driver', 'id': self.driver.__class__.__name__}) else: - volume_stats = self.driver.get_volume_stats(refresh=True) + slowmsg = "The " + self.driver.__class__.__name__ + " volume " \ + "driver's get_volume_stats operation ran for " \ + "%(seconds).1f seconds. This may indicate a " \ + "performance problem with the backend which can lead " \ + "to instability." + + @timeutils.time_it( + LOG, log_level=logging.WARN, message=slowmsg, + min_duration=CONF.backend_stats_polling_interval / 2) + def get_stats(): + return self.driver.get_volume_stats(refresh=True) + + volume_stats = get_stats() + if self.extra_capabilities: volume_stats.update(self.extra_capabilities) if volume_stats: @@ -4543,6 +4557,9 @@ self.db.volume_attachment_update(ctxt, attachment.id, values) connection_info['attachment_id'] = attachment.id + LOG.debug("Connection info returned from driver %(connection_info)s", + {'connection_info': + strutils.mask_dict_password(connection_info)}) return connection_info def attachment_update(self, @@ -4687,21 +4704,9 @@ if has_shared_connection is not None and not has_shared_connection: self.driver.remove_export(context.elevated(), vref) except Exception: - # FIXME(jdg): Obviously our volume object is going to need some - # changes to deal with multi-attach and figuring out how to - # represent a single failed attach out of multiple attachments - - # TODO(jdg): object method here - self.db.volume_attachment_update( - context, attachment.get('id'), - {'attach_status': fields.VolumeAttachStatus.ERROR_DETACHING}) - else: - self.db.volume_detached(context.elevated(), vref.id, - attachment.get('id')) - self.db.volume_admin_metadata_delete(context.elevated(), - vref.id, - 'attached_mode') - self._notify_about_volume_usage(context, vref, "detach.end") + # Failures on detach_volume and remove_export are not considered + # failures in terms of detaching the volume. + pass # Replication group API (Tiramisu) def enable_replication(self, ctxt, group): diff -Nru cinder-17.0.1/cinder/volume/rpcapi.py cinder-17.4.0/cinder/volume/rpcapi.py --- cinder-17.0.1/cinder/volume/rpcapi.py 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/cinder/volume/rpcapi.py 2022-04-20 17:38:33.000000000 +0000 @@ -231,9 +231,12 @@ return cctxt.call(ctxt, 'terminate_connection', volume_id=volume['id'], connector=connector, force=force) - def remove_export(self, ctxt, volume): + def remove_export(self, ctxt, volume, sync=False): cctxt = self._get_cctxt(volume.service_topic_queue) - cctxt.cast(ctxt, 'remove_export', volume_id=volume['id']) + if sync: + cctxt.call(ctxt, 'remove_export', volume_id=volume.id) + else: + cctxt.cast(ctxt, 'remove_export', volume_id=volume.id) def publish_service_capabilities(self, ctxt): cctxt = self._get_cctxt(fanout=True) diff -Nru cinder-17.0.1/debian/changelog cinder-17.4.0/debian/changelog --- cinder-17.0.1/debian/changelog 2023-01-18 08:06:59.000000000 +0000 +++ cinder-17.4.0/debian/changelog 2023-05-15 07:36:50.000000000 +0000 @@ -1,12 +1,39 @@ -cinder (2:17.0.1-1+deb11u1) bullseye-security; urgency=high +cinder (2:17.4.0-1~deb11u2) bullseye-security; urgency=medium + + * CVE-2023-2088: Unauthorized volume access through deleted volume + attachments. Applied upstream patch: Reject unsafe delete attachment calls + (Closes: #1035961). + * Above patch temporarily disabled until breackage is fixed. + * Add add-params-thin_provisioning-equal-one.patch. + * CVE-2024-32498: Arbitrary file access through custom QCOW2 external data. + Add upstream patch (Closes: #1074763): + - CVE-2024-32498_0_1_Use_the_json_format_output_of_qemu-img_info.patch + - CVE-2024-32498_1_Check_for_external_qcow2_data_file.patch + * Fix cinder-common.config.in reading glance_api_servers. + * Build-depends on qemu-utils to be able to run additional tests. + * (build-)depends on oslo.utils >= 4.6.1, needed for the CVE fix. + * Correctly calls manage_glance_api_servers() in config script. + + -- Thomas Goirand Mon, 15 May 2023 09:36:50 +0200 + +cinder (2:17.4.0-1~deb11u1) bullseye-security; urgency=high * CVE-2022-47951: By supplying a specially created VMDK flat image which references a specific backing file path, an authenticated user may convince systems to return a copy of that file's contents from the server resulting in unauthorized access to potentially sensitive data. Add upstream patch - cve-2022-47951-cinder-stable-victoria.patch (Closes: #1029562). + cve-2022-47951-cinder-stable-victoria.patch (Closes: #1029561). + + -- Thomas Goirand Wed, 18 Jan 2023 08:36:08 +0100 + +cinder (2:17.4.0-1) unstable; urgency=medium + + * Tune cinder-api-uwsgi.ini for performance. + * New upstream release. + * Fix minimum version of oslo.serialization, as per upstream requirements. + * - -- Thomas Goirand Wed, 18 Jan 2023 09:06:59 +0100 + -- Thomas Goirand Fri, 13 May 2022 12:19:46 +0200 cinder (2:17.0.1-1) unstable; urgency=medium diff -Nru cinder-17.0.1/debian/cinder-api-uwsgi.ini cinder-17.4.0/debian/cinder-api-uwsgi.ini --- cinder-17.0.1/debian/cinder-api-uwsgi.ini 2023-01-18 08:06:59.000000000 +0000 +++ cinder-17.4.0/debian/cinder-api-uwsgi.ini 2023-05-15 07:36:50.000000000 +0000 @@ -12,11 +12,6 @@ # This is running standalone master = true -# Threads and processes -enable-threads = true - -processes = 4 - # uwsgi recommends this to prevent thundering herd on accept. thunder-lock = true @@ -34,6 +29,23 @@ # exit instead of brutal reload on SIGTERM die-on-term = true +########################## +### Performance tuning ### +########################## +# Threads and processes +enable-threads = true + +# For max perf, set this to number of core*2 +processes = 8 + +# This was benchmarked as a good value +threads = 32 + +# This is the number of sockets in the queue. +# It improves a lot performances. This is comparable +# to the Apache ServerLimit/MaxClients option. +listen = 100 + ################################## ### OpenStack service specific ### ################################## diff -Nru cinder-17.0.1/debian/cinder-common.config.in cinder-17.4.0/debian/cinder-common.config.in --- cinder-17.0.1/debian/cinder-common.config.in 2023-01-18 08:06:59.000000000 +0000 +++ cinder-17.4.0/debian/cinder-common.config.in 2023-05-15 07:36:50.000000000 +0000 @@ -32,7 +32,7 @@ manage_glance_api_servers () { pkgos_inifile get ${CINDER_CONF} DEFAULT glance_api_servers - if [ -n "${RET}" ] && [ ! "${RET}" = "NOT_FOUND" ] ; then + if [ -n "${RET}" ] && [ "${RET}" != "NOT_FOUND" ] ; then db_set cinder/glance-api-servers "${RET}" fi db_input high cinder/glance-api-servers || true @@ -58,6 +58,7 @@ fi fi fi +manage_glance_api_servers db_input high cinder/volume_group || true db_go manage_cinder_my_ip diff -Nru cinder-17.0.1/debian/control cinder-17.4.0/debian/control --- cinder-17.0.1/debian/control 2023-01-18 08:06:59.000000000 +0000 +++ cinder-17.4.0/debian/control 2023-05-15 07:36:50.000000000 +0000 @@ -57,10 +57,10 @@ python3-oslo.privsep (>= 2.3.0), python3-oslo.reports, python3-oslo.rootwrap, - python3-oslo.serialization, + python3-oslo.serialization (>= 4.0.2), python3-oslo.service (>= 2.0.0), python3-oslo.upgradecheck, - python3-oslo.utils (>= 3.40.2), + python3-oslo.utils (>= 4.6.1), python3-oslo.versionedobjects, python3-oslo.vmware, python3-oslotest, @@ -95,6 +95,7 @@ python3-tz, python3-webob, python3-zstd, + qemu-utils, subunit, Standards-Version: 4.4.1 Vcs-Browser: https://salsa.debian.org/openstack-team/services/cinder @@ -276,10 +277,10 @@ python3-oslo.privsep (>= 2.3.0), python3-oslo.reports, python3-oslo.rootwrap, - python3-oslo.serialization, + python3-oslo.serialization (>= 4.0.2), python3-oslo.service (>= 2.0.0), python3-oslo.upgradecheck, - python3-oslo.utils (>= 3.40.2), + python3-oslo.utils (>= 5.6.1), python3-oslo.versionedobjects, python3-oslo.vmware, python3-osprofiler, diff -Nru cinder-17.0.1/debian/patches/add-params-thin_provisioning-equal-one.patch cinder-17.4.0/debian/patches/add-params-thin_provisioning-equal-one.patch --- cinder-17.0.1/debian/patches/add-params-thin_provisioning-equal-one.patch 1970-01-01 00:00:00.000000000 +0000 +++ cinder-17.4.0/debian/patches/add-params-thin_provisioning-equal-one.patch 2023-05-15 07:36:50.000000000 +0000 @@ -0,0 +1,17 @@ +Description: Add "params thin_provisioning=1" in tgt export + Add 'params thin_provisioning=1' in iscsi volume export + to allow fstrim to work with the lvm backend. +Author: Thomas Goirand +Forwarded: https://review.opendev.org/c/openstack/cinder/+/900513 +Last-Update: 2023-12-04 + +--- cinder-23.0.0.orig/cinder/volume/targets/tgt.py ++++ cinder-23.0.0/cinder/volume/targets/tgt.py +@@ -42,6 +42,7 @@ class TgtAdm(iscsi.ISCSITarget): + %(chap_auth)s + %(target_flags)s + write-cache %(write_cache)s ++ params thin_provisioning=1 + + """) + diff -Nru cinder-17.0.1/debian/patches/cve-2022-47951-cinder-stable-victoria.patch cinder-17.4.0/debian/patches/cve-2022-47951-cinder-stable-victoria.patch --- cinder-17.0.1/debian/patches/cve-2022-47951-cinder-stable-victoria.patch 2023-01-18 08:06:59.000000000 +0000 +++ cinder-17.4.0/debian/patches/cve-2022-47951-cinder-stable-victoria.patch 2023-05-15 07:36:50.000000000 +0000 @@ -1,14 +1,14 @@ -Description: CVE-2022-47951: Check VMDK subformat against an allowed list +From: Brian Rosmaita +Date: Sat, 14 Jan 2023 07:06:27 -0500 +Description: [PATCH] Check VMDK subformat against an allowed list Also add a more general check to convert_image that the image format reported by qemu-img matches what the caller says it is. . -From: Brian Rosmaita -Date: Sat, 14 Jan 2023 07:06:27 -0500 Change-Id: I3c60ee4c0795aadf03108ed9b5a46ecd116894af -Bug: #1996188 -Debian-Bug: https://bugs.debian.org/1029562 +Bug: https://launchpad.net/bugs/1996188 +Debian-Bug: https://bugs.debian.org/1029561 Origin: upstream, https://review.opendev.org/c/openstack/cinder/+/871628 -Last-Update: 2022-01-30 +Last-Update: 2022-01-18 Index: cinder/cinder/image/image_utils.py =================================================================== @@ -469,7 +469,7 @@ data.backing_file = None data.virtual_size = 1234 tmp = mock_temp.return_value.__enter__.return_value -@@ -1153,7 +1221,9 @@ class TestFetchToVolumeFormat(test.TestC +@@ -1154,7 +1222,9 @@ class TestFetchToVolumeFormat(test.TestC mock_convert.assert_called_once_with(tmp, dest, volume_format, out_subformat=out_subformat, run_as_root=True, @@ -480,7 +480,7 @@ @mock.patch('cinder.image.image_utils.check_virtual_size') @mock.patch('cinder.image.image_utils.check_available_space') -@@ -1171,7 +1241,9 @@ class TestFetchToVolumeFormat(test.TestC +@@ -1172,7 +1242,9 @@ class TestFetchToVolumeFormat(test.TestC mock_is_xen, mock_repl_xen, mock_copy, mock_convert, mock_check_space, mock_check_size): ctxt = mock.sentinel.context @@ -491,7 +491,7 @@ image_id = mock.sentinel.image_id dest = mock.sentinel.dest volume_format = mock.sentinel.volume_format -@@ -1183,7 +1255,7 @@ class TestFetchToVolumeFormat(test.TestC +@@ -1184,7 +1256,7 @@ class TestFetchToVolumeFormat(test.TestC run_as_root = mock.sentinel.run_as_root data = mock_info.return_value @@ -500,7 +500,7 @@ data.backing_file = None data.virtual_size = 1234 tmp = mock_temp.return_value.__enter__.return_value -@@ -1205,7 +1277,9 @@ class TestFetchToVolumeFormat(test.TestC +@@ -1207,7 +1279,9 @@ class TestFetchToVolumeFormat(test.TestC mock_convert.assert_called_once_with(tmp, dest, volume_format, out_subformat=out_subformat, run_as_root=run_as_root, @@ -511,7 +511,7 @@ mock_check_size.assert_called_once_with(data.virtual_size, size, image_id) -@@ -1236,12 +1310,14 @@ class TestFetchToVolumeFormat(test.TestC +@@ -1238,12 +1312,14 @@ class TestFetchToVolumeFormat(test.TestC size = 4321 run_as_root = mock.sentinel.run_as_root @@ -528,7 +528,7 @@ expect_format = 'vpc' output = image_utils.fetch_to_volume_format( -@@ -1261,7 +1337,9 @@ class TestFetchToVolumeFormat(test.TestC +@@ -1264,7 +1340,9 @@ class TestFetchToVolumeFormat(test.TestC mock_convert.assert_called_once_with(tmp, dest, volume_format, out_subformat=out_subformat, run_as_root=run_as_root, @@ -539,7 +539,7 @@ @mock.patch('cinder.image.image_utils.check_virtual_size') @mock.patch('cinder.image.image_utils.check_available_space') -@@ -1288,12 +1366,14 @@ class TestFetchToVolumeFormat(test.TestC +@@ -1291,12 +1369,14 @@ class TestFetchToVolumeFormat(test.TestC size = 4321 run_as_root = mock.sentinel.run_as_root @@ -556,7 +556,7 @@ expect_format = 'raw' output = image_utils.fetch_to_volume_format( -@@ -1312,7 +1392,9 @@ class TestFetchToVolumeFormat(test.TestC +@@ -1316,7 +1396,9 @@ class TestFetchToVolumeFormat(test.TestC mock_convert.assert_called_once_with(tmp, dest, volume_format, out_subformat=out_subformat, run_as_root=run_as_root, @@ -567,7 +567,7 @@ @mock.patch('cinder.image.image_utils.check_available_space', new=mock.Mock()) -@@ -1331,7 +1413,9 @@ class TestFetchToVolumeFormat(test.TestC +@@ -1335,7 +1417,9 @@ class TestFetchToVolumeFormat(test.TestC mock_copy, mock_convert): ctxt = mock.sentinel.context ctxt.user_id = mock.sentinel.user_id @@ -578,7 +578,7 @@ image_id = mock.sentinel.image_id dest = mock.sentinel.dest volume_format = mock.sentinel.volume_format -@@ -1339,7 +1423,7 @@ class TestFetchToVolumeFormat(test.TestC +@@ -1343,7 +1427,7 @@ class TestFetchToVolumeFormat(test.TestC blocksize = mock.sentinel.blocksize data = mock_info.return_value @@ -587,7 +587,7 @@ data.backing_file = None data.virtual_size = 1234 tmp = mock.sentinel.tmp -@@ -1367,7 +1451,9 @@ class TestFetchToVolumeFormat(test.TestC +@@ -1371,7 +1455,9 @@ class TestFetchToVolumeFormat(test.TestC mock_convert.assert_called_once_with(tmp, dest, volume_format, out_subformat=out_subformat, run_as_root=True, @@ -598,7 +598,7 @@ @mock.patch('cinder.image.image_utils.convert_image') @mock.patch('cinder.image.image_utils.volume_utils.copy_volume') -@@ -1672,7 +1758,9 @@ class TestFetchToVolumeFormat(test.TestC +@@ -1682,7 +1768,9 @@ class TestFetchToVolumeFormat(test.TestC mock_copy, mock_convert, mock_check_space, mock_check_size): ctxt = mock.sentinel.context @@ -609,7 +609,7 @@ image_id = mock.sentinel.image_id dest = mock.sentinel.dest volume_format = mock.sentinel.volume_format -@@ -1683,7 +1771,7 @@ class TestFetchToVolumeFormat(test.TestC +@@ -1693,7 +1781,7 @@ class TestFetchToVolumeFormat(test.TestC run_as_root = mock.sentinel.run_as_root data = mock_info.return_value @@ -618,7 +618,7 @@ data.backing_file = None data.virtual_size = 1234 tmp = mock_temp.return_value.__enter__.return_value -@@ -1705,7 +1793,9 @@ class TestFetchToVolumeFormat(test.TestC +@@ -1716,7 +1804,9 @@ class TestFetchToVolumeFormat(test.TestC mock_convert.assert_called_once_with(tmp, dest, volume_format, out_subformat=None, run_as_root=run_as_root, @@ -629,7 +629,7 @@ @mock.patch('cinder.image.image_utils.fetch') @mock.patch('cinder.image.image_utils.qemu_img_info', -@@ -1831,7 +1921,9 @@ class TestFetchToVolumeFormat(test.TestC +@@ -1842,7 +1932,9 @@ class TestFetchToVolumeFormat(test.TestC ctxt = mock.sentinel.context ctxt.user_id = mock.sentinel.user_id @@ -640,7 +640,7 @@ image_id = mock.sentinel.image_id dest = mock.sentinel.dest volume_format = mock.sentinel.volume_format -@@ -1839,7 +1931,7 @@ class TestFetchToVolumeFormat(test.TestC +@@ -1850,7 +1942,7 @@ class TestFetchToVolumeFormat(test.TestC blocksize = mock.sentinel.blocksize data = mock_info.return_value @@ -649,7 +649,7 @@ data.backing_file = None data.virtual_size = 1234 tmp = mock_temp.return_value.__enter__.return_value -@@ -1863,7 +1955,9 @@ class TestFetchToVolumeFormat(test.TestC +@@ -1875,7 +1967,9 @@ class TestFetchToVolumeFormat(test.TestC mock_convert.assert_called_once_with(tmp, dest, volume_format, out_subformat=out_subformat, run_as_root=True, @@ -660,7 +660,7 @@ mock_engine.decompress_img.assert_called() -@@ -2162,3 +2256,300 @@ class TestImageUtils(test.TestCase): +@@ -2174,3 +2268,300 @@ class TestImageUtils(test.TestCase): 'ivgen_alg': 'essiv'} result = image_utils.decode_cipher('aes-xts-essiv', 256) self.assertEqual(expected, result) diff -Nru cinder-17.0.1/debian/patches/CVE-2023-2088_Reject_unsafe_delete_attachment_calls.patch cinder-17.4.0/debian/patches/CVE-2023-2088_Reject_unsafe_delete_attachment_calls.patch --- cinder-17.0.1/debian/patches/CVE-2023-2088_Reject_unsafe_delete_attachment_calls.patch 1970-01-01 00:00:00.000000000 +0000 +++ cinder-17.4.0/debian/patches/CVE-2023-2088_Reject_unsafe_delete_attachment_calls.patch 2023-05-15 07:36:50.000000000 +0000 @@ -0,0 +1,956 @@ +Author: Gorka Eguileor +Date: Thu, 16 Feb 2023 15:57:15 +0100 +Subject: CVE-2023-2088 Reject unsafe delete attachment calls + Due to how the Linux SCSI kernel driver works there are some storage + systems, such as iSCSI with shared targets, where a normal user can + access other projects' volume data connected to the same compute host + using the attachments REST API. + . + This affects both single and multi-pathed connections. + . + To prevent users from doing this, unintentionally or maliciously, + cinder-api will now reject some delete attachment requests that are + deemed unsafe. + . + Cinder will process the delete attachment request normally in the + following cases: + . + - The request comes from an OpenStack service that is sending the + service token that has one of the roles in `service_token_roles`. + - Attachment doesn't have an instance_uuid value + - The instance for the attachment doesn't exist in Nova + - According to Nova the volume is not connected to the instance + - Nova is not using this attachment record + . + There are 3 operations in the actions REST API endpoint that can be used + for an attack: + . + - `os-terminate_connection`: Terminate volume attachment + - `os-detach`: Detach a volume + - `os-force_detach`: Force detach a volume + . + In this endpoint we just won't allow most requests not coming from a + service. The rules we apply are the same as for attachment delete + explained earlier, but in this case we may not have the attachment id + and be more restrictive. This should not be a problem for normal + operations because: + . + - Cinder backup doesn't use the REST API but RPC calls via RabbitMQ + - Glance doesn't use this interface + . + Checking whether it's a service or not is done at the cinder-api level + by checking that the service user that made the call has at least one of + the roles in the `service_token_roles` configuration. These roles are + retrieved from keystone by the keystone middleware using the value of + the "X-Service-Token" header. + . + If Cinder is configured with `service_token_roles_required = true` and + an attacker provides non-service valid credentials the service will + return a 401 error, otherwise it'll return 409 as if a normal user had + made the call without the service token. +Bug: https://launchpad.net/bugs/2004555 +Bug-Debian: https://bugs.debian.org/1035961 +Change-Id: I612905a1bf4a1706cce913c0d8a6df7a240d599a +Origin: upstream, https://review.opendev.org/c/openstack/cinder/+/882839 +Last-Update: 2023-05-12 + +Index: cinder/api-ref/source/v3/attachments.inc +=================================================================== +--- cinder.orig/api-ref/source/v3/attachments.inc ++++ cinder/api-ref/source/v3/attachments.inc +@@ -41,6 +41,20 @@ Delete attachment + + Deletes an attachment. + ++For security reasons (see bug `#2004555 ++`_) the Block Storage API rejects ++REST API calls manually made from users with a 409 status code if there is a ++Nova instance currently using the attachment, which happens when all the ++following conditions are met: ++ ++- Attachment has an instance uuid ++- VM exists in Nova ++- Instance has the volume attached ++- Attached volume in instance is using the attachment ++ ++Calls coming from other OpenStack services (like the Compute Service) are ++always accepted. ++ + Available starting in the 3.27 microversion. + + Response codes +@@ -54,6 +68,7 @@ Response codes + + - 400 + - 404 ++ - 409 + + + Request +Index: cinder/api-ref/source/v3/volumes-v3-volumes-actions.inc +=================================================================== +--- cinder.orig/api-ref/source/v3/volumes-v3-volumes-actions.inc ++++ cinder/api-ref/source/v3/volumes-v3-volumes-actions.inc +@@ -337,6 +337,21 @@ Preconditions + + - Volume status must be ``in-use``. + ++For security reasons (see bug `#2004555 ++`_), regardless of the policy ++defaults, the Block Storage API rejects REST API calls manually made from ++users with a 409 status code if completing the request could pose a risk, which ++happens if all of these happen: ++ ++- The request comes from a user ++- There's an instance uuid in provided attachment or in the volume's attachment ++- VM exists in Nova ++- Instance has the volume attached ++- Attached volume in instance is using the attachment ++ ++Calls coming from other OpenStack services (like the Compute Service) are ++always accepted. ++ + Response codes + -------------- + +@@ -344,6 +359,9 @@ Response codes + + - 202 + ++.. rest_status_code:: error ../status.yaml ++ ++ - 409 + + Request + ------- +@@ -415,6 +433,21 @@ perform this operation. Cloud providers + through the ``volume_extension:volume_admin_actions:force_detach`` rule in + the policy configuration file. + ++For security reasons (see bug `#2004555 ++`_), regardless of the policy ++defaults, the Block Storage API rejects REST API calls manually made from ++users with a 409 status code if completing the request could pose a risk, which ++happens if all of these happen: ++ ++- The request comes from a user ++- There's an instance uuid in provided attachment or in the volume's attachment ++- VM exists in Nova ++- Instance has the volume attached ++- Attached volume in instance is using the attachment ++ ++Calls coming from other OpenStack services (like the Compute Service) are ++always accepted. ++ + Response codes + -------------- + +@@ -422,6 +455,9 @@ Response codes + + - 202 + ++.. rest_status_code:: error ../status.yaml ++ ++ - 409 + + Request + ------- +@@ -883,6 +919,22 @@ Preconditions + + - Volume status must be ``in-use``. + ++For security reasons (see bug `#2004555 ++`_), regardless of the policy ++defaults, the Block Storage API rejects REST API calls manually made from ++users with a 409 status code if completing the request could pose a risk, which ++happens if all of these happen: ++ ++- The request comes from a user ++- There's an instance uuid in the volume's attachment ++- VM exists in Nova ++- Instance has the volume attached ++- Attached volume in instance is using the attachment ++ ++Calls coming from other OpenStack services (like the Compute Service) are ++always accepted. ++ ++ + Response codes + -------------- + +@@ -890,6 +942,9 @@ Response codes + + - 202 + ++.. rest_status_code:: error ../status.yaml ++ ++ - 409 + + Request + ------- +Index: cinder/cinder/compute/nova.py +=================================================================== +--- cinder.orig/cinder/compute/nova.py ++++ cinder/cinder/compute/nova.py +@@ -133,6 +133,7 @@ def novaclient(context, privileged_user= + + class API(base.Base): + """API for interacting with novaclient.""" ++ NotFound = nova_exceptions.NotFound + + def __init__(self): + self.message_api = message_api.API() +@@ -218,3 +219,9 @@ class API(base.Base): + resource_uuid=volume_id, + detail=message_field.Detail.NOTIFY_COMPUTE_SERVICE_FAILED) + return result ++ ++ @staticmethod ++ def get_server_volume(context, server_id, volume_id): ++ # Use microversion that includes attachment_id ++ nova = novaclient(context, api_version='2.89') ++ return nova.volumes.get_server_volume(server_id, volume_id) +Index: cinder/cinder/exception.py +=================================================================== +--- cinder.orig/cinder/exception.py ++++ cinder/cinder/exception.py +@@ -1089,3 +1089,10 @@ class CinderAcceleratorError(CinderExcep + class SnapshotLimitReached(CinderException): + message = _("Exceeded the configured limit of " + "%(set_limit)s snapshots per volume.") ++ ++ ++class ConflictNovaUsingAttachment(CinderException): ++ message = _("Detach volume from instance %(instance_id)s using the " ++ "Compute API") ++ code = 409 ++ safe = True +Index: cinder/cinder/tests/unit/api/contrib/test_admin_actions.py +=================================================================== +--- cinder.orig/cinder/tests/unit/api/contrib/test_admin_actions.py ++++ cinder/cinder/tests/unit/api/contrib/test_admin_actions.py +@@ -972,6 +972,8 @@ class AdminActionsAttachDetachTest(BaseA + super(AdminActionsAttachDetachTest, self).setUp() + # start service to handle rpc messages for attach requests + self.svc = self.start_service('volume', host='test') ++ self.mock_deletion_allowed = self.mock_object( ++ volume_api.API, 'attachment_deletion_allowed', return_value=None) + + def tearDown(self): + self.svc.stop() +@@ -1027,6 +1029,16 @@ class AdminActionsAttachDetachTest(BaseA + admin_metadata = volume.admin_metadata + self.assertEqual(1, len(admin_metadata)) + self.assertEqual('False', admin_metadata['readonly']) ++ # One call is for the terminate_connection and the other is for the ++ # detach ++ self.assertEqual(2, self.mock_deletion_allowed.call_count) ++ self.mock_deletion_allowed.assert_has_calls( ++ [mock.call(self.ctx, None, mock.ANY), ++ mock.call(self.ctx, attachment['id'], mock.ANY)]) ++ for i in (0, 1): ++ self.assertIsInstance( ++ self.mock_deletion_allowed.call_args_list[i][0][2], ++ objects.Volume) + + def test_force_detach_host_attached_volume(self): + # current status is available +@@ -1078,6 +1090,16 @@ class AdminActionsAttachDetachTest(BaseA + admin_metadata = volume['admin_metadata'] + self.assertEqual(1, len(admin_metadata)) + self.assertEqual('False', admin_metadata['readonly']) ++ # One call is for the terminate_connection and the other is for the ++ # detach ++ self.assertEqual(2, self.mock_deletion_allowed.call_count) ++ self.mock_deletion_allowed.assert_has_calls( ++ [mock.call(self.ctx, None, mock.ANY), ++ mock.call(self.ctx, attachment['id'], mock.ANY)]) ++ for i in (0, 1): ++ self.assertIsInstance( ++ self.mock_deletion_allowed.call_args_list[i][0][2], ++ objects.Volume) + + def test_volume_force_detach_raises_remote_error(self): + # current status is available +@@ -1121,6 +1143,10 @@ class AdminActionsAttachDetachTest(BaseA + resp = req.get_response(app()) + self.assertEqual(http_client.BAD_REQUEST, resp.status_int) + ++ self.mock_deletion_allowed.assert_called_once_with( ++ self.ctx, None, volume) ++ self.mock_deletion_allowed.reset_mock() ++ + # test for VolumeBackendAPIException + volume_remote_error = ( + messaging.RemoteError(exc_type='VolumeBackendAPIException')) +@@ -1140,6 +1166,8 @@ class AdminActionsAttachDetachTest(BaseA + self.assertRaises(messaging.RemoteError, + req.get_response, + app()) ++ self.mock_deletion_allowed.assert_called_once_with( ++ self.ctx, None, volume) + + def test_volume_force_detach_raises_db_error(self): + # In case of DB error 500 error code is returned to user +@@ -1185,6 +1213,8 @@ class AdminActionsAttachDetachTest(BaseA + self.assertRaises(messaging.RemoteError, + req.get_response, + app()) ++ self.mock_deletion_allowed.assert_called_once_with( ++ self.ctx, None, volume) + + def test_volume_force_detach_missing_connector(self): + # current status is available +@@ -1225,6 +1255,8 @@ class AdminActionsAttachDetachTest(BaseA + # make request + resp = req.get_response(app()) + self.assertEqual(http_client.ACCEPTED, resp.status_int) ++ self.mock_deletion_allowed.assert_called_once_with( ++ self.ctx, None, volume) + + def test_attach_in_used_volume_by_instance(self): + """Test that attaching to an in-use volume fails.""" +Index: cinder/cinder/tests/unit/api/v3/test_attachments.py +=================================================================== +--- cinder.orig/cinder/tests/unit/api/v3/test_attachments.py ++++ cinder/cinder/tests/unit/api/v3/test_attachments.py +@@ -190,6 +190,8 @@ class AttachmentsAPITestCase(test.TestCa + @ddt.data('reserved', 'attached') + @mock.patch.object(volume_rpcapi.VolumeAPI, 'attachment_delete') + def test_delete_attachment(self, status, mock_delete): ++ self.patch('cinder.volume.api.API.attachment_deletion_allowed', ++ return_value=None) + volume1 = self._create_volume(display_name='fake_volume_1', + project_id=fake.PROJECT_ID) + attachment = self._create_attachment( +Index: cinder/cinder/tests/unit/attachments/test_attachments_api.py +=================================================================== +--- cinder.orig/cinder/tests/unit/attachments/test_attachments_api.py ++++ cinder/cinder/tests/unit/attachments/test_attachments_api.py +@@ -15,6 +15,7 @@ from unittest import mock + from oslo_config import cfg + from oslo_policy import policy as oslo_policy + ++from cinder.compute import nova + from cinder import context + from cinder import db + from cinder import exception +@@ -24,6 +25,7 @@ from cinder.policies import base as base + from cinder import policy + from cinder.tests.unit.api.v2 import fakes as v2_fakes + from cinder.tests.unit import fake_constants as fake ++from cinder.tests.unit import fake_volume + from cinder.tests.unit import test + from cinder.tests.unit import utils as tests_utils + from cinder.volume import api as volume_api +@@ -86,10 +88,13 @@ class AttachmentManagerTestCase(test.Tes + attachment.id) + self.assertEqual(connection_info, new_attachment.connection_info) + ++ @mock.patch.object(volume_api.API, 'attachment_deletion_allowed') + @mock.patch('cinder.volume.rpcapi.VolumeAPI.attachment_delete') + def test_attachment_delete_reserved(self, +- mock_rpc_attachment_delete): ++ mock_rpc_attachment_delete, ++ mock_allowed): + """Test attachment_delete with reserved.""" ++ mock_allowed.return_value = None + volume_params = {'status': 'available'} + + vref = tests_utils.create_volume(self.context, **volume_params) +@@ -102,18 +107,22 @@ class AttachmentManagerTestCase(test.Tes + self.assertEqual(vref.id, aref.volume_id) + self.volume_api.attachment_delete(self.context, + aobj) ++ mock_allowed.assert_called_once_with(self.context, aobj) + + # Since it's just reserved and never finalized, we should never make an + # rpc call + mock_rpc_attachment_delete.assert_not_called() + ++ @mock.patch.object(volume_api.API, 'attachment_deletion_allowed') + @mock.patch('cinder.volume.rpcapi.VolumeAPI.attachment_delete') + @mock.patch('cinder.volume.rpcapi.VolumeAPI.attachment_update') + def test_attachment_create_update_and_delete( + self, + mock_rpc_attachment_update, +- mock_rpc_attachment_delete): ++ mock_rpc_attachment_delete, ++ mock_allowed): + """Test attachment_delete.""" ++ mock_allowed.return_value = None + volume_params = {'status': 'available'} + connection_info = {'fake_key': 'fake_value', + 'fake_key2': ['fake_value1', 'fake_value2']} +@@ -150,6 +159,7 @@ class AttachmentManagerTestCase(test.Tes + self.volume_api.attachment_delete(self.context, + aref) + ++ mock_allowed.assert_called_once_with(self.context, aref) + mock_rpc_attachment_delete.assert_called_once_with(self.context, + aref.id, + mock.ANY) +@@ -181,10 +191,13 @@ class AttachmentManagerTestCase(test.Tes + vref.id) + self.assertEqual(2, len(vref.volume_attachment)) + ++ @mock.patch.object(volume_api.API, 'attachment_deletion_allowed') + @mock.patch('cinder.volume.rpcapi.VolumeAPI.attachment_update') + def test_attachment_create_reserve_delete( + self, +- mock_rpc_attachment_update): ++ mock_rpc_attachment_update, ++ mock_allowed): ++ mock_allowed.return_value = None + volume_params = {'status': 'available'} + connector = { + "initiator": "iqn.1993-08.org.debian:01:cad181614cec", +@@ -219,12 +232,15 @@ class AttachmentManagerTestCase(test.Tes + # attachments reserve + self.volume_api.attachment_delete(self.context, + aref) ++ mock_allowed.assert_called_once_with(self.context, aref) + vref = objects.Volume.get_by_id(self.context, + vref.id) + self.assertEqual('reserved', vref.status) + +- def test_reserve_reserve_delete(self): ++ @mock.patch.object(volume_api.API, 'attachment_deletion_allowed') ++ def test_reserve_reserve_delete(self, mock_allowed): + """Test that we keep reserved status across multiple reserves.""" ++ mock_allowed.return_value = None + volume_params = {'status': 'available'} + + vref = tests_utils.create_volume(self.context, **volume_params) +@@ -243,6 +259,7 @@ class AttachmentManagerTestCase(test.Tes + self.assertEqual('reserved', vref.status) + self.volume_api.attachment_delete(self.context, + aref) ++ mock_allowed.assert_called_once_with(self.context, aref) + vref = objects.Volume.get_by_id(self.context, + vref.id) + self.assertEqual('reserved', vref.status) +@@ -372,3 +389,169 @@ class AttachmentManagerTestCase(test.Tes + self.context, + vref, + fake.UUID1) ++ ++ def _get_attachment(self, with_instance_id=True): ++ volume = fake_volume.fake_volume_obj(self.context, id=fake.VOLUME_ID) ++ volume.volume_attachment = objects.VolumeAttachmentList() ++ attachment = fake_volume.volume_attachment_ovo( ++ self.context, ++ volume_id=fake.VOLUME_ID, ++ instance_uuid=fake.INSTANCE_ID if with_instance_id else None, ++ connection_info='{"a": 1}') ++ attachment.volume = volume ++ return attachment ++ ++ @mock.patch('cinder.compute.nova.API.get_server_volume') ++ def test_attachment_deletion_allowed_service_call(self, mock_get_server): ++ """Service calls are never redirected.""" ++ self.context.service_roles = ['reader', 'service'] ++ attachment = self._get_attachment() ++ self.volume_api.attachment_deletion_allowed(self.context, attachment) ++ mock_get_server.assert_not_called() ++ ++ @mock.patch('cinder.compute.nova.API.get_server_volume') ++ def test_attachment_deletion_allowed_service_call_different_service_name( ++ self, mock_get_server): ++ """Service calls are never redirected and role can be different. ++ ++ In this test we support 2 different service roles, the standard service ++ and a custom one called captain_awesome, and passing the custom one ++ works as expected. ++ """ ++ self.override_config('service_token_roles', ++ ['service', 'captain_awesome'], ++ group='keystone_authtoken') ++ ++ self.context.service_roles = ['reader', 'captain_awesome'] ++ attachment = self._get_attachment() ++ self.volume_api.attachment_deletion_allowed(self.context, attachment) ++ mock_get_server.assert_not_called() ++ ++ @mock.patch('cinder.compute.nova.API.get_server_volume') ++ def test_attachment_deletion_allowed_no_instance(self, mock_get_server): ++ """Attachments with no instance id are never redirected.""" ++ attachment = self._get_attachment(with_instance_id=False) ++ self.volume_api.attachment_deletion_allowed(self.context, attachment) ++ mock_get_server.assert_not_called() ++ ++ @mock.patch('cinder.compute.nova.API.get_server_volume') ++ def test_attachment_deletion_allowed_no_conn_info(self, mock_get_server): ++ """Attachments with no connection information are never redirected.""" ++ attachment = self._get_attachment(with_instance_id=False) ++ attachment.connection_info = None ++ self.volume_api.attachment_deletion_allowed(self.context, attachment) ++ ++ mock_get_server.assert_not_called() ++ ++ def test_attachment_deletion_allowed_no_attachment(self): ++ """For users don't allow operation with no attachment reference.""" ++ self.assertRaises(exception.ConflictNovaUsingAttachment, ++ self.volume_api.attachment_deletion_allowed, ++ self.context, None) ++ ++ @mock.patch('cinder.objects.VolumeAttachment.get_by_id', ++ side_effect=exception.VolumeAttachmentNotFound()) ++ def test_attachment_deletion_allowed_attachment_id_not_found(self, ++ mock_get): ++ """For users don't allow if attachment cannot be found.""" ++ attachment = self._get_attachment(with_instance_id=False) ++ attachment.connection_info = None ++ self.assertRaises(exception.ConflictNovaUsingAttachment, ++ self.volume_api.attachment_deletion_allowed, ++ self.context, fake.ATTACHMENT_ID) ++ mock_get.assert_called_once_with(self.context, fake.ATTACHMENT_ID) ++ ++ def test_attachment_deletion_allowed_volume_no_attachments(self): ++ """For users allow if volume has no attachments.""" ++ volume = tests_utils.create_volume(self.context) ++ self.volume_api.attachment_deletion_allowed(self.context, None, volume) ++ ++ def test_attachment_deletion_allowed_multiple_attachment(self): ++ """For users don't allow if volume has multiple attachments.""" ++ attachment = self._get_attachment() ++ volume = attachment.volume ++ volume.volume_attachment = objects.VolumeAttachmentList( ++ objects=[attachment, attachment]) ++ self.assertRaises(exception.ConflictNovaUsingAttachment, ++ self.volume_api.attachment_deletion_allowed, ++ self.context, None, volume) ++ ++ @mock.patch('cinder.compute.nova.API.get_server_volume') ++ def test_attachment_deletion_allowed_vm_not_found(self, mock_get_server): ++ """Don't reject if instance doesn't exist""" ++ mock_get_server.side_effect = nova.API.NotFound(404) ++ attachment = self._get_attachment() ++ self.volume_api.attachment_deletion_allowed(self.context, attachment) ++ ++ mock_get_server.assert_called_once_with(self.context, fake.INSTANCE_ID, ++ fake.VOLUME_ID) ++ ++ @mock.patch('cinder.compute.nova.API.get_server_volume') ++ def test_attachment_deletion_allowed_attachment_from_volume( ++ self, mock_get_server): ++ """Don't reject if instance doesn't exist""" ++ mock_get_server.side_effect = nova.API.NotFound(404) ++ attachment = self._get_attachment() ++ volume = attachment.volume ++ volume.volume_attachment = objects.VolumeAttachmentList( ++ objects=[attachment]) ++ self.volume_api.attachment_deletion_allowed(self.context, None, volume) ++ ++ mock_get_server.assert_called_once_with(self.context, fake.INSTANCE_ID, ++ volume.id) ++ ++ @mock.patch('cinder.objects.VolumeAttachment.get_by_id') ++ def test_attachment_deletion_allowed_mismatched_volume_and_attach_id( ++ self, mock_get_attatchment): ++ """Reject if volume and attachment don't match.""" ++ attachment = self._get_attachment() ++ volume = attachment.volume ++ volume.volume_attachment = objects.VolumeAttachmentList( ++ objects=[attachment]) ++ attachment2 = self._get_attachment() ++ attachment2.volume_id = attachment.volume.id = fake.VOLUME2_ID ++ self.assertRaises(exception.InvalidInput, ++ self.volume_api.attachment_deletion_allowed, ++ self.context, attachment2.id, volume) ++ mock_get_attatchment.assert_called_once_with(self.context, ++ attachment2.id) ++ ++ @mock.patch('cinder.objects.VolumeAttachment.get_by_id') ++ @mock.patch('cinder.compute.nova.API.get_server_volume') ++ def test_attachment_deletion_allowed_not_found_attachment_id( ++ self, mock_get_server, mock_get_attachment): ++ """Don't reject if instance doesn't exist""" ++ mock_get_server.side_effect = nova.API.NotFound(404) ++ mock_get_attachment.return_value = self._get_attachment() ++ ++ self.volume_api.attachment_deletion_allowed(self.context, ++ fake.ATTACHMENT_ID) ++ ++ mock_get_attachment.assert_called_once_with(self.context, ++ fake.ATTACHMENT_ID) ++ ++ mock_get_server.assert_called_once_with(self.context, fake.INSTANCE_ID, ++ fake.VOLUME_ID) ++ ++ @mock.patch('cinder.compute.nova.API.get_server_volume') ++ def test_attachment_deletion_allowed_mismatch_id(self, mock_get_server): ++ """Don't reject if attachment id on nova doesn't match""" ++ mock_get_server.return_value.attachment_id = fake.ATTACHMENT2_ID ++ attachment = self._get_attachment() ++ self.volume_api.attachment_deletion_allowed(self.context, attachment) ++ ++ mock_get_server.assert_called_once_with(self.context, fake.INSTANCE_ID, ++ fake.VOLUME_ID) ++ ++ @mock.patch('cinder.compute.nova.API.get_server_volume') ++ def test_attachment_deletion_allowed_user_call_fails(self, ++ mock_get_server): ++ """Fail user calls""" ++ attachment = self._get_attachment() ++ mock_get_server.return_value.attachment_id = attachment.id ++ self.assertRaises(exception.ConflictNovaUsingAttachment, ++ self.volume_api.attachment_deletion_allowed, ++ self.context, attachment) ++ ++ mock_get_server.assert_called_once_with(self.context, fake.INSTANCE_ID, ++ fake.VOLUME_ID) +Index: cinder/cinder/tests/unit/policies/test_volume_actions.py +=================================================================== +--- cinder.orig/cinder/tests/unit/policies/test_volume_actions.py ++++ cinder/cinder/tests/unit/policies/test_volume_actions.py +@@ -261,6 +261,7 @@ class VolumeProtectionTests(test_base.Ci + self.assertEqual(http_client.ACCEPTED, response.status_int) + + body = {"os-detach": {}} ++ # Detach for user call succeeds because the volume has no attachments + response = self._get_request_response(admin_context, path, 'POST', + body=body) + self.assertEqual(http_client.ACCEPTED, response.status_int) +@@ -281,6 +282,7 @@ class VolumeProtectionTests(test_base.Ci + body=body) + self.assertEqual(http_client.ACCEPTED, response.status_int) + ++ # Succeeds for a user call because there are no attachments + body = {"os-detach": {}} + response = self._get_request_response(user_context, path, 'POST', + body=body) +@@ -377,6 +379,7 @@ class VolumeProtectionTests(test_base.Ci + 'terminate_connection') + def test_admin_can_initialize_terminate_conn(self, mock_t, mock_i): + admin_context = self.admin_context ++ admin_context.service_roles = ['service'] + + volume = self._create_fake_volume(admin_context) + path = '/v3/%(project_id)s/volumes/%(volume_id)s/action' % { +@@ -399,6 +402,7 @@ class VolumeProtectionTests(test_base.Ci + 'terminate_connection') + def test_owner_can_initialize_terminate_conn(self, mock_t, mock_i): + user_context = self.user_context ++ user_context.service_roles = ['service'] + + volume = self._create_fake_volume(user_context) + path = '/v3/%(project_id)s/volumes/%(volume_id)s/action' % { +Index: cinder/cinder/tests/unit/volume/test_connection.py +=================================================================== +--- cinder.orig/cinder/tests/unit/volume/test_connection.py ++++ cinder/cinder/tests/unit/volume/test_connection.py +@@ -1334,7 +1334,8 @@ class VolumeAttachDetachTestCase(base.Ba + self.context, + volume, None, None, None, None) + +- def test_volume_detach_in_maintenance(self): ++ @mock.patch('cinder.volume.api.API.attachment_deletion_allowed') ++ def test_volume_detach_in_maintenance(self, mock_attachment_deletion): + """Test detach the volume in maintenance.""" + test_meta1 = {'fake_key1': 'fake_value1', 'fake_key2': 'fake_value2'} + volume = tests_utils.create_volume(self.context, metadata=test_meta1, +@@ -1345,3 +1346,5 @@ class VolumeAttachDetachTestCase(base.Ba + volume_api.detach, + self.context, + volume, None) ++ mock_attachment_deletion.assert_called_once_with(self.context, ++ None, volume) +Index: cinder/cinder/volume/api.py +=================================================================== +--- cinder.orig/cinder/volume/api.py ++++ cinder/cinder/volume/api.py +@@ -31,6 +31,7 @@ import six + + from cinder.api import common + from cinder.common import constants ++from cinder import compute + from cinder import context + from cinder import coordination + from cinder import db +@@ -787,11 +788,14 @@ class API(base.Base): + def detach(self, context, volume, attachment_id): + context.authorize(vol_action_policy.DETACH_POLICY, + target_obj=volume) ++ self.attachment_deletion_allowed(context, attachment_id, volume) ++ + if volume['status'] == 'maintenance': + LOG.info('Unable to detach volume, ' + 'because it is in maintenance.', resource=volume) + msg = _("The volume cannot be detached in maintenance mode.") + raise exception.InvalidVolume(reason=msg) ++ + detach_results = self.volume_rpcapi.detach_volume(context, volume, + attachment_id) + LOG.info("Detach volume completed successfully.", +@@ -815,9 +819,24 @@ class API(base.Base): + resource=volume) + return init_results + ++ @staticmethod ++ def is_service_request(ctxt: 'context.RequestContext') -> bool: ++ """Check if a request is coming from a service ++ ++ A request is coming from a service if it has a service token and the ++ service user has one of the roles configured in the ++ `service_token_roles` configuration option in the ++ `[keystone_authtoken]` section (defaults to `service`). ++ """ ++ roles = ctxt.service_roles ++ service_roles = set(CONF.keystone_authtoken.service_token_roles) ++ return bool(roles and service_roles.intersection(roles)) ++ + def terminate_connection(self, context, volume, connector, force=False): + context.authorize(vol_action_policy.TERMINATE_POLICY, + target_obj=volume) ++ self.attachment_deletion_allowed(context, None, volume) ++ + self.volume_rpcapi.terminate_connection(context, + volume, + connector, +@@ -2258,9 +2277,88 @@ class API(base.Base): + attachment_ref.save() + return attachment_ref + ++ def attachment_deletion_allowed(self, ++ ctxt: context.RequestContext, ++ attachment_or_attachment_id, ++ volume=None): ++ """Check if deleting an attachment is allowed (Bug #2004555) ++ ++ Allowed is based on the REST API policy, the status of the attachment, ++ where it is used, and who is making the request. ++ ++ Deleting an attachment on the Cinder side while leaving the volume ++ connected to the nova host results in leftover devices that can lead to ++ data leaks/corruption. ++ ++ OS-Brick may have code to detect it, but in some cases it is detected ++ after it has already been exposed, so it's better to prevent users from ++ being able to intentionally triggering the issue. ++ """ ++ # It's ok to delete an attachment if the request comes from a service ++ if self.is_service_request(ctxt): ++ return ++ ++ if not attachment_or_attachment_id and volume: ++ if not volume.volume_attachment: ++ return ++ if len(volume.volume_attachment) == 1: ++ attachment_or_attachment_id = volume.volume_attachment[0] ++ ++ if isinstance(attachment_or_attachment_id, str): ++ try: ++ attachment = objects.VolumeAttachment.get_by_id( ++ ctxt, attachment_or_attachment_id) ++ except exception.VolumeAttachmentNotFound: ++ attachment = None ++ else: ++ attachment = attachment_or_attachment_id ++ ++ if attachment: ++ if volume: ++ if volume.id != attachment.volume_id: ++ raise exception.InvalidInput( ++ reason='Mismatched volume and attachment') ++ ++ server_id = attachment.instance_uuid ++ # It's ok to delete if it's not connected to a vm. ++ if not server_id or not attachment.connection_info: ++ return ++ ++ volume = volume or attachment.volume ++ nova = compute.API() ++ LOG.info('Attachment connected to vm %s, checking data on nova', ++ server_id) ++ # If nova is down the client raises 503 and we report that ++ try: ++ nova_volume = nova.get_server_volume(ctxt, server_id, ++ volume.id) ++ except nova.NotFound: ++ LOG.warning('Instance or volume not found on Nova, deleting ' ++ 'attachment locally, which may leave leftover ' ++ 'devices on Nova compute') ++ return ++ ++ if nova_volume.attachment_id != attachment.id: ++ LOG.warning('Mismatch! Nova has different attachment id (%s) ' ++ 'for the volume, deleting attachment locally. ' ++ 'May leave leftover devices in a compute node', ++ nova_volume.attachment_id) ++ return ++ else: ++ server_id = '' ++ ++ LOG.error('Detected user call to delete in-use attachment. Call must ' ++ 'come from the nova service and nova must be configured to ' ++ 'send the service token. Bug #2004555') ++ raise exception.ConflictNovaUsingAttachment(instance_id=server_id) ++ + def attachment_delete(self, ctxt, attachment): ++ # Check if policy allows user to delete attachment + ctxt.authorize(attachment_policy.DELETE_POLICY, + target_obj=attachment) ++ ++ self.attachment_deletion_allowed(ctxt, attachment) ++ + volume = objects.Volume.get_by_id(ctxt, attachment.volume_id) + + if attachment.attach_status == fields.VolumeAttachStatus.RESERVED: +Index: cinder/doc/source/configuration/block-storage/service-token.rst +=================================================================== +--- cinder.orig/doc/source/configuration/block-storage/service-token.rst ++++ cinder/doc/source/configuration/block-storage/service-token.rst +@@ -1,6 +1,6 @@ +-========================================================= +-Using service tokens to prevent long-running job failures +-========================================================= ++==================== ++Using service tokens ++==================== + + When a user initiates a request whose processing involves multiple services + (for example, a boot-from-volume request to the Compute Service will require +@@ -8,20 +8,32 @@ processing by the Block Storage Service, + Image Service), the user's token is handed from service to service. This + ensures that the requestor is tracked correctly for audit purposes and also + guarantees that the requestor has the appropriate permissions to do what needs +-to be done by the other services. If the chain of operations takes a long +-time, however, the user's token may expire before the action is completed, +-leading to the failure of the user's original request. +- +-One way to deal with this is to set a long token life in Keystone, and this may +-be what you are currently doing. But this can be problematic for installations +-whose security policies prefer short user token lives. Beginning with the +-Queens release, an alternative solution is available. You have the ability to +-configure some services (particularly Nova and Cinder) to send a "service +-token" along with the user's token. When properly configured, the Identity +-Service will validate an expired user token *when it is accompanied by a valid +-service token*. Thus if the user's token expires somewhere during a long +-running chain of operations among various OpenStack services, the operations +-can continue. ++to be done by the other services. ++ ++There are several instances where we want to differentiate between a request ++coming from the user to one coming from another OpenStack service on behalf of ++the user: ++ ++- **For security reasons** There are some operations in the Block Storage ++ service, required for normal operations, that could be exploited by a ++ malicious user to gain access to resources belonging to other users. By ++ differentiating when the request comes directly from a user and when from ++ another OpenStack service the Cinder service can protect the deployment. ++ ++- To prevent long-running job failures: If the chain of operations takes a long ++ time, the user's token may expire before the action is completed, leading to ++ the failure of the user's original request. ++ ++ One way to deal with this is to set a long token life in Keystone, and this ++ may be what you are currently doing. But this can be problematic for ++ installations whose security policies prefer short user token lives. ++ Beginning with the Queens release, an alternative solution is available. You ++ have the ability to configure some services (particularly Nova and Cinder) to ++ send a "service token" along with the user's token. When properly ++ configured, the Identity Service will validate an expired user token *when it ++ is accompanied by a valid service token*. Thus if the user's token expires ++ somewhere during a long running chain of operations among various OpenStack ++ services, the operations can continue. + + .. note:: + There's nothing special about a service token. It's a regular token +Index: cinder/doc/source/configuration/index.rst +=================================================================== +--- cinder.orig/doc/source/configuration/index.rst ++++ cinder/doc/source/configuration/index.rst +@@ -6,6 +6,7 @@ Cinder Service Configuration + :maxdepth: 1 + + block-storage/block-storage-overview.rst ++ block-storage/service-token.rst + block-storage/volume-drivers.rst + block-storage/backup-drivers.rst + block-storage/schedulers.rst +@@ -15,10 +16,16 @@ Cinder Service Configuration + block-storage/fc-zoning.rst + block-storage/nested-quota.rst + block-storage/volume-encryption.rst +- block-storage/service-token.rst + block-storage/config-options.rst + block-storage/samples/index.rst + ++.. warning:: ++ ++ For security reasons **Service Tokens must to be configured** in OpenStack ++ for Cinder to operate securely. Pay close attention to the :doc:`specific ++ section describing it: `. See ++ https://bugs.launchpad.net/nova/+bug/2004555 for details. ++ + .. note:: + + The examples of common configurations for shared +Index: cinder/doc/source/install/index.rst +=================================================================== +--- cinder.orig/doc/source/install/index.rst ++++ cinder/doc/source/install/index.rst +@@ -35,6 +35,13 @@ Adding Cinder to your OpenStack Environm + + The following links describe how to install the Cinder Block Storage Service: + ++.. warning:: ++ ++ For security reasons **Service Tokens must to be configured** in OpenStack ++ for Cinder to operate securely. Pay close attention to the :doc:`specific ++ section describing it: <../configuration/block-storage/service-token>`. See ++ https://bugs.launchpad.net/nova/+bug/2004555 for details. ++ + .. toctree:: + + get-started-block-storage +Index: cinder/releasenotes/notes/redirect-detach-nova-4b7b7902d7d182e0.yaml +=================================================================== +--- /dev/null ++++ cinder/releasenotes/notes/redirect-detach-nova-4b7b7902d7d182e0.yaml +@@ -0,0 +1,43 @@ ++--- ++critical: ++ - | ++ Detaching volumes will fail if Nova is not `configured to send service ++ tokens `_, ++ please read the upgrade section for more information. (`Bug #2004555 ++ `_). ++upgrade: ++ - | ++ Nova must be `configured to send service tokens ++ `_ ++ **and** cinder must be configured to recognize at least one of the roles ++ that the nova service user has been assigned in keystone. By default, ++ cinder will recognize the ``service`` role, so if the nova service user ++ is assigned a differently named role in your cloud, you must adjust your ++ cinder configuration file (``service_token_roles`` configuration option ++ in the ``keystone_authtoken`` section). If nova and cinder are not ++ configured correctly in this regard, detaching volumes will no longer ++ work (`Bug #2004555 `_). ++security: ++ - | ++ As part of the fix for `Bug #2004555 ++ `_, cinder now rejects ++ user attachment delete requests for attachments that are being used by nova ++ instances to ensure that no leftover devices are produced on the compute ++ nodes which could be used to access another project's volumes. Terminate ++ connection, detach, and force detach volume actions (calls that are not ++ usually made by users directly) are, in most cases, not allowed for users. ++fixes: ++ - | ++ `Bug #2004555 `_: Fixed ++ issue where a user manually deleting an attachment, calling terminate ++ connection, detach, or force detach, for a volume that is still used by a ++ nova instance resulted in leftover devices on the compute node. These ++ operations will now fail when it is believed to be a problem. ++issues: ++ - | ++ For security reasons (`Bug #2004555 ++ `_) manually deleting an ++ attachment, manually doing the ``os-terminate_connection``, ``os-detach`` ++ or ``os-force_detach`` actions will no longer be allowed in most cases ++ unless the request is coming from another OpenStack service on behalf of a ++ user. diff -Nru cinder-17.0.1/debian/patches/CVE-2024-32498_0_1_Use_the_json_format_output_of_qemu-img_info.patch cinder-17.4.0/debian/patches/CVE-2024-32498_0_1_Use_the_json_format_output_of_qemu-img_info.patch --- cinder-17.0.1/debian/patches/CVE-2024-32498_0_1_Use_the_json_format_output_of_qemu-img_info.patch 1970-01-01 00:00:00.000000000 +0000 +++ cinder-17.4.0/debian/patches/CVE-2024-32498_0_1_Use_the_json_format_output_of_qemu-img_info.patch 2023-05-15 07:36:50.000000000 +0000 @@ -0,0 +1,556 @@ +Description: Use the json format output of qemu-img info + Support for the human format by oslo_utils.imageutils.QemuImgInfo was + deprecated since oslo.utils 4.9.1 [1]. This change replaces the human + format with the json format which will be used by default. + . + [1] 73eb0673f627aad382e08a816191b637af436465 + . + Backport note: The json format is preferable because it allows access + to format specific details (since oslo.utils 4.1.0). These details + are not present when the default 'human' format is used. See change + I133da07a5a9628b8a9 for details. +Author: Takashi Kajinami +Date: Tue, 06 Jul 2021 21:48:44 +0900 +Closes-Bug: #1940540 +Change-Id: Ia0353204abf849467106ee08982d1271de23101a +Origin: upstream, https://review.opendev.org/c/openstack/cinder/+/869985 +Bug: https://launchpad.net/bugs/2059809 +Bug-Debian: https://bugs.debian.org/1074763 +Last-Update: 2024-06-26 + +Index: cinder/cinder/image/image_utils.py +=================================================================== +--- cinder.orig/cinder/image/image_utils.py ++++ cinder/cinder/image/image_utils.py +@@ -132,7 +132,7 @@ def from_qemu_img_disk_format(disk_forma + + def qemu_img_info(path, run_as_root=True, force_share=False): + """Return an object containing the parsed output from qemu-img info.""" +- cmd = ['env', 'LC_ALL=C', 'qemu-img', 'info'] ++ cmd = ['env', 'LC_ALL=C', 'qemu-img', 'info', '--output=json'] + if force_share: + if qemu_img_supports_force_share(): + cmd.append('--force-share') +@@ -146,7 +146,7 @@ def qemu_img_info(path, run_as_root=True + cmd = cmd[2:] + out, _err = utils.execute(*cmd, run_as_root=run_as_root, + prlimit=QEMU_IMG_LIMITS) +- info = imageutils.QemuImgInfo(out) ++ info = imageutils.QemuImgInfo(out, format='json') + + # The fix for bug #1996188 requires using the format_specific + # attribute of QemuImgInfo introduced in oslo.utils 4.1.0. +Index: cinder/cinder/tests/unit/test_image_utils.py +=================================================================== +--- cinder.orig/cinder/tests/unit/test_image_utils.py ++++ cinder/cinder/tests/unit/test_image_utils.py +@@ -42,7 +42,8 @@ class TestQemuImgInfo(test.TestCase): + + output = image_utils.qemu_img_info(test_path) + mock_exec.assert_called_once_with('env', 'LC_ALL=C', 'qemu-img', +- 'info', test_path, run_as_root=True, ++ 'info', '--output=json', test_path, ++ run_as_root=True, + prlimit=image_utils.QEMU_IMG_LIMITS) + self.assertEqual(mock_info.return_value, output) + +@@ -59,7 +60,8 @@ class TestQemuImgInfo(test.TestCase): + force_share=False, + run_as_root=False) + mock_exec.assert_called_once_with('env', 'LC_ALL=C', 'qemu-img', +- 'info', test_path, run_as_root=False, ++ 'info', '--output=json', test_path, ++ run_as_root=False, + prlimit=image_utils.QEMU_IMG_LIMITS) + self.assertEqual(mock_info.return_value, output) + +@@ -74,8 +76,8 @@ class TestQemuImgInfo(test.TestCase): + mock_os.name = 'nt' + + output = image_utils.qemu_img_info(test_path) +- mock_exec.assert_called_once_with('qemu-img', 'info', test_path, +- run_as_root=True, ++ mock_exec.assert_called_once_with('qemu-img', 'info', '--output=json', ++ test_path, run_as_root=True, + prlimit=image_utils.QEMU_IMG_LIMITS) + self.assertEqual(mock_info.return_value, output) + +Index: cinder/cinder/tests/unit/volume/drivers/test_nfs.py +=================================================================== +--- cinder.orig/cinder/tests/unit/volume/drivers/test_nfs.py ++++ cinder/cinder/tests/unit/volume/drivers/test_nfs.py +@@ -335,102 +335,112 @@ NFS_CONFIG4 = {'max_over_subscription_ra + 'nas_secure_file_permissions': 'false', + 'nas_secure_file_operations': 'true'} + +-QEMU_IMG_INFO_OUT1 = """image: %(volid)s +- file format: raw +- virtual size: %(size_gb)sG (%(size_b)s bytes) +- disk size: 173K +- """ +- +-QEMU_IMG_INFO_OUT2 = """image: %(volid)s +-file format: qcow2 +-virtual size: %(size_gb)sG (%(size_b)s bytes) +-disk size: 196K +-cluster_size: 65536 +-Format specific information: +- compat: 1.1 +- lazy refcounts: false +- refcount bits: 16 +- corrupt: false +- """ +- +-QEMU_IMG_INFO_OUT3 = """image: volume-%(volid)s.%(snapid)s +-file format: qcow2 +-virtual size: %(size_gb)sG (%(size_b)s bytes) +-disk size: 196K +-cluster_size: 65536 +-backing file: volume-%(volid)s +-backing file format: qcow2 +-Format specific information: +- compat: 1.1 +- lazy refcounts: false +- refcount bits: 16 +- corrupt: false +- """ +- +-QEMU_IMG_INFO_OUT4 = """image: volume-%(volid)s.%(snapid)s +-file format: raw +-virtual size: %(size_gb)sG (%(size_b)s bytes) +-disk size: 196K +-cluster_size: 65536 +-backing file: volume-%(volid)s +-backing file format: raw +-Format specific information: +- compat: 1.1 +- lazy refcounts: false +- refcount bits: 16 +- corrupt: false +- """ +- +-QEMU_IMG_INFO_OUT5 = """image: volume-%(volid)s.%(snapid)s +-file format: qcow2 +-virtual size: %(size_gb)sG (%(size_b)s bytes) +-disk size: 196K +-encrypted: yes +-cluster_size: 65536 +-backing file: volume-%(volid)s +-backing file format: raw +-Format specific information: +- compat: 1.1 +- lazy refcounts: false +- refcount bits: 16 +- encrypt: +- ivgen alg: plain64 +- hash alg: sha256 +- cipher alg: aes-256 +- uuid: 386f8626-33f0-4683-a517-78ddfe385e33 +- format: luks +- cipher mode: xts +- slots: +- [0]: +- active: true +- iters: 1892498 +- key offset: 4096 +- stripes: 4000 +- [1]: +- active: false +- key offset: 262144 +- [2]: +- active: false +- key offset: 520192 +- [3]: +- active: false +- key offset: 778240 +- [4]: +- active: false +- key offset: 1036288 +- [5]: +- active: false +- key offset: 1294336 +- [6]: +- active: false +- key offset: 1552384 +- [7]: +- active: false +- key offset: 1810432 +- payload offset: 2068480 +- master key iters: 459347 +- corrupt: false +-""" ++QEMU_IMG_INFO_OUT1 = """{ ++ "filename": "%(volid)s", ++ "format": "raw", ++ "virtual-size": %(size_b)s, ++ "actual-size": 173000 ++}""" ++ ++QEMU_IMG_INFO_OUT2 = """{ ++ "filename": "%(volid)s", ++ "format": "qcow2", ++ "virtual-size": %(size_b)s, ++ "actual-size": 196000, ++ "cluster-size": 65536, ++ "format-specific": { ++ "compat": "1.1", ++ "lazy-refcounts": false, ++ "refcount-bits": 16, ++ "corrupt": false ++ } ++}""" ++ ++QEMU_IMG_INFO_OUT3 = """{ ++ "filename": "volume-%(volid)s.%(snapid)s", ++ "format": "qcow2", ++ "virtual-size": %(size_b)s, ++ "actual-size": 196000, ++ "cluster-size": 65536, ++ "backing-filename": "volume-%(volid)s", ++ "backing-filename-format": "qcow2", ++ "format-specific": { ++ "compat": "1.1", ++ "lazy-refcounts": false, ++ "refcount-bits": 16, ++ "corrupt": false ++ } ++}""" ++ ++QEMU_IMG_INFO_OUT4 = """{ ++ "filename": "volume-%(volid)s.%(snapid)s", ++ "format": "raw", ++ "virtual-size": %(size_b)s, ++ "actual-size": 196000, ++ "cluster-size": 65536, ++ "backing-filename": "volume-%(volid)s", ++ "backing-filename-format": "raw" ++}""" ++ ++QEMU_IMG_INFO_OUT5 = """{ ++ "filename": "volume-%(volid)s.%(snapid)s", ++ "format": "qcow2", ++ "virtual-size": %(size_b)s, ++ "actual-size": 196000, ++ "encrypted": true, ++ "cluster-size": 65536, ++ "backing-filename": "volume-%(volid)s", ++ "backing-filename-format": "raw", ++ "format-specific": { ++ "type": "luks", ++ "data": { ++ "ivgen-alg": "plain64", ++ "hash-alg": "sha256", ++ "cipher-alg": "aes-256", ++ "uuid": "386f8626-33f0-4683-a517-78ddfe385e33", ++ "cipher-mode": "xts", ++ "slots": [ ++ { ++ "active": true, ++ "iters": 1892498, ++ "key offset": 4096, ++ "stripes": 4000 ++ }, ++ { ++ "active": false, ++ "key offset": 262144 ++ }, ++ { ++ "active": false, ++ "key offset": 520192 ++ }, ++ { ++ "active": false, ++ "key offset": 778240 ++ }, ++ { ++ "active": false, ++ "key offset": 1036288 ++ }, ++ { ++ "active": false, ++ "key offset": 1294336 ++ }, ++ { ++ "active": false, ++ "key offset": 1552384 ++ }, ++ { ++ "active": false, ++ "key offset": 1810432 ++ } ++ ], ++ "payload-offset": 2068480, ++ "master-key-iters": 459347 ++ }, ++ "corrupt": false ++ } ++}""" + + + @ddt.ddt +@@ -1312,7 +1322,7 @@ class NfsDriverTestCase(test.TestCase): + 'size_gb': src_volume.size, + 'size_b': src_volume.size * units.Gi} + +- img_info = imageutils.QemuImgInfo(img_out) ++ img_info = imageutils.QemuImgInfo(img_out, format='json') + mock_img_info = self.mock_object(image_utils, 'qemu_img_info') + mock_img_info.return_value = img_info + mock_convert_image = self.mock_object(image_utils, 'convert_image') +@@ -1392,7 +1402,7 @@ class NfsDriverTestCase(test.TestCase): + 'snapid': fake_snap.id, + 'size_gb': src_volume.size, + 'size_b': src_volume.size * units.Gi} +- img_info = imageutils.QemuImgInfo(img_out) ++ img_info = imageutils.QemuImgInfo(img_out, format='json') + mock_img_info = self.mock_object(image_utils, 'qemu_img_info') + mock_img_info.return_value = img_info + +@@ -1458,7 +1468,8 @@ class NfsDriverTestCase(test.TestCase): + mock_img_utils = self.mock_object(image_utils, 'qemu_img_info') + img_out = qemu_img_info % {'volid': volume.id, 'size_gb': volume.size, + 'size_b': volume.size * units.Gi} +- mock_img_utils.return_value = imageutils.QemuImgInfo(img_out) ++ mock_img_utils.return_value = imageutils.QemuImgInfo(img_out, ++ format='json') + self.mock_object(drv, '_read_info_file', + return_value={'active': "volume-%s" % volume.id}) + +@@ -1478,12 +1489,14 @@ class NfsDriverTestCase(test.TestCase): + drv = self._driver + volume = self._simple_volume() + +- qemu_img_output = """image: %s +- file format: iso +- virtual size: 1.0G (1073741824 bytes) +- disk size: 173K +- """ % volume['name'] +- mock_img_info.return_value = imageutils.QemuImgInfo(qemu_img_output) ++ qemu_img_output = """{ ++ "filename": "%s", ++ "format": "iso", ++ "virtual-size": 1073741824, ++ "actual-size": 173000 ++}""" % volume['name'] ++ mock_img_info.return_value = imageutils.QemuImgInfo(qemu_img_output, ++ format='json') + + self.assertRaises(exception.InvalidVolume, + drv.initialize_connection, +Index: cinder/cinder/tests/unit/volume/drivers/test_quobyte.py +=================================================================== +--- cinder.orig/cinder/tests/unit/volume/drivers/test_quobyte.py ++++ cinder/cinder/tests/unit/volume/drivers/test_quobyte.py +@@ -940,13 +940,14 @@ class QuobyteDriverTestCase(test.TestCas + self.TEST_QUOBYTE_VOLUME), + self.VOLUME_UUID) + +- qemu_img_info_output = """image: volume-%s +- file format: qcow2 +- virtual size: 1.0G (1073741824 bytes) +- disk size: 473K +- """ % self.VOLUME_UUID ++ qemu_img_info_output = """{ ++ "filename": "volume-%s", ++ "format": "qcow2", ++ "virtual-size": 1073741824, ++ "actual-size": 473000 ++}""" % self.VOLUME_UUID + +- img_info = imageutils.QemuImgInfo(qemu_img_info_output) ++ img_info = imageutils.QemuImgInfo(qemu_img_info_output, format='json') + + image_utils.qemu_img_info = mock.Mock(return_value=img_info) + image_utils.resize_image = mock.Mock() +@@ -986,13 +987,14 @@ class QuobyteDriverTestCase(test.TestCas + + size = dest_volume['size'] + +- qemu_img_output = """image: %s +- file format: raw +- virtual size: 1.0G (1073741824 bytes) +- disk size: 173K +- backing file: %s +- """ % (snap_file, src_volume['name']) +- img_info = imageutils.QemuImgInfo(qemu_img_output) ++ qemu_img_output = """{ ++ "filename": "%s", ++ "format": "raw", ++ "virtual-size": 1073741824, ++ "actual-size": 173000, ++ "backing-filename": "%s" ++}""" % (snap_file, src_volume['name']) ++ img_info = imageutils.QemuImgInfo(qemu_img_output, format='json') + + # mocking and testing starts here + image_utils.convert_image = mock.Mock() +@@ -1042,13 +1044,14 @@ class QuobyteDriverTestCase(test.TestCas + + size = dest_volume['size'] + +- qemu_img_output = """image: %s +- file format: raw +- virtual size: 1.0G (1073741824 bytes) +- disk size: 173K +- backing file: %s +- """ % (snap_file, src_volume['name']) +- img_info = imageutils.QemuImgInfo(qemu_img_output) ++ qemu_img_output = """{ ++ "filename": "%s", ++ "format": "raw", ++ "virtual-size": 1073741824, ++ "actual-size": 173000, ++ "backing-filename": "%s" ++}""" % (snap_file, src_volume['name']) ++ img_info = imageutils.QemuImgInfo(qemu_img_output, format='json') + + # mocking and testing starts here + image_utils.convert_image = mock.Mock() +@@ -1103,13 +1106,14 @@ class QuobyteDriverTestCase(test.TestCas + + size = dest_volume['size'] + +- qemu_img_output = """image: %s +- file format: raw +- virtual size: 1.0G (1073741824 bytes) +- disk size: 173K +- backing file: %s +- """ % (snap_file, src_volume['name']) +- img_info = imageutils.QemuImgInfo(qemu_img_output) ++ qemu_img_output = """{ ++ "filename": "%s", ++ "format": "raw", ++ "virtual-size": 1073741824, ++ "actual-size": 173000, ++ "backing-filename": "%s" ++}""" % (snap_file, src_volume['name']) ++ img_info = imageutils.QemuImgInfo(qemu_img_output, format='json') + + # mocking and testing starts here + image_utils.convert_image = mock.Mock() +@@ -1167,13 +1171,14 @@ class QuobyteDriverTestCase(test.TestCas + + size = dest_volume['size'] + +- qemu_img_output = """image: %s +- file format: raw +- virtual size: 1.0G (1073741824 bytes) +- disk size: 173K +- backing file: %s +- """ % (snap_file, src_volume['name']) +- img_info = imageutils.QemuImgInfo(qemu_img_output) ++ qemu_img_output = """{ ++ "filename": "%s", ++ "format": "raw", ++ "virtual-size": 1073741824, ++ "actual-size": 173000, ++ "backing-filename": "%s" ++}""" % (snap_file, src_volume['name']) ++ img_info = imageutils.QemuImgInfo(qemu_img_output, format='json') + + # mocking and testing starts here + image_utils.convert_image = mock.Mock() +@@ -1244,12 +1249,13 @@ class QuobyteDriverTestCase(test.TestCas + drv._get_hash_str(self.TEST_QUOBYTE_VOLUME)) + vol_path = os.path.join(vol_dir, volume['name']) + +- qemu_img_output = """image: %s +- file format: raw +- virtual size: 1.0G (1073741824 bytes) +- disk size: 173K +- """ % volume['name'] +- img_info = imageutils.QemuImgInfo(qemu_img_output) ++ qemu_img_output = """{ ++ "filename": "%s", ++ "format": "raw", ++ "virtual-size": 1073741824, ++ "actual-size": 173000 ++}""" % volume['name'] ++ img_info = imageutils.QemuImgInfo(qemu_img_output, format='json') + + drv.get_active_image_from_info = mock.Mock(return_value=volume['name']) + image_utils.qemu_img_info = mock.Mock(return_value=img_info) +@@ -1293,12 +1299,13 @@ class QuobyteDriverTestCase(test.TestCas + + mock_create_temporary_file.return_value = self.TEST_TMP_FILE + +- qemu_img_output = """image: %s +- file format: raw +- virtual size: 1.0G (1073741824 bytes) +- disk size: 173K +- """ % volume['name'] +- img_info = imageutils.QemuImgInfo(qemu_img_output) ++ qemu_img_output = """{ ++ "filename": "%s", ++ "format": "raw", ++ "virtual-size": 1073741824, ++ "actual-size": 173000 ++}""" % volume['name'] ++ img_info = imageutils.QemuImgInfo(qemu_img_output, format='json') + mock_qemu_img_info.return_value = img_info + + upload_path = volume_path +@@ -1345,12 +1352,13 @@ class QuobyteDriverTestCase(test.TestCas + + mock_create_temporary_file.return_value = self.TEST_TMP_FILE + +- qemu_img_output = """image: %s +- file format: qcow2 +- virtual size: 1.0G (1073741824 bytes) +- disk size: 173K +- """ % volume['name'] +- img_info = imageutils.QemuImgInfo(qemu_img_output) ++ qemu_img_output = """{ ++ "filename": "%s", ++ "format": "qcow2", ++ "virtual-size": 1073741824, ++ "actual-size": 173000 ++}""" % volume['name'] ++ img_info = imageutils.QemuImgInfo(qemu_img_output, format='json') + mock_qemu_img_info.return_value = img_info + + upload_path = self.TEST_TMP_FILE +@@ -1400,13 +1408,14 @@ class QuobyteDriverTestCase(test.TestCas + + mock_create_temporary_file.return_value = self.TEST_TMP_FILE + +- qemu_img_output = """image: volume-%s.%s +- file format: qcow2 +- virtual size: 1.0G (1073741824 bytes) +- disk size: 173K +- backing file: %s +- """ % (self.VOLUME_UUID, self.SNAP_UUID, volume_filename) +- img_info = imageutils.QemuImgInfo(qemu_img_output) ++ qemu_img_output = """{ ++ "filename": "volume-%s.%s", ++ "format": "qcow2", ++ "virtual-size": 1073741824, ++ "actual-size": 173000, ++ "backing-filename": "%s" ++}""" % (self.VOLUME_UUID, self.SNAP_UUID, volume_filename) ++ img_info = imageutils.QemuImgInfo(qemu_img_output, format='json') + mock_qemu_img_info.return_value = img_info + + upload_path = self.TEST_TMP_FILE +Index: cinder/cinder/volume/flows/manager/create_volume.py +=================================================================== +--- cinder.orig/cinder/volume/flows/manager/create_volume.py ++++ cinder/cinder/volume/flows/manager/create_volume.py +@@ -21,6 +21,7 @@ from oslo_log import log as logging + from oslo_utils import excutils + from oslo_utils import fileutils + from oslo_utils import netutils ++from oslo_utils import strutils + from oslo_utils import timeutils + from oslo_utils import uuidutils + import six +@@ -552,7 +553,8 @@ class CreateVolumeFromSpecTask(flow_util + volume, + encryption) + +- if image_info.encrypted == 'yes': ++ # see Bug #1942682 and Change I949f07582a708 for why we do this ++ if strutils.bool_from_string(image_info.encrypted): + key_str = source_pass + "\n" + new_pass + "\n" + del source_pass + diff -Nru cinder-17.0.1/debian/patches/CVE-2024-32498_1_Check_for_external_qcow2_data_file.patch cinder-17.4.0/debian/patches/CVE-2024-32498_1_Check_for_external_qcow2_data_file.patch --- cinder-17.0.1/debian/patches/CVE-2024-32498_1_Check_for_external_qcow2_data_file.patch 1970-01-01 00:00:00.000000000 +0000 +++ cinder-17.4.0/debian/patches/CVE-2024-32498_1_Check_for_external_qcow2_data_file.patch 2023-05-15 07:36:50.000000000 +0000 @@ -0,0 +1,2271 @@ +Description: [PATCH] CVE-2024-32498: Check for external qcow2 data file + Adds code to image_utils to check for a qcow2 external data + file, a recent feature of qemu which we do not support and + which can be used maliciously. + . + Advice from the qemu-img community is that it is dangerous + to call qemu-img info on untrusted files, so we copy over + the format_inspector module from Glance. This performs basic + analysis on the image data file so we can detect problematic + images before we call qemu-img info to get all the image + attributes. It is expected that this code will eventually be + added to oslo so it can be consumed by Glance, Cinder, and + Nova. + . + Because cinder itself may create qcow2 format images with a + backing file in nfs-based backends, the glance format_inspector + has been modified to optionally allow such files. Since we are + monkeying with the format_inspector code, we also copy over + its unit tests to prevent regressions and to add tests for the + changed code. + . + Includes an additional fix to prevent an issue where a user + could mount a raw volume and write a qcow2 header with a larger + virtual size on it. On reattaching the volume it would have the + new larger virtual size avaialable without actually changing + the size value in cinder. While we cannot prevent this we can + prevent the user from using this volume again, which makes this + exploit pointless. +Author: Brian Rosmaita +Date: Wed, 26 Jun 2024 14:09:30 -0400 +Co-authored-by: Dan Smith +Co-authored-by: Felix Huettner +Change-Id: I65857288b797cde573e7443ac6e7e6f57fedde01 +Bug: https://launchpad.net/bugs/2059809 +Bug-Debian: https://bugs.debian.org/1074763 + +Index: cinder/cinder/image/format_inspector.py +=================================================================== +--- /dev/null ++++ cinder/cinder/image/format_inspector.py +@@ -0,0 +1,938 @@ ++# Copyright 2020 Red Hat, Inc ++# All Rights Reserved. ++# ++# Licensed under the Apache License, Version 2.0 (the "License"); you may ++# not use this file except in compliance with the License. You may obtain ++# a copy of the License at ++# ++# http://www.apache.org/licenses/LICENSE-2.0 ++# ++# Unless required by applicable law or agreed to in writing, software ++# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT ++# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the ++# License for the specific language governing permissions and limitations ++# under the License. ++ ++""" ++This is a python implementation of virtual disk format inspection routines ++gathered from various public specification documents, as well as qemu disk ++driver code. It attempts to store and parse the minimum amount of data ++required, and in a streaming-friendly manner to collect metadata about ++complex-format images. ++""" ++ ++import struct ++ ++from oslo_log import log as logging ++ ++LOG = logging.getLogger(__name__) ++ ++ ++def chunked_reader(fileobj, chunk_size=512): ++ while True: ++ chunk = fileobj.read(chunk_size) ++ if not chunk: ++ break ++ yield chunk ++ ++ ++class CaptureRegion(object): ++ """Represents a region of a file we want to capture. ++ ++ A region of a file we want to capture requires a byte offset into ++ the file and a length. This is expected to be used by a data ++ processing loop, calling capture() with the most recently-read ++ chunk. This class handles the task of grabbing the desired region ++ of data across potentially multiple fractional and unaligned reads. ++ ++ :param offset: Byte offset into the file starting the region ++ :param length: The length of the region ++ """ ++ def __init__(self, offset, length): ++ self.offset = offset ++ self.length = length ++ self.data = b'' ++ ++ @property ++ def complete(self): ++ """Returns True when we have captured the desired data.""" ++ return self.length == len(self.data) ++ ++ def capture(self, chunk, current_position): ++ """Process a chunk of data. ++ ++ This should be called for each chunk in the read loop, at least ++ until complete returns True. ++ ++ :param chunk: A chunk of bytes in the file ++ :param current_position: The position of the file processed by the ++ read loop so far. Note that this will be ++ the position in the file *after* the chunk ++ being presented. ++ """ ++ read_start = current_position - len(chunk) ++ if (read_start <= self.offset <= current_position or ++ self.offset <= read_start <= (self.offset + self.length)): ++ if read_start < self.offset: ++ lead_gap = self.offset - read_start ++ else: ++ lead_gap = 0 ++ self.data += chunk[lead_gap:] ++ self.data = self.data[:self.length] ++ ++ ++class ImageFormatError(Exception): ++ """An unrecoverable image format error that aborts the process.""" ++ pass ++ ++ ++class TraceDisabled(object): ++ """A logger-like thing that swallows tracing when we do not want it.""" ++ def debug(self, *a, **k): ++ pass ++ ++ info = debug ++ warning = debug ++ error = debug ++ ++ ++class FileInspector(object): ++ """A stream-based disk image inspector. ++ ++ This base class works on raw images and is subclassed for more ++ complex types. It is to be presented with the file to be examined ++ one chunk at a time, during read processing and will only store ++ as much data as necessary to determine required attributes of ++ the file. ++ """ ++ ++ def __init__(self, tracing=False): ++ self._total_count = 0 ++ ++ # NOTE(danms): The logging in here is extremely verbose for a reason, ++ # but should never really be enabled at that level at runtime. To ++ # retain all that work and assist in future debug, we have a separate ++ # debug flag that can be passed from a manual tool to turn it on. ++ if tracing: ++ self._log = logging.getLogger(str(self)) ++ else: ++ self._log = TraceDisabled() ++ self._capture_regions = {} ++ ++ def _capture(self, chunk, only=None): ++ for name, region in self._capture_regions.items(): ++ if only and name not in only: ++ continue ++ if not region.complete: ++ region.capture(chunk, self._total_count) ++ ++ def eat_chunk(self, chunk): ++ """Call this to present chunks of the file to the inspector.""" ++ pre_regions = set(self._capture_regions.keys()) ++ ++ # Increment our position-in-file counter ++ self._total_count += len(chunk) ++ ++ # Run through the regions we know of to see if they want this ++ # data ++ self._capture(chunk) ++ ++ # Let the format do some post-read processing of the stream ++ self.post_process() ++ ++ # Check to see if the post-read processing added new regions ++ # which may require the current chunk. ++ new_regions = set(self._capture_regions.keys()) - pre_regions ++ if new_regions: ++ self._capture(chunk, only=new_regions) ++ ++ def post_process(self): ++ """Post-read hook to process what has been read so far. ++ ++ This will be called after each chunk is read and potentially captured ++ by the defined regions. If any regions are defined by this call, ++ those regions will be presented with the current chunk in case it ++ is within one of the new regions. ++ """ ++ pass ++ ++ def region(self, name): ++ """Get a CaptureRegion by name.""" ++ return self._capture_regions[name] ++ ++ def new_region(self, name, region): ++ """Add a new CaptureRegion by name.""" ++ if self.has_region(name): ++ # This is a bug, we tried to add the same region twice ++ raise ImageFormatError('Inspector re-added region %s' % name) ++ self._capture_regions[name] = region ++ ++ def has_region(self, name): ++ """Returns True if named region has been defined.""" ++ return name in self._capture_regions ++ ++ @property ++ def format_match(self): ++ """Returns True if the file appears to be the expected format.""" ++ return True ++ ++ @property ++ def virtual_size(self): ++ """Returns the virtual size of the disk image, or zero if unknown.""" ++ return self._total_count ++ ++ @property ++ def actual_size(self): ++ """Returns the total size of the file. ++ ++ This is usually smaller than virtual_size. NOTE: this will only be ++ accurate if the entire file is read and processed. ++ """ ++ return self._total_count ++ ++ @property ++ def complete(self): ++ """Returns True if we have all the information needed.""" ++ return all(r.complete for r in self._capture_regions.values()) ++ ++ def __str__(self): ++ """The string name of this file format.""" ++ return 'raw' ++ ++ @property ++ def context_info(self): ++ """Return info on amount of data held in memory for auditing. ++ ++ This is a dict of region:sizeinbytes items that the inspector ++ uses to examine the file. ++ """ ++ return {name: len(region.data) for name, region in ++ self._capture_regions.items()} ++ ++ @classmethod ++ def from_file(cls, filename): ++ """Read as much of a file as necessary to complete inspection. ++ ++ NOTE: Because we only read as much of the file as necessary, the ++ actual_size property will not reflect the size of the file, but the ++ amount of data we read before we satisfied the inspector. ++ ++ Raises ImageFormatError if we cannot parse the file. ++ """ ++ inspector = cls() ++ with open(filename, 'rb') as f: ++ for chunk in chunked_reader(f): ++ inspector.eat_chunk(chunk) ++ if inspector.complete: ++ # No need to eat any more data ++ break ++ if not inspector.complete or not inspector.format_match: ++ raise ImageFormatError('File is not in requested format') ++ return inspector ++ ++ def safety_check(self): ++ """Perform some checks to determine if this file is safe. ++ ++ Returns True if safe, False otherwise. It may raise ImageFormatError ++ if safety cannot be guaranteed because of parsing or other errors. ++ """ ++ return True ++ ++ ++# The qcow2 format consists of a big-endian 72-byte header, of which ++# only a small portion has information we care about: ++# ++# Dec Hex Name ++# 0 0x00 Magic 4-bytes 'QFI\xfb' ++# 4 0x04 Version (uint32_t, should always be 2 for modern files) ++# . . . ++# 8 0x08 Backing file offset (uint64_t) ++# 24 0x18 Size in bytes (unint64_t) ++# . . . ++# 72 0x48 Incompatible features bitfield (6 bytes) ++# ++# https://gitlab.com/qemu-project/qemu/-/blob/master/docs/interop/qcow2.txt ++class QcowInspector(FileInspector): ++ """QEMU QCOW2 Format ++ ++ This should only require about 32 bytes of the beginning of the file ++ to determine the virtual size, and 104 bytes to perform the safety check. ++ """ ++ ++ BF_OFFSET = 0x08 ++ BF_OFFSET_LEN = 8 ++ I_FEATURES = 0x48 ++ I_FEATURES_LEN = 8 ++ I_FEATURES_DATAFILE_BIT = 3 ++ I_FEATURES_MAX_BIT = 4 ++ ++ def __init__(self, *a, **k): ++ super(QcowInspector, self).__init__(*a, **k) ++ self.new_region('header', CaptureRegion(0, 512)) ++ ++ def _qcow_header_data(self): ++ magic, version, bf_offset, bf_sz, cluster_bits, size = ( ++ struct.unpack('>4sIQIIQ', self.region('header').data[:32])) ++ return magic, size ++ ++ @property ++ def has_header(self): ++ return self.region('header').complete ++ ++ @property ++ def virtual_size(self): ++ if not self.region('header').complete: ++ return 0 ++ if not self.format_match: ++ return 0 ++ magic, size = self._qcow_header_data() ++ return size ++ ++ @property ++ def format_match(self): ++ if not self.region('header').complete: ++ return False ++ magic, size = self._qcow_header_data() ++ return magic == b'QFI\xFB' ++ ++ @property ++ def has_backing_file(self): ++ if not self.region('header').complete: ++ return None ++ if not self.format_match: ++ return False ++ bf_offset_bytes = self.region('header').data[ ++ self.BF_OFFSET:self.BF_OFFSET + self.BF_OFFSET_LEN] ++ # nonzero means "has a backing file" ++ bf_offset, = struct.unpack('>Q', bf_offset_bytes) ++ return bf_offset != 0 ++ ++ @property ++ def has_unknown_features(self): ++ if not self.region('header').complete: ++ return None ++ if not self.format_match: ++ return False ++ i_features = self.region('header').data[ ++ self.I_FEATURES:self.I_FEATURES + self.I_FEATURES_LEN] ++ ++ # This is the maximum byte number we should expect any bits to be set ++ max_byte = self.I_FEATURES_MAX_BIT // 8 ++ ++ # The flag bytes are in big-endian ordering, so if we process ++ # them in index-order, they're reversed ++ for i, byte_num in enumerate(reversed(range(self.I_FEATURES_LEN))): ++ if byte_num == max_byte: ++ # If we're in the max-allowed byte, allow any bits less than ++ # the maximum-known feature flag bit to be set ++ allow_mask = ((1 << self.I_FEATURES_MAX_BIT) - 1) ++ elif byte_num > max_byte: ++ # If we're above the byte with the maximum known feature flag ++ # bit, then we expect all zeroes ++ allow_mask = 0x0 ++ else: ++ # Any earlier-than-the-maximum byte can have any of the flag ++ # bits set ++ allow_mask = 0xFF ++ ++ if i_features[i] & ~allow_mask: ++ LOG.warning('Found unknown feature bit in byte %i: %s/%s', ++ byte_num, bin(i_features[byte_num] & ~allow_mask), ++ bin(allow_mask)) ++ return True ++ ++ return False ++ ++ @property ++ def has_data_file(self): ++ if not self.region('header').complete: ++ return None ++ if not self.format_match: ++ return False ++ i_features = self.region('header').data[ ++ self.I_FEATURES:self.I_FEATURES + self.I_FEATURES_LEN] ++ ++ # First byte of bitfield, which is i_features[7] ++ byte = self.I_FEATURES_LEN - 1 - self.I_FEATURES_DATAFILE_BIT // 8 ++ # Third bit of bitfield, which is 0x04 ++ bit = 1 << (self.I_FEATURES_DATAFILE_BIT - 1 % 8) ++ return bool(i_features[byte] & bit) ++ ++ def __str__(self): ++ return 'qcow2' ++ ++ def safety_check(self): ++ return (not self.has_backing_file and ++ not self.has_data_file and ++ not self.has_unknown_features) ++ ++ def safety_check_allow_backing_file(self): ++ return (not self.has_data_file and ++ not self.has_unknown_features) ++ ++ ++class QEDInspector(FileInspector): ++ def __init__(self, tracing=False): ++ super().__init__(tracing) ++ self.new_region('header', CaptureRegion(0, 512)) ++ ++ @property ++ def format_match(self): ++ if not self.region('header').complete: ++ return False ++ return self.region('header').data.startswith(b'QED\x00') ++ ++ def safety_check(self): ++ # QED format is not supported by anyone, but we want to detect it ++ # and mark it as just always unsafe. ++ return False ++ ++ ++# The VHD (or VPC as QEMU calls it) format consists of a big-endian ++# 512-byte "footer" at the beginning of the file with various ++# information, most of which does not matter to us: ++# ++# Dec Hex Name ++# 0 0x00 Magic string (8-bytes, always 'conectix') ++# 40 0x28 Disk size (uint64_t) ++# ++# https://github.com/qemu/qemu/blob/master/block/vpc.c ++class VHDInspector(FileInspector): ++ """Connectix/MS VPC VHD Format ++ ++ This should only require about 512 bytes of the beginning of the file ++ to determine the virtual size. ++ """ ++ def __init__(self, *a, **k): ++ super(VHDInspector, self).__init__(*a, **k) ++ self.new_region('header', CaptureRegion(0, 512)) ++ ++ @property ++ def format_match(self): ++ return self.region('header').data.startswith(b'conectix') ++ ++ @property ++ def virtual_size(self): ++ if not self.region('header').complete: ++ return 0 ++ ++ if not self.format_match: ++ return 0 ++ ++ return struct.unpack('>Q', self.region('header').data[40:48])[0] ++ ++ def __str__(self): ++ return 'vhd' ++ ++ ++# The VHDX format consists of a complex dynamic little-endian ++# structure with multiple regions of metadata and data, linked by ++# offsets with in the file (and within regions), identified by MSFT ++# GUID strings. The header is a 320KiB structure, only a few pieces of ++# which we actually need to capture and interpret: ++# ++# Dec Hex Name ++# 0 0x00000 Identity (Technically 9-bytes, padded to 64KiB, the first ++# 8 bytes of which are 'vhdxfile') ++# 196608 0x30000 The Region table (64KiB of a 32-byte header, followed ++# by up to 2047 36-byte region table entry structures) ++# ++# The region table header includes two items we need to read and parse, ++# which are: ++# ++# 196608 0x30000 4-byte signature ('regi') ++# 196616 0x30008 Entry count (uint32-t) ++# ++# The region table entries follow the region table header immediately ++# and are identified by a 16-byte GUID, and provide an offset of the ++# start of that region. We care about the "metadata region", identified ++# by the METAREGION class variable. The region table entry is (offsets ++# from the beginning of the entry, since it could be in multiple places): ++# ++# 0 0x00000 16-byte MSFT GUID ++# 16 0x00010 Offset of the actual metadata region (uint64_t) ++# ++# When we find the METAREGION table entry, we need to grab that offset ++# and start examining the region structure at that point. That ++# consists of a metadata table of structures, which point to places in ++# the data in an unstructured space that follows. The header is ++# (offsets relative to the region start): ++# ++# 0 0x00000 8-byte signature ('metadata') ++# . . . ++# 16 0x00010 2-byte entry count (up to 2047 entries max) ++# ++# This header is followed by the specified number of metadata entry ++# structures, identified by GUID: ++# ++# 0 0x00000 16-byte MSFT GUID ++# 16 0x00010 4-byte offset (uint32_t, relative to the beginning of ++# the metadata region) ++# ++# We need to find the "Virtual Disk Size" metadata item, identified by ++# the GUID in the VIRTUAL_DISK_SIZE class variable, grab the offset, ++# add it to the offset of the metadata region, and examine that 8-byte ++# chunk of data that follows. ++# ++# The "Virtual Disk Size" is a naked uint64_t which contains the size ++# of the virtual disk, and is our ultimate target here. ++# ++# https://docs.microsoft.com/en-us/openspecs/windows_protocols/ms-vhdx/83e061f8-f6e2-4de1-91bd-5d518a43d477 ++class VHDXInspector(FileInspector): ++ """MS VHDX Format ++ ++ This requires some complex parsing of the stream. The first 256KiB ++ of the image is stored to get the header and region information, ++ and then we capture the first metadata region to read those ++ records, find the location of the virtual size data and parse ++ it. This needs to store the metadata table entries up until the ++ VDS record, which may consist of up to 2047 32-byte entries at ++ max. Finally, it must store a chunk of data at the offset of the ++ actual VDS uint64. ++ ++ """ ++ METAREGION = '8B7CA206-4790-4B9A-B8FE-575F050F886E' ++ VIRTUAL_DISK_SIZE = '2FA54224-CD1B-4876-B211-5DBED83BF4B8' ++ VHDX_METADATA_TABLE_MAX_SIZE = 32 * 2048 # From qemu ++ ++ def __init__(self, *a, **k): ++ super(VHDXInspector, self).__init__(*a, **k) ++ self.new_region('ident', CaptureRegion(0, 32)) ++ self.new_region('header', CaptureRegion(192 * 1024, 64 * 1024)) ++ ++ def post_process(self): ++ # After reading a chunk, we may have the following conditions: ++ # ++ # 1. We may have just completed the header region, and if so, ++ # we need to immediately read and calculate the location of ++ # the metadata region, as it may be starting in the same ++ # read we just did. ++ # 2. We may have just completed the metadata region, and if so, ++ # we need to immediately calculate the location of the ++ # "virtual disk size" record, as it may be starting in the ++ # same read we just did. ++ if self.region('header').complete and not self.has_region('metadata'): ++ region = self._find_meta_region() ++ if region: ++ self.new_region('metadata', region) ++ elif self.has_region('metadata') and not self.has_region('vds'): ++ region = self._find_meta_entry(self.VIRTUAL_DISK_SIZE) ++ if region: ++ self.new_region('vds', region) ++ ++ @property ++ def format_match(self): ++ return self.region('ident').data.startswith(b'vhdxfile') ++ ++ @staticmethod ++ def _guid(buf): ++ """Format a MSFT GUID from the 16-byte input buffer.""" ++ guid_format = '= 2048: ++ raise ImageFormatError('Region count is %i (limit 2047)' % count) ++ ++ # Process the regions until we find the metadata one; grab the ++ # offset and return ++ self._log.debug('Region entry first is %x', region_entry_first) ++ self._log.debug('Region entries %i', count) ++ meta_offset = 0 ++ for i in range(0, count): ++ entry_start = region_entry_first + (i * 32) ++ entry_end = entry_start + 32 ++ entry = self.region('header').data[entry_start:entry_end] ++ self._log.debug('Entry offset is %x', entry_start) ++ ++ # GUID is the first 16 bytes ++ guid = self._guid(entry[:16]) ++ if guid == self.METAREGION: ++ # This entry is the metadata region entry ++ meta_offset, meta_len, meta_req = struct.unpack( ++ '= 2048: ++ raise ImageFormatError( ++ 'Metadata item count is %i (limit 2047)' % count) ++ ++ for i in range(0, count): ++ entry_offset = 32 + (i * 32) ++ guid = self._guid(meta_buffer[entry_offset:entry_offset + 16]) ++ if guid == desired_guid: ++ # Found the item we are looking for by id. ++ # Stop our region from capturing ++ item_offset, item_length, _reserved = struct.unpack( ++ ' imageutils.QemuImgInfo: + """Return an object containing the parsed output from qemu-img info.""" +- cmd = ['env', 'LC_ALL=C', 'qemu-img', 'info', '--output=json'] ++ ++ inspector = format_inspector.detect_file_format(path) ++ format_name = str(inspector) ++ safe = inspector.safety_check() ++ if not safe and format_name == 'qcow2' and allow_qcow2_backing_file: ++ safe = inspector.safety_check_allow_backing_file() ++ if not safe: ++ LOG.warning('Image/Volume %s failed safety check', path) ++ # NOTE(danms): This is the same exception as would be raised ++ # by qemu_img_info() if the disk format was unreadable or ++ # otherwise unsuitable. ++ raise exception.Invalid( ++ reason=_('Image/Volume failed safety check')) ++ ++ cmd = ['env', 'LC_ALL=C', 'qemu-img', 'info', ++ '-f', format_name, '--output=json'] + if force_share: + if qemu_img_supports_force_share(): + cmd.append('--force-share') +@@ -156,8 +176,32 @@ def qemu_img_info(path, run_as_root=True + qemu_info = json.loads(out) + info.format_specific = qemu_info.get('format-specific') + ++ # FIXME: figure out a more elegant way to do this ++ if info.file_format == 'raw': ++ # The format_inspector will detect a luks image as 'raw', and then when ++ # we call qemu-img info -f raw above, we don't get any of the luks ++ # format-specific info (some of which is used in the create_volume ++ # flow). So we need to check if this is really a luks container. ++ # (We didn't have to do this in the past because we called ++ # qemu-img info without -f.) ++ cmd = ['env', 'LC_ALL=C', 'qemu-img', 'info', ++ '-f', 'luks', '--output=json'] ++ if force_share: ++ cmd.append('--force-share') ++ cmd.append(path) ++ if os.name == 'nt': ++ cmd = cmd[2:] ++ try: ++ out, _err = utils.execute(*cmd, run_as_root=run_as_root, ++ prlimit=QEMU_IMG_LIMITS) ++ info = imageutils.QemuImgInfo(out, format='json') ++ except processutils.ProcessExecutionError: ++ # we'll just use the info object we already got earlier ++ pass ++ + # From Cinder's point of view, any 'luks' formatted images +- # should be treated as 'raw'. ++ # should be treated as 'raw'. (This changes the file_format, but ++ # not any of the format-specific information.) + if info.file_format == 'luks': + info.file_format = 'raw' + +@@ -613,6 +657,35 @@ def get_qemu_data(image_id, has_meta, di + return data + + ++def check_qcow2_image(image_id: str, data: imageutils.QemuImgInfo) -> None: ++ """Check some rules about qcow2 images. ++ ++ Does not check for a backing_file, because cinder has some legitimate ++ use cases for qcow2 backing files. ++ ++ Makes sure the image: ++ ++ - does not have a data_file ++ ++ :param image_id: the image id ++ :param data: an imageutils.QemuImgInfo object ++ :raises ImageUnacceptable: when the image fails the check ++ """ ++ try: ++ data_file = data.format_specific['data'].get('data-file') ++ except (KeyError, TypeError): ++ LOG.debug('Unexpected response from qemu-img info when processing ' ++ 'image %s: missing format-specific info for a qcow2 image', ++ image_id) ++ msg = _('Cannot determine format-specific information') ++ raise exception.ImageUnacceptable(image_id=image_id, reason=msg) ++ if data_file: ++ LOG.warning("Refusing to process qcow2 file with data-file '%s'", ++ data_file) ++ msg = _('A qcow2 format image is not allowed to have a data file') ++ raise exception.ImageUnacceptable(image_id=image_id, reason=msg) ++ ++ + def check_vmdk_image(image_id, data): + """Check some rules about VMDK images. + +@@ -703,6 +776,8 @@ def check_image_format(source, + + if data.file_format == 'vmdk': + check_vmdk_image(image_id, data) ++ if data.file_format == 'qcow2': ++ check_qcow2_image(image_id, data) + + + def fetch_verify_image(context, image_service, image_id, dest, +@@ -750,6 +825,11 @@ def fetch_verify_image(context, image_se + if fmt == 'vmdk': + check_vmdk_image(image_id, data) + ++ # Bug #2059809: a qcow2 can have a data file that's similar ++ # to a backing file and is also unacceptable ++ if fmt == 'qcow2': ++ check_qcow2_image(image_id, data) ++ + + def fetch_to_vhd(context, image_service, + image_id, dest, blocksize, volume_subformat=None, +Index: cinder/cinder/tests/unit/image/test_format_inspector.py +=================================================================== +--- /dev/null ++++ cinder/cinder/tests/unit/image/test_format_inspector.py +@@ -0,0 +1,515 @@ ++# Copyright 2020 Red Hat, Inc ++# All Rights Reserved. ++# ++# Licensed under the Apache License, Version 2.0 (the "License"); you may ++# not use this file except in compliance with the License. You may obtain ++# a copy of the License at ++# ++# http://www.apache.org/licenses/LICENSE-2.0 ++# ++# Unless required by applicable law or agreed to in writing, software ++# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT ++# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the ++# License for the specific language governing permissions and limitations ++# under the License. ++ ++import io ++import os ++import re ++import struct ++import subprocess ++import tempfile ++from unittest import mock ++ ++from oslo_utils import units ++ ++from cinder.image import format_inspector ++from cinder.tests.unit import test ++ ++ ++def get_size_from_qemu_img(filename): ++ output = subprocess.check_output('qemu-img info "%s"' % filename, ++ shell=True) ++ for line in output.split(b'\n'): ++ m = re.search(b'^virtual size: .* .([0-9]+) bytes', line.strip()) ++ if m: ++ return int(m.group(1)) ++ ++ raise Exception('Could not find virtual size with qemu-img') ++ ++ ++class TestFormatInspectors(test.TestCase): ++ def setUp(self): ++ super(TestFormatInspectors, self).setUp() ++ self._created_files = [] ++ ++ def tearDown(self): ++ super(TestFormatInspectors, self).tearDown() ++ for fn in self._created_files: ++ try: ++ os.remove(fn) ++ except Exception: ++ pass ++ ++ def _create_img(self, fmt, size, subformat=None, options=None, ++ backing_file=None): ++ if fmt == 'vhd': ++ # QEMU calls the vhd format vpc ++ fmt = 'vpc' ++ ++ if options is None: ++ options = {} ++ opt = '' ++ prefix = 'glance-unittest-formatinspector-' ++ ++ if subformat: ++ options['subformat'] = subformat ++ prefix += subformat + '-' ++ ++ if options: ++ opt += '-o ' + ','.join('%s=%s' % (k, v) ++ for k, v in options.items()) ++ ++ if backing_file is not None: ++ opt += ' -b %s -F raw' % backing_file ++ ++ fn = tempfile.mktemp(prefix=prefix, ++ suffix='.%s' % fmt) ++ self._created_files.append(fn) ++ subprocess.check_output( ++ 'qemu-img create -f %s %s %s %i' % (fmt, opt, fn, size), ++ shell=True) ++ return fn ++ ++ def _create_allocated_vmdk(self, size_mb, subformat=None): ++ # We need a "big" VMDK file to exercise some parts of the code of the ++ # format_inspector. A way to create one is to first create an empty ++ # file, and then to convert it with the -S 0 option. ++ ++ if subformat is None: ++ # Matches qemu-img default, see `qemu-img convert -O vmdk -o help` ++ subformat = 'monolithicSparse' ++ ++ prefix = 'glance-unittest-formatinspector-%s-' % subformat ++ fn = tempfile.mktemp(prefix=prefix, suffix='.vmdk') ++ self._created_files.append(fn) ++ raw = tempfile.mktemp(prefix=prefix, suffix='.raw') ++ self._created_files.append(raw) ++ ++ # Create a file with pseudo-random data, otherwise it will get ++ # compressed in the streamOptimized format ++ subprocess.check_output( ++ 'dd if=/dev/urandom of=%s bs=1M count=%i' % (raw, size_mb), ++ shell=True) ++ ++ # Convert it to VMDK ++ subprocess.check_output( ++ 'qemu-img convert -f raw -O vmdk -o subformat=%s -S 0 %s %s' % ( ++ subformat, raw, fn), ++ shell=True) ++ return fn ++ ++ def _test_format_at_block_size(self, format_name, img, block_size): ++ fmt = format_inspector.get_inspector(format_name)() ++ self.assertIsNotNone(fmt, ++ 'Did not get format inspector for %s' % ( ++ format_name)) ++ wrapper = format_inspector.InfoWrapper(open(img, 'rb'), fmt) ++ ++ while True: ++ chunk = wrapper.read(block_size) ++ if not chunk: ++ break ++ ++ wrapper.close() ++ return fmt ++ ++ def _test_format_at_image_size(self, format_name, image_size, ++ subformat=None): ++ img = self._create_img(format_name, image_size, subformat=subformat) ++ ++ # Some formats have internal alignment restrictions making this not ++ # always exactly like image_size, so get the real value for comparison ++ virtual_size = get_size_from_qemu_img(img) ++ ++ # Read the format in various sizes, some of which will read whole ++ # sections in a single read, others will be completely unaligned, etc. ++ for block_size in (64 * units.Ki, 512, 17, 1 * units.Mi): ++ fmt = self._test_format_at_block_size(format_name, img, block_size) ++ self.assertTrue(fmt.format_match, ++ 'Failed to match %s at size %i block %i' % ( ++ format_name, image_size, block_size)) ++ self.assertEqual(virtual_size, fmt.virtual_size, ++ ('Failed to calculate size for %s at size %i ' ++ 'block %i') % (format_name, image_size, ++ block_size)) ++ memory = sum(fmt.context_info.values()) ++ self.assertLess(memory, 512 * units.Ki, ++ 'Format used more than 512KiB of memory: %s' % ( ++ fmt.context_info)) ++ ++ def _test_format(self, format_name, subformat=None): ++ # Try a few different image sizes, including some odd and very small ++ # sizes ++ for image_size in (512, 513, 2057, 7): ++ self._test_format_at_image_size(format_name, image_size * units.Mi, ++ subformat=subformat) ++ ++ def test_qcow2(self): ++ self._test_format('qcow2') ++ ++ def test_vhd(self): ++ self._test_format('vhd') ++ ++ def test_vhdx(self): ++ self._test_format('vhdx') ++ ++ def test_vmdk(self): ++ self._test_format('vmdk') ++ ++ def test_vmdk_stream_optimized(self): ++ self._test_format('vmdk', 'streamOptimized') ++ ++ def test_from_file_reads_minimum(self): ++ img = self._create_img('qcow2', 10 * units.Mi) ++ file_size = os.stat(img).st_size ++ fmt = format_inspector.QcowInspector.from_file(img) ++ # We know everything we need from the first 512 bytes of a QCOW image, ++ # so make sure that we did not read the whole thing when we inspect ++ # a local file. ++ self.assertLess(fmt.actual_size, file_size) ++ ++ def test_qed_always_unsafe(self): ++ img = self._create_img('qed', 10 * units.Mi) ++ fmt = format_inspector.get_inspector('qed').from_file(img) ++ self.assertTrue(fmt.format_match) ++ self.assertFalse(fmt.safety_check()) ++ ++ def _test_vmdk_bad_descriptor_offset(self, subformat=None): ++ format_name = 'vmdk' ++ image_size = 10 * units.Mi ++ descriptorOffsetAddr = 0x1c ++ BAD_ADDRESS = 0x400 ++ img = self._create_img(format_name, image_size, subformat=subformat) ++ ++ # Corrupt the header ++ fd = open(img, 'r+b') ++ fd.seek(descriptorOffsetAddr) ++ fd.write(struct.pack('`_ for details. ++fixes: ++ - | ++ `Bug #2059809 `_: ++ Fixed issue where a qcow2 format image with an external data file ++ could expose host information. Such an image is now rejected with ++ an ``ImageUnacceptable`` error if it is used to create a volume. ++ Given that qcow2 external data files were never supported by ++ Cinder, the only use for such an image previously was to attempt ++ to steal host information, and hence this change should have no ++ impact on users. diff -Nru cinder-17.0.1/debian/patches/series cinder-17.4.0/debian/patches/series --- cinder-17.0.1/debian/patches/series 2023-01-18 08:06:59.000000000 +0000 +++ cinder-17.4.0/debian/patches/series 2023-05-15 07:36:50.000000000 +0000 @@ -1,3 +1,7 @@ install-missing-files.patch add-a-healthcheck-url.patch cve-2022-47951-cinder-stable-victoria.patch +#CVE-2023-2088_Reject_unsafe_delete_attachment_calls.patch +add-params-thin_provisioning-equal-one.patch +CVE-2024-32498_0_1_Use_the_json_format_output_of_qemu-img_info.patch +CVE-2024-32498_1_Check_for_external_qcow2_data_file.patch diff -Nru cinder-17.0.1/doc/source/configuration/block-storage/drivers/ceph-rbd-volume-driver.rst cinder-17.4.0/doc/source/configuration/block-storage/drivers/ceph-rbd-volume-driver.rst --- cinder-17.0.1/doc/source/configuration/block-storage/drivers/ceph-rbd-volume-driver.rst 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/doc/source/configuration/block-storage/drivers/ceph-rbd-volume-driver.rst 2022-04-20 17:38:33.000000000 +0000 @@ -81,6 +81,37 @@ Linux kernel and QEMU block devices that stripe data across multiple objects. +RBD pool +~~~~~~~~ + +The RBD pool used by the Cinder backend is configured with option ``rbd_pool``, +and by default the driver expects exclusive management access to that pool, as +in being the only system creating and deleting resources in it, since that's +the recommended deployment choice. + +Pool sharing is strongly discouraged, and if we were to share the pool with +other services, within OpenStack (Nova, Glance, another Cinder backend) or +outside of OpenStack (oVirt), then the stats returned by the driver to the +scheduler would not be entirely accurate. + +The inaccuracy would be that the actual size in use by the cinder volumes would +be lower than the reported one, since it would be also including the used space +by the other services. + +We can set the ``rbd_exclusive_cinder_pool`` configuration option to ``false`` +to fix this inaccuracy, but this has a performance impact. + +.. warning:: + + Setting ``rbd_exclusive_cinder_pool`` to ``false`` will increase the burden + on the Cinder driver and the Ceph cluster, since a request will be made for + each existing image, to retrieve its size, during the stats gathering + process. + + For deployments with large amount of volumes it is recommended to leave the + default value of ``true``, and accept the inaccuracy, as it should not be + particularly problematic. + Driver options ~~~~~~~~~~~~~~ diff -Nru cinder-17.0.1/doc/source/configuration/block-storage/drivers/dell-emc-powerstore-driver.rst cinder-17.4.0/doc/source/configuration/block-storage/drivers/dell-emc-powerstore-driver.rst --- cinder-17.0.1/doc/source/configuration/block-storage/drivers/dell-emc-powerstore-driver.rst 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/doc/source/configuration/block-storage/drivers/dell-emc-powerstore-driver.rst 2022-04-20 17:38:33.000000000 +0000 @@ -77,3 +77,19 @@ The driver creates thin provisioned compressed volumes by default. Thick provisioning is not supported. + +CHAP authentication support +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The driver supports one-way (Single mode) CHAP authentication. +To use CHAP authentication CHAP Single mode has to be enabled on the storage +side. + +.. note:: When enabling CHAP, any previously added hosts will need to be updated + with CHAP configuration since there will be I/O disruption for those hosts. + It is recommended that before adding hosts to the cluster, + decide what type of CHAP configuration is required, if any. + +CHAP configuration is retrieved from the storage during driver initialization, +no additional configuration is needed. +Secrets are generated automatically. diff -Nru cinder-17.0.1/doc/source/configuration/block-storage/drivers/dell-emc-unity-driver.rst cinder-17.4.0/doc/source/configuration/block-storage/drivers/dell-emc-unity-driver.rst --- cinder-17.0.1/doc/source/configuration/block-storage/drivers/dell-emc-unity-driver.rst 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/doc/source/configuration/block-storage/drivers/dell-emc-unity-driver.rst 2022-04-20 17:38:33.000000000 +0000 @@ -48,7 +48,7 @@ Driver configuration ~~~~~~~~~~~~~~~~~~~~ -.. note:: The following instructions should all be performed on Black Storage +.. note:: The following instructions should all be performed on Block Storage nodes. #. Install `storops` from pypi: diff -Nru cinder-17.0.1/doc/source/configuration/block-storage/drivers/dell-emc-xtremio-driver.rst cinder-17.4.0/doc/source/configuration/block-storage/drivers/dell-emc-xtremio-driver.rst --- cinder-17.0.1/doc/source/configuration/block-storage/drivers/dell-emc-xtremio-driver.rst 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/doc/source/configuration/block-storage/drivers/dell-emc-xtremio-driver.rst 2022-04-20 17:38:33.000000000 +0000 @@ -233,6 +233,16 @@ password) are generated automatically by the Block Storage driver. Therefore, there is no need to configure the initial CHAP credentials manually in XMS. +Configuring ports filtering +~~~~~~~~~~~~~~~~~~~~~~~~~~~ +The XtremIO Block Storage driver supports ports filtering to define a list +of iSCSI IP-addresses or FC WWNs which will be used to attach volumes. +If option is not set all ports are allowed. + +.. code-block:: ini + + xtremio_ports = iSCSI IPs or FC WWNs + .. _emc_extremio_configuration_example: Configuration example @@ -250,6 +260,7 @@ volume_driver = cinder.volume.drivers.dell_emc.xtremio.XtremIOFibreChannelDriver san_ip = XMS_IP xtremio_cluster_name = Cluster01 + xtremio_ports = 21:00:00:24:ff:57:b2:36,21:00:00:24:ff:57:b2:55 san_login = XMS_USER san_password = XMS_PASSWD volume_backend_name = XtremIOAFA diff -Nru cinder-17.0.1/doc/source/configuration/block-storage/drivers/hpe-3par-driver.rst cinder-17.4.0/doc/source/configuration/block-storage/drivers/hpe-3par-driver.rst --- cinder-17.0.1/doc/source/configuration/block-storage/drivers/hpe-3par-driver.rst 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/doc/source/configuration/block-storage/drivers/hpe-3par-driver.rst 2022-04-20 17:38:33.000000000 +0000 @@ -378,15 +378,15 @@ san_password=3parpass # FIBRE CHANNEL DRIVER - # Note: For Primera, only FC driver is supported as of now. # (uncomment the next line to enable the FC driver) #volume_driver=cinder.volume.drivers.hpe.hpe_3par_fc.HPE3PARFCDriver # iSCSI DRIVER # If you enable the iSCSI driver, you must also set values # for hpe3par_iscsi_ips or iscsi_ip_address in this file. - # Note: Primera currently requires the FC driver. If you - # configure iSCSI with Primera, the driver will fail to start. + # Note: The iSCSI driver is supported with 3PAR (all versions) + # and Primera (version 4.2 or higher). If you configure iSCSI + # with Primera 4.0 or 4.1, the driver will fail to start. # (uncomment the next line to enable the iSCSI driver) #volume_driver=cinder.volume.drivers.hpe.hpe_3par_iscsi.HPE3PARISCSIDriver diff -Nru cinder-17.0.1/doc/source/reference/support-matrix.ini cinder-17.4.0/doc/source/reference/support-matrix.ini --- cinder-17.0.1/doc/source/reference/support-matrix.ini 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/doc/source/reference/support-matrix.ini 2022-04-20 17:38:33.000000000 +0000 @@ -698,7 +698,7 @@ driver.macrosan=complete driver.nec=complete driver.netapp_ontap=missing -driver.netapp_solidfire=missing +driver.netapp_solidfire=complete driver.nexenta=missing driver.nfs=missing driver.nimble=missing @@ -902,7 +902,7 @@ driver.nfs=missing driver.nimble=missing driver.prophetstor=missing -driver.pure=missing +driver.pure=complete driver.qnap=missing driver.quobyte=missing driver.rbd=complete diff -Nru cinder-17.0.1/lower-constraints.txt cinder-17.4.0/lower-constraints.txt --- cinder-17.0.1/lower-constraints.txt 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/lower-constraints.txt 2022-04-20 17:38:33.000000000 +0000 @@ -1,6 +1,6 @@ alabaster==0.7.10 alembic==1.0.0 -amqp==2.2.2 +amqp==2.6.0 appdirs==1.4.3 asn1crypto==0.24.0 automaton==1.17.0 @@ -14,7 +14,7 @@ cmd2==0.8.1 contextlib2==0.5.5 coverage==4.1 -cryptography==2.1.4 +cryptography==2.5 cursive==0.2.1 ddt==1.2.1 debtcollector==1.22.0 @@ -51,7 +51,7 @@ Mako==1.0.7 MarkupSafe==1.1.0 mock==2.0.0 -msgpack==0.5.6 +msgpack==0.6.0 netaddr==0.7.19 netifaces==0.10.7 networkx==2.1.0 @@ -73,7 +73,7 @@ oslo.privsep==2.3.0 oslo.reports==1.18.0 oslo.rootwrap==5.8.0 -oslo.serialization==2.25.0 +oslo.serialization==4.0.2 oslo.service==2.0.0 oslo.utils==3.40.2 oslo.versionedobjects==1.31.2 diff -Nru cinder-17.0.1/playbooks/cinder-multibackend-matrix.yaml cinder-17.4.0/playbooks/cinder-multibackend-matrix.yaml --- cinder-17.0.1/playbooks/cinder-multibackend-matrix.yaml 1970-01-01 00:00:00.000000000 +0000 +++ cinder-17.4.0/playbooks/cinder-multibackend-matrix.yaml 2022-04-20 17:38:33.000000000 +0000 @@ -0,0 +1,35 @@ +# Playbook originally inspired by +# https://opendev.org/openstack/tempest/src/tag/23.0.0/playbooks/devstack-tempest.yaml + +# Changes that run through devstack-tempest are likely to have an impact on +# the devstack part of the job, so we keep devstack in the main play to +# avoid zuul retrying on legitimate failures. +- hosts: all + roles: + - orchestrate-devstack + +# We run tests only on one node, regardless how many nodes are in the system +- hosts: tempest + vars: + migration_backends: + - lvm + - ceph + - nfs + migration_test_results: [] + migration_tempest_conf: "/opt/stack/tempest/etc/tempest.conf" + tasks: + - include_role: + name: setup-tempest-run-dir + - include_role: + name: setup-tempest-data-dir + - include_role: + name: acl-devstack-files + - include_role: + name: configure-run-migration-tests + vars: + migration_source_backend: "{{ item[0] }}" + migration_destination_backend: "{{ item[1] }}" + loop: "{{ migration_backends|product(migration_backends)|list }}" + when: item[0] != item[1] + - include_role: + name: save-cinder-migration-results diff -Nru cinder-17.0.1/releasenotes/notes/bug-1870103-013e314e9a5b8e08.yaml cinder-17.4.0/releasenotes/notes/bug-1870103-013e314e9a5b8e08.yaml --- cinder-17.0.1/releasenotes/notes/bug-1870103-013e314e9a5b8e08.yaml 1970-01-01 00:00:00.000000000 +0000 +++ cinder-17.4.0/releasenotes/notes/bug-1870103-013e314e9a5b8e08.yaml 2022-04-20 17:38:33.000000000 +0000 @@ -0,0 +1,7 @@ +--- +fixes: + - | + Pure Storage driver `bug 1870103 + `_: + Ensure that unmanaged volumes do not exceed maximum + character length on FlashArray. diff -Nru cinder-17.0.1/releasenotes/notes/bug-1888951-backup-from-nfs-snapshot-2e06235eb318b852.yaml cinder-17.4.0/releasenotes/notes/bug-1888951-backup-from-nfs-snapshot-2e06235eb318b852.yaml --- cinder-17.0.1/releasenotes/notes/bug-1888951-backup-from-nfs-snapshot-2e06235eb318b852.yaml 1970-01-01 00:00:00.000000000 +0000 +++ cinder-17.4.0/releasenotes/notes/bug-1888951-backup-from-nfs-snapshot-2e06235eb318b852.yaml 2022-04-20 17:38:33.000000000 +0000 @@ -0,0 +1,6 @@ +--- +fixes: + - | + `Bug #1888951 `_: + Fixed an issue with creating a backup from snapshot with NFS volume + driver. diff -Nru cinder-17.0.1/releasenotes/notes/bug-1890254-clone-fcmap-is-not-deleting-in-cleanup-f5bbb467be1b889d.yaml cinder-17.4.0/releasenotes/notes/bug-1890254-clone-fcmap-is-not-deleting-in-cleanup-f5bbb467be1b889d.yaml --- cinder-17.0.1/releasenotes/notes/bug-1890254-clone-fcmap-is-not-deleting-in-cleanup-f5bbb467be1b889d.yaml 1970-01-01 00:00:00.000000000 +0000 +++ cinder-17.4.0/releasenotes/notes/bug-1890254-clone-fcmap-is-not-deleting-in-cleanup-f5bbb467be1b889d.yaml 2022-04-20 17:38:33.000000000 +0000 @@ -0,0 +1,8 @@ +--- +fixes: + - | + IBM Spectrum Virtualize driver `Bug #1890254 + `_: + Fix check_vdisk_fc_mappings is not deleting all flashcopy + mappings while deleting source volume, when multiple clones + and snapshots are created using common source volume. diff -Nru cinder-17.0.1/releasenotes/notes/bug-1890591-Pool-information-is-not-saved-in-stats-22f302d941cd9fe2.yaml cinder-17.4.0/releasenotes/notes/bug-1890591-Pool-information-is-not-saved-in-stats-22f302d941cd9fe2.yaml --- cinder-17.0.1/releasenotes/notes/bug-1890591-Pool-information-is-not-saved-in-stats-22f302d941cd9fe2.yaml 1970-01-01 00:00:00.000000000 +0000 +++ cinder-17.4.0/releasenotes/notes/bug-1890591-Pool-information-is-not-saved-in-stats-22f302d941cd9fe2.yaml 2022-04-20 17:38:33.000000000 +0000 @@ -0,0 +1,7 @@ +--- +fixes: + - | + `Bug #1890591 `_: + IBM Spectrum Virtualize Family: Fixed issue in do_setup of + StorwizeSVCCommonDriver to save pool information in stats + during initialisation. diff -Nru cinder-17.0.1/releasenotes/notes/bug-1897598-powerflex-volume-type-conversion.yaml cinder-17.4.0/releasenotes/notes/bug-1897598-powerflex-volume-type-conversion.yaml --- cinder-17.0.1/releasenotes/notes/bug-1897598-powerflex-volume-type-conversion.yaml 1970-01-01 00:00:00.000000000 +0000 +++ cinder-17.4.0/releasenotes/notes/bug-1897598-powerflex-volume-type-conversion.yaml 2022-04-20 17:38:33.000000000 +0000 @@ -0,0 +1,9 @@ +--- + +fixes: + - | + PowerFlex driver `bug #1897598 + `_: Fixed bug with + PowerFlex storage-assisted volume migration when volume migration was + performed without conversion of volume type in cases where it should have + been converted to/from thin/thick provisioned. diff -Nru cinder-17.0.1/releasenotes/notes/bug-1900979-powerstore-chap-support.yaml cinder-17.4.0/releasenotes/notes/bug-1900979-powerstore-chap-support.yaml --- cinder-17.0.1/releasenotes/notes/bug-1900979-powerstore-chap-support.yaml 1970-01-01 00:00:00.000000000 +0000 +++ cinder-17.4.0/releasenotes/notes/bug-1900979-powerstore-chap-support.yaml 2022-04-20 17:38:33.000000000 +0000 @@ -0,0 +1,5 @@ +--- +fixes: + - | + `Bug #1900979 `_: + Fix bug with using PowerStore with enabled CHAP as a storage backend. diff -Nru cinder-17.0.1/releasenotes/notes/bug-1900979-xtremio-ports-filtering-e68f90d47f17a7d9.yaml cinder-17.4.0/releasenotes/notes/bug-1900979-xtremio-ports-filtering-e68f90d47f17a7d9.yaml --- cinder-17.0.1/releasenotes/notes/bug-1900979-xtremio-ports-filtering-e68f90d47f17a7d9.yaml 1970-01-01 00:00:00.000000000 +0000 +++ cinder-17.4.0/releasenotes/notes/bug-1900979-xtremio-ports-filtering-e68f90d47f17a7d9.yaml 2022-04-20 17:38:33.000000000 +0000 @@ -0,0 +1,5 @@ +--- +fixes: + - | + `Bug #1915800 `_: + Add support for ports filtering in XtremIO driver. diff -Nru cinder-17.0.1/releasenotes/notes/bug-1904892-ipv6-nfs-manage-391118115dfaaf54.yaml cinder-17.4.0/releasenotes/notes/bug-1904892-ipv6-nfs-manage-391118115dfaaf54.yaml --- cinder-17.0.1/releasenotes/notes/bug-1904892-ipv6-nfs-manage-391118115dfaaf54.yaml 1970-01-01 00:00:00.000000000 +0000 +++ cinder-17.4.0/releasenotes/notes/bug-1904892-ipv6-nfs-manage-391118115dfaaf54.yaml 2022-04-20 17:38:33.000000000 +0000 @@ -0,0 +1,7 @@ +--- +fixes: + - | + `Bug #1904892 `_: + Fix cinder manage operations for NFS backends using IPv6 addresses + in the NFS server address. These were previously rejected by the + Cinder API. diff -Nru cinder-17.0.1/releasenotes/notes/bug-1905564-e7dcf28fd734d3b2.yaml cinder-17.4.0/releasenotes/notes/bug-1905564-e7dcf28fd734d3b2.yaml --- cinder-17.0.1/releasenotes/notes/bug-1905564-e7dcf28fd734d3b2.yaml 1970-01-01 00:00:00.000000000 +0000 +++ cinder-17.4.0/releasenotes/notes/bug-1905564-e7dcf28fd734d3b2.yaml 2022-04-20 17:38:33.000000000 +0000 @@ -0,0 +1,7 @@ +--- +fixes: + - | + PowerMax Driver `bug #1905564 + `_: Fix + Fix remote SRP not being assigned to volume's Host when + performing retype during failover-promotion. diff -Nru cinder-17.0.1/releasenotes/notes/bug-1907964-9277e5ddec2abeda.yaml cinder-17.4.0/releasenotes/notes/bug-1907964-9277e5ddec2abeda.yaml --- cinder-17.0.1/releasenotes/notes/bug-1907964-9277e5ddec2abeda.yaml 1970-01-01 00:00:00.000000000 +0000 +++ cinder-17.4.0/releasenotes/notes/bug-1907964-9277e5ddec2abeda.yaml 2022-04-20 17:38:33.000000000 +0000 @@ -0,0 +1,11 @@ +--- +fixes: + - | + RBD driver `bug #1907964 + `_: Add support + for fast-diff on backup images stored in Ceph. + Provided fast-diff is supported by the backend it will automatically be + enabled and used. + With fast-diff enabled, the generation of diffs between images and + snapshots as well as determining the actual data usage of a snapshot + is speed up significantly. \ No newline at end of file diff -Nru cinder-17.0.1/releasenotes/notes/bug-1908315-020fea3e244d49bb.yaml cinder-17.4.0/releasenotes/notes/bug-1908315-020fea3e244d49bb.yaml --- cinder-17.0.1/releasenotes/notes/bug-1908315-020fea3e244d49bb.yaml 1970-01-01 00:00:00.000000000 +0000 +++ cinder-17.4.0/releasenotes/notes/bug-1908315-020fea3e244d49bb.yaml 2022-04-20 17:38:33.000000000 +0000 @@ -0,0 +1,38 @@ +--- +upgrade: + - | + This release contains a fix for `Bug #1908315 + `_, which changes the + default value of the policy governing the Block Storage API action + `Reset group snapshot status + `_ + to make the action administrator-only. This policy was inadvertently + changed to be admin-or-owner during the Queens development cycle. + + The policy is named ``group:reset_group_snapshot_status``. + + * If you have a custom value for this policy in your cinder policy + configuration file, this change to the default value will not affect + you. + * If you have been aware of this regression and like the current + (incorrect) behavior, you may add the following line to your cinder + policy configuration file to restore that behavior:: + + "group:reset_group_snapshot_status": "rule:admin_or_owner" + + This setting is *not recommended* by the Cinder project team, as it + may allow end users to put a group snapshot into an invalid status with + indeterminate consequences. + + For more information about the cinder policy configuration file, see the + `policy.yaml + `_ + section of the Cinder Configuration Guide. +fixes: + - | + `Bug #1908315 `_: Corrected + the default checkstring for the ``group:reset_group_snapshot_status`` + policy to make it admin-only. This policy governs the Block Storage API + action `Reset group snapshot status + `_, + which by default is supposed to be an adminstrator-only action. diff -Nru cinder-17.0.1/releasenotes/notes/bug-1912564-strowize-hyperswap-volume-is-not-deleting-a94291248f8f59cd.yaml cinder-17.4.0/releasenotes/notes/bug-1912564-strowize-hyperswap-volume-is-not-deleting-a94291248f8f59cd.yaml --- cinder-17.0.1/releasenotes/notes/bug-1912564-strowize-hyperswap-volume-is-not-deleting-a94291248f8f59cd.yaml 1970-01-01 00:00:00.000000000 +0000 +++ cinder-17.4.0/releasenotes/notes/bug-1912564-strowize-hyperswap-volume-is-not-deleting-a94291248f8f59cd.yaml 2022-04-20 17:38:33.000000000 +0000 @@ -0,0 +1,6 @@ +--- +fixes: + - | + IBM Spectrum Virtualize Family driver `Bug #1912564 + `_: Fixed HyperSwap + volume deletion issue. diff -Nru cinder-17.0.1/releasenotes/notes/bug-1913449-4796b366ae7e871b.yaml cinder-17.4.0/releasenotes/notes/bug-1913449-4796b366ae7e871b.yaml --- cinder-17.0.1/releasenotes/notes/bug-1913449-4796b366ae7e871b.yaml 1970-01-01 00:00:00.000000000 +0000 +++ cinder-17.4.0/releasenotes/notes/bug-1913449-4796b366ae7e871b.yaml 2022-04-20 17:38:33.000000000 +0000 @@ -0,0 +1,7 @@ +--- +fixes: + - | + `Bug 1913449 `_: + Fix RBD driver _update_volume_stats() failing when using Ceph + Pacific python rados libraries. This failed because we + were passing a str instead of bytes to cluster.mon_command() diff -Nru cinder-17.0.1/releasenotes/notes/bug-1920237-backup-remove-export-race-941e2ab1f056e54c.yaml cinder-17.4.0/releasenotes/notes/bug-1920237-backup-remove-export-race-941e2ab1f056e54c.yaml --- cinder-17.0.1/releasenotes/notes/bug-1920237-backup-remove-export-race-941e2ab1f056e54c.yaml 1970-01-01 00:00:00.000000000 +0000 +++ cinder-17.4.0/releasenotes/notes/bug-1920237-backup-remove-export-race-941e2ab1f056e54c.yaml 2022-04-20 17:38:33.000000000 +0000 @@ -0,0 +1,8 @@ +--- +fixes: + - | + `Bug #1920237 `_: The + backup manager calls volume remove_export() but does not wait for it to + complete when detaching a volume after backup. This caused problems + when a subsequent operation started on that volume before it had fully + detached. diff -Nru cinder-17.0.1/releasenotes/notes/bug-1920729-powerstore-iscsi-targets-filtering-9623ac03da5c6721.yaml cinder-17.4.0/releasenotes/notes/bug-1920729-powerstore-iscsi-targets-filtering-9623ac03da5c6721.yaml --- cinder-17.0.1/releasenotes/notes/bug-1920729-powerstore-iscsi-targets-filtering-9623ac03da5c6721.yaml 1970-01-01 00:00:00.000000000 +0000 +++ cinder-17.4.0/releasenotes/notes/bug-1920729-powerstore-iscsi-targets-filtering-9623ac03da5c6721.yaml 2022-04-20 17:38:33.000000000 +0000 @@ -0,0 +1,8 @@ +--- +fixes: + - | + PowerStore driver `Bug #1920729 + `_: Fix + iSCSI targets not being returned from the REST API call if + targets are used for multiple purposes + (iSCSI target, Replication target, etc.). diff -Nru cinder-17.0.1/releasenotes/notes/bug1929429-e749f5e5a242a599.yaml cinder-17.4.0/releasenotes/notes/bug1929429-e749f5e5a242a599.yaml --- cinder-17.0.1/releasenotes/notes/bug1929429-e749f5e5a242a599.yaml 1970-01-01 00:00:00.000000000 +0000 +++ cinder-17.4.0/releasenotes/notes/bug1929429-e749f5e5a242a599.yaml 2022-04-20 17:38:33.000000000 +0000 @@ -0,0 +1,8 @@ +--- +fixes: + - | + PowerMax driver `bug #1929429 + `_: Fixes + child/parent storage group check so that a pattern match + is not case sensitive. For example, myStorageGroup should + equal MYSTORAGEGROUP and mystoragegroup. diff -Nru cinder-17.0.1/releasenotes/notes/bug-193688-bb045badcd5aecad.yaml cinder-17.4.0/releasenotes/notes/bug-193688-bb045badcd5aecad.yaml --- cinder-17.0.1/releasenotes/notes/bug-193688-bb045badcd5aecad.yaml 1970-01-01 00:00:00.000000000 +0000 +++ cinder-17.4.0/releasenotes/notes/bug-193688-bb045badcd5aecad.yaml 2022-04-20 17:38:33.000000000 +0000 @@ -0,0 +1,12 @@ +--- +fixes: + - | + `Bug #1935688 `_: + Cinder only supports uploading a volume of an encrypted volume type as an + image to the Image service in ``raw`` format using a ``bare`` container + type. Previously, ``os-volume_upload_image`` action requests to the Block + Storage API specifying different format option values were accepted, but + would result in a later failure. This condition is now checked at the API + layer, and ``os-volume_upload_image`` action requests on a volume of an + encrypted type that specify unsupported values for ``disk_format`` or + ``container_format`` now result in a 400 (Bad Request) response. diff -Nru cinder-17.0.1/releasenotes/notes/bug-1939139-02ab552420813e70.yaml cinder-17.4.0/releasenotes/notes/bug-1939139-02ab552420813e70.yaml --- cinder-17.0.1/releasenotes/notes/bug-1939139-02ab552420813e70.yaml 1970-01-01 00:00:00.000000000 +0000 +++ cinder-17.4.0/releasenotes/notes/bug-1939139-02ab552420813e70.yaml 2022-04-20 17:38:33.000000000 +0000 @@ -0,0 +1,7 @@ +--- +fixes: + - | + PowerMax driver `bug #1939139 + `_: Fix + on create snapshot operation that exists when using PowerMax OS + 5978.711 and later. diff -Nru cinder-17.0.1/releasenotes/notes/bug-1947518-rbd-open-readonly-ba523c4b0ddbba76.yaml cinder-17.4.0/releasenotes/notes/bug-1947518-rbd-open-readonly-ba523c4b0ddbba76.yaml --- cinder-17.0.1/releasenotes/notes/bug-1947518-rbd-open-readonly-ba523c4b0ddbba76.yaml 1970-01-01 00:00:00.000000000 +0000 +++ cinder-17.4.0/releasenotes/notes/bug-1947518-rbd-open-readonly-ba523c4b0ddbba76.yaml 2022-04-20 17:38:33.000000000 +0000 @@ -0,0 +1,9 @@ +--- +fixes: + - | + RBD driver `bug #1947518 + `_: + Corrected a regression caused by the fix for `Bug #1931004 + `_ that was attempting + to access the glance images RBD pool with write privileges when creating + a volume from an image. diff -Nru cinder-17.0.1/releasenotes/notes/change-default-rbd_exclusive_cinder_pool-e59c528c7f728780.yaml cinder-17.4.0/releasenotes/notes/change-default-rbd_exclusive_cinder_pool-e59c528c7f728780.yaml --- cinder-17.0.1/releasenotes/notes/change-default-rbd_exclusive_cinder_pool-e59c528c7f728780.yaml 1970-01-01 00:00:00.000000000 +0000 +++ cinder-17.4.0/releasenotes/notes/change-default-rbd_exclusive_cinder_pool-e59c528c7f728780.yaml 2022-04-20 17:38:33.000000000 +0000 @@ -0,0 +1,17 @@ +--- +upgrade: + - | + Ceph/RBD volume backends will now assume exclusive cinder pools, as if they + had ``rbd_exclusive_cinder_pool = true`` in their configuration. + + This helps deployments with a large number of volumes and prevent issues on + deployments with a growing number of volumes at the small cost of a + slightly less accurate stats being reported to the scheduler. +fixes: + - | + Ceph/RBD: Fix cinder taking a long time to start for Ceph/RBD backends. + (`Related-Bug #1704106 `_) + - | + Ceph/RBD: Fix Cinder becoming non-responsive and stats gathering taking + longer that its period. (`Related-Bug #1704106 + `_) diff -Nru cinder-17.0.1/releasenotes/notes/detach-notification-31ae15dafdef36c1.yaml cinder-17.4.0/releasenotes/notes/detach-notification-31ae15dafdef36c1.yaml --- cinder-17.0.1/releasenotes/notes/detach-notification-31ae15dafdef36c1.yaml 1970-01-01 00:00:00.000000000 +0000 +++ cinder-17.4.0/releasenotes/notes/detach-notification-31ae15dafdef36c1.yaml 2022-04-20 17:38:33.000000000 +0000 @@ -0,0 +1,9 @@ +--- +fixes: + - | + `Bug #1916980 `_: Fixed + stale volume notification information on volume detach. + - | + `Bug #1935011 `_: Fixed + missing detach.start notification when deleting an attachment in reserved + state. diff -Nru cinder-17.0.1/releasenotes/notes/detach-race-delete-012820ad9c8dbe16.yaml cinder-17.4.0/releasenotes/notes/detach-race-delete-012820ad9c8dbe16.yaml --- cinder-17.0.1/releasenotes/notes/detach-race-delete-012820ad9c8dbe16.yaml 1970-01-01 00:00:00.000000000 +0000 +++ cinder-17.4.0/releasenotes/notes/detach-race-delete-012820ad9c8dbe16.yaml 2022-04-20 17:38:33.000000000 +0000 @@ -0,0 +1,6 @@ +--- +fixes: + - | + `Bug #1937084 `_: Fixed + race condition between delete attachment and delete volume that can leave + deleted volumes stuck as attached to instances. diff -Nru cinder-17.0.1/releasenotes/notes/fix-schema-validation-attachment-create-3488914cb52d44d2.yaml cinder-17.4.0/releasenotes/notes/fix-schema-validation-attachment-create-3488914cb52d44d2.yaml --- cinder-17.0.1/releasenotes/notes/fix-schema-validation-attachment-create-3488914cb52d44d2.yaml 1970-01-01 00:00:00.000000000 +0000 +++ cinder-17.4.0/releasenotes/notes/fix-schema-validation-attachment-create-3488914cb52d44d2.yaml 2022-04-20 17:38:33.000000000 +0000 @@ -0,0 +1,10 @@ +--- +fixes: + - | + Fixed the schema validation for attachment create API + to make instance uuid an optional field. It had mistakenly + been defined as a required field when schema validation + was added in an earlier release. + Also updated the schema to allow specification of the ``mode`` + parameter, which has been available since microversion >= 3.54, + but which was not recognized as a legitimate request field. diff -Nru cinder-17.0.1/releasenotes/notes/fix-sub-clone-operation-f42a84ab17930f24.yaml cinder-17.4.0/releasenotes/notes/fix-sub-clone-operation-f42a84ab17930f24.yaml --- cinder-17.0.1/releasenotes/notes/fix-sub-clone-operation-f42a84ab17930f24.yaml 1970-01-01 00:00:00.000000000 +0000 +++ cinder-17.4.0/releasenotes/notes/fix-sub-clone-operation-f42a84ab17930f24.yaml 2022-04-20 17:38:33.000000000 +0000 @@ -0,0 +1,7 @@ +--- +fixes: + - | + `Bug #1924643 `_: Fixed + the NetApp cinder driver sub-clone operation that might be used + by extend operation in case the extended size is greater than the max + LUN geometry. diff -Nru cinder-17.0.1/releasenotes/notes/fix-transfer-accept-policy-7594806372b14284.yaml cinder-17.4.0/releasenotes/notes/fix-transfer-accept-policy-7594806372b14284.yaml --- cinder-17.0.1/releasenotes/notes/fix-transfer-accept-policy-7594806372b14284.yaml 1970-01-01 00:00:00.000000000 +0000 +++ cinder-17.4.0/releasenotes/notes/fix-transfer-accept-policy-7594806372b14284.yaml 2022-04-20 17:38:33.000000000 +0000 @@ -0,0 +1,8 @@ +--- +fixes: + - | + `Bug #1950474 `_: Fixed + policy authorization for transfer accept API. Previously, if an operator + had overridden the default transfer accept policy to something project + specific in policy.yaml file, it would break the transfer accept API + which is fixed in this release. diff -Nru cinder-17.0.1/releasenotes/notes/hpe-3par-primera-add-iscsi-5af339643dfa0928.yaml cinder-17.4.0/releasenotes/notes/hpe-3par-primera-add-iscsi-5af339643dfa0928.yaml --- cinder-17.0.1/releasenotes/notes/hpe-3par-primera-add-iscsi-5af339643dfa0928.yaml 1970-01-01 00:00:00.000000000 +0000 +++ cinder-17.4.0/releasenotes/notes/hpe-3par-primera-add-iscsi-5af339643dfa0928.yaml 2022-04-20 17:38:33.000000000 +0000 @@ -0,0 +1,5 @@ +--- +features: + - | + HPE 3PAR Driver: Add support of iSCSI driver for Primera 4.2 + or higher versions. diff -Nru cinder-17.0.1/releasenotes/notes/incorrect-host-config-option-347e60f957458d54_new.yaml cinder-17.4.0/releasenotes/notes/incorrect-host-config-option-347e60f957458d54_new.yaml --- cinder-17.0.1/releasenotes/notes/incorrect-host-config-option-347e60f957458d54_new.yaml 1970-01-01 00:00:00.000000000 +0000 +++ cinder-17.4.0/releasenotes/notes/incorrect-host-config-option-347e60f957458d54_new.yaml 2022-04-20 17:38:33.000000000 +0000 @@ -0,0 +1,7 @@ +--- +fixes: + - | + `Bug #1941068 `_: Fixed + type of the ``host`` configuration option. It was limited to valid FQDN + values when we document that it isn't. This may result in the + ``cinder-manage db sync`` command failing. diff -Nru cinder-17.0.1/releasenotes/notes/lvm-delete-error-f12da00c1b3859dc.yaml cinder-17.4.0/releasenotes/notes/lvm-delete-error-f12da00c1b3859dc.yaml --- cinder-17.0.1/releasenotes/notes/lvm-delete-error-f12da00c1b3859dc.yaml 1970-01-01 00:00:00.000000000 +0000 +++ cinder-17.4.0/releasenotes/notes/lvm-delete-error-f12da00c1b3859dc.yaml 2022-04-20 17:38:33.000000000 +0000 @@ -0,0 +1,6 @@ +--- +fixes: + - | + LVM driver `bug #1901783 + `_: Fix unexpected delete + volume failure due to unexpected exit code 139 on ``lvs`` command call. diff -Nru cinder-17.0.1/releasenotes/notes/netapp-migrated-qos-c0c8aae50d010c75.yaml cinder-17.4.0/releasenotes/notes/netapp-migrated-qos-c0c8aae50d010c75.yaml --- cinder-17.0.1/releasenotes/notes/netapp-migrated-qos-c0c8aae50d010c75.yaml 1970-01-01 00:00:00.000000000 +0000 +++ cinder-17.4.0/releasenotes/notes/netapp-migrated-qos-c0c8aae50d010c75.yaml 2022-04-20 17:38:33.000000000 +0000 @@ -0,0 +1,7 @@ +--- +fixes: + - | + NetApp ONTAP `bug #1906291 + `_: Fix volume losing its + QoS policy on the backend after moving it (migrate or retype with migrate) + to a NetApp NFS backend. diff -Nru cinder-17.0.1/releasenotes/notes/nfs-online-snapshot-c05e6c8113bbded6.yaml cinder-17.4.0/releasenotes/notes/nfs-online-snapshot-c05e6c8113bbded6.yaml --- cinder-17.0.1/releasenotes/notes/nfs-online-snapshot-c05e6c8113bbded6.yaml 1970-01-01 00:00:00.000000000 +0000 +++ cinder-17.4.0/releasenotes/notes/nfs-online-snapshot-c05e6c8113bbded6.yaml 2022-04-20 17:38:33.000000000 +0000 @@ -0,0 +1,6 @@ +--- +fixes: + - | + NFS driver `bug #1860913 + `_: Fixed instance uses + base image file when it is rebooted after online snapshot creation. diff -Nru cinder-17.0.1/releasenotes/notes/promotion_offline_r1_fix-f7a008d0d13a3eff.yaml cinder-17.4.0/releasenotes/notes/promotion_offline_r1_fix-f7a008d0d13a3eff.yaml --- cinder-17.0.1/releasenotes/notes/promotion_offline_r1_fix-f7a008d0d13a3eff.yaml 1970-01-01 00:00:00.000000000 +0000 +++ cinder-17.4.0/releasenotes/notes/promotion_offline_r1_fix-f7a008d0d13a3eff.yaml 2022-04-20 17:38:33.000000000 +0000 @@ -0,0 +1,8 @@ +--- +fixes: + - | + PowerMax Driver - `bug #1908920 + `_: This offline r1 + promotion fix resets replication enabled and configuration metadata + during promotion retype with offline r1 array. It also gets management + storage group name from source extra_specs during promotion. diff -Nru cinder-17.0.1/releasenotes/notes/promotion_rdfg_num_fix-65a5838277ac8edf.yaml cinder-17.4.0/releasenotes/notes/promotion_rdfg_num_fix-65a5838277ac8edf.yaml --- cinder-17.0.1/releasenotes/notes/promotion_rdfg_num_fix-65a5838277ac8edf.yaml 1970-01-01 00:00:00.000000000 +0000 +++ cinder-17.4.0/releasenotes/notes/promotion_rdfg_num_fix-65a5838277ac8edf.yaml 2022-04-20 17:38:33.000000000 +0000 @@ -0,0 +1,6 @@ +--- +fixes: + - | + PowerMax Driver - Promotion RDF Group number fix uses remote array + SID when finding rdf group number when performing retype during + failover. diff -Nru cinder-17.0.1/releasenotes/notes/pure-active-active-support-dbd0d3da3ab64e64.yaml cinder-17.4.0/releasenotes/notes/pure-active-active-support-dbd0d3da3ab64e64.yaml --- cinder-17.0.1/releasenotes/notes/pure-active-active-support-dbd0d3da3ab64e64.yaml 1970-01-01 00:00:00.000000000 +0000 +++ cinder-17.4.0/releasenotes/notes/pure-active-active-support-dbd0d3da3ab64e64.yaml 2022-04-20 17:38:33.000000000 +0000 @@ -0,0 +1,7 @@ +--- +features: + - | + Pure Storage FlashArray driver: Enabled support for Active/Active + to both the iSCSI and FC driver. This allows users to configure + Pure Storage backends in clustered environments. + diff -Nru cinder-17.0.1/releasenotes/notes/pure_storage_multiattach-f4aee3576757b2ff.yaml cinder-17.4.0/releasenotes/notes/pure_storage_multiattach-f4aee3576757b2ff.yaml --- cinder-17.0.1/releasenotes/notes/pure_storage_multiattach-f4aee3576757b2ff.yaml 1970-01-01 00:00:00.000000000 +0000 +++ cinder-17.4.0/releasenotes/notes/pure_storage_multiattach-f4aee3576757b2ff.yaml 2022-04-20 17:38:33.000000000 +0000 @@ -0,0 +1,7 @@ +--- +fixes: + - | + Pure Storage `bug #1930748 + `_: Fixed issues + with multiattched volumes being diconnected from a backend when + still listed as an attachment to an instance. diff -Nru cinder-17.0.1/releasenotes/notes/pure_tempest_cg_fix-913d405f7487de00.yaml cinder-17.4.0/releasenotes/notes/pure_tempest_cg_fix-913d405f7487de00.yaml --- cinder-17.0.1/releasenotes/notes/pure_tempest_cg_fix-913d405f7487de00.yaml 1970-01-01 00:00:00.000000000 +0000 +++ cinder-17.4.0/releasenotes/notes/pure_tempest_cg_fix-913d405f7487de00.yaml 2022-04-20 17:38:33.000000000 +0000 @@ -0,0 +1,5 @@ +--- +fixes: + - | + Pure Storage FlashArray driver fix to ensure cinder_tempest_plugin consistency + group tests pass. diff -Nru cinder-17.0.1/releasenotes/notes/rbd-choose-correct-stripe-unit-9d317f4717533fb4.yaml cinder-17.4.0/releasenotes/notes/rbd-choose-correct-stripe-unit-9d317f4717533fb4.yaml --- cinder-17.0.1/releasenotes/notes/rbd-choose-correct-stripe-unit-9d317f4717533fb4.yaml 1970-01-01 00:00:00.000000000 +0000 +++ cinder-17.4.0/releasenotes/notes/rbd-choose-correct-stripe-unit-9d317f4717533fb4.yaml 2022-04-20 17:38:33.000000000 +0000 @@ -0,0 +1,6 @@ +--- +fixes: + - | + `Bug #1931004 `_: Fixed + use of incorrect stripe unit in RBD image clone causing volume-from-image + to fail when using raw images backed by Ceph. diff -Nru cinder-17.0.1/releasenotes/notes/remove_export_failure_leaves_attachment-24e0c648269b0177.yaml cinder-17.4.0/releasenotes/notes/remove_export_failure_leaves_attachment-24e0c648269b0177.yaml --- cinder-17.0.1/releasenotes/notes/remove_export_failure_leaves_attachment-24e0c648269b0177.yaml 1970-01-01 00:00:00.000000000 +0000 +++ cinder-17.4.0/releasenotes/notes/remove_export_failure_leaves_attachment-24e0c648269b0177.yaml 2022-04-20 17:38:33.000000000 +0000 @@ -0,0 +1,6 @@ +--- +fixes: + - | + `Bug #1935057 `_: Fixed + sometimes on a detach volume may end in available and detached yet have an + attachment in error_detaching. diff -Nru cinder-17.0.1/releasenotes/notes/sf-fix-duplicate-volume-request-lost-adefacda1298dc62.yaml cinder-17.4.0/releasenotes/notes/sf-fix-duplicate-volume-request-lost-adefacda1298dc62.yaml --- cinder-17.0.1/releasenotes/notes/sf-fix-duplicate-volume-request-lost-adefacda1298dc62.yaml 1970-01-01 00:00:00.000000000 +0000 +++ cinder-17.4.0/releasenotes/notes/sf-fix-duplicate-volume-request-lost-adefacda1298dc62.yaml 2022-04-20 17:38:33.000000000 +0000 @@ -0,0 +1,14 @@ +--- +fixes: + - | + NetApp SolidFire driver `Bug #1896112 + `_: + Fixes an issue that may duplicate volumes during creation, in case + the SolidFire backend successfully processes a request and creates + the volume, but fails to deliver the result back to the driver (the + response is lost). When this scenario occurs, the SolidFire driver + will retry the operation, which previously resulted in the creation + of a duplicate volume. This fix adds the ``sf_volume_create_timeout`` + configuration option (default value: 60 seconds) which specifies an + additional length of time that the driver will wait for the volume to + become active on the backend before raising an exception. diff -Nru cinder-17.0.1/releasenotes/notes/sf-fix-error-on-cluster-rebalancing-515bf41104cd181a.yaml cinder-17.4.0/releasenotes/notes/sf-fix-error-on-cluster-rebalancing-515bf41104cd181a.yaml --- cinder-17.0.1/releasenotes/notes/sf-fix-error-on-cluster-rebalancing-515bf41104cd181a.yaml 1970-01-01 00:00:00.000000000 +0000 +++ cinder-17.4.0/releasenotes/notes/sf-fix-error-on-cluster-rebalancing-515bf41104cd181a.yaml 2022-04-20 17:38:33.000000000 +0000 @@ -0,0 +1,8 @@ +--- +fixes: + - | + NetApp SolidFire driver `Bug #1891914 + `_: + Fix an error that might occur on cluster workload rebalancing or + system upgrade, when an operation is made to a volume at the same + time its connection is being moved to a secondary node. diff -Nru cinder-17.0.1/releasenotes/notes/slow-get-volume-stats-91b84c6e661dc605.yaml cinder-17.4.0/releasenotes/notes/slow-get-volume-stats-91b84c6e661dc605.yaml --- cinder-17.0.1/releasenotes/notes/slow-get-volume-stats-91b84c6e661dc605.yaml 1970-01-01 00:00:00.000000000 +0000 +++ cinder-17.4.0/releasenotes/notes/slow-get-volume-stats-91b84c6e661dc605.yaml 2022-04-20 17:38:33.000000000 +0000 @@ -0,0 +1,7 @@ +--- +features: + - | + Log a warning from the volume service when a volume driver's + get_volume_stats() call takes a long time to return. This can help + deployers troubleshoot a cinder-volume service misbehaving due to a + driver/backend performance issue. diff -Nru cinder-17.0.1/releasenotes/notes/support-images-api-2.11-3699b20670db1843.yaml cinder-17.4.0/releasenotes/notes/support-images-api-2.11-3699b20670db1843.yaml --- cinder-17.0.1/releasenotes/notes/support-images-api-2.11-3699b20670db1843.yaml 1970-01-01 00:00:00.000000000 +0000 +++ cinder-17.4.0/releasenotes/notes/support-images-api-2.11-3699b20670db1843.yaml 2022-04-20 17:38:33.000000000 +0000 @@ -0,0 +1,12 @@ +--- +fixes: + - | + `Bug #1898075 + `_: When Glance added + support for multiple cinder stores, Images API version 2.11 modified + the format of the image location URI, which Cinder reads in order + to try to use an optimized data path when creating a volume from an + image. Unfortunately, Cinder did not understand the new format and + when Glance multiple cinder stores were used, Cinder could not use + the optimized data path, and instead downloaded image data from + the Image service. Cinder now supports Images API version 2.11. diff -Nru cinder-17.0.1/requirements.txt cinder-17.4.0/requirements.txt --- cinder-17.0.1/requirements.txt 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/requirements.txt 2022-04-20 17:38:33.000000000 +0000 @@ -25,7 +25,7 @@ oslo.privsep>=2.3.0 # Apache-2.0 oslo.reports>=1.18.0 # Apache-2.0 oslo.rootwrap>=5.8.0 # Apache-2.0 -oslo.serialization>=2.25.0 # Apache-2.0 +oslo.serialization>=4.0.2 # Apache-2.0 oslo.service>=2.0.0 # Apache-2.0 oslo.upgradecheck>=0.1.0 # Apache-2.0 oslo.utils>=3.40.2 # Apache-2.0 @@ -61,6 +61,6 @@ tooz>=1.58.0 # Apache-2.0 google-api-python-client>=1.4.2 # Apache-2.0 castellan>=1.3.0 # Apache-2.0 -cryptography>=2.1.4 # BSD/Apache-2.0 +cryptography>=2.5 # BSD/Apache-2.0 cursive>=0.2.1 # Apache-2.0 zstd>=1.4.5.0 # BSD diff -Nru cinder-17.0.1/roles/configure-run-migration-tests/defaults/main.yaml cinder-17.4.0/roles/configure-run-migration-tests/defaults/main.yaml --- cinder-17.0.1/roles/configure-run-migration-tests/defaults/main.yaml 1970-01-01 00:00:00.000000000 +0000 +++ cinder-17.4.0/roles/configure-run-migration-tests/defaults/main.yaml 2022-04-20 17:38:33.000000000 +0000 @@ -0,0 +1,6 @@ +--- +migration_source_backend: lvm +migration_destination_backend: lvm +migration_test_regex: "(.*test_volume_retype_with_migration.*|.*test_volume_migrate_attached.*)" +migration_test_results: [] +tempest_run_result: {} diff -Nru cinder-17.0.1/roles/configure-run-migration-tests/tasks/main.yaml cinder-17.4.0/roles/configure-run-migration-tests/tasks/main.yaml --- cinder-17.0.1/roles/configure-run-migration-tests/tasks/main.yaml 1970-01-01 00:00:00.000000000 +0000 +++ cinder-17.4.0/roles/configure-run-migration-tests/tasks/main.yaml 2022-04-20 17:38:33.000000000 +0000 @@ -0,0 +1,33 @@ +--- +- name: Reconfigure tempest.conf + ini_file: + path: "{{ migration_tempest_conf }}" + section: volume + option: backend_names + value: "{{ migration_source_backend }},{{ migration_destination_backend }}" + become: true + become_user: tempest + +- set_fact: + tempest_run_result: {} + +- name: Run migration ({{ migration_source_backend }} -> {{ migration_destination_backend }}) + include_role: + name: run-tempest + apply: + # ignore the errors for this run, otherwise the other migration tests + # won't be executed + ignore_errors: yes + vars: + tempest_test_regex: "{{ migration_test_regex }}" + tox_envlist: all + +- set_fact: + _migration_result_item: + source: "{{ migration_source_backend }}" + destination: "{{ migration_destination_backend }}" + result: "{{ tempest_run_result.get('rc', 1) }}" + +- name: Update the migration test results + set_fact: + migration_test_results: "{{ migration_test_results + [ _migration_result_item ] }}" diff -Nru cinder-17.0.1/roles/save-cinder-migration-results/defaults/main.yaml cinder-17.4.0/roles/save-cinder-migration-results/defaults/main.yaml --- cinder-17.0.1/roles/save-cinder-migration-results/defaults/main.yaml 1970-01-01 00:00:00.000000000 +0000 +++ cinder-17.4.0/roles/save-cinder-migration-results/defaults/main.yaml 2022-04-20 17:38:33.000000000 +0000 @@ -0,0 +1,3 @@ +--- +devstack_base_dir: /opt/stack +tempest_work_dir: "{{ devstack_base_dir }}/tempest" diff -Nru cinder-17.0.1/roles/save-cinder-migration-results/tasks/main.yaml cinder-17.4.0/roles/save-cinder-migration-results/tasks/main.yaml --- cinder-17.0.1/roles/save-cinder-migration-results/tasks/main.yaml 1970-01-01 00:00:00.000000000 +0000 +++ cinder-17.4.0/roles/save-cinder-migration-results/tasks/main.yaml 2022-04-20 17:38:33.000000000 +0000 @@ -0,0 +1,14 @@ +--- +- block: + - template: + src: migration_results_reporter.py.j2 + dest: "{{ tempest_work_dir }}/migration_results_reporter.py" + + - name: Generate the results using stestr + shell: | + stestr run --no-discover --test-path . migration_results_reporter + args: + chdir: "{{ tempest_work_dir }}" + + become: true + become_user: tempest diff -Nru cinder-17.0.1/roles/save-cinder-migration-results/templates/migration_results_reporter.py.j2 cinder-17.4.0/roles/save-cinder-migration-results/templates/migration_results_reporter.py.j2 --- cinder-17.0.1/roles/save-cinder-migration-results/templates/migration_results_reporter.py.j2 1970-01-01 00:00:00.000000000 +0000 +++ cinder-17.4.0/roles/save-cinder-migration-results/templates/migration_results_reporter.py.j2 2022-04-20 17:38:33.000000000 +0000 @@ -0,0 +1,10 @@ +import unittest + + +class CinderMigrationsMatrixTest(unittest.TestCase): + +{% for test_case in migration_test_results %} + def test__{{ test_case.source }}_to_{{ test_case.destination }}(self): + self.assertEqual({{ test_case.result }}, 0) + +{% endfor %} diff -Nru cinder-17.0.1/tools/hooks/README cinder-17.4.0/tools/hooks/README --- cinder-17.0.1/tools/hooks/README 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/tools/hooks/README 1970-01-01 00:00:00.000000000 +0000 @@ -1,4 +0,0 @@ -These are hooks to be used by the OpenStack infra test system. These scripts -may be called by certain jobs at important times to do extra testing, setup, -etc. They are really only relevant within the scope of the OpenStack infra -system and are not expected to be useful to anyone else. \ No newline at end of file diff -Nru cinder-17.0.1/tools/hooks/run_multi_backend_matrix.sh cinder-17.4.0/tools/hooks/run_multi_backend_matrix.sh --- cinder-17.0.1/tools/hooks/run_multi_backend_matrix.sh 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/tools/hooks/run_multi_backend_matrix.sh 1970-01-01 00:00:00.000000000 +0000 @@ -1,94 +0,0 @@ -#!/bin/bash - -# Copyright (c) 2016, Hitachi, Erlon Cruz -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -set -x -export TEMPEST_USER=${TEMPEST_USER:-tempest} -chmod +w $BASE/new/tempest -cd $BASE/new/tempest -source $BASE/new/devstack/functions -source $BASE/new/devstack/functions-common -source $WORKSPACE/devstack-gate/functions.sh -source $BASE/new/cinder/tools/hooks/utils.sh -export TEMPEST_CONFIG=$BASE/new/tempest/etc/tempest.conf - -# Disable bash verbose so we have a cleaner output. Also, exit on error must -# be disable as we will run several tests that can return error. -set +x +e - -function configure_tempest_backends { - be1=$1 - be2=$2 - echo "Configuring tempest conf in ${TEMPEST_CONFIG}" - iniset -sudo $TEMPEST_CONFIG 'volume' 'backend_names' ${be1},${be2} - -} - -BACKENDS='lvm ceph nfs' -RGEX="(.*test_volume_retype_with_migration.*|.*test_volume_migrate_attached.*)" -final_result=0 -final_message='Migrations tests finished SUCCESSFULLY!' -declare -A TEST_RESULTS -start_time=`date +%s` -for be1 in ${BACKENDS}; do - for be2 in ${BACKENDS}; do - if [ ${be1} != ${be2} ]; then - configure_tempest_backends ${be1} ${be2} - echo "============================================================" - echo "Testing multibackend features: ${be1} vs ${be2}" - echo "============================================================" - run_tempest "${be1} vs ${be2}" ${RGEX} - result=$? - # If any of the test fail, we keep running but return failure as - # the final result - if [ ${result} -ne 0 ]; then - TEST_RESULTS[${be1},${be2}]="FAILURE" - final_message='Migrations tests FAILED!' - final_result=1 - else - TEST_RESULTS[${be1},${be2}]="SUCCESS" - fi - fi - done -done -end_time=`date +%s` -elapsed=$(expr $(expr ${end_time} - ${start_time}) / 60) - -# Print the results -num_rows=$(echo $BACKENDS | wc -w) -fmt=" %15s" -echo "============================================================" -echo " ${final_message} In ${elapsed} minutes." -echo "============================================================" - -printf "$fmt" '' -for be1 in ${BACKENDS}; do - printf "$fmt" ${be1} -done -echo -for be1 in ${BACKENDS}; do - printf "$fmt" ${be1} - for be2 in ${BACKENDS}; do - if [ ${be1} == ${be2} ]; then - printf "$fmt" '---' - else - printf "$fmt" ${TEST_RESULTS[${be1},${be2}]} - fi - done - echo -done - -exit ${final_result} diff -Nru cinder-17.0.1/tools/hooks/utils.sh cinder-17.4.0/tools/hooks/utils.sh --- cinder-17.0.1/tools/hooks/utils.sh 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/tools/hooks/utils.sh 1970-01-01 00:00:00.000000000 +0000 @@ -1,10 +0,0 @@ -#!/bin/bash - -function run_tempest { - local message=$1 - local tempest_regex=$2 - sudo -H -u ${TEMPEST_USER} tox -eall -- $tempest_regex \ - --concurrency=${TEMPEST_CONCURRENCY} - exitcode=$? - return ${exitcode} -} diff -Nru cinder-17.0.1/.zuul.yaml cinder-17.4.0/.zuul.yaml --- cinder-17.0.1/.zuul.yaml 2020-11-29 22:25:43.000000000 +0000 +++ cinder-17.4.0/.zuul.yaml 2022-04-20 17:38:33.000000000 +0000 @@ -1,6 +1,5 @@ - project: templates: - - openstack-lower-constraints-jobs - openstack-python3-victoria-jobs - publish-openstack-docs-pti - periodic-stable-jobs @@ -68,6 +67,11 @@ irrelevant-files: *gate-irrelevant-files - tempest-ipv6-only: irrelevant-files: *gate-irrelevant-files + - openstacksdk-functional-devstack: + required-projects: + - name: opendev.org/openstack/openstacksdk + override-branch: stable/victoria + irrelevant-files: *gate-irrelevant-files gate: jobs: - cinder-grenade-mn-sub-volbak: @@ -80,11 +84,16 @@ irrelevant-files: *gate-irrelevant-files - tempest-ipv6-only: irrelevant-files: *gate-irrelevant-files + - openstacksdk-functional-devstack: + required-projects: + - name: opendev.org/openstack/openstacksdk + override-branch: stable/victoria + irrelevant-files: *gate-irrelevant-files experimental: jobs: - tempest-cinder-v2-api: irrelevant-files: *gate-irrelevant-files - - legacy-tempest-dsvm-multibackend-matrix: + - cinder-multibackend-matrix-migration: irrelevant-files: *gate-irrelevant-files - cinder-grenade-mn-sub-volschbak: irrelevant-files: *gate-irrelevant-files @@ -162,8 +171,7 @@ - job: name: cinder-plugin-ceph-tempest-mn-aa - nodeset: openstack-two-node-bionic - parent: devstack-plugin-ceph-tempest-py3 + parent: devstack-plugin-ceph-multinode-tempest-py3 roles: - zuul: opendev.org/openstack/cinderlib - zuul: opendev.org/openstack/cinder-tempest-plugin @@ -174,15 +182,13 @@ vars: zuul_additional_subunit_dirs: - "{{ ansible_user_dir }}/{{ zuul.projects['opendev.org/openstack/cinderlib'].src_dir }}" + devstack_localrc: + TEMPEST_VOLUME_REVERT_TO_SNAPSHOT: True devstack_local_conf: post-config: $CINDER_CONF: DEFAULT: cluster: ceph - test-config: - $TEMPEST_CONFIG: - volume-feature-enabled: - volume_revert: True - job: name: cinder-grenade-mn-sub-bak @@ -277,3 +283,31 @@ CINDER_ENABLED_BACKENDS: 'lvm:lvmdriver-1,lvm:lvmdriver-2' CINDER_VOLUME_CLEAR: none irrelevant-files: *gate-irrelevant-files + +- job: + name: cinder-multibackend-matrix-migration + parent: devstack-tempest + description: | + Run migration tests between several combinations of backends + (LVM, Ceph, NFS) + Former names for this job were: + * legacy-tempest-dsvm-multibackend-matrix + timeout: 10800 + required-projects: + - opendev.org/openstack/devstack-plugin-ceph + - opendev.org/openstack/devstack-plugin-nfs + run: playbooks/cinder-multibackend-matrix.yaml + host-vars: + controller: + devstack_plugins: + devstack-plugin-ceph: https://opendev.org/openstack/devstack-plugin-ceph + devstack-plugin-nfs: https://opendev.org/openstack/devstack-plugin-nfs + vars: + devstack_localrc: + CINDER_ENABLED_BACKENDS: lvm:lvm,nfs:nfs,ceph:ceph + ENABLE_NFS_CINDER: true + devstack_local_conf: + test-config: + $TEMPEST_CONFIG: + volume: + build_timeout: 900