diff -Nru glance-21.0.0/debian/changelog glance-21.1.0/debian/changelog --- glance-21.0.0/debian/changelog 2023-01-18 09:14:44.000000000 +0000 +++ glance-21.1.0/debian/changelog 2024-06-21 13:46:47.000000000 +0000 @@ -1,12 +1,25 @@ -glance (2:21.0.0-2+deb11u1) bullseye-security; urgency=medium +glance (2:21.1.0-1+deb11u2) bullseye-security; urgency=high + * CVE-2024-32498: Arbitrary file access through custom QCOW2 external data. + Add upstream patch (Closes: #1074761): + - CVE-2024-32498_train-ish-backport.patch + * (build-)depends on oslo.utils >= 4.6.1. + + -- Thomas Goirand Fri, 21 Jun 2024 15:46:47 +0200 + +glance (2:21.1.0-1+deb11u1) bullseye-security; urgency=high + + * Disable using uwsgi for now, until upstream repairs the transfer-encoding: + chucked with uwsgi. + * New upstream release. + * Fixed depends for this release. * CVE-2022-47951: By supplying a specially created VMDK flat image which references a specific backing file path, an authenticated user may convince systems to return a copy of that file's contents from the server resulting in unauthorized access to potentially sensitive data. Add upstream patch cve-2022-47951-glance-stable-victoria.patch (Closes: #1029563). - -- Thomas Goirand Wed, 18 Jan 2023 10:14:44 +0100 + -- Thomas Goirand Wed, 18 Jan 2023 09:54:30 +0100 glance (2:21.0.0-2) unstable; urgency=medium diff -Nru glance-21.0.0/debian/control glance-21.1.0/debian/control --- glance-21.0.0/debian/control 2023-01-18 09:14:44.000000000 +0000 +++ glance-21.1.0/debian/control 2024-06-21 13:46:47.000000000 +0000 @@ -22,6 +22,7 @@ python3-alembic, python3-babel, python3-boto, + python3-boto3, python3-castellan, python3-coverage, python3-cryptography, @@ -46,9 +47,10 @@ python3-openssl, python3-openstackdocstheme, python3-os-api-ref, + python3-os-win (>= 4.0.1), python3-oslo.concurrency, python3-oslo.config, - python3-oslo.context, + python3-oslo.context (>= 2.22.0), python3-oslo.db (>= 5.0.0), python3-oslo.i18n, python3-oslo.log, @@ -57,7 +59,7 @@ python3-oslo.policy (>= 2.4.1), python3-oslo.reports, python3-oslo.upgradecheck, - python3-oslo.utils, + python3-oslo.utils (>= 4.6.1), python3-oslotest, python3-osprofiler, python3-paste, @@ -79,7 +81,7 @@ python3-stestr, python3-stevedore, python3-swiftclient, - python3-taskflow, + python3-taskflow (>= 4.0.0), python3-testresources, python3-testscenarios, python3-testtools, @@ -198,9 +200,10 @@ python3-monotonic, python3-mysqldb, python3-openssl, + python3-os-win (>= 4.0.1), python3-oslo.concurrency, python3-oslo.config, - python3-oslo.context, + python3-oslo.context (>= 2.22.0), python3-oslo.db (>= 5.0.0), python3-oslo.i18n, python3-oslo.log, @@ -209,7 +212,7 @@ python3-oslo.policy (>= 2.4.1), python3-oslo.reports, python3-oslo.upgradecheck, - python3-oslo.utils, + python3-oslo.utils (>= 4.6.1), python3-osprofiler, python3-paste, python3-pastedeploy, @@ -225,7 +228,7 @@ python3-sqlparse, python3-stevedore, python3-swiftclient, - python3-taskflow, + python3-taskflow (>= 4.0.0), python3-webob, python3-wsme, python3-xattr, diff -Nru glance-21.0.0/debian/glance-api.init.in glance-21.1.0/debian/glance-api.init.in --- glance-21.0.0/debian/glance-api.init.in 2023-01-18 09:14:44.000000000 +0000 +++ glance-21.1.0/debian/glance-api.init.in 2024-06-21 13:46:47.000000000 +0000 @@ -16,6 +16,8 @@ PROJECT_NAME=glance NAME=${PROJECT_NAME}-api CONFIG_FILE=/etc/${PROJECT_NAME}/glance-api.conf -UWSGI_PORT=9292 -UWSGI_INI_PATH=/etc/glance/glance-api-uwsgi.ini -UWSGI_INI_APP=/usr/bin/glance-wsgi-api +# Glance appears broken with uwsgi. +# Let's disable uwsgi for now. +#UWSGI_PORT=9292 +#UWSGI_INI_PATH=/etc/glance/glance-api-uwsgi.ini +#UWSGI_INI_APP=/usr/bin/glance-wsgi-api diff -Nru glance-21.0.0/debian/patches/cve-2022-47951-glance-stable-victoria.patch glance-21.1.0/debian/patches/cve-2022-47951-glance-stable-victoria.patch --- glance-21.0.0/debian/patches/cve-2022-47951-glance-stable-victoria.patch 2023-01-18 09:14:44.000000000 +0000 +++ glance-21.1.0/debian/patches/cve-2022-47951-glance-stable-victoria.patch 2024-06-21 13:46:47.000000000 +0000 @@ -16,7 +16,7 @@ and even in the case where image_conversion is not enabled. Author: Dan Smith Date: Mon, 19 Dec 2022 15:00:35 +0000 -Bug: https://launchpad.net/bugs/1996188 +Bug: #1996188 Debian-Bug: https://bugs.debian.org/1029563 Change-Id: Idf561f6306cebf756c787d8eefdc452ce44bd5e0 Origin: upstream, https://review.opendev.org/c/openstack/glance/+/871623 diff -Nru glance-21.0.0/debian/patches/CVE-2024-32498_train-ish-backport.patch glance-21.1.0/debian/patches/CVE-2024-32498_train-ish-backport.patch --- glance-21.0.0/debian/patches/CVE-2024-32498_train-ish-backport.patch 1970-01-01 00:00:00.000000000 +0000 +++ glance-21.1.0/debian/patches/CVE-2024-32498_train-ish-backport.patch 2024-06-21 13:46:47.000000000 +0000 @@ -0,0 +1,760 @@ +Author: Dan Smith +Date: Mon, 1 Apr 2024 08:06:31 -0700 +Description: [PATCH 1/3] Reject qcow files with data-file attributes + This patch concatenates 3 patchesm and was backported by Dan for + Train, then forward ported to Victoria by Thomas Goirand. +Change-Id: I6326a3e85c1ba4cb1da944a4323769f2399ed2c1 +Bug: https://launchpad.net/bugs/2059809 +Bug-Debian: https://bugs.debian.org/1074761 +Last-Update: 2024-06-30 + +Index: glance/glance/async_/flows/base_import.py +=================================================================== +--- glance.orig/glance/async_/flows/base_import.py ++++ glance/glance/async_/flows/base_import.py +@@ -182,6 +182,16 @@ class _ImportToFS(task.Task): + 'bfile': backing_file} + raise RuntimeError(msg) + ++ try: ++ data_file = metadata['format-specific']['data']['data-file'] ++ except KeyError: ++ data_file = None ++ if data_file is not None: ++ msg = _("File %(path)s has invalid data-file " ++ "%(dfile)s, aborting.") % {"path": path, ++ "dfile": data_file} ++ raise RuntimeError(msg) ++ + return path + + def revert(self, image_id, result, **kwargs): +Index: glance/glance/async_/flows/plugins/image_conversion.py +=================================================================== +--- glance.orig/glance/async_/flows/plugins/image_conversion.py ++++ glance/glance/async_/flows/plugins/image_conversion.py +@@ -25,6 +25,7 @@ from taskflow.patterns import linear_flo + from taskflow import task + + from glance.async_ import utils ++from glance.common import format_inspector + from glance.i18n import _ + + LOG = logging.getLogger(__name__) +@@ -84,8 +85,41 @@ class _ConvertImage(task.Task): + 'target': target_format} + self.dest_path = dest_path + ++ image = self.image_repo.get(self.image_id) ++ source_format = image.disk_format ++ inspector_cls = format_inspector.get_inspector(source_format) ++ if not inspector_cls: ++ # We cannot convert from disk_format types that qemu-img doesn't ++ # support (like iso, ploop, etc). The ones it supports overlaps ++ # with the ones we have inspectors for, so reject conversion for ++ # any format we don't have an inspector for. ++ raise RuntimeError( ++ 'Unable to convert from format %s' % source_format) ++ ++ # Use our own cautious inspector module (if we have one for this ++ # format) to make sure a file is the format the submitter claimed ++ # it is and that it passes some basic safety checks _before_ we run ++ # qemu-img on it. ++ # See https://bugs.launchpad.net/nova/+bug/2059809 for details. ++ try: ++ inspector = inspector_cls.from_file(src_path) ++ if not inspector.safety_check(): ++ LOG.error('Image failed %s safety check; aborting conversion', ++ source_format) ++ raise RuntimeError('Image has disallowed configuration') ++ except RuntimeError: ++ raise ++ except format_inspector.ImageFormatError as e: ++ LOG.error('Image claimed to be %s format failed format ' ++ 'inspection: %s', source_format, e) ++ raise RuntimeError('Image format detection failed') ++ except Exception as e: ++ LOG.exception('Unknown error inspecting image format: %s', e) ++ raise RuntimeError('Unable to inspect image') ++ + try: + stdout, stderr = putils.trycmd("qemu-img", "info", ++ "-f", source_format, + "--output=json", + src_path, + prlimit=utils.QEMU_IMG_PROC_LIMITS, +@@ -102,9 +136,12 @@ class _ConvertImage(task.Task): + raise RuntimeError(stderr) + + metadata = json.loads(stdout) +- source_format = metadata.get('format') ++ if metadata.get('format') != source_format: ++ LOG.error('Image claiming to be %s reported as %s by qemu-img', ++ source_format, metadata.get('format', 'unknown')) ++ raise RuntimeError('Image metadata disagrees about format') ++ + virtual_size = metadata.get('virtual-size', 0) +- image = self.image_repo.get(self.image_id) + image.virtual_size = virtual_size + + if 'backing-filename' in metadata: +@@ -112,6 +149,14 @@ class _ConvertImage(task.Task): + raise RuntimeError( + 'QCOW images with backing files are not allowed') + ++ try: ++ data_file = metadata['format-specific']['data']['data-file'] ++ except KeyError: ++ data_file = None ++ if data_file is not None: ++ raise RuntimeError( ++ 'QCOW images with data-file set are not allowed') ++ + if metadata.get('format') == 'vmdk': + create_type = metadata.get( + 'format-specific', {}).get( +Index: glance/glance/tests/unit/async_/flows/plugins/test_image_conversion.py +=================================================================== +--- glance.orig/glance/tests/unit/async_/flows/plugins/test_image_conversion.py ++++ glance/glance/tests/unit/async_/flows/plugins/test_image_conversion.py +@@ -13,6 +13,7 @@ + # License for the specific language governing permissions and limitations + # under the License. + ++import fixtures + import json + import os + from unittest import mock +@@ -22,6 +23,8 @@ from oslo_concurrency import processutil + from oslo_config import cfg + + import glance.async_.flows.plugins.image_conversion as image_conversion ++from glance.async_ import utils as async_utils ++from glance.common import format_inspector + from glance.common import utils + from glance import domain + from glance import gateway +@@ -78,8 +81,14 @@ class TestConvertImageTask(test_utils.Ba + task_time_to_live=task_ttl, + task_input=task_input) + ++ self.inspector_mock = mock.MagicMock() ++ self.useFixture(fixtures.MockPatch('glance.common.format_inspector.' ++ 'get_inspector', ++ self.inspector_mock)) ++ ++ @mock.patch.object(os, 'stat') + @mock.patch.object(os, 'remove') +- def test_image_convert_success(self, mock_os_remove): ++ def test_image_convert_success(self, mock_os_remove, mock_stat): + mock_os_remove.return_value = None + image_convert = image_conversion._ConvertImage(self.context, + self.task.task_id, +@@ -89,7 +98,9 @@ class TestConvertImageTask(test_utils.Ba + + self.task_repo.get.return_value = self.task + image = mock.MagicMock(image_id=self.image_id, virtual_size=None, +- disk_format='qcow2') ++ extra_properties={ ++ 'os_glance_import_task': self.task.task_id}, ++ disk_format='raw') + self.img_repo.get.return_value = image + + with mock.patch.object(processutils, 'execute') as exc_mock: +@@ -105,7 +116,10 @@ class TestConvertImageTask(test_utils.Ba + self.assertIn('-f', exc_mock.call_args[0]) + self.assertEqual("qcow2", image.disk_format) + +- def _setup_image_convert_info_fail(self): ++ self.assertEqual('bare', image.container_format) ++ self.assertEqual('qcow2', image.disk_format) ++ ++ def _setup_image_convert_info_fail(self, disk_format='qcow2'): + image_convert = image_conversion._ConvertImage(self.context, + self.task.task_id, + self.task_type, +@@ -116,10 +130,42 @@ class TestConvertImageTask(test_utils.Ba + image = mock.MagicMock(image_id=self.image_id, virtual_size=None, + extra_properties={ + 'os_glance_import_task': self.task.task_id}, +- disk_format='qcow2') ++ disk_format=disk_format) + self.img_repo.get.return_value = image + return image_convert + ++ def test_image_convert_fails_inspection(self): ++ convert = self._setup_image_convert_info_fail() ++ with mock.patch.object(processutils, 'execute') as exc_mock: ++ exc_mock.side_effect = OSError('fail') ++ self.assertRaises(OSError, ++ convert.execute, 'file:///test/path.raw') ++ exc_mock.assert_called_once_with( ++ 'qemu-img', 'info', ++ '-f', 'qcow2', ++ '--output=json', ++ '/test/path.raw', ++ prlimit=async_utils.QEMU_IMG_PROC_LIMITS, ++ log_errors=processutils.LOG_ALL_ERRORS) ++ # Make sure we did not update the image ++ self.img_repo.save.assert_not_called() ++ ++ def test_image_convert_inspection_reports_error(self): ++ convert = self._setup_image_convert_info_fail() ++ with mock.patch.object(processutils, 'execute') as exc_mock: ++ exc_mock.return_value = '', 'some error' ++ self.assertRaises(RuntimeError, ++ convert.execute, 'file:///test/path.raw') ++ exc_mock.assert_called_once_with( ++ 'qemu-img', 'info', ++ '-f', 'qcow2', ++ '--output=json', ++ '/test/path.raw', ++ prlimit=async_utils.QEMU_IMG_PROC_LIMITS, ++ log_errors=processutils.LOG_ALL_ERRORS) ++ # Make sure we did not update the image ++ self.img_repo.save.assert_not_called() ++ + def test_image_convert_invalid_qcow(self): + data = {'format': 'qcow2', + 'backing-filename': '/etc/hosts'} +@@ -132,6 +178,52 @@ class TestConvertImageTask(test_utils.Ba + self.assertEqual('QCOW images with backing files are not allowed', + str(e)) + ++ def test_image_convert_invalid_qcow_data_file(self): ++ data = {'format': 'qcow2', ++ 'format-specific': { ++ 'data': { ++ 'data-file': '/etc/hosts', ++ }, ++ }} ++ ++ convert = self._setup_image_convert_info_fail() ++ with mock.patch.object(processutils, 'execute') as exc_mock: ++ exc_mock.return_value = json.dumps(data), '' ++ e = self.assertRaises(RuntimeError, ++ convert.execute, 'file:///test/path.qcow') ++ self.assertEqual('QCOW images with data-file set are not allowed', ++ str(e)) ++ ++ def test_image_convert_no_inspector_match(self): ++ convert = self._setup_image_convert_info_fail() ++ self.inspector_mock.return_value = None ++ self.assertRaisesRegex(RuntimeError, ++ 'Unable to convert from format', ++ convert.execute, 'file:///test/path.hpfs') ++ ++ def test_image_convert_fails_inspection_safety_check(self): ++ convert = self._setup_image_convert_info_fail() ++ inspector = self.inspector_mock.return_value.from_file.return_value ++ inspector.safety_check.return_value = False ++ self.assertRaisesRegex(RuntimeError, ++ 'Image has disallowed configuration', ++ convert.execute, 'file:///test/path.qcow') ++ ++ def test_image_convert_fails_inspection_format_check(self): ++ convert = self._setup_image_convert_info_fail() ++ self.inspector_mock.return_value.from_file.side_effect = ( ++ format_inspector.ImageFormatError()) ++ self.assertRaisesRegex(RuntimeError, ++ 'Image format detection failed', ++ convert.execute, 'file:///test/path.qcow') ++ ++ def test_image_convert_fails_inspection_error(self): ++ convert = self._setup_image_convert_info_fail() ++ self.inspector_mock.return_value.from_file.side_effect = ValueError ++ self.assertRaisesRegex(RuntimeError, ++ 'Unable to inspect image', ++ convert.execute, 'file:///test/path.qcow') ++ + def _test_image_convert_invalid_vmdk(self): + data = {'format': 'vmdk', + 'format-specific': { +@@ -139,7 +231,7 @@ class TestConvertImageTask(test_utils.Ba + 'create-type': 'monolithicFlat', + }}} + +- convert = self._setup_image_convert_info_fail() ++ convert = self._setup_image_convert_info_fail(disk_format='vmdk') + with mock.patch.object(processutils, 'execute') as exc_mock: + exc_mock.return_value = json.dumps(data), '' + convert.execute('file:///test/path.vmdk') +@@ -167,6 +259,82 @@ class TestConvertImageTask(test_utils.Ba + self.assertRaises(FileNotFoundError, + self._test_image_convert_invalid_vmdk) + ++ def test_image_convert_fails(self): ++ convert = self._setup_image_convert_info_fail(disk_format='raw') ++ with mock.patch.object(processutils, 'execute') as exc_mock: ++ exc_mock.side_effect = [('{"format":"raw"}', ''), ++ OSError('convert_fail')] ++ self.assertRaises(OSError, ++ convert.execute, 'file:///test/path.raw') ++ exc_mock.assert_has_calls( ++ [mock.call('qemu-img', 'info', ++ '-f', 'raw', ++ '--output=json', ++ '/test/path.raw', ++ prlimit=async_utils.QEMU_IMG_PROC_LIMITS, ++ log_errors=processutils.LOG_ALL_ERRORS), ++ mock.call('qemu-img', 'convert', '-f', 'raw', '-O', 'qcow2', ++ '/test/path.raw', '/test/path.raw.qcow2', ++ log_errors=processutils.LOG_ALL_ERRORS)]) ++ # Make sure we did not update the image ++ self.img_repo.save.assert_not_called() ++ ++ def test_image_convert_reports_fail(self): ++ convert = self._setup_image_convert_info_fail(disk_format='raw') ++ with mock.patch.object(processutils, 'execute') as exc_mock: ++ exc_mock.side_effect = [('{"format":"raw"}', ''), ++ ('', 'some error')] ++ self.assertRaises(RuntimeError, ++ convert.execute, 'file:///test/path.raw') ++ exc_mock.assert_has_calls( ++ [mock.call('qemu-img', 'info', ++ '-f', 'raw', ++ '--output=json', ++ '/test/path.raw', ++ prlimit=async_utils.QEMU_IMG_PROC_LIMITS, ++ log_errors=processutils.LOG_ALL_ERRORS), ++ mock.call('qemu-img', 'convert', '-f', 'raw', '-O', 'qcow2', ++ '/test/path.raw', '/test/path.raw.qcow2', ++ log_errors=processutils.LOG_ALL_ERRORS)]) ++ # Make sure we did not update the image ++ self.img_repo.save.assert_not_called() ++ ++ def test_image_convert_fails_source_format(self): ++ convert = self._setup_image_convert_info_fail() ++ with mock.patch.object(processutils, 'execute') as exc_mock: ++ exc_mock.return_value = ('{}', '') ++ exc = self.assertRaises(RuntimeError, ++ convert.execute, 'file:///test/path.raw') ++ self.assertIn('Image metadata disagrees about format', str(exc)) ++ exc_mock.assert_called_once_with( ++ 'qemu-img', 'info', ++ '-f', 'qcow2', ++ '--output=json', ++ '/test/path.raw', ++ prlimit=async_utils.QEMU_IMG_PROC_LIMITS, ++ log_errors=processutils.LOG_ALL_ERRORS) ++ # Make sure we did not update the image ++ self.img_repo.save.assert_not_called() ++ ++ def test_image_convert_same_format_does_nothing(self): ++ convert = self._setup_image_convert_info_fail() ++ with mock.patch.object(processutils, 'execute') as exc_mock: ++ exc_mock.return_value = ( ++ '{"format": "qcow2", "virtual-size": 123}', '') ++ convert.execute('file:///test/path.qcow') ++ # Make sure we only called qemu-img for inspection, not conversion ++ exc_mock.assert_called_once_with( ++ 'qemu-img', 'info', ++ '-f', 'qcow2', ++ '--output=json', ++ '/test/path.qcow', ++ prlimit=async_utils.QEMU_IMG_PROC_LIMITS, ++ log_errors=processutils.LOG_ALL_ERRORS) ++ ++ # Make sure we set the virtual_size before we exited ++ image = self.img_repo.get.return_value ++ self.assertEqual(123, image.virtual_size) ++ + @mock.patch.object(os, 'remove') + def test_image_convert_revert_success(self, mock_os_remove): + mock_os_remove.return_value = None +Index: glance/glance/tests/unit/async_/flows/test_import.py +=================================================================== +--- glance.orig/glance/tests/unit/async_/flows/test_import.py ++++ glance/glance/tests/unit/async_/flows/test_import.py +@@ -13,6 +13,7 @@ + # License for the specific language governing permissions and limitations + # under the License. + ++import io + import json + import os + from unittest import mock +@@ -175,6 +176,39 @@ class TestImportTask(test_utils.BaseTest + self.assertFalse(os.path.exists(tmp_image_path)) + self.assertTrue(os.path.exists(image_path)) + ++ def test_import_flow_invalid_data_file(self): ++ self.config(engine_mode='serial', ++ group='taskflow_executor') ++ ++ img_factory = mock.MagicMock() ++ ++ executor = taskflow_executor.TaskExecutor( ++ self.context, ++ self.task_repo, ++ self.img_repo, ++ img_factory) ++ ++ self.task_repo.get.return_value = self.task ++ ++ def create_image(*args, **kwargs): ++ kwargs['image_id'] = UUID1 ++ return self.img_factory.new_image(*args, **kwargs) ++ ++ self.img_repo.get.return_value = self.image ++ img_factory.new_image.side_effect = create_image ++ ++ with mock.patch.object(script_utils, 'get_image_data_iter') as dmock: ++ dmock.return_value = io.BytesIO(b"TEST_IMAGE") ++ ++ with mock.patch.object(putils, 'trycmd') as tmock: ++ out = json.dumps({'format-specific': ++ {'data': {'data-file': 'somefile'}}}) ++ tmock.return_value = (out, '') ++ e = self.assertRaises(RuntimeError, ++ executor.begin_processing, ++ self.task.task_id) ++ self.assertIn('somefile', str(e)) ++ + def test_import_flow_revert_import_to_fs(self): + self.config(engine_mode='serial', group='taskflow_executor') + +Index: glance/glance/common/format_inspector.py +=================================================================== +--- glance.orig/glance/common/format_inspector.py ++++ glance/glance/common/format_inspector.py +@@ -28,6 +28,14 @@ from oslo_log import log as logging + LOG = logging.getLogger(__name__) + + ++def chunked_reader(fileobj, chunk_size=512): ++ while True: ++ chunk = fileobj.read(chunk_size) ++ if not chunk: ++ break ++ yield chunk ++ ++ + class CaptureRegion(object): + """Represents a region of a file we want to capture. + +@@ -176,10 +184,16 @@ class FileInspector(object): + @property + def actual_size(self): + """Returns the total size of the file, usually smaller than +- virtual_size. ++ virtual_size. NOTE: this will only be accurate if the entire ++ file is read and processed. + """ + return self._total_count + ++ @property ++ def complete(self): ++ """Returns True if we have all the information needed.""" ++ return all(r.complete for r in self._capture_regions.values()) ++ + def __str__(self): + """The string name of this file format.""" + return 'raw' +@@ -194,6 +208,35 @@ class FileInspector(object): + return {name: len(region.data) for name, region in + self._capture_regions.items()} + ++ @classmethod ++ def from_file(cls, filename): ++ """Read as much of a file as necessary to complete inspection. ++ ++ NOTE: Because we only read as much of the file as necessary, the ++ actual_size property will not reflect the size of the file, but the ++ amount of data we read before we satisfied the inspector. ++ ++ Raises ImageFormatError if we cannot parse the file. ++ """ ++ inspector = cls() ++ with open(filename, 'rb') as f: ++ for chunk in chunked_reader(f): ++ inspector.eat_chunk(chunk) ++ if inspector.complete: ++ # No need to eat any more data ++ break ++ if not inspector.complete or not inspector.format_match: ++ raise ImageFormatError('File is not in requested format') ++ return inspector ++ ++ def safety_check(self): ++ """Perform some checks to determine if this file is safe. ++ ++ Returns True if safe, False otherwise. It may raise ImageFormatError ++ if safety cannot be guaranteed because of parsing or other errors. ++ """ ++ return True ++ + + # The qcow2 format consists of a big-endian 72-byte header, of which + # only a small portion has information we care about: +@@ -202,15 +245,26 @@ class FileInspector(object): + # 0 0x00 Magic 4-bytes 'QFI\xfb' + # 4 0x04 Version (uint32_t, should always be 2 for modern files) + # . . . ++# 8 0x08 Backing file offset (uint64_t) + # 24 0x18 Size in bytes (unint64_t) ++# . . . ++# 72 0x48 Incompatible features bitfield (6 bytes) + # +-# https://people.gnome.org/~markmc/qcow-image-format.html ++# https://gitlab.com/qemu-project/qemu/-/blob/master/docs/interop/qcow2.txt + class QcowInspector(FileInspector): + """QEMU QCOW2 Format + + This should only require about 32 bytes of the beginning of the file +- to determine the virtual size. ++ to determine the virtual size, and 104 bytes to perform the safety check. + """ ++ ++ BF_OFFSET = 0x08 ++ BF_OFFSET_LEN = 8 ++ I_FEATURES = 0x48 ++ I_FEATURES_LEN = 8 ++ I_FEATURES_DATAFILE_BIT = 3 ++ I_FEATURES_MAX_BIT = 4 ++ + def __init__(self, *a, **k): + super(QcowInspector, self).__init__(*a, **k) + self.new_region('header', CaptureRegion(0, 512)) +@@ -221,6 +275,10 @@ class QcowInspector(FileInspector): + return magic, size + + @property ++ def has_header(self): ++ return self.region('header').complete ++ ++ @property + def virtual_size(self): + if not self.region('header').complete: + return 0 +@@ -236,12 +294,80 @@ class QcowInspector(FileInspector): + magic, size = self._qcow_header_data() + return magic == b'QFI\xFB' + ++ @property ++ def has_backing_file(self): ++ if not self.region('header').complete: ++ return None ++ if not self.format_match: ++ return False ++ bf_offset_bytes = self.region('header').data[ ++ self.BF_OFFSET:self.BF_OFFSET + self.BF_OFFSET_LEN] ++ # nonzero means "has a backing file" ++ bf_offset, = struct.unpack('>Q', bf_offset_bytes) ++ return bf_offset != 0 ++ ++ @property ++ def has_unknown_features(self): ++ if not self.region('header').complete: ++ return None ++ if not self.format_match: ++ return False ++ i_features = self.region('header').data[ ++ self.I_FEATURES:self.I_FEATURES + self.I_FEATURES_LEN] ++ ++ # This is the maximum byte number we should expect any bits to be set ++ max_byte = self.I_FEATURES_MAX_BIT // 8 ++ ++ # The flag bytes are in big-endian ordering, so if we process ++ # them in index-order, they're reversed ++ for i, byte_num in enumerate(reversed(range(self.I_FEATURES_LEN))): ++ if byte_num == max_byte: ++ # If we're in the max-allowed byte, allow any bits less than ++ # the maximum-known feature flag bit to be set ++ allow_mask = ((1 << self.I_FEATURES_MAX_BIT) - 1) ++ elif byte_num > max_byte: ++ # If we're above the byte with the maximum known feature flag ++ # bit, then we expect all zeroes ++ allow_mask = 0x0 ++ else: ++ # Any earlier-than-the-maximum byte can have any of the flag ++ # bits set ++ allow_mask = 0xFF ++ ++ if i_features[i] & ~allow_mask: ++ LOG.warning('Found unknown feature bit in byte %i: %s/%s', ++ byte_num, bin(i_features[byte_num] & ~allow_mask), ++ bin(allow_mask)) ++ return True ++ ++ return False ++ ++ @property ++ def has_data_file(self): ++ if not self.region('header').complete: ++ return None ++ if not self.format_match: ++ return False ++ i_features = self.region('header').data[ ++ self.I_FEATURES:self.I_FEATURES + self.I_FEATURES_LEN] ++ ++ # First byte of bitfield, which is i_features[7] ++ byte = self.I_FEATURES_LEN - 1 - self.I_FEATURES_DATAFILE_BIT // 8 ++ # Third bit of bitfield, which is 0x04 ++ bit = 1 << (self.I_FEATURES_DATAFILE_BIT - 1 % 8) ++ return bool(i_features[byte] & bit) ++ + def __str__(self): + return 'qcow2' + ++ def safety_check(self): ++ return (not self.has_backing_file and ++ not self.has_data_file and ++ not self.has_unknown_features) ++ + + # The VHD (or VPC as QEMU calls it) format consists of a big-endian +-# 512-byte "footer" at the beginning fo the file with various ++# 512-byte "footer" at the beginning of the file with various + # information, most of which does not matter to us: + # + # Dec Hex Name +@@ -345,6 +471,7 @@ class VHDXInspector(FileInspector): + """ + METAREGION = '8B7CA206-4790-4B9A-B8FE-575F050F886E' + VIRTUAL_DISK_SIZE = '2FA54224-CD1B-4876-B211-5DBED83BF4B8' ++ VHDX_METADATA_TABLE_MAX_SIZE = 32 * 2048 # From qemu + + def __init__(self, *a, **k): + super(VHDXInspector, self).__init__(*a, **k) +@@ -459,6 +586,8 @@ class VHDXInspector(FileInspector): + item_offset, item_length, _reserved = struct.unpack( + ' +Forwarded: no +Last-Update: 2024-06-28 + +Index: glance/glance/tests/unit/async_/flows/plugins/test_image_conversion.py +=================================================================== +--- glance.orig/glance/tests/unit/async_/flows/plugins/test_image_conversion.py ++++ glance/glance/tests/unit/async_/flows/plugins/test_image_conversion.py +@@ -146,6 +146,7 @@ class TestConvertImageTask(test_utils.Ba + '--output=json', + '/test/path.raw', + prlimit=async_utils.QEMU_IMG_PROC_LIMITS, ++ python_exec='/usr/bin/python3.9', + log_errors=processutils.LOG_ALL_ERRORS) + # Make sure we did not update the image + self.img_repo.save.assert_not_called() +@@ -162,6 +163,7 @@ class TestConvertImageTask(test_utils.Ba + '--output=json', + '/test/path.raw', + prlimit=async_utils.QEMU_IMG_PROC_LIMITS, ++ python_exec='/usr/bin/python3.9', + log_errors=processutils.LOG_ALL_ERRORS) + # Make sure we did not update the image + self.img_repo.save.assert_not_called() +@@ -272,6 +274,7 @@ class TestConvertImageTask(test_utils.Ba + '--output=json', + '/test/path.raw', + prlimit=async_utils.QEMU_IMG_PROC_LIMITS, ++ python_exec='/usr/bin/python3.9', + log_errors=processutils.LOG_ALL_ERRORS), + mock.call('qemu-img', 'convert', '-f', 'raw', '-O', 'qcow2', + '/test/path.raw', '/test/path.raw.qcow2', +@@ -292,6 +295,7 @@ class TestConvertImageTask(test_utils.Ba + '--output=json', + '/test/path.raw', + prlimit=async_utils.QEMU_IMG_PROC_LIMITS, ++ python_exec='/usr/bin/python3.9', + log_errors=processutils.LOG_ALL_ERRORS), + mock.call('qemu-img', 'convert', '-f', 'raw', '-O', 'qcow2', + '/test/path.raw', '/test/path.raw.qcow2', +@@ -312,6 +316,7 @@ class TestConvertImageTask(test_utils.Ba + '--output=json', + '/test/path.raw', + prlimit=async_utils.QEMU_IMG_PROC_LIMITS, ++ python_exec='/usr/bin/python3.9', + log_errors=processutils.LOG_ALL_ERRORS) + # Make sure we did not update the image + self.img_repo.save.assert_not_called() +@@ -329,6 +334,7 @@ class TestConvertImageTask(test_utils.Ba + '--output=json', + '/test/path.qcow', + prlimit=async_utils.QEMU_IMG_PROC_LIMITS, ++ python_exec='/usr/bin/python3.9', + log_errors=processutils.LOG_ALL_ERRORS) + + # Make sure we set the virtual_size before we exited diff -Nru glance-21.0.0/debian/patches/series glance-21.1.0/debian/patches/series --- glance-21.0.0/debian/patches/series 2023-01-18 09:14:44.000000000 +0000 +++ glance-21.1.0/debian/patches/series 2024-06-21 13:46:47.000000000 +0000 @@ -1,3 +1,5 @@ sql_conn-registry.patch missing-files.patch cve-2022-47951-glance-stable-victoria.patch +CVE-2024-32498_train-ish-backport.patch +fix-unit-tests.patch diff -Nru glance-21.0.0/doc/source/configuration/configuring.rst glance-21.1.0/doc/source/configuration/configuring.rst --- glance-21.0.0/doc/source/configuration/configuring.rst 2020-09-30 22:45:36.000000000 +0000 +++ glance-21.1.0/doc/source/configuration/configuring.rst 2021-09-13 14:50:26.000000000 +0000 @@ -198,28 +198,6 @@ Optional. Default: ``64`` (Limited by max_header_line default: 16384) -Configuring SSL Support -~~~~~~~~~~~~~~~~~~~~~~~ - -``cert_file=PATH`` - Path to the certificate file the server should use when binding to an - SSL-wrapped socket. - - Optional. Default: not enabled. - -``key_file=PATH`` - Path to the private key file the server should use when binding to an - SSL-wrapped socket. - - Optional. Default: not enabled. - -``ca_file=PATH`` - Path to the CA certificate file the server should use to validate client - certificates provided during an SSL handshake. This is ignored if - ``cert_file`` and ''key_file`` are not set. - - Optional. Default: not enabled. - Configuring Registry Access ~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff -Nru glance-21.0.0/.gitreview glance-21.1.0/.gitreview --- glance-21.0.0/.gitreview 2020-09-30 22:45:36.000000000 +0000 +++ glance-21.1.0/.gitreview 2021-09-13 14:50:26.000000000 +0000 @@ -2,3 +2,4 @@ host=review.opendev.org port=29418 project=openstack/glance.git +defaultbranch=stable/victoria diff -Nru glance-21.0.0/glance/async_/flows/api_image_import.py glance-21.1.0/glance/async_/flows/api_image_import.py --- glance-21.0.0/glance/async_/flows/api_image_import.py 2020-09-30 22:45:36.000000000 +0000 +++ glance-21.1.0/glance/async_/flows/api_image_import.py 2021-09-13 14:50:26.000000000 +0000 @@ -730,7 +730,8 @@ flow.add(_ImageLock(task_id, task_type, action_wrapper)) if import_method in ['web-download', 'copy-image']: - internal_plugin = internal_plugins.get_import_plugin(**kwargs) + internal_plugin = internal_plugins.get_import_plugin( + **dict(kwargs, action_wrapper=action_wrapper)) flow.add(internal_plugin) if CONF.enabled_backends: separator, staging_dir = store_utils.get_dir_separator() diff -Nru glance-21.0.0/glance/async_/flows/_internal_plugins/copy_image.py glance-21.1.0/glance/async_/flows/_internal_plugins/copy_image.py --- glance-21.0.0/glance/async_/flows/_internal_plugins/copy_image.py 2020-09-30 22:45:36.000000000 +0000 +++ glance-21.1.0/glance/async_/flows/_internal_plugins/copy_image.py 2021-09-13 14:50:26.000000000 +0000 @@ -33,11 +33,12 @@ default_provides = 'file_uri' - def __init__(self, task_id, task_type, image_repo, image_id): + def __init__(self, task_id, task_type, image_repo, action_wrapper): self.task_id = task_id self.task_type = task_type self.image_repo = image_repo - self.image_id = image_id + self.image_id = action_wrapper.image_id + self.action_wrapper = action_wrapper super(_CopyImage, self).__init__( name='%s-CopyImage-%s' % (task_type, task_id)) @@ -131,13 +132,14 @@ :param task_id: Task ID. :param task_type: Type of the task. :param image_repo: Image repository used. - :param uri: URI the image data is downloaded from. + :param image_id: Image ID. + :param action_wrapper: An api_image_import.ActionWrapper. """ task_id = kwargs.get('task_id') task_type = kwargs.get('task_type') image_repo = kwargs.get('image_repo') - image_id = kwargs.get('image_id') + action_wrapper = kwargs.get('action_wrapper') return lf.Flow(task_type).add( - _CopyImage(task_id, task_type, image_repo, image_id), + _CopyImage(task_id, task_type, image_repo, action_wrapper), ) diff -Nru glance-21.0.0/glance/async_/flows/_internal_plugins/web_download.py glance-21.1.0/glance/async_/flows/_internal_plugins/web_download.py --- glance-21.0.0/glance/async_/flows/_internal_plugins/web_download.py 2020-09-30 22:45:36.000000000 +0000 +++ glance-21.1.0/glance/async_/flows/_internal_plugins/web_download.py 2021-09-13 14:50:26.000000000 +0000 @@ -35,12 +35,13 @@ default_provides = 'file_uri' - def __init__(self, task_id, task_type, image_repo, image_id, uri): + def __init__(self, task_id, task_type, uri, action_wrapper, stores): self.task_id = task_id self.task_type = task_type - self.image_repo = image_repo - self.image_id = image_id + self.image_id = action_wrapper.image_id self.uri = uri + self.action_wrapper = action_wrapper + self.stores = stores self._path = None super(_WebDownload, self).__init__( name='%s-WebDownload-%s' % (task_type, task_id)) @@ -140,9 +141,13 @@ 'image_id': self.image_id}) # NOTE(abhishekk): Revert image state back to 'queued' as # something went wrong. - image = self.image_repo.get(self.image_id) - image.status = 'queued' - self.image_repo.save(image) + # NOTE(danms): If we failed to stage the image, then none + # of the _ImportToStore() tasks could have run, so we need + # to move all stores out of "importing" and into "failed". + with self.action_wrapper as action: + action.set_image_status('queued') + action.remove_importing_stores(self.stores) + action.add_failed_stores(self.stores) # NOTE(abhishekk): Deleting partial image data from staging area if self._path is not None: @@ -166,13 +171,14 @@ :param task_type: Type of the task. :param image_repo: Image repository used. :param uri: URI the image data is downloaded from. + :param action_wrapper: An api_image_import.ActionWrapper. """ task_id = kwargs.get('task_id') task_type = kwargs.get('task_type') - image_repo = kwargs.get('image_repo') - image_id = kwargs.get('image_id') uri = kwargs.get('import_req')['method'].get('uri') + action_wrapper = kwargs.get('action_wrapper') + stores = kwargs.get('backend', [None]) return lf.Flow(task_type).add( - _WebDownload(task_id, task_type, image_repo, image_id, uri), + _WebDownload(task_id, task_type, uri, action_wrapper, stores), ) diff -Nru glance-21.0.0/glance/common/auth.py glance-21.1.0/glance/common/auth.py --- glance-21.0.0/glance/common/auth.py 2020-09-30 22:45:36.000000000 +0000 +++ glance-21.1.0/glance/common/auth.py 2021-09-13 14:50:26.000000000 +0000 @@ -32,7 +32,7 @@ """ import httplib2 -from keystoneauth1.access import service_catalog as ks_service_catalog +from keystoneclient import service_catalog as ks_service_catalog from oslo_serialization import jsonutils from six.moves import http_client as http # NOTE(jokke): simplified transition to py3, behaves like py2 xrange @@ -326,10 +326,11 @@ otherwise we will raise an exception. """ endpoints = ks_service_catalog.ServiceCatalogV2( - service_catalog).get_urls(interface=endpoint_type, - service_type=service_type, - region_name=endpoint_region) - if len(endpoints) == 0: + {'serviceCatalog': service_catalog} + ).get_urls(service_type=service_type, + region_name=endpoint_region, + endpoint_type=endpoint_type) + if endpoints is None: raise exception.NoServiceEndpoint() elif len(endpoints) == 1: return endpoints[0] diff -Nru glance-21.0.0/glance/common/trust_auth.py glance-21.1.0/glance/common/trust_auth.py --- glance-21.0.0/glance/common/trust_auth.py 2020-09-30 22:45:36.000000000 +0000 +++ glance-21.1.0/glance/common/trust_auth.py 2021-09-13 14:50:26.000000000 +0000 @@ -14,6 +14,7 @@ from keystoneauth1 import exceptions as ka_exceptions from keystoneauth1 import loading as ka_loading +from keystoneclient.v3 import client as ks_client from oslo_config import cfg from oslo_log import log as logging @@ -31,17 +32,17 @@ # step 1: create trust to ensure that we can always update token # trustor = user who made the request - trustor_client = self._load_session(user_plugin) - trustor_id = trustor_client.get_user_id() + trustor_client = self._load_client(user_plugin) + trustor_id = trustor_client.session.get_user_id() # get trustee user client that impersonates main user trustee_user_auth = ka_loading.load_auth_from_conf_options( CONF, 'keystone_authtoken') # save service user client because we need new service token # to refresh trust-scoped client later - self.trustee_user_client = self._load_session(trustee_user_auth) + self.trustee_user_client = self._load_client(trustee_user_auth) - trustee_id = self.trustee_user_client.get_user_id() + trustee_id = self.trustee_user_client.session.get_user_id() self.trust_id = trustor_client.trusts.create(trustor_user=trustor_id, trustee_user=trustee_id, @@ -63,12 +64,12 @@ if self.trustee_client is None: self.trustee_client = self._refresh_trustee_client() try: - return self.trustee_client.get_token() + return self.trustee_client.session.get_token() except ka_exceptions.Unauthorized: # in case of Unauthorized exceptions try to refresh client because # service user token may expired self.trustee_client = self._refresh_trustee_client() - return self.trustee_client.get_token() + return self.trustee_client.session.get_token() def release_resources(self): """Release keystone resources required for refreshing""" @@ -98,11 +99,11 @@ trustee_auth = ka_loading.load_auth_from_conf_options( CONF, 'keystone_authtoken', **kwargs) - return self._load_session(trustee_auth) + return self._load_client(trustee_auth) @staticmethod - def _load_session(plugin): - # load ksa session from auth settings and user plugin + def _load_client(plugin): + # load client from auth settings and user plugin sess = ka_loading.load_session_from_conf_options( CONF, 'keystone_authtoken', auth=plugin) - return sess + return ks_client.Client(session=sess) diff -Nru glance-21.0.0/glance/common/wsgi_app.py glance-21.1.0/glance/common/wsgi_app.py --- glance-21.0.0/glance/common/wsgi_app.py 2020-09-30 22:45:36.000000000 +0000 +++ glance-21.1.0/glance/common/wsgi_app.py 2021-09-13 14:50:26.000000000 +0000 @@ -12,6 +12,7 @@ import atexit import os +import threading import glance_store from oslo_config import cfg @@ -22,6 +23,7 @@ import glance.async_ from glance.common import config from glance.common import store_utils +from glance import housekeeping from glance.i18n import _ from glance import notifier @@ -79,6 +81,17 @@ pool_model.pool.shutdown() +def run_staging_cleanup(): + cleaner = housekeeping.StagingStoreCleaner(glance.db.get_api()) + # NOTE(danms): Start thread as a daemon. It is still a + # single-shot, but this will not block our shutdown if it is + # running. + cleanup_thread = threading.Thread( + target=cleaner.clean_orphaned_staging_residue, + daemon=True) + cleanup_thread.start() + + def init_app(): config.set_config_defaults() config_files = _get_config_files() @@ -111,5 +124,7 @@ glance_store.create_stores(CONF) glance_store.verify_default_store() + run_staging_cleanup() + _setup_os_profiler() return config.load_paste_app('glance-api') diff -Nru glance-21.0.0/glance/common/wsgi.py glance-21.1.0/glance/common/wsgi.py --- glance-21.0.0/glance/common/wsgi.py 2020-09-30 22:45:36.000000000 +0000 +++ glance-21.1.0/glance/common/wsgi.py 2021-09-13 14:50:26.000000000 +0000 @@ -54,6 +54,8 @@ from glance.common import exception from glance.common import store_utils from glance.common import utils +import glance.db +from glance import housekeeping from glance import i18n from glance.i18n import _, _LE, _LI, _LW @@ -501,14 +503,18 @@ self.default_port = default_port self.configure() self.start_wsgi() + + cleaner = housekeeping.StagingStoreCleaner(glance.db.get_api()) + self.pool.spawn_n(cleaner.clean_orphaned_staging_residue) + if self.initialize_prefetcher: self.cache_images() def start_wsgi(self): workers = get_num_workers() + self.pool = self.create_pool() if workers == 0: # Useful for profiling, test, debug etc. - self.pool = self.create_pool() self.pool.spawn_n(self._single_run, self.application, self.sock) return else: diff -Nru glance-21.0.0/glance/housekeeping.py glance-21.1.0/glance/housekeeping.py --- glance-21.0.0/glance/housekeeping.py 1970-01-01 00:00:00.000000000 +0000 +++ glance-21.1.0/glance/housekeeping.py 2021-09-13 14:50:26.000000000 +0000 @@ -0,0 +1,126 @@ +# Copyright 2021 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os + +from oslo_config import cfg +from oslo_log import log as logging +from oslo_utils import uuidutils + +from glance.common import exception +from glance.common import store_utils +from glance import context +from glance.i18n import _LE + +LOG = logging.getLogger(__name__) +CONF = cfg.CONF + + +def staging_store_path(): + """Return the local path to the staging store. + + :raises: GlanceException if staging store is not configured to be + a file:// URI + """ + if CONF.enabled_backends: + separator, staging_dir = store_utils.get_dir_separator() + else: + staging_dir = CONF.node_staging_uri + expected_prefix = 'file://' + if not staging_dir.startswith(expected_prefix): + raise exception.GlanceException( + 'Unexpected scheme in staging store; ' + 'unable to scan for residue') + return staging_dir[len(expected_prefix):] + + +class StagingStoreCleaner: + def __init__(self, db): + self.db = db + self.context = context.get_admin_context() + + @staticmethod + def get_image_id(filename): + if '.' in filename: + filename, ext = filename.split('.', 1) + if uuidutils.is_uuid_like(filename): + return filename + + def is_valid_image(self, image_id): + try: + image = self.db.image_get(self.context, image_id) + # FIXME(danms): Maybe check that it's not deleted or + # something else like state, size, etc + return not image['deleted'] + except exception.ImageNotFound: + return False + + @staticmethod + def delete_file(path): + try: + os.remove(path) + except FileNotFoundError: + # NOTE(danms): We must have raced with something else, so this + # is not a problem + pass + except Exception as e: + LOG.error(_LE('Failed to delete stale staging ' + 'path %(path)r: %(err)s'), + {'path': path, 'err': str(e)}) + return False + return True + + def clean_orphaned_staging_residue(self): + try: + files = os.listdir(staging_store_path()) + except FileNotFoundError: + # NOTE(danms): If we cannot list the staging dir, there is + # clearly nothing left from a previous run, so nothing to + # clean up. + files = [] + if not files: + return + + LOG.debug('Found %i files in staging directory for potential cleanup', + len(files)) + cleaned = ignored = error = 0 + for filename in files: + image_id = self.get_image_id(filename) + if not image_id: + # NOTE(danms): We should probably either have a config option + # that decides what to do here (i.e. reap or ignore), or decide + # that this is not okay and just nuke anything we find. + LOG.debug('Staging directory contains unexpected non-image ' + 'file %r; ignoring', + filename) + ignored += 1 + continue + if self.is_valid_image(image_id): + # NOTE(danms): We found a non-deleted image for this + # file, so leave it in place. + ignored += 1 + continue + path = os.path.join(staging_store_path(), filename) + LOG.debug('Stale staging residue found for image ' + '%(uuid)s: %(file)r; deleting now.', + {'uuid': image_id, 'file': path}) + if self.delete_file(path): + cleaned += 1 + else: + error += 1 + + LOG.debug('Cleaned %(cleaned)i stale staging files, ' + '%(ignored)i ignored (%(error)i errors)', + {'cleaned': cleaned, 'ignored': ignored, 'error': error}) diff -Nru glance-21.0.0/glance/tests/functional/__init__.py glance-21.1.0/glance/tests/functional/__init__.py --- glance-21.0.0/glance/tests/functional/__init__.py 2020-09-30 22:45:36.000000000 +0000 +++ glance-21.1.0/glance/tests/functional/__init__.py 2021-09-13 14:50:26.000000000 +0000 @@ -424,6 +424,9 @@ self.send_identity_headers = False + self.node_staging_uri = 'file://%s' % os.path.join( + self.test_dir, 'staging') + self.conf_base = """[DEFAULT] debug = %(debug)s default_log_levels = eventlet.wsgi.server=DEBUG,stevedore.extension=INFO @@ -453,6 +456,7 @@ image_location_quota=%(image_location_quota)s location_strategy=%(location_strategy)s allow_additional_image_properties = True +node_staging_uri=%(node_staging_uri)s [oslo_policy] policy_file = %(policy_file)s policy_default_rule = %(policy_default_rule)s diff -Nru glance-21.0.0/glance/tests/functional/test_wsgi.py glance-21.1.0/glance/tests/functional/test_wsgi.py --- glance-21.0.0/glance/tests/functional/test_wsgi.py 2020-09-30 22:45:36.000000000 +0000 +++ glance-21.1.0/glance/tests/functional/test_wsgi.py 2021-09-13 14:50:26.000000000 +0000 @@ -16,9 +16,14 @@ """Tests for `glance.wsgi`.""" import os +from six.moves import http_client as http import socket import time +from oslo_serialization import jsonutils +from oslo_utils.fixture import uuidsentinel as uuids +import requests + from glance.common import wsgi from glance.tests import functional @@ -57,3 +62,116 @@ self.assertIn(greetings, get_request()) # Should fail - connection timed out so we get nothing from the server self.assertFalse(get_request(delay=1.1)) + + +class StagingCleanupBase: + def _url(self, path): + return 'http://127.0.0.1:%d%s' % (self.api_port, path) + + def _headers(self, custom_headers=None): + base_headers = { + 'X-Identity-Status': 'Confirmed', + 'X-Auth-Token': '932c5c84-02ac-4fe5-a9ba-620af0e2bb96', + 'X-User-Id': 'f9a41d13-0c13-47e9-bee2-ce4e8bfe958e', + 'X-Tenant-Id': uuids.tenant1, + 'X-Roles': 'member', + } + base_headers.update(custom_headers or {}) + return base_headers + + def test_clean_on_start(self): + staging = os.path.join(self.test_dir, 'staging') + + # Start the server + self.start_servers(**self.__dict__.copy()) + + # Create an image + path = self._url('/v2/images') + headers = self._headers({'content-type': 'application/json'}) + data = jsonutils.dumps({'name': 'image-1', 'type': 'kernel', + 'disk_format': 'aki', + 'container_format': 'aki'}) + response = requests.post(path, headers=headers, data=data) + self.assertEqual(http.CREATED, response.status_code) + image = jsonutils.loads(response.text) + image_id = image['id'] + + # Stage data for the image + path = self._url('/v2/images/%s/stage' % image_id) + headers = self._headers({'Content-Type': 'application/octet-stream'}) + image_data = b'ZZZZZ' + response = requests.put(path, headers=headers, data=image_data) + self.assertEqual(http.NO_CONTENT, response.status_code) + + # Stop the server + self.my_api_server.stop() + + # Create more files in staging, one unrecognized one, and one + # uuid that matches nothing in the database, and some residue + # like we would see from failed conversions and decompressions + # for the image we created above. + open(os.path.join(staging, 'foo'), 'w') + open(os.path.join(staging, uuids.stale), 'w') + open(os.path.join(staging, uuids.converting), 'w') + converting_fn = os.path.join(staging, '%s.qcow2' % uuids.converting) + decompressing_fn = os.path.join(staging, '%s.uc' % uuids.decompressing) + open(converting_fn, 'w') + open(decompressing_fn, 'w') + + # Restart the server. We set "needs_database" to False here to avoid + # recreating the database during startup, thus causing the server to + # think there are no valid images and deleting everything. + self.my_api_server.needs_database = False + self.start_with_retry(self.my_api_server, + 'api_port', 3, **self.__dict__.copy()) + + # Poll to give it time to come up and do the work. Use the presence + # of the extra files to determine if the cleanup has run yet. + for i in range(0, 10): + try: + requests.get(self._url('/v2/images')) + except Exception: + # Not even answering queries yet + pass + else: + files = os.listdir(staging) + if len(files) == 2: + break + + time.sleep(1) + + # We should still find the not-an-image file... + self.assertTrue(os.path.exists(os.path.join(staging, 'foo'))) + # ...and make sure the actually-staged image file is still present.... + self.assertTrue(os.path.exists(os.path.join(staging, image_id))) + # ... but the stale image should be gone, + self.assertFalse(os.path.exists(os.path.join(staging, + uuids.stale))) + # ... along with the residue of the conversion ... + self.assertFalse(os.path.exists(converting_fn)) + # ... and the residue of the decompression. + self.assertFalse(os.path.exists(decompressing_fn)) + + self.stop_servers() + + +class TestStagingCleanupMultistore(functional.MultipleBackendFunctionalTest, + StagingCleanupBase): + """Test for staging store cleanup on API server startup. + + This tests the multistore case. + """ + def setUp(self): + super(TestStagingCleanupMultistore, self).setUp() + self.my_api_server = self.api_server_multiple_backend + + +class TestStagingCleanupSingleStore(functional.FunctionalTest, + StagingCleanupBase): + """Test for staging store cleanup on API server startup. + + This tests the single store case. + """ + def setUp(self): + super(TestStagingCleanupSingleStore, self).setUp() + self.my_api_server = self.api_server diff -Nru glance-21.0.0/glance/tests/functional/v2/test_images.py glance-21.1.0/glance/tests/functional/v2/test_images.py --- glance-21.0.0/glance/tests/functional/v2/test_images.py 2020-09-30 22:45:36.000000000 +0000 +++ glance-21.1.0/glance/tests/functional/v2/test_images.py 2021-09-13 14:50:26.000000000 +0000 @@ -235,7 +235,6 @@ self.stop_servers() def test_image_import_using_web_download(self): - self.config(node_staging_uri="file:///tmp/staging/") self.start_servers(**self.__dict__.copy()) # Image list should be empty @@ -4817,7 +4816,6 @@ self.stop_servers() def test_image_import_using_web_download(self): - self.config(node_staging_uri="file:///tmp/staging/") self.start_servers(**self.__dict__.copy()) # Image list should be empty @@ -4980,7 +4978,6 @@ self.stop_servers() def test_image_import_using_web_download_different_backend(self): - self.config(node_staging_uri="file:///tmp/staging/") self.start_servers(**self.__dict__.copy()) # Image list should be empty @@ -5144,7 +5141,6 @@ self.stop_servers() def test_image_import_multi_stores(self): - self.config(node_staging_uri="file:///tmp/staging/") self.start_servers(**self.__dict__.copy()) # Image list should be empty @@ -5759,7 +5755,6 @@ self.stop_servers() def test_image_import_multi_stores_specifying_all_stores(self): - self.config(node_staging_uri="file:///tmp/staging/") self.start_servers(**self.__dict__.copy()) # Image list should be empty diff -Nru glance-21.0.0/glance/tests/unit/async_/flows/test_copy_image.py glance-21.1.0/glance/tests/unit/async_/flows/test_copy_image.py --- glance-21.0.0/glance/tests/unit/async_/flows/test_copy_image.py 2020-09-30 22:45:36.000000000 +0000 +++ glance-21.1.0/glance/tests/unit/async_/flows/test_copy_image.py 2021-09-13 14:50:26.000000000 +0000 @@ -21,6 +21,7 @@ from oslo_config import cfg from glance.async_.flows._internal_plugins import copy_image +from glance.async_.flows import api_image_import import glance.common.exception as exception from glance import domain import glance.tests.unit.utils as unit_test_utils @@ -93,6 +94,11 @@ self.task = self.task_factory.new_task(self.task_type, TENANT1, task_time_to_live=task_ttl, task_input=task_input) + self.task_id = self.task.task_id + self.action_wrapper = api_image_import.ImportActionWrapper( + self.image_repo, self.image_id, self.task_id) + self.image_repo.get.return_value = mock.MagicMock( + extra_properties={'os_glance_import_task': self.task_id}) stores = {'cheap': 'file', 'fast': 'file'} self.config(enabled_backends=stores) @@ -126,7 +132,7 @@ mock_store_api.return_value = self.staging_store copy_image_task = copy_image._CopyImage( self.task.task_id, self.task_type, self.image_repo, - self.image_id) + self.action_wrapper) with mock.patch.object(self.image_repo, 'get') as get_mock: get_mock.return_value = mock.MagicMock( image_id=self.images[0]['id'], @@ -152,7 +158,7 @@ copy_image_task = copy_image._CopyImage( self.task.task_id, self.task_type, self.image_repo, - self.image_id) + self.action_wrapper) with mock.patch.object(self.image_repo, 'get') as get_mock: get_mock.return_value = mock.MagicMock( image_id=self.images[0]['id'], @@ -182,7 +188,7 @@ copy_image_task = copy_image._CopyImage( self.task.task_id, self.task_type, self.image_repo, - self.image_id) + self.action_wrapper) with mock.patch.object(self.image_repo, 'get') as get_mock: get_mock.return_value = mock.MagicMock( image_id=self.images[0]['id'], @@ -206,7 +212,7 @@ mock_store_api.return_value = self.staging_store copy_image_task = copy_image._CopyImage( self.task.task_id, self.task_type, self.image_repo, - self.image_id) + self.action_wrapper) with mock.patch.object(self.image_repo, 'get') as get_mock: get_mock.side_effect = exception.NotFound() diff -Nru glance-21.0.0/glance/tests/unit/async_/flows/test_web_download.py glance-21.1.0/glance/tests/unit/async_/flows/test_web_download.py --- glance-21.0.0/glance/tests/unit/async_/flows/test_web_download.py 2020-09-30 22:45:36.000000000 +0000 +++ glance-21.1.0/glance/tests/unit/async_/flows/test_web_download.py 2021-09-13 14:50:26.000000000 +0000 @@ -40,7 +40,7 @@ super(TestWebDownloadTask, self).setUp() self.config(node_staging_uri='/tmp/staging') - self.task_repo = mock.MagicMock() + self.image_repo = mock.MagicMock() self.image_id = mock.MagicMock() self.uri = mock.MagicMock() self.task_factory = domain.TaskFactory() @@ -60,11 +60,17 @@ task_time_to_live=task_ttl, task_input=task_input) + self.task_id = self.task.task_id + self.action_wrapper = api_image_import.ImportActionWrapper( + self.image_repo, self.image_id, self.task_id) + self.image_repo.get.return_value = mock.MagicMock( + extra_properties={'os_glance_import_task': self.task_id}) + @mock.patch.object(filesystem.Store, 'add') def test_web_download(self, mock_add): web_download_task = web_download._WebDownload( - self.task.task_id, self.task_type, self.task_repo, - self.image_id, self.uri) + self.task.task_id, self.task_type, self.uri, self.action_wrapper, + ['foo']) with mock.patch.object(script_utils, 'get_image_data_iter') as mock_iter: mock_add.return_value = ["path", 4] @@ -76,8 +82,8 @@ @mock.patch.object(filesystem.Store, 'add') def test_web_download_with_content_length(self, mock_add): web_download_task = web_download._WebDownload( - self.task.task_id, self.task_type, self.task_repo, - self.image_id, self.uri) + self.task.task_id, self.task_type, self.uri, self.action_wrapper, + ['foo']) with mock.patch.object(script_utils, 'get_image_data_iter') as mock_iter: mock_iter.return_value.headers = {'content-length': '4'} @@ -89,8 +95,8 @@ @mock.patch.object(filesystem.Store, 'add') def test_web_download_with_invalid_content_length(self, mock_add): web_download_task = web_download._WebDownload( - self.task.task_id, self.task_type, self.task_repo, - self.image_id, self.uri) + self.task.task_id, self.task_type, self.uri, self.action_wrapper, + ['foo']) with mock.patch.object(script_utils, 'get_image_data_iter') as mock_iter: mock_iter.return_value.headers = {'content-length': "not_valid"} @@ -102,8 +108,8 @@ @mock.patch.object(filesystem.Store, 'add') def test_web_download_fails_when_data_size_different(self, mock_add): web_download_task = web_download._WebDownload( - self.task.task_id, self.task_type, self.task_repo, - self.image_id, self.uri) + self.task.task_id, self.task_type, self.uri, self.action_wrapper, + ['foo']) with mock.patch.object(script_utils, 'get_image_data_iter') as mock_iter: mock_iter.return_value.headers = {'content-length': '4'} @@ -116,8 +122,8 @@ self.config(node_staging_uri=None) self.assertRaises(glance.common.exception.BadTaskConfiguration, web_download._WebDownload, self.task.task_id, - self.task_type, self.task_repo, self.image_id, - self.uri) + self.task_type, self.uri, self.action_wrapper, + ['foo']) @mock.patch.object(cfg.ConfigOpts, "set_override") def test_web_download_node_store_initialization_failed(self, @@ -126,14 +132,14 @@ mock_load_store.return_value = None self.assertRaises(glance.common.exception.BadTaskConfiguration, web_download._WebDownload, self.task.task_id, - self.task_type, self.task_repo, self.image_id, - self.uri) + self.task_type, self.uri, self.action_wrapper, + ['foo']) mock_override.assert_called() def test_web_download_failed(self): web_download_task = web_download._WebDownload( - self.task.task_id, self.task_type, self.task_repo, - self.image_id, self.uri) + self.task.task_id, self.task_type, self.uri, self.action_wrapper, + ['foo']) with mock.patch.object(script_utils, "get_image_data_iter") as mock_iter: mock_iter.side_effect = glance.common.exception.NotFound @@ -181,9 +187,12 @@ @mock.patch("glance.async_.flows._internal_plugins.web_download.store_api") def test_web_download_revert_with_failure(self, mock_store_api, mock_add): + image = self.image_repo.get.return_value + image.extra_properties['os_glance_importing_to_stores'] = 'foo' + image.extra_properties['os_glance_failed_import'] = '' web_download_task = web_download._WebDownload( - self.task.task_id, self.task_type, self.task_repo, - self.image_id, self.uri) + self.task.task_id, self.task_type, self.uri, self.action_wrapper, + ['foo']) with mock.patch.object(script_utils, 'get_image_data_iter') as mock_iter: mock_iter.return_value.headers = {'content-length': '4'} @@ -195,6 +204,13 @@ web_download_task.revert(None) mock_store_api.delete_from_backend.assert_called_once_with( "/path/to_downloaded_data") + # NOTE(danms): Since we told revert that we were not at fault, + # we should not have updated the image. + self.image_repo.save.assert_not_called() + self.assertEqual( + 'foo', image.extra_properties['os_glance_importing_to_stores']) + self.assertEqual( + '', image.extra_properties['os_glance_failed_import']) @mock.patch("glance.async_.flows._internal_plugins.web_download.store_api") def test_web_download_revert_without_failure_multi_store(self, @@ -205,8 +221,8 @@ } self.config(enabled_backends=enabled_backends) web_download_task = web_download._WebDownload( - self.task.task_id, self.task_type, self.task_repo, - self.image_id, self.uri) + self.task.task_id, self.task_type, self.uri, self.action_wrapper, + ['foo']) web_download_task._path = "/path/to_downloaded_data" web_download_task.revert("/path/to_downloaded_data") mock_store_api.delete.assert_called_once_with( @@ -215,21 +231,35 @@ @mock.patch("glance.async_.flows._internal_plugins.web_download.store_api") def test_web_download_revert_with_failure_without_path(self, mock_store_api): + image = self.image_repo.get.return_value + image.status = 'importing' + image.extra_properties['os_glance_importing_to_stores'] = 'foo' + image.extra_properties['os_glance_failed_import'] = '' result = failure.Failure.from_exception( glance.common.exception.ImportTaskError()) web_download_task = web_download._WebDownload( - self.task.task_id, self.task_type, self.task_repo, - self.image_id, self.uri) + self.task.task_id, self.task_type, self.uri, self.action_wrapper, + ['foo']) web_download_task.revert(result) mock_store_api.delete_from_backend.assert_not_called() + # NOTE(danms): Since we told revert that we were the problem, + # we should have updated the image status and moved the stores + # to the failed list. + self.image_repo.save.assert_called_once_with(image, 'importing') + self.assertEqual('queued', image.status) + self.assertEqual( + '', image.extra_properties['os_glance_importing_to_stores']) + self.assertEqual( + 'foo', image.extra_properties['os_glance_failed_import']) + @mock.patch("glance.async_.flows._internal_plugins.web_download.store_api") def test_web_download_revert_with_failure_with_path(self, mock_store_api): result = failure.Failure.from_exception( glance.common.exception.ImportTaskError()) web_download_task = web_download._WebDownload( - self.task.task_id, self.task_type, self.task_repo, - self.image_id, self.uri) + self.task.task_id, self.task_type, self.uri, self.action_wrapper, + ['foo']) web_download_task._path = "/path/to_downloaded_data" web_download_task.revert(result) mock_store_api.delete_from_backend.assert_called_once_with( @@ -241,8 +271,8 @@ glance.common.exception.ImportTaskError()) mock_store_api.delete_from_backend.side_effect = Exception web_download_task = web_download._WebDownload( - self.task.task_id, self.task_type, self.task_repo, - self.image_id, self.uri) + self.task.task_id, self.task_type, self.uri, self.action_wrapper, + ['foo']) web_download_task._path = "/path/to_downloaded_data" # this will verify that revert does not break because of failure # while deleting data in staging area diff -Nru glance-21.0.0/glance/tests/unit/common/test_wsgi_app.py glance-21.1.0/glance/tests/unit/common/test_wsgi_app.py --- glance-21.0.0/glance/tests/unit/common/test_wsgi_app.py 2020-09-30 22:45:36.000000000 +0000 +++ glance-21.1.0/glance/tests/unit/common/test_wsgi_app.py 2021-09-13 14:50:26.000000000 +0000 @@ -63,3 +63,17 @@ # Make sure that shutdown() was called on the tasks_pool # ThreadPoolExecutor mock_shutdown.assert_called_once_with() + + @mock.patch('glance.async_._THREADPOOL_MODEL', new=None) + @mock.patch('glance.common.config.load_paste_app') + @mock.patch('glance.common.wsgi_app._get_config_files') + @mock.patch('threading.Thread') + @mock.patch('glance.housekeeping.StagingStoreCleaner') + def test_runs_staging_cleanup(self, mock_cleaner, mock_Thread, mock_conf, + mock_load): + mock_conf.return_value = [] + wsgi_app.init_app() + mock_Thread.assert_called_once_with( + target=mock_cleaner().clean_orphaned_staging_residue, + daemon=True) + mock_Thread.return_value.start.assert_called_once_with() diff -Nru glance-21.0.0/glance/tests/unit/test_housekeeping.py glance-21.1.0/glance/tests/unit/test_housekeeping.py --- glance-21.0.0/glance/tests/unit/test_housekeeping.py 1970-01-01 00:00:00.000000000 +0000 +++ glance-21.1.0/glance/tests/unit/test_housekeeping.py 2021-09-13 14:50:26.000000000 +0000 @@ -0,0 +1,235 @@ +# Copyright 2021 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +from unittest import mock + +import glance_store +from oslo_config import cfg +from oslo_utils.fixture import uuidsentinel as uuids + +from glance.common import exception +from glance import context +from glance import housekeeping +import glance.tests.unit.utils as unit_test_utils +import glance.tests.utils as test_utils + +CONF = cfg.CONF + + +class TestStagingStoreHousekeeping(test_utils.BaseTestCase): + def _store_dir(self, store): + return os.path.join(self.test_dir, store) + + def setUp(self): + super(TestStagingStoreHousekeeping, self).setUp() + + self.config(enabled_backends={'store1': 'file'}) + glance_store.register_store_opts( + CONF, + reserved_stores={'os_glance_staging_store': 'file'}) + + self.config(default_backend='store1', + group='glance_store') + self.config(filesystem_store_datadir=self._store_dir('store1'), + group='store1') + self.config(filesystem_store_datadir=self._store_dir('staging'), + group='os_glance_staging_store') + + glance_store.create_multi_stores( + CONF, + reserved_stores={'os_glance_staging_store': 'file'}) + + self.db = unit_test_utils.FakeDB(initialize=False) + self.cleaner = housekeeping.StagingStoreCleaner(self.db) + self.context = context.get_admin_context() + + def test_get_staging_path(self): + expected = os.path.join(self.test_dir, 'staging') + self.assertEqual(expected, housekeeping.staging_store_path()) + + def test_get_staging_path_single_store(self): + self.config(enabled_backends={}) + expected = '/tmp/staging/' + self.assertEqual(expected, housekeeping.staging_store_path()) + + @mock.patch('glance.common.store_utils.get_dir_separator') + def test_assert_staging_scheme(self, mock_get_dir_separator): + # NOTE(danms): This cannot happen now, but since we need to be + # opinionated about the fact that the URL is a file path, better + # to check for it, in case it changes in the future. + mock_get_dir_separator.return_value = ('/', 'http://foo') + self.assertRaises(exception.GlanceException, + lambda: housekeeping.staging_store_path()) + + def test_assert_staging_scheme_on_init(self): + # NOTE(danms): Make this a single-store scenario, which will cover + # our assertion about node_staging_uri while we test for the + # assert-on-init behavior. + self.config(enabled_backends={}, + node_staging_uri='http://good.luck') + self.assertRaises(exception.GlanceException, + housekeeping.staging_store_path) + + def test_get_image_id(self): + self.assertEqual(uuids.some_random_uuid, + self.cleaner.get_image_id(uuids.some_random_uuid)) + self.assertEqual(uuids.some_random_uuid, + self.cleaner.get_image_id( + '%s.qcow2' % uuids.some_random_uuid)) + self.assertEqual(uuids.some_random_uuid, + self.cleaner.get_image_id( + '%s.uc' % uuids.some_random_uuid)) + self.assertEqual(uuids.some_random_uuid, + self.cleaner.get_image_id( + '%s.blah' % uuids.some_random_uuid)) + + self.assertIsNone(self.cleaner.get_image_id('foo')) + self.assertIsNone(self.cleaner.get_image_id('foo.bar')) + + def test_is_valid_image(self): + image = self.db.image_create(self.context, {'status': 'queued'}) + self.assertTrue(self.cleaner.is_valid_image(image['id'])) + self.assertFalse(self.cleaner.is_valid_image('foo')) + + def test_is_valid_image_deleted(self): + image = self.db.image_create(self.context, {'status': 'queued'}) + self.db.image_destroy(self.context, image['id']) + self.assertFalse(self.cleaner.is_valid_image(image['id'])) + + @mock.patch('os.remove') + def test_delete_file(self, mock_remove): + self.assertTrue(self.cleaner.delete_file('foo')) + os.remove.assert_called_once_with('foo') + + @mock.patch('os.remove') + @mock.patch.object(housekeeping, 'LOG') + def test_delete_file_not_found(self, mock_LOG, mock_remove): + os.remove.side_effect = FileNotFoundError('foo is gone') + # We should ignore a file-not-found error + self.assertTrue(self.cleaner.delete_file('foo')) + os.remove.assert_called_once_with('foo') + mock_LOG.error.assert_not_called() + + @mock.patch('os.remove') + @mock.patch.object(housekeeping, 'LOG') + def test_delete_file_failed(self, mock_LOG, mock_remove): + # Any other error should report failure and log + os.remove.side_effect = Exception('insufficient plutonium') + self.assertFalse(self.cleaner.delete_file('foo')) + os.remove.assert_called_once_with('foo') + mock_LOG.error.assert_called_once_with( + 'Failed to delete stale staging path %(path)r: %(err)s', + {'path': 'foo', 'err': 'insufficient plutonium'}) + + @mock.patch('os.listdir') + @mock.patch('os.remove') + @mock.patch.object(housekeeping, 'LOG') + def test_clean_orphaned_staging_residue_empty(self, mock_LOG, mock_remove, + mock_listdir): + mock_listdir.return_value = [] + self.cleaner.clean_orphaned_staging_residue() + mock_listdir.assert_called_once_with(housekeeping.staging_store_path()) + mock_remove.assert_not_called() + mock_LOG.assert_not_called() + + @mock.patch('os.remove') + @mock.patch('os.listdir') + @mock.patch.object(housekeeping, 'LOG') + def test_clean_orphaned_staging_residue(self, mock_LOG, mock_listdir, + mock_remove): + staging = housekeeping.staging_store_path() + + image = self.db.image_create(self.context, {'status': 'queued'}) + + mock_listdir.return_value = ['notanimageid', image['id'], uuids.stale, + uuids.midconvert, + '%s.qcow2' % uuids.midconvert] + self.cleaner.clean_orphaned_staging_residue() + + # NOTE(danms): We should have deleted the stale image file + expected_stale = os.path.join(staging, uuids.stale) + + # NOTE(danms): We should have deleted the mid-convert base image and + # the target file + expected_mc = os.path.join(staging, uuids.midconvert) + expected_mc_target = os.path.join(staging, + '%s.qcow2' % uuids.midconvert) + + mock_remove.assert_has_calls([ + mock.call(expected_stale), + mock.call(expected_mc), + mock.call(expected_mc_target), + ]) + + # NOTE(danms): We should have cleaned the one (which we os.remove()'d) + # above, and ignore the invalid and active ones. No errors this time. + mock_LOG.debug.assert_has_calls([ + mock.call('Found %i files in staging directory for potential ' + 'cleanup', 5), + mock.call('Staging directory contains unexpected non-image file ' + '%r; ignoring', + 'notanimageid'), + mock.call('Stale staging residue found for image %(uuid)s: ' + '%(file)r; deleting now.', + {'uuid': uuids.stale, 'file': expected_stale}), + mock.call('Stale staging residue found for image %(uuid)s: ' + '%(file)r; deleting now.', + {'uuid': uuids.midconvert, 'file': expected_mc}), + mock.call('Stale staging residue found for image %(uuid)s: ' + '%(file)r; deleting now.', + {'uuid': uuids.midconvert, 'file': expected_mc_target}), + mock.call('Cleaned %(cleaned)i stale staging files, ' + '%(ignored)i ignored (%(error)i errors)', + {'cleaned': 3, 'ignored': 2, 'error': 0}), + ]) + + @mock.patch('os.listdir') + @mock.patch('os.remove') + @mock.patch.object(housekeeping, 'LOG') + def test_clean_orphaned_staging_residue_handles_errors(self, mock_LOG, + mock_remove, + mock_listdir): + staging = housekeeping.staging_store_path() + + mock_listdir.return_value = [uuids.gone, uuids.error] + mock_remove.side_effect = [FileNotFoundError('gone'), + PermissionError('not yours')] + self.cleaner.clean_orphaned_staging_residue() + + # NOTE(danms): We should only have logged an error for the + # permission failure + mock_LOG.error.assert_called_once_with( + 'Failed to delete stale staging path %(path)r: %(err)s', + {'path': os.path.join(staging, uuids.error), + 'err': 'not yours'}) + + # NOTE(danms): We should report the permission failure as an error, + # but not the already-gone or invalid ones. + mock_LOG.debug.assert_has_calls([ + mock.call('Found %i files in staging directory for potential ' + 'cleanup', 2), + mock.call('Stale staging residue found for image %(uuid)s: ' + '%(file)r; deleting now.', + {'uuid': uuids.gone, + 'file': os.path.join(staging, uuids.gone)}), + mock.call('Stale staging residue found for image %(uuid)s: ' + '%(file)r; deleting now.', + {'uuid': uuids.error, + 'file': os.path.join(staging, uuids.error)}), + mock.call('Cleaned %(cleaned)i stale staging files, ' + '%(ignored)i ignored (%(error)i errors)', + {'cleaned': 1, 'ignored': 0, 'error': 1}), + ]) diff -Nru glance-21.0.0/lower-constraints.txt glance-21.1.0/lower-constraints.txt --- glance-21.0.0/lower-constraints.txt 2020-09-30 22:45:36.000000000 +0000 +++ glance-21.1.0/lower-constraints.txt 1970-01-01 00:00:00.000000000 +0000 @@ -1,127 +0,0 @@ -alembic==0.8.10 -amqp==2.2.2 -appdirs==1.4.3 -asn1crypto==0.24.0 -automaton==1.14.0 -Babel==2.3.4 -boto3==1.9.199 -cachetools==2.0.1 -castellan==0.17.0 -cffi==1.13.2 -chardet==3.0.4 -cliff==2.11.0 -cmd2==0.8.1 -contextlib2==0.5.5 -coverage==4.0 -cryptography==2.1 -cursive==0.2.1 -ddt==1.0.1 -debtcollector==1.2.0 -decorator==4.2.1 -defusedxml==0.6.0 -dnspython==1.16.0 -doc8==0.6.0 -docutils==0.14 -entrypoints==0.3 -eventlet==0.25.1 -extras==1.0.0 -fasteners==0.14.1 -fixtures==3.0.0 -future==0.16.0 -futurist==1.2.0 -glance-store==2.3.0 -greenlet==0.4.13 -httplib2==0.9.1 -idna==2.6 -iso8601==0.1.11 -Jinja2==2.10 -jsonschema==3.2.0 -keystoneauth1==3.4.0 -keystonemiddleware==4.17.0 -kombu==4.3.0 -linecache2==1.0.0 -lxml==4.1.1 -Mako==1.0.7 -MarkupSafe==1.0 -mock==2.0.0 -monotonic==1.5 -mox3==0.25.0 -msgpack==0.5.6 -netaddr==0.7.19 -netifaces==0.10.6 -networkx==1.11 -os-client-config==1.29.0 -os-win==3.0.0 -oslo.concurrency==3.26.0 -oslo.config==5.2.0 -oslo.context==2.19.2 -oslo.db==5.0.0 -oslo.i18n==3.15.3 -oslo.log==3.36.0 -oslo.messaging==5.29.0 -oslo.middleware==3.31.0 -oslo.policy==2.4.1 -oslo.reports==1.18.0 -oslo.serialization==2.25.0 -oslo.service==1.30.0 -oslo.upgradecheck==0.1.0 -oslo.utils==3.33.0 -oslotest==3.2.0 -osprofiler==1.4.0 -Paste==2.0.2 -PasteDeploy==1.5.0 -pbr==2.0.0 -pika==0.10.0 -pika-pool==0.1.3 -positional==1.2.1 -prettytable==0.7.1 -psutil==3.2.2 -psycopg2==2.8.4 -pycadf==2.7.0 -pycparser==2.18 -Pygments==2.2.0 -pyinotify==0.9.6 -PyMySQL==0.7.6 -pyOpenSSL==17.1.0 -pyparsing==2.2.0 -pyperclip==1.8.0 -pysendfile==2.0.0 -python-barbicanclient==4.6.0 -python-dateutil==2.7.0 -python-editor==1.0.3 -python-keystoneclient==3.8.0 -python-mimeparse==1.6.0 -python-subunit==1.2.0 -python-swiftclient==3.2.0 -pytz==2018.3 -PyYAML==3.12 -repoze.lru==0.7 -requests==2.14.2 -requestsexceptions==1.4.0 -restructuredtext-lint==1.1.3 -retrying==1.2.3 -rfc3986==1.1.0 -Routes==2.3.1 -simplegeneric==0.8.1 -six==1.10.0 -SQLAlchemy==1.0.10 -sqlalchemy-migrate==0.11.0 -sqlparse==0.2.2 -statsd==3.2.2 -stestr==2.0.0 -stevedore==1.20.0 -taskflow==2.16.0 -Tempita==0.5.2 -tenacity==4.9.0 -testrepository==0.0.18 -testresources==2.0.0 -testscenarios==0.4 -testtools==2.2.0 -traceback2==1.4.0 -unittest2==1.1.0 -vine==1.1.4 -voluptuous==0.11.1 -WebOb==1.8.1 -wrapt==1.10.11 -WSME==0.8.0 -xattr==0.9.2 diff -Nru glance-21.0.0/releasenotes/notes/victoria-21.0.1-3e2d24edcaba77b1.yaml glance-21.1.0/releasenotes/notes/victoria-21.0.1-3e2d24edcaba77b1.yaml --- glance-21.0.0/releasenotes/notes/victoria-21.0.1-3e2d24edcaba77b1.yaml 1970-01-01 00:00:00.000000000 +0000 +++ glance-21.1.0/releasenotes/notes/victoria-21.0.1-3e2d24edcaba77b1.yaml 2021-09-13 14:50:26.000000000 +0000 @@ -0,0 +1,15 @@ +--- +fixes: + - | + Bug 1913625_: Glance will leak staging data + - | + Bug 1914826_: web download with invalid url does not report error + - | + Bug 1916052_: Unable to Create trust errors in glance-api + - | + Bug 1930597_: Doc for "Configuring SSL Support" outdated in glance + + .. _1913625: https://code.launchpad.net/bugs/1913625 + .. _1914826: https://code.launchpad.net/bugs/1914826 + .. _1916052: https://code.launchpad.net/bugs/1916052 + .. _1930597: https://code.launchpad.net/bugs/1930597 diff -Nru glance-21.0.0/requirements.txt glance-21.1.0/requirements.txt --- glance-21.0.0/requirements.txt 2020-09-30 22:45:36.000000000 +0000 +++ glance-21.1.0/requirements.txt 2021-09-13 14:50:26.000000000 +0000 @@ -13,16 +13,16 @@ WebOb>=1.8.1 # MIT sqlalchemy-migrate>=0.11.0 # Apache-2.0 sqlparse>=0.2.2 # BSD -alembic>=0.8.10 # MIT +alembic>=0.9.6 # MIT httplib2>=0.9.1 # MIT oslo.config>=5.2.0 # Apache-2.0 oslo.concurrency>=3.26.0 # Apache-2.0 -oslo.context>=2.19.2 # Apache-2.0 +oslo.context>=2.22.0 # Apache-2.0 oslo.upgradecheck>=0.1.0 # Apache-2.0 -oslo.utils>=3.33.0 # Apache-2.0 +oslo.utils>=3.40.2 # Apache-2.0 stevedore!=3.0.0,>=1.20.0 # Apache-2.0 futurist>=1.2.0 # Apache-2.0 -taskflow>=2.16.0 # Apache-2.0 +taskflow>=4.0.0 # Apache-2.0 keystoneauth1>=3.4.0 # Apache-2.0 keystonemiddleware>=4.17.0 # Apache-2.0 WSME>=0.8.0 # MIT @@ -32,9 +32,10 @@ Paste>=2.0.2 # MIT jsonschema>=3.2.0 # MIT +python-keystoneclient>=3.8.0 # Apache-2.0 pyOpenSSL>=17.1.0 # Apache-2.0 # Required by openstack.common libraries -six>=1.10.0 # MIT +six>=1.11.0 # MIT oslo.db>=5.0.0 # Apache-2.0 oslo.i18n>=3.15.3 # Apache-2.0 @@ -52,12 +53,12 @@ debtcollector>=1.2.0 # Apache-2.0 -cryptography>=2.1 # BSD/Apache-2.0 +cryptography>=2.6.1 # BSD/Apache-2.0 cursive>=0.2.1 # Apache-2.0 # timeutils iso8601>=0.1.11 # MIT -os-win>=3.0.0 # Apache-2.0 +os-win>=4.0.1 # Apache-2.0 castellan>=0.17.0 # Apache-2.0 diff -Nru glance-21.0.0/tox.ini glance-21.1.0/tox.ini --- glance-21.0.0/tox.ini 2020-09-30 22:45:36.000000000 +0000 +++ glance-21.1.0/tox.ini 2021-09-13 14:50:26.000000000 +0000 @@ -25,7 +25,7 @@ OS_TEST_DBAPI_ADMIN_CONNECTION=sqlite:////tmp/placeholder-never-created-nor-used.db usedevelop = True install_command = pip install {opts} {packages} -deps = -c{env:UPPER_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} +deps = -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/victoria} -r{toxinidir}/test-requirements.txt -r{toxinidir}/requirements.txt commands = @@ -95,7 +95,7 @@ # This environment is called from CI scripts to test and publish # the API Ref to docs.openstack.org. deps = - -c{env:UPPER_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} + -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/victoria} -r{toxinidir}/doc/requirements.txt commands = rm -rf api-ref/build @@ -147,7 +147,7 @@ [testenv:docs] deps = - -c{env:UPPER_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} + -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/victoria} -r{toxinidir}/doc/requirements.txt commands = rm -fr doc/build @@ -162,7 +162,7 @@ [testenv:releasenotes] deps = - -c{env:UPPER_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} + -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/victoria} -r{toxinidir}/doc/requirements.txt commands = sphinx-build -a -E -W -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html @@ -192,12 +192,3 @@ [testenv:debug-py38] commands = oslo_debug_helper {posargs} - -[testenv:lower-constraints] -deps = - -c{toxinidir}/lower-constraints.txt - -r{toxinidir}/test-requirements.txt - -r{toxinidir}/requirements.txt -commands = - {[testenv]commands} - stestr run --slowest {posargs} diff -Nru glance-21.0.0/.zuul.yaml glance-21.1.0/.zuul.yaml --- glance-21.0.0/.zuul.yaml 2020-09-30 22:45:36.000000000 +0000 +++ glance-21.1.0/.zuul.yaml 2021-09-13 14:50:26.000000000 +0000 @@ -7,7 +7,6 @@ tox_envlist: gateonly irrelevant-files: - ^(test-|)requirements.txt$ - - ^lower-constraints.txt$ - ^.*\.rst$ - ^api-ref/.*$ - ^doc/.*$ @@ -277,7 +276,6 @@ templates: - check-requirements - integrated-gate-storage - - openstack-lower-constraints-jobs - openstack-python3-victoria-jobs - periodic-stable-jobs - publish-openstack-docs-pti @@ -291,7 +289,6 @@ voting: false irrelevant-files: &tempest-irrelevant-files - ^(test-|)requirements.txt$ - - ^lower-constraints.txt$ - ^.*\.rst$ - ^api-ref/.*$ - ^glance/hacking/.*$