Merge pull request #116 from stackhpc/upstream/yoga-2024-10-07

Synchronise yoga with upstream

Author: priteau

.zuul.yaml

@@ -197,24 +197,11 @@
     parent: devstack-tempest
     description: |
       Run tempest compute API tests using LVM image backend. This only runs
-      against nova/virt/libvirt/* changes.
-    # Copy irrelevant-files from nova-dsvm-multinode-base and then exclude
-    # anything that is not in nova/virt/libvirt/* or nova/privsep/*.
-    irrelevant-files:
-      - ^(?!.zuul.yaml)(?!nova/virt/libvirt/)(?!nova/privsep/).*$
-      - ^api-.*$
-      - ^(test-|)requirements.txt$
-      - ^.*\.rst$
-      - ^.git.*$
-      - ^doc/.*$
-      - ^nova/hacking/.*$
-      - ^nova/locale/.*$
-      - ^nova/tests/.*$
-      - ^nova/test.py$
-      - ^releasenotes/.*$
-      - ^setup.cfg$
-      - ^tools/.*$
-      - ^tox.ini$
+      against nova/virt/libvirt/*, nova/privsep/* and .zuul.yaml changes.
+    files:
+      - ^nova/virt/libvirt/.*$
+      - ^nova/privsep/.*$
+      - .zuul.yaml
     vars:
       # We use the "all" environment for tempest_test_regex and
       # tempest_exclude_regex.
@@ -253,22 +240,11 @@
     # NOTE(chateaulav): due to constraints with no IDE support for aarch64,
     # tests have been limited to eliminate any items that are incompatible.
     # This is to be re-evaluated as greater support is added and defined.
-    irrelevant-files:
-      - ^(?!.zuul.yaml)(?!nova/virt/libvirt/)(?!nova/objects/)(?!nova/scheduler/).*$
-      - ^api-.*$
-      - ^(test-|)requirements.txt$
-      - ^.*\.rst$
-      - ^.git.*$
-      - ^doc/.*$
-      - ^nova/hacking/.*$
-      - ^nova/locale/.*$
-      - ^nova/policies/.*$
-      - ^nova/tests/.*$
-      - ^nova/test.py$
-      - ^releasenotes/.*$
-      - ^setup.cfg$
-      - ^tools/.*$
-      - ^tox.ini$
+    files:
+      - ^nova/virt/libvirt/.*$
+      - ^nova/objects/.*$
+      - ^nova/scheduler/.*$
+      - .zuul.yaml
     vars:
       tox_envlist: all
       tempest_test_regex: ^tempest\.(api\.compute\.servers|scenario\.test_network_basic_ops)
@@ -580,6 +556,16 @@
         GLANCE_STANDALONE: True
         GLANCE_USE_IMPORT_WORKFLOW: True
         DEVSTACK_PARALLEL: True
+        MYSQL_REDUCE_MEMORY: True
+      # NOTE(danms): This job is pretty heavy as it is, so we disable some
+      # services that are not relevant to the nova-glance-ceph scenario
+      # that this job is intended to validate.
+      devstack_services:
+        c-bak: false
+        s-account: false
+        s-container: false
+        s-object: false
+        s-proxy: false
       devstack_local_conf:
         post-config:
           $NOVA_CONF:
@@ -632,11 +618,12 @@
         - nova-ceph-multistore:
             irrelevant-files: *nova-base-irrelevant-files
         - neutron-linuxbridge-tempest:
-            irrelevant-files:
+            files:
               # NOTE(mriedem): This job has its own irrelevant-files section
               # so that we only run it on changes to networking and libvirt/vif
               # code; we don't need to run this on all changes.
-              - ^(?!nova/network/.*)(?!nova/virt/libvirt/vif.py).*$
+              - ^nova/network/.*$
+              - nova/virt/libvirt/vif.py
         - nova-live-migration
         - nova-live-migration-ceph
         - nova-lvm
@@ -680,11 +667,6 @@
         - barbican-tempest-plugin-simple-crypto:
             irrelevant-files: *nova-base-irrelevant-files
             voting: false
-        - tempest-integrated-compute-centos-8-stream:
-            irrelevant-files: *nova-base-irrelevant-files
-        - tempest-centos8-stream-fips:
-            irrelevant-files: *nova-base-irrelevant-files
-            voting: false
     gate:
       jobs:
         - nova-live-migration
@@ -697,11 +679,12 @@
         - nova-ceph-multistore:
             irrelevant-files: *nova-base-irrelevant-files
         - neutron-linuxbridge-tempest:
-            irrelevant-files:
+            files:
               # NOTE(mriedem): This job has its own irrelevant-files section
               # so that we only run it on changes to networking and libvirt/vif
               # code; we don't need to run this on all changes.
-              - ^(?!nova/network/.*)(?!nova/virt/libvirt/vif.py).*$
+              - ^nova/network/.*$
+              - nova/virt/libvirt/vif.py
         - tempest-integrated-compute:
            irrelevant-files: *policies-irrelevant-files
         - nova-grenade-multinode:
@@ -710,8 +693,6 @@
             irrelevant-files: *nova-base-irrelevant-files
         - openstacksdk-functional-devstack:
             irrelevant-files: *nova-base-irrelevant-files
-        - tempest-integrated-compute-centos-8-stream:
-            irrelevant-files: *nova-base-irrelevant-files
     experimental:
       jobs:
         - ironic-tempest-bfv:

nova/compute/manager.py

@@ -6646,9 +6646,9 @@ def _shelve_offload_instance(self, context, instance, clean_shutdown,
 
         instance.power_state = current_power_state
         # NOTE(mriedem): The vm_state has to be set before updating the
-        # resource tracker, see vm_states.ALLOW_RESOURCE_REMOVAL. The host/node
-        # values cannot be nulled out until after updating the resource tracker
-        # though.
+        # resource tracker, see vm_states.allow_resource_removal(). The
+        # host/node values cannot be nulled out until after updating the
+        # resource tracker though.
         instance.vm_state = vm_states.SHELVED_OFFLOADED
         instance.task_state = None
         instance.save(expected_task_state=[task_states.SHELVING,

nova/compute/resource_tracker.py

@@ -1495,7 +1495,8 @@ def _update_usage_from_instance(self, context, instance, nodename,
         # NOTE(sfinucan): Both brand new instances as well as instances that
         # are being unshelved will have is_new_instance == True
         is_removed_instance = not is_new_instance and (is_removed or
-            instance['vm_state'] in vm_states.ALLOW_RESOURCE_REMOVAL)
+            vm_states.allow_resource_removal(
+                vm_state=instance['vm_state'], task_state=instance.task_state))
 
         if is_new_instance:
             self.tracked_instances.add(uuid)
@@ -1554,7 +1555,9 @@ def _update_usage_from_instances(self, context, instances, nodename):
 
         instance_by_uuid = {}
         for instance in instances:
-            if instance.vm_state not in vm_states.ALLOW_RESOURCE_REMOVAL:
+            if not vm_states.allow_resource_removal(
+                    vm_state=instance['vm_state'],
+                    task_state=instance.task_state):
                 self._update_usage_from_instance(context, instance, nodename)
             instance_by_uuid[instance.uuid] = instance
         return instance_by_uuid

nova/compute/stats.py

@@ -105,7 +105,8 @@ def update_stats_for_instance(self, instance, is_removed=False):
         (vm_state, task_state, os_type, project_id) = \
                 self._extract_state_from_instance(instance)
 
-        if is_removed or vm_state in vm_states.ALLOW_RESOURCE_REMOVAL:
+        if is_removed or vm_states.allow_resource_removal(
+                vm_state=vm_state, task_state=task_state):
             self._decrement("num_instances")
             self.states.pop(uuid)
         else:

nova/compute/vm_states.py

@@ -27,6 +27,7 @@
 See http://wiki.openstack.org/VMState
 """
 
+from nova.compute import task_states
 from nova.objects import fields
 
 
@@ -74,5 +75,11 @@
 # states we allow to trigger crash dump
 ALLOW_TRIGGER_CRASH_DUMP = [ACTIVE, PAUSED, RESCUED, RESIZED, ERROR]
 
-# states we allow resources to be freed in
-ALLOW_RESOURCE_REMOVAL = [DELETED, SHELVED_OFFLOADED]
+
+def allow_resource_removal(vm_state, task_state=None):
+    """(vm_state, task_state) combinations we allow resources to be freed in"""
+
+    return (
+        vm_state == DELETED or
+        vm_state == SHELVED_OFFLOADED and task_state != task_states.SPAWNING
+    )

nova/conf/workarounds.py

@@ -416,6 +416,16 @@
         help="""
 When this is enabled, it will skip version-checking of hypervisors
 during live migration.
+"""),
+    cfg.BoolOpt(
+        'disable_deep_image_inspection',
+        default=False,
+        help="""
+This disables the additional deep image inspection that the compute node does
+when downloading from glance. This includes backing-file, data-file, and
+known-features detection *before* passing the image to qemu-img. Generally,
+this inspection should be enabled for maximum safety, but this workaround
+option allows disabling it if there is a compatibility concern.
 """),
     cfg.BoolOpt(
         'skip_reserve_in_use_ironic_nodes',
@@ -431,16 +441,6 @@
 Howerver, if you don't use automatic cleaning, it can cause an
 extra delay before and Ironic node is available for building a
 new Nova instance.
-"""),
-    cfg.BoolOpt(
-        'disable_deep_image_inspection',
-        default=False,
-        help="""
-This disables the additional deep image inspection that the compute node does
-when downloading from glance. This includes backing-file, data-file, and
-known-features detection *before* passing the image to qemu-img. Generally,
-this inspection should be enabled for maximum safety, but this workaround
-option allows disabling it if there is a compatibility concern.
 """),
 ]
 

nova/tests/functional/regressions/test_bug_2025480.py

@@ -0,0 +1,86 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+from unittest import mock
+
+from nova import context
+from nova.objects import compute_node
+from nova import test
+from nova.tests import fixtures as nova_fixtures
+from nova.tests.functional import fixtures as func_fixtures
+from nova.tests.functional import integrated_helpers
+
+
+class UnshelveUpdateAvailableResourcesPeriodicRace(
+        test.TestCase, integrated_helpers.InstanceHelperMixin):
+    def setUp(self):
+        super(UnshelveUpdateAvailableResourcesPeriodicRace, self).setUp()
+
+        placement = func_fixtures.PlacementFixture()
+        self.useFixture(placement)
+        self.placement = placement.api
+        self.neutron = nova_fixtures.NeutronFixture(self)
+        self.useFixture(self.neutron)
+        self.useFixture(nova_fixtures.GlanceFixture(self))
+        # Start nova services.
+        self.api = self.useFixture(nova_fixtures.OSAPIFixture(
+            api_version='v2.1')).admin_api
+        self.api.microversion = 'latest'
+        self.notifier = self.useFixture(
+            nova_fixtures.NotificationFixture(self))
+
+        self.start_service('conductor')
+        self.start_service('scheduler')
+
+    def test_unshelve_spawning_update_available_resources(self):
+        compute = self._start_compute('compute1')
+
+        server = self._create_server(
+            networks=[{'port': self.neutron.port_1['id']}])
+
+        node = compute_node.ComputeNode.get_by_nodename(
+            context.get_admin_context(), 'compute1')
+        self.assertEqual(1, node.vcpus_used)
+
+        # with default config shelve means immediate offload as well
+        req = {
+            'shelve': {}
+        }
+        self.api.post_server_action(server['id'], req)
+        self._wait_for_server_parameter(
+            server, {'status': 'SHELVED_OFFLOADED',
+                     'OS-EXT-SRV-ATTR:host': None})
+
+        node = compute_node.ComputeNode.get_by_nodename(
+            context.get_admin_context(), 'compute1')
+        self.assertEqual(0, node.vcpus_used)
+
+        def fake_spawn(*args, **kwargs):
+            self._run_periodics()
+
+        with mock.patch.object(
+                compute.driver, 'spawn', side_effect=fake_spawn):
+            req = {'unshelve': None}
+            self.api.post_server_action(server['id'], req)
+            self.notifier.wait_for_versioned_notifications(
+                'instance.unshelve.start')
+            self._wait_for_server_parameter(
+                server,
+                {
+                    'status': 'ACTIVE',
+                    'OS-EXT-STS:task_state': None,
+                    'OS-EXT-SRV-ATTR:host': 'compute1',
+                })
+
+        node = compute_node.ComputeNode.get_by_nodename(
+            context.get_admin_context(), 'compute1')
+        # After the fix, the instance should have resources claimed
+        self.assertEqual(1, node.vcpus_used)

nova/tests/unit/compute/test_stats.py

@@ -208,6 +208,22 @@ def test_update_stats_for_instance_offloaded(self):
         self.assertEqual(0, self.stats.num_os_type("Linux"))
         self.assertEqual(0, self.stats["num_vm_" + vm_states.BUILDING])
 
+    def test_update_stats_for_instance_being_unshelved(self):
+        instance = self._create_instance()
+        self.stats.update_stats_for_instance(instance)
+        self.assertEqual(1, self.stats.num_instances_for_project("1234"))
+
+        instance["vm_state"] = vm_states.SHELVED_OFFLOADED
+        instance["task_state"] = task_states.SPAWNING
+        self.stats.update_stats_for_instance(instance)
+
+        self.assertEqual(1, self.stats.num_instances)
+        self.assertEqual(1, self.stats.num_instances_for_project(1234))
+        self.assertEqual(1, self.stats["num_os_type_Linux"])
+        self.assertEqual(1, self.stats["num_vm_%s" %
+                                       vm_states.SHELVED_OFFLOADED])
+        self.assertEqual(1, self.stats["num_task_%s" % task_states.SPAWNING])
+
     def test_io_workload(self):
         vms = [vm_states.ACTIVE, vm_states.BUILDING, vm_states.PAUSED]
         tasks = [task_states.RESIZE_MIGRATING, task_states.REBUILDING,

nova/tests/unit/image/test_format_inspector.py

@@ -426,12 +426,14 @@ def test_qcow2_safety_checks(self):
         inspector = format_inspector.QcowInspector.from_file(fn)
         self.assertFalse(inspector.safety_check())
 
+        # Note(lajoskatona): This image create fails on bionic due to
+        # old qemu-img utilities, let's skip this only test from yoga
         # A data-file makes it unsafe
-        fn = self._create_img('qcow2', 5 * units.Mi,
-                              options={'data_file': data_fn,
-                                       'data_file_raw': 'on'})
-        inspector = format_inspector.QcowInspector.from_file(fn)
-        self.assertFalse(inspector.safety_check())
+        # fn = self._create_img('qcow2', 5 * units.Mi,
+        #                       options={'data_file': data_fn,
+        #                                'data_file_raw': 'on'})
+        # inspector = format_inspector.QcowInspector.from_file(fn)
+        # self.assertFalse(inspector.safety_check())
 
         # Trying to load a non-QCOW file is an error
         self.assertRaises(format_inspector.ImageFormatError,

nova/tests/unit/virt/libvirt/test_driver.py

@@ -8848,7 +8848,7 @@ def test_unquiesce(self, mock_has_min_version):
 
     def test_create_snapshot_metadata(self):
         base = objects.ImageMeta.from_dict(
-            {'disk_format': 'raw'})
+            {'disk_format': 'qcow2'})
         instance_data = {'kernel_id': 'kernel',
                     'project_id': 'prj_id',
                     'ramdisk_id': 'ram_id',
@@ -8880,10 +8880,12 @@ def test_create_snapshot_metadata(self):
             {'disk_format': 'ami',
              'container_format': 'test_container'})
         expected['properties']['os_type'] = instance['os_type']
-        expected['disk_format'] = base.disk_format
+        # The disk_format of the snapshot should be the *actual* format of the
+        # thing we upload, regardless of what type of image we booted from.
+        expected['disk_format'] = img_fmt
         expected['container_format'] = base.container_format
         ret = drvr._create_snapshot_metadata(base, instance, img_fmt, snp_name)
-        self.assertEqual(ret, expected)
+        self.assertEqual(expected, ret)
 
     def test_get_volume_driver(self):
         conn = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), False)
@@ -28225,7 +28227,8 @@ def test_ami(self):
           utils.get_system_metadata_from_image(
             {'disk_format': 'ami'})
 
-        self._test_snapshot(disk_format='ami')
+        # If we're uploading a qcow2, we must set the disk_format as such
+        self._test_snapshot(disk_format='qcow2')
 
     @mock.patch('nova.virt.libvirt.utils.get_disk_type_from_path',
                 new=mock.Mock(return_value=None))