merge LTS branch 4.22 into main

Author: Daan Hoogland

agent/conf/uefi.properties.in

@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Configuration file for UEFI
+
+guest.nvram.template.legacy=@GUESTNVRAMTEMPLATELEGACY@
+guest.loader.legacy=@GUESTLOADERLEGACY@
+guest.nvram.template.secure=@GUESTNVRAMTEMPLATESECURE@
+guest.loader.secure=@GUESTLOADERSECURE@
+guest.nvram.path=@GUESTNVRAMPATH@

api/src/main/java/org/apache/cloudstack/api/command/user/address/ListPublicIpAddressesCmd.java

@@ -53,7 +53,7 @@ public class ListPublicIpAddressesCmd extends BaseListRetrieveOnlyResourceCountC
     @Parameter(name = ApiConstants.ALLOCATED_ONLY, type = CommandType.BOOLEAN, description = "limits search results to allocated public IP addresses")
     private Boolean allocatedOnly;
 
-    @Parameter(name = ApiConstants.STATE, type = CommandType.STRING, description = "lists all public IP addresses by state")
+    @Parameter(name = ApiConstants.STATE, type = CommandType.STRING, description = "lists all public IP addresses by state. A comma-separated list of states can be passed")
     private String state;
 
     @Parameter(name = ApiConstants.FOR_VIRTUAL_NETWORK, type = CommandType.BOOLEAN, description = "the virtual network for the IP address")

debian/cloudstack-agent.install

@@ -16,6 +16,7 @@
 # under the License.
 
 /etc/cloudstack/agent/agent.properties
+/etc/cloudstack/agent/uefi.properties
 /etc/cloudstack/agent/environment.properties
 /etc/cloudstack/agent/log4j-cloud.xml
 /etc/default/cloudstack-agent

debian/cloudstack-agent.postinst

@@ -23,7 +23,7 @@ case "$1" in
     configure)
         OLDCONFDIR="/etc/cloud/agent"
         NEWCONFDIR="/etc/cloudstack/agent"
-        CONFFILES="agent.properties log4j.xml log4j-cloud.xml"
+        CONFFILES="agent.properties uefi.properties log4j.xml log4j-cloud.xml"
 
         mkdir -m 0755 -p /usr/share/cloudstack-agent/tmp
 

debian/control

@@ -24,7 +24,7 @@ Description: CloudStack server library
 
 Package: cloudstack-agent
 Architecture: all
-Depends: ${python:Depends}, ${python3:Depends}, openjdk-17-jre-headless | java17-runtime-headless | java17-runtime | zulu-17, cloudstack-common (= ${source:Version}), lsb-base (>= 9), openssh-client, qemu-kvm (>= 2.5) | qemu-system-x86 (>= 5.2), libvirt-bin (>= 1.3) | libvirt-daemon-system (>= 3.0), iproute2, ebtables, vlan, ipset, python3-libvirt, ethtool, iptables, cryptsetup, rng-tools, rsync, lsb-release, ufw, apparmor, cpu-checker, libvirt-daemon-driver-storage-rbd, sysstat
+Depends: ${python:Depends}, ${python3:Depends}, openjdk-17-jre-headless | java17-runtime-headless | java17-runtime | zulu-17, cloudstack-common (= ${source:Version}), lsb-base (>= 9), openssh-client, qemu-kvm (>= 2.5) | qemu-system-x86 (>= 5.2), libvirt-bin (>= 1.3) | libvirt-daemon-system (>= 3.0), iproute2, ebtables, vlan, ipset, python3-libvirt, ethtool, iptables, cryptsetup, rng-tools, rsync, ovmf, swtpm, lsb-release, ufw, apparmor, cpu-checker, libvirt-daemon-driver-storage-rbd, sysstat
 Recommends: init-system-helpers
 Conflicts: cloud-agent, cloud-agent-libs, cloud-agent-deps, cloud-agent-scripts
 Description: CloudStack agent

engine/components-api/src/main/java/com/cloud/event/UsageEventUtils.java

@@ -94,6 +94,14 @@ public static void publishUsageEvent(String usageType, long accountId, long zone
 
     }
 
+    public static void publishUsageEvent(String usageType, long accountId, long zoneId, long resourceId, String resourceName, Long offeringId, Long templateId,
+                                         Long size, String entityType, String entityUUID, Long vmId, boolean displayResource) {
+        if (displayResource) {
+            saveUsageEvent(usageType, accountId, zoneId, resourceId, offeringId, templateId, size, vmId, resourceName);
+        }
+        publishUsageEvent(usageType, accountId, zoneId, entityType, entityUUID);
+    }
+
     public static void publishUsageEvent(String usageType, long accountId, long zoneId, long resourceId, String resourceName, Long offeringId, Long templateId,
                                          Long size, Long virtualSize, String entityType, String entityUUID, Map<String, String> details) {
         saveUsageEvent(usageType, accountId, zoneId, resourceId, resourceName, offeringId, templateId, size, virtualSize, details);
@@ -202,6 +210,10 @@ public static void saveUsageEvent(String usageType, long accountId, long zoneId,
         s_usageEventDao.persist(new UsageEventVO(usageType, accountId, zoneId, vmId, securityGroupId));
     }
 
+    public static void saveUsageEvent(String usageType, long accountId, long zoneId, long resourceId, Long offeringId, Long templateId, Long size, Long vmId, String resourceName) {
+        s_usageEventDao.persist(new UsageEventVO(usageType, accountId, zoneId, resourceId, offeringId, templateId, size, vmId, resourceName));
+    }
+
     private static void publishUsageEvent(String usageEventType, Long accountId, Long zoneId, String resourceType, String resourceUUID) {
         String configKey = "publish.usage.events";
         String value = s_configDao.getValue(configKey);

engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java

@@ -903,7 +903,7 @@ public DiskProfile allocateRawVolume(Type type, String name, DiskOffering offeri
         // Save usage event and update resource count for user vm volumes
         if (vm.getType() == VirtualMachine.Type.User) {
             UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_CREATE, vol.getAccountId(), vol.getDataCenterId(), vol.getId(), vol.getName(), offering.getId(), null, size,
-                    Volume.class.getName(), vol.getUuid(), vol.isDisplayVolume());
+                    Volume.class.getName(), vol.getUuid(), vol.getInstanceId(), vol.isDisplayVolume());
             _resourceLimitMgr.incrementVolumeResourceCount(vm.getAccountId(), vol.isDisplayVolume(), vol.getSize(), offering);
         }
         DiskProfile diskProfile = toDiskProfile(vol, offering);
@@ -981,7 +981,7 @@ private DiskProfile allocateTemplatedVolume(Type type, String name, DiskOffering
             }
 
             UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_CREATE, vol.getAccountId(), vol.getDataCenterId(), vol.getId(), vol.getName(), offeringId, vol.getTemplateId(), size,
-                    Volume.class.getName(), vol.getUuid(), vol.isDisplayVolume());
+                    Volume.class.getName(), vol.getUuid(), vol.getInstanceId(), vol.isDisplayVolume());
 
             _resourceLimitMgr.incrementVolumeResourceCount(vm.getAccountId(), vol.isDisplayVolume(), vol.getSize(), offering);
         }

engine/schema/src/main/java/com/cloud/event/UsageEventVO.java

@@ -75,6 +75,9 @@ public enum DynamicParameters {
     @Column(name = "virtual_size")
     private Long virtualSize;
 
+    @Column(name = "vm_id")
+    private Long vmId;
+
     public UsageEventVO() {
     }
 
@@ -143,6 +146,18 @@ public UsageEventVO(String usageType, long accountId, long zoneId, long vmId, lo
         this.offeringId = securityGroupId;
     }
 
+    public UsageEventVO(String usageType, long accountId, long zoneId, long resourceId, Long offeringId, Long templateId, Long size, Long vmId, String resourceName) {
+        this.type = usageType;
+        this.accountId = accountId;
+        this.zoneId = zoneId;
+        this.resourceId = resourceId;
+        this.offeringId = offeringId;
+        this.templateId = templateId;
+        this.size = size;
+        this.vmId = vmId;
+        this.resourceName = resourceName;
+    }
+
     @Override
     public long getId() {
         return id;
@@ -248,4 +263,11 @@ public void setVirtualSize(Long virtualSize) {
         this.virtualSize = virtualSize;
     }
 
+    public Long getVmId() {
+        return vmId;
+    }
+
+    public void setVmId(Long vmId) {
+        this.vmId = vmId;
+    }
 }

engine/schema/src/main/java/com/cloud/event/dao/UsageEventDaoImpl.java

@@ -45,11 +45,11 @@ public class UsageEventDaoImpl extends GenericDaoBase<UsageEventVO, Long> implem
     private final SearchBuilder<UsageEventVO> latestEventsSearch;
     private final SearchBuilder<UsageEventVO> IpeventsSearch;
     private static final String COPY_EVENTS =
-        "INSERT INTO cloud_usage.usage_event (id, type, account_id, created, zone_id, resource_id, resource_name, offering_id, template_id, size, resource_type, virtual_size) "
-            + "SELECT id, type, account_id, created, zone_id, resource_id, resource_name, offering_id, template_id, size, resource_type, virtual_size FROM cloud.usage_event vmevt WHERE vmevt.id > ? and vmevt.id <= ? ";
+        "INSERT INTO cloud_usage.usage_event (id, type, account_id, created, zone_id, resource_id, resource_name, offering_id, template_id, size, resource_type, virtual_size, vm_id) "
+            + "SELECT id, type, account_id, created, zone_id, resource_id, resource_name, offering_id, template_id, size, resource_type, virtual_size, vm_id FROM cloud.usage_event vmevt WHERE vmevt.id > ? and vmevt.id <= ? ";
     private static final String COPY_ALL_EVENTS =
-        "INSERT INTO cloud_usage.usage_event (id, type, account_id, created, zone_id, resource_id, resource_name, offering_id, template_id, size, resource_type, virtual_size) "
-            + "SELECT id, type, account_id, created, zone_id, resource_id, resource_name, offering_id, template_id, size, resource_type, virtual_size FROM cloud.usage_event vmevt WHERE vmevt.id <= ?";
+        "INSERT INTO cloud_usage.usage_event (id, type, account_id, created, zone_id, resource_id, resource_name, offering_id, template_id, size, resource_type, virtual_size, vm_id) "
+            + "SELECT id, type, account_id, created, zone_id, resource_id, resource_name, offering_id, template_id, size, resource_type, virtual_size, vm_id FROM cloud.usage_event vmevt WHERE vmevt.id <= ?";
     private static final String COPY_EVENT_DETAILS = "INSERT INTO cloud_usage.usage_event_details (id, usage_event_id, name, value) "
             + "SELECT id, usage_event_id, name, value FROM cloud.usage_event_details vmevtDetails WHERE vmevtDetails.usage_event_id > ? and vmevtDetails.usage_event_id <= ? ";
     private static final String COPY_ALL_EVENT_DETAILS = "INSERT INTO cloud_usage.usage_event_details (id, usage_event_id, name, value) "

engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java

@@ -89,10 +89,11 @@
 import com.cloud.upgrade.dao.Upgrade41910to42000;
 import com.cloud.upgrade.dao.Upgrade42000to42010;
 import com.cloud.upgrade.dao.Upgrade42010to42100;
-import com.cloud.upgrade.dao.Upgrade420to421;
 import com.cloud.upgrade.dao.Upgrade42100to42200;
+import com.cloud.upgrade.dao.Upgrade42200to42210;
+import com.cloud.upgrade.dao.Upgrade420to421;
 import com.cloud.upgrade.dao.Upgrade421to430;
-import com.cloud.upgrade.dao.Upgrade42200to42300;
+import com.cloud.upgrade.dao.Upgrade42210to42300;
 import com.cloud.upgrade.dao.Upgrade430to440;
 import com.cloud.upgrade.dao.Upgrade431to440;
 import com.cloud.upgrade.dao.Upgrade432to440;
@@ -237,7 +238,8 @@ public DatabaseUpgradeChecker() {
                 .next("4.20.0.0", new Upgrade42000to42010())
                 .next("4.20.1.0", new Upgrade42010to42100())
                 .next("4.21.0.0", new Upgrade42100to42200())
-                .next("4.22.0.0", new Upgrade42200to42300())
+                .next("4.22.0.0", new Upgrade42200to42210())
+                .next("4.22.1.0", new Upgrade42210to42300())
                 .build();
     }
 
@@ -315,20 +317,20 @@ protected void upgrade(CloudStackVersion dbVersion, CloudStackVersion currentVer
     }
 
     protected void executeProcedureScripts() {
-        LOGGER.info(String.format("Executing Stored Procedure scripts that are under resource directory [%s].", PROCEDURES_DIRECTORY));
+        LOGGER.info("Executing Stored Procedure scripts that are under resource directory [{}].", PROCEDURES_DIRECTORY);
         List<String> filesPathUnderViewsDirectory = FileUtil.getFilesPathsUnderResourceDirectory(PROCEDURES_DIRECTORY);
 
         try (TransactionLegacy txn = TransactionLegacy.open("execute-procedure-scripts")) {
             Connection conn = txn.getConnection();
 
             for (String filePath : filesPathUnderViewsDirectory) {
-                LOGGER.debug(String.format("Executing PROCEDURE script [%s].", filePath));
+                LOGGER.debug("Executing PROCEDURE script [{}].", filePath);
 
                 InputStream viewScript = Thread.currentThread().getContextClassLoader().getResourceAsStream(filePath);
                 runScript(conn, viewScript);
             }
 
-            LOGGER.info(String.format("Finished execution of PROCEDURE scripts that are under resource directory [%s].", PROCEDURES_DIRECTORY));
+            LOGGER.info("Finished execution of PROCEDURE scripts that are under resource directory [{}].", PROCEDURES_DIRECTORY);
         } catch (SQLException e) {
             String message = String.format("Unable to execute PROCEDURE scripts due to [%s].", e.getMessage());
             LOGGER.error(message, e);
@@ -337,7 +339,7 @@ protected void executeProcedureScripts() {
     }
 
     private DbUpgrade[] executeUpgrades(CloudStackVersion dbVersion, CloudStackVersion currentVersion) {
-        LOGGER.info("Database upgrade must be performed from " + dbVersion + " to " + currentVersion);
+        LOGGER.info("Database upgrade must be performed from {} to {}", dbVersion, currentVersion);
 
         final DbUpgrade[] upgrades = calculateUpgradePath(dbVersion, currentVersion);
 
@@ -350,8 +352,8 @@ private DbUpgrade[] executeUpgrades(CloudStackVersion dbVersion, CloudStackVersi
 
     private VersionVO executeUpgrade(DbUpgrade upgrade) {
         VersionVO version;
-        LOGGER.debug("Running upgrade " + upgrade.getClass().getSimpleName() + " to upgrade from " + upgrade.getUpgradableVersionRange()[0] + "-" + upgrade
-            .getUpgradableVersionRange()[1] + " to " + upgrade.getUpgradedVersion());
+        LOGGER.debug("Running upgrade {} to upgrade from {}-{} to {}", upgrade.getClass().getSimpleName(), upgrade.getUpgradableVersionRange()[0], upgrade
+                .getUpgradableVersionRange()[1], upgrade.getUpgradedVersion());
         TransactionLegacy txn = TransactionLegacy.open("Upgrade");
         txn.start();
         try {
@@ -394,8 +396,8 @@ private void executeUpgradeCleanup(DbUpgrade upgrade, VersionVO version) {
         // Run the corresponding '-cleanup.sql' script
         txn = TransactionLegacy.open("Cleanup");
         try {
-            LOGGER.info("Cleanup upgrade " + upgrade.getClass().getSimpleName() + " to upgrade from " + upgrade.getUpgradableVersionRange()[0] + "-" + upgrade
-                .getUpgradableVersionRange()[1] + " to " + upgrade.getUpgradedVersion());
+            LOGGER.info("Cleanup upgrade {} to upgrade from {}-{} to {}", upgrade.getClass().getSimpleName(), upgrade.getUpgradableVersionRange()[0], upgrade
+                    .getUpgradableVersionRange()[1], upgrade.getUpgradedVersion());
 
             txn.start();
             Connection conn;
@@ -410,7 +412,7 @@ private void executeUpgradeCleanup(DbUpgrade upgrade, VersionVO version) {
             if (scripts != null) {
                 for (InputStream script : scripts) {
                     runScript(conn, script);
-                    LOGGER.debug("Cleanup script " + upgrade.getClass().getSimpleName() + " is executed successfully");
+                    LOGGER.debug("Cleanup script {} is executed successfully", upgrade.getClass().getSimpleName());
                 }
             }
             txn.commit();
@@ -420,27 +422,27 @@ private void executeUpgradeCleanup(DbUpgrade upgrade, VersionVO version) {
             version.setUpdated(new Date());
             _dao.update(version.getId(), version);
             txn.commit();
-            LOGGER.debug("Upgrade completed for version " + version.getVersion());
+            LOGGER.debug("Upgrade completed for version {}", version.getVersion());
         } finally {
             txn.close();
         }
     }
 
     protected void executeViewScripts() {
-        LOGGER.info(String.format("Executing VIEW scripts that are under resource directory [%s].", VIEWS_DIRECTORY));
+        LOGGER.info("Executing VIEW scripts that are under resource directory [{}].", VIEWS_DIRECTORY);
         List<String> filesPathUnderViewsDirectory = FileUtil.getFilesPathsUnderResourceDirectory(VIEWS_DIRECTORY);
 
         try (TransactionLegacy txn = TransactionLegacy.open("execute-view-scripts")) {
             Connection conn = txn.getConnection();
 
             for (String filePath : filesPathUnderViewsDirectory) {
-                LOGGER.debug(String.format("Executing VIEW script [%s].", filePath));
+                LOGGER.debug("Executing VIEW script [{}].", filePath);
 
                 InputStream viewScript = Thread.currentThread().getContextClassLoader().getResourceAsStream(filePath);
                 runScript(conn, viewScript);
             }
 
-            LOGGER.info(String.format("Finished execution of VIEW scripts that are under resource directory [%s].", VIEWS_DIRECTORY));
+            LOGGER.info("Finished execution of VIEW scripts that are under resource directory [{}].", VIEWS_DIRECTORY);
         } catch (SQLException e) {
             String message = String.format("Unable to execute VIEW scripts due to [%s].", e.getMessage());
             LOGGER.error(message, e);
@@ -470,10 +472,10 @@ public void check() {
                 String csVersion = SystemVmTemplateRegistration.parseMetadataFile();
                 final CloudStackVersion sysVmVersion = CloudStackVersion.parse(csVersion);
                 final  CloudStackVersion currentVersion = CloudStackVersion.parse(currentVersionValue);
-                SystemVmTemplateRegistration.CS_MAJOR_VERSION  = String.valueOf(sysVmVersion.getMajorRelease()) + "." + String.valueOf(sysVmVersion.getMinorRelease());
+                SystemVmTemplateRegistration.CS_MAJOR_VERSION  = sysVmVersion.getMajorRelease() + "." + sysVmVersion.getMinorRelease();
                 SystemVmTemplateRegistration.CS_TINY_VERSION = String.valueOf(sysVmVersion.getPatchRelease());
 
-                LOGGER.info("DB version = " + dbVersion + " Code Version = " + currentVersion);
+                LOGGER.info("DB version = {} Code Version = {}", dbVersion, currentVersion);
 
                 if (dbVersion.compareTo(currentVersion) > 0) {
                     throw new CloudRuntimeException("Database version " + dbVersion + " is higher than management software version " + currentVersionValue);
@@ -522,7 +524,7 @@ private void decryptInit(Connection conn) throws SQLException {
              ResultSet result = pstmt.executeQuery()) {
             if (result.next()) {
                 String init = result.getString(1);
-                LOGGER.info("init = " + DBEncryptionUtil.decrypt(init));
+                LOGGER.info("init = {}", DBEncryptionUtil.decrypt(init));
             }
         }
     }
@@ -553,21 +555,11 @@ public String getUpgradedVersion() {
             return upgradedVersion;
         }
 
-        @Override
-        public boolean supportsRollingUpgrade() {
-            return false;
-        }
-
         @Override
         public InputStream[] getPrepareScripts() {
             return new InputStream[0];
         }
 
-        @Override
-        public void performDataMigration(Connection conn) {
-
-        }
-
         @Override
         public InputStream[] getCleanupScripts() {
             return new InputStream[0];