kvm: support Host HA on Ceph RBD primary storage

debian/control

@@ -24,7 +24,7 @@ Description: CloudStack server library
 
 Package: cloudstack-agent
 Architecture: all
-Depends: ${python:Depends}, ${python3:Depends}, openjdk-17-jre-headless | java17-runtime-headless | java17-runtime | zulu-17, cloudstack-common (= ${source:Version}), lsb-base (>= 9), openssh-client, qemu-kvm (>= 2.5) | qemu-system-x86 (>= 5.2), libvirt-bin (>= 1.3) | libvirt-daemon-system (>= 3.0), iproute2, ebtables, vlan, ipset, python3-libvirt, ethtool, iptables, cryptsetup, rng-tools, rsync, ovmf, swtpm, lsb-release, ufw, apparmor, cpu-checker, libvirt-daemon-driver-storage-rbd, sysstat, python3-libnbd, socat
+Depends: ${python:Depends}, ${python3:Depends}, openjdk-17-jre-headless | java17-runtime-headless | java17-runtime | zulu-17, cloudstack-common (= ${source:Version}), lsb-base (>= 9), openssh-client, qemu-kvm (>= 2.5) | qemu-system-x86 (>= 5.2), libvirt-bin (>= 1.3) | libvirt-daemon-system (>= 3.0), iproute2, ebtables, vlan, ipset, python3-libvirt, ethtool, iptables, cryptsetup, rng-tools, rsync, ovmf, swtpm, lsb-release, ufw, apparmor, cpu-checker, libvirt-daemon-driver-storage-rbd, sysstat, python3-libnbd, python3-rados, python3-rbd, socat
 Recommends: init-system-helpers
 Conflicts: cloud-agent, cloud-agent-libs, cloud-agent-deps, cloud-agent-scripts
 Description: CloudStack agent

engine/components-api/src/main/java/com/cloud/ha/HighAvailabilityManager.java

@@ -33,7 +33,8 @@
  */
 public interface HighAvailabilityManager extends Manager {
 
-    List<StoragePoolType> LIBVIRT_STORAGE_POOL_TYPES_WITH_HA_SUPPORT = List.of(StoragePoolType.NetworkFilesystem, StoragePoolType.SharedMountPoint);
+    List<StoragePoolType> LIBVIRT_STORAGE_POOL_TYPES_WITH_HA_SUPPORT = List.of(StoragePoolType.NetworkFilesystem, StoragePoolType.SharedMountPoint,
+            StoragePoolType.RBD);
 
     ConfigKey<Boolean> ForceHA = new ConfigKey<>("Advanced", Boolean.class, "force.ha", "false",
         "Force High-Availability to happen even if the VM says no.", true, Cluster);

packaging/el8/cloud.spec

@@ -128,6 +128,8 @@ Requires: (libgcrypt > 1.8.3 or libgcrypt20)
 Requires: (selinux-tools if selinux-tools)
 Requires: sysstat
 Requires: python3-libnbd
+Requires: python3-rados
+Requires: python3-rbd
 Requires: socat
 Provides: cloud-agent
 Group: System Environment/Libraries

packaging/suse15/cloud.spec

@@ -120,6 +120,8 @@ Requires: cifs-utils
 Requires: (edk2-ovmf or qemu-ovmf-x86_64)
 Requires: swtpm
 Requires: (python3-libvirt or python3-libvirt-python)
+Requires: (python3-rados or python3-ceph-common)
+Requires: (python3-rbd or python3-ceph-common)
 Requires: (qemu-img or qemu-tools)
 Requires: qemu-kvm
 Requires: cryptsetup

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAChecker.java

@@ -53,7 +53,7 @@ public Boolean hasHeartBeat() {
         }
 
         if (!heartBeatCheckResult) {
-            logger.warn("All checks with KVMHAChecker for {} considered it as dead. It may cause a shutdown of the host.", hostAndPools);
+            logger.warn("All checks with KVMHAChecker for {} considered it as dead, which may trigger HA actions, such as fencing it and restarting its VMs on other hosts.", hostAndPools);
         }
 
         return heartBeatCheckResult;

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java

@@ -20,6 +20,7 @@
 import com.cloud.agent.properties.AgentPropertiesFileHandler;
 import com.cloud.ha.HighAvailabilityManager;
 import com.cloud.utils.script.Script;
+import org.apache.commons.lang3.StringUtils;
 import org.libvirt.Connect;
 import org.libvirt.LibvirtException;
 import org.libvirt.StoragePool;
@@ -54,7 +55,11 @@ public void removeStoragePool(String uuid) {
         synchronized (haStoragePools) {
             HAStoragePool pool = haStoragePools.get(uuid);
             if (pool != null) {
-                Script.runSimpleBashScript("umount " + pool.getMountDestPath());
+                if (StringUtils.isNotEmpty(pool.getMountDestPath())) {
+                    Script umount = new Script("umount", logger);
+                    umount.add(pool.getMountDestPath());
+                    umount.execute();
+                }
                 haStoragePools.remove(uuid);
             }
         }
@@ -130,7 +135,7 @@ private void checkForNotExistingLibvirtStoragePools(Set<String> removedPools, St
                 removedPools.add(uuid);
             }
 
-            logger.debug("Found NFS storage pool [{}] in libvirt, continuing.", uuid);
+            logger.debug("Found storage pool [{}] in libvirt, continuing.", uuid);
 
         } catch (LibvirtException e) {
             logger.debug("Failed to lookup libvirt storage pool [{}].", uuid, e);

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVMActivityOnStoragePoolCommandWrapper.java

@@ -22,6 +22,8 @@
 import com.cloud.agent.api.Answer;
 import com.cloud.agent.api.CheckVMActivityOnStoragePoolCommand;
 import com.cloud.agent.api.to.StorageFilerTO;
+import com.cloud.agent.properties.AgentProperties;
+import com.cloud.agent.properties.AgentPropertiesFileHandler;
 import com.cloud.hypervisor.kvm.resource.KVMHABase.HAStoragePool;
 import com.cloud.hypervisor.kvm.storage.KVMStoragePool;
 import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager;
@@ -30,6 +32,8 @@
 import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource;
 import com.cloud.resource.CommandWrapper;
 import com.cloud.resource.ResourceWrapper;
+import com.cloud.storage.Storage.StoragePoolType;
+import com.cloud.utils.script.Script;
 
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
@@ -50,7 +54,11 @@ public Answer execute(final CheckVMActivityOnStoragePoolCommand command, final L
 
         if (primaryPool.isPoolSupportHA()) {
             final HAStoragePool haPool = monitor.getStoragePool(pool.getUuid());
-            final KVMHAVMActivityChecker ha = new KVMHAVMActivityChecker(haPool, command.getHost(), command.getVolumeList(), libvirtComputingResource.getVmActivityCheckPath(), command.getSuspectTimeInSeconds());
+            final String vmActivityCheckPath = getVmActivityCheckPath(pool.getType(), libvirtComputingResource);
+            if (vmActivityCheckPath == null) {
+                return new Answer(command, false, "CheckVMActivityOnStoragePoolCommand: unable to find the script which checks the VM activity on a storage pool of type " + pool.getType());
+            }
+            final KVMHAVMActivityChecker ha = new KVMHAVMActivityChecker(haPool, command.getHost(), command.getVolumeList(), vmActivityCheckPath, command.getSuspectTimeInSeconds());
             final Future<Boolean> future = executors.submit(ha);
             try {
                 final Boolean result = future.get();
@@ -67,4 +75,16 @@ public Answer execute(final CheckVMActivityOnStoragePoolCommand command, final L
         }
         return new Answer(command, false, "Unsupported Storage");
     }
+
+    /**
+     * The script which checks the VM activity depends on the type of the storage pool: a RBD pool has no shared mount point to look at,
+     * so its script checks the watchers of the volumes, instead of the files of the volumes as the script of the other pools.
+     */
+    private String getVmActivityCheckPath(final StoragePoolType poolType, final LibvirtComputingResource libvirtComputingResource) {
+        if (StoragePoolType.RBD.equals(poolType)) {
+            final String kvmScriptsDir = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_SCRIPTS_DIR);
+            return Script.findScript(kvmScriptsDir, "kvmvmactivity_rbd.py");
+        }
+        return libvirtComputingResource.getVmActivityCheckPath();
+    }
 }

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java

@@ -37,6 +37,7 @@
 
 import com.cloud.agent.api.to.DiskTO;
 import com.cloud.agent.api.to.VirtualMachineTO;
+import com.cloud.ha.HighAvailabilityManager;
 import com.cloud.hypervisor.kvm.resource.KVMHABase;
 import com.cloud.hypervisor.kvm.resource.KVMHABase.PoolType;
 import com.cloud.hypervisor.kvm.resource.KVMHAMonitor;
@@ -445,7 +446,7 @@ public boolean disconnectPhysicalDisk(StoragePoolType type, String poolUuid, Str
 
     public synchronized boolean deleteStoragePool(StoragePoolType type, String uuid) {
         StorageAdaptor adaptor = getStorageAdaptor(type);
-        if (type == StoragePoolType.NetworkFilesystem) {
+        if (HighAvailabilityManager.LIBVIRT_STORAGE_POOL_TYPES_WITH_HA_SUPPORT.contains(type)) {
             _haMonitor.removeStoragePool(uuid);
         }
         boolean deleteStatus = adaptor.deleteStoragePool(uuid);;
@@ -457,7 +458,7 @@ public synchronized boolean deleteStoragePool(StoragePoolType type, String uuid)
 
     public boolean deleteStoragePool(StoragePoolType type, String uuid, Map<String, String> details) {
         StorageAdaptor adaptor = getStorageAdaptor(type);
-        if (type == StoragePoolType.NetworkFilesystem) {
+        if (HighAvailabilityManager.LIBVIRT_STORAGE_POOL_TYPES_WITH_HA_SUPPORT.contains(type)) {
             _haMonitor.removeStoragePool(uuid);
         }
         boolean deleteStatus = adaptor.deleteStoragePool(uuid, details);

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStoragePool.java

@@ -17,10 +17,14 @@
 package com.cloud.hypervisor.kvm.storage;
 
 import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
 import java.util.List;
 import java.util.Map;
 
 import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.joda.time.Duration;
@@ -35,11 +39,15 @@
 import com.cloud.hypervisor.kvm.resource.KVMHABase.HAStoragePool;
 import com.cloud.storage.Storage;
 import com.cloud.storage.Storage.StoragePoolType;
+import com.cloud.utils.RbdUtils;
 import com.cloud.utils.exception.CloudRuntimeException;
 import com.cloud.utils.script.OutputInterpreter;
 import com.cloud.utils.script.Script;
 
 public class LibvirtStoragePool implements KVMStoragePool {
+    /** The environment variable which gives the Ceph key to the RBD heartbeat and VM-activity scripts. */
+    protected static final String RBD_KEY_ENVIRONMENT_VARIABLE = "CEPH_KEY";
+
     protected Logger logger = LogManager.getLogger(getClass());
     protected String uuid;
     protected long capacity;
@@ -330,37 +338,105 @@ public boolean isPoolSupportHA() {
 
     public String getHearthBeatPath() {
         if (StoragePoolType.NetworkFilesystem.equals(type)) {
-            String kvmScriptsDir = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_SCRIPTS_DIR);
-            String scriptPath = Script.findScript(kvmScriptsDir, "kvmheartbeat.sh");
-            if (scriptPath == null) {
-                throw new CloudRuntimeException("Unable to find heartbeat script 'kvmheartbeat.sh' in directory: " + kvmScriptsDir);
-            }
-            return scriptPath;
+            return findKvmHaScript("kvmheartbeat.sh");
         } else if (StoragePoolType.SharedMountPoint.equals(type)) {
-            String kvmScriptsDir = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_SCRIPTS_DIR);
-            String scriptPath = Script.findScript(kvmScriptsDir, "kvmsmpheartbeat.sh");
-            if (scriptPath == null) {
-                throw new CloudRuntimeException("Unable to find heartbeat script 'kvmsmpheartbeat.sh' in directory: " + kvmScriptsDir);
-            }
-            return scriptPath;
+            return findKvmHaScript("kvmsmpheartbeat.sh");
+        } else if (StoragePoolType.RBD.equals(type)) {
+            return findKvmHaScript("kvmheartbeat_rbd.py");
         }
         return null;
     }
 
+    private String findKvmHaScript(String scriptName) {
+        String kvmScriptsDir = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_SCRIPTS_DIR);
+        String scriptPath = Script.findScript(kvmScriptsDir, scriptName);
+        if (scriptPath == null) {
+            throw new CloudRuntimeException(String.format("Unable to find script '%s' in directory: %s", scriptName, kvmScriptsDir));
+        }
+        return scriptPath;
+    }
+
+    /**
+     * Returns the environment of a heartbeat/VM-activity check script: for a RBD storage pool with cephx, the
+     * Ceph key, which is not an argument of the script as it would be visible in the process list.
+     */
+    protected Map<String, String> getHeartBeatScriptEnvironment() {
+        if (StoragePoolType.RBD.equals(type) && StringUtils.isNotBlank(authSecret)) {
+            return Map.of(RBD_KEY_ENVIRONMENT_VARIABLE, authSecret);
+        }
+        return null;
+    }
+
+    /**
+     * Adds the Ceph cluster connection details (monitors, pool and, if cephx is enabled, the user; the key is in
+     * {@link #getHeartBeatScriptEnvironment()}) to a heartbeat/VM-activity check {@link Script} for a RBD storage pool. Mirrors the "mon_host"/"id"/"key"
+     * options that qemu itself uses to talk to RBD (see {@link KVMPhysicalDisk#RBDStringBuilder}).
+     */
+    protected void addRbdConnectionArgs(Script cmd) {
+        boolean hasUser = StringUtils.isNotBlank(authUsername);
+        boolean hasSecret = StringUtils.isNotBlank(authSecret);
+        if (hasUser != hasSecret) {
+            throw new CloudRuntimeException(String.format("The Ceph user and key must be either both set or both unset for the storage pool %s", uuid));
+        }
+        cmd.add("-s", RbdUtils.getRbdMonitors(sourceHost, sourcePort));
+        cmd.add("-o", sourceDir);
+        if (hasUser) {
+            cmd.add("-n", authUsername);
+        }
+    }
+
+    /**
+     * Adds the arguments identifying the storage to a heartbeat/VM-activity check {@link Script}:
+     * the Ceph connection details for a RBD pool, or the NFS server, path and mount point otherwise.
+     */
+    private void addPoolConnectionArgs(Script cmd, HAStoragePool pool) {
+        if (StoragePoolType.RBD.equals(type)) {
+            addRbdConnectionArgs(cmd);
+        } else {
+            cmd.add("-i", pool.getPoolIp());
+            cmd.add("-p", pool.getPoolMountSourcePath());
+            cmd.add("-m", pool.getMountDestPath());
+        }
+    }
+
+    /**
+     * Adds the volumes of the host to a VM-activity check {@link Script}. For a RBD storage pool they are given in
+     * a file, as they can be too many for an argument of a command (the limit is 128 KiB, about 3500 volumes),
+     * which would also be written to the log, with all the volumes, at each check. Otherwise, or if the file can not
+     * be written, they are an argument.
+     *
+     * @return the file, which has to be deleted after the script ran, or null if there is none
+     */
+    protected File addVolumeListArgs(Script cmd, String volumes) {
+        if (StoragePoolType.RBD.equals(type)) {
+            File volumeListFile = null;
+            try {
+                volumeListFile = File.createTempFile("kvmha-volumes-", ".txt");
+                Files.write(volumeListFile.toPath(), volumes.getBytes(StandardCharsets.UTF_8));
+                cmd.add("-U", volumeListFile.getAbsolutePath());
+                return volumeListFile;
+            } catch (IOException e) {
+                logger.warn("Unable to write the list of volumes to a file, giving it to the script as an argument: {}", e.getMessage());
+                if (volumeListFile != null && !volumeListFile.delete()) {
+                    logger.debug("Unable to delete the file {}", volumeListFile);
+                }
+            }
+        }
+        cmd.add("-u", volumes);
+        return null;
+    }
 
     public String createHeartBeatCommand(HAStoragePool primaryStoragePool, String hostPrivateIp, boolean hostValidation) {
         Script cmd = new Script(primaryStoragePool.getPool().getHearthBeatPath(), HeartBeatUpdateTimeoutInMs, logger);
-        cmd.add("-i", primaryStoragePool.getPoolIp());
-        cmd.add("-p", primaryStoragePool.getPoolMountSourcePath());
-        cmd.add("-m", primaryStoragePool.getMountDestPath());
+        addPoolConnectionArgs(cmd, primaryStoragePool);
 
         if (hostValidation) {
             cmd.add("-h", hostPrivateIp);
         } else {
             cmd.add("-c");
         }
 
-        return cmd.execute();
+        return cmd.execute(new OutputInterpreter.OutputLogger(logger), getHeartBeatScriptEnvironment());
     }
 
     @Override
@@ -377,20 +453,18 @@ public String getStorageNodeId() {
     public Boolean hasHeartBeat(HAStoragePool pool, HostTO host) {
         String hostIp = host.getPrivateNetwork().getIp();
         Script cmd = new Script(getHearthBeatPath(), HeartBeatCheckerTimeoutInMs, logger);
-        cmd.add("-i", pool.getPoolIp());
-        cmd.add("-p", pool.getPoolMountSourcePath());
-        cmd.add("-m", pool.getMountDestPath());
+        addPoolConnectionArgs(cmd, pool);
         cmd.add("-h", hostIp);
         cmd.add("-r");
         cmd.add("-t", String.valueOf(HeartBeatUpdateFreqInMs / 1000));
         OutputInterpreter.OneLineParser parser = new OutputInterpreter.OneLineParser();
-        String result = cmd.execute(parser);
+        String result = cmd.execute(parser, getHeartBeatScriptEnvironment());
         String parsedLine = parser.getLine();
 
         logger.debug("Checking heart beat for host IP {} with KVMHAChecker [{command=\"{}\", result: \"{}\", log: \"{}\", pool: \"{}\"}].", hostIp, cmd.toString(), result, parsedLine, pool.getPoolIp());
 
         if (result == null && parsedLine.contains("DEAD")) {
-            logger.warn("Checking heart beat for host IP {} with KVMHAChecker command [{}] returned [{}]. It may cause a shutdown of the host.", hostIp, cmd.toString(), parsedLine);
+            logger.warn("Checking heart beat for host IP {} with KVMHAChecker command [{}] returned [{}]. The host is considered dead, which may trigger HA actions, such as fencing it and restarting its VMs on other hosts.", hostIp, cmd.toString(), parsedLine);
             return false;
         } else {
             logger.debug("Checking heart beat for host IP {} with KVMHAChecker command [{}] succeeded.", hostIp, cmd.toString());
@@ -402,22 +476,27 @@ public Boolean hasHeartBeat(HAStoragePool pool, HostTO host) {
     public Boolean hasVmActivity(HAStoragePool pool, HostTO host, Duration activityScriptTimeout, String volumeUUIDListString, String vmActivityCheckPath, long duration) {
         String hostIp = host.getPrivateNetwork().getIp();
         Script cmd = new Script(vmActivityCheckPath, activityScriptTimeout.getStandardSeconds(), logger);
-        cmd.add("-i", pool.getPoolIp());
-        cmd.add("-p", pool.getPoolMountSourcePath());
-        cmd.add("-m", pool.getMountDestPath());
+        addPoolConnectionArgs(cmd, pool);
         cmd.add("-h", hostIp);
-        cmd.add("-u", volumeUUIDListString);
+        File volumeListFile = addVolumeListArgs(cmd, volumeUUIDListString);
         cmd.add("-t", String.valueOf(System.currentTimeMillis() / 1000));
         cmd.add("-d", String.valueOf(duration));
         OutputInterpreter.OneLineParser parser = new OutputInterpreter.OneLineParser();
 
-        String result = cmd.execute(parser);
+        String result;
+        try {
+            result = cmd.execute(parser, getHeartBeatScriptEnvironment());
+        } finally {
+            if (volumeListFile != null && !volumeListFile.delete()) {
+                logger.debug("Unable to delete the file {}", volumeListFile);
+            }
+        }
         String parsedLine = parser.getLine();
 
         logger.debug("Checking VM activity for host IP {} with KVMHAVMActivityChecker [{command=\"{}\", result: \"{}\", log: \"{}\", pool: \"{}\"}].", hostIp, cmd.toString(), result, parsedLine, pool.getPoolIp());
 
         if (result == null && parsedLine.contains("DEAD")) {
-            logger.warn("Checking VM activity for host IP {} with KVMHAVMActivityChecker command [{}] returned [{}]. It may cause a shutdown of the host.", hostIp, cmd.toString(), parsedLine);
+            logger.warn("Checking VM activity for host IP {} with KVMHAVMActivityChecker command [{}] returned [{}]. The host is considered dead, which may trigger HA actions, such as fencing it and restarting its VMs on other hosts.", hostIp, cmd.toString(), parsedLine);
             return false;
         } else {
             logger.debug("Checking VM activity for host IP {} with KVMHAVMActivityChecker command [{}] succeeded.", hostIp, cmd.toString());