Skip to content

Commit 4359198

Browse files
sureshanapartisureshanaparti
authored
KVM Host HA improvements - Fix to not cancel VM HA items when Host HA inspection in progress, and some code improvements (#13088)
* Host HA code improvements * Fix to not cancel VM HA items when Host HA is enabled & inspection in progress, and some code improvements - When Host HA inspection in progress, the investigor returns the Host Status as Up which cancels the VM HA items - Don't cancel the VM HA items, instead reschedule them to try again later * Changes to consider Recovered/Available Host HA state along with the agent connection status to determine the Host HA inspection in progress or not, and some code improvements
1 parent ddcc0c8 commit 4359198

41 files changed

Lines changed: 481 additions & 440 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎api/src/main/java/com/cloud/ha/Investigator.java‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -26,17 +26,19 @@ public interface Investigator extends Adapter {
2626
* Returns if the vm is still alive.
2727
*
2828
* @param vm to work on.
29+
* @return true if vm is alive, otherwise false
2930
*/
30-
public boolean isVmAlive(VirtualMachine vm, Host host) throws UnknownVM;
31+
boolean isVmAlive(VirtualMachine vm, Host host) throws UnknownVM;
3132

32-
public Status isAgentAlive(Host agent);
33+
/**
34+
* Returns the agent status of the host.
35+
*
36+
* @param host
37+
* @return status of the host agent
38+
*/
39+
Status getHostAgentStatus(Host host);
3340

3441
class UnknownVM extends Exception {
35-
36-
/**
37-
*
38-
*/
3942
private static final long serialVersionUID = 1L;
40-
4143
};
4244
}

‎api/src/main/java/org/apache/cloudstack/api/command/admin/ha/ConfigureHAForHostCmd.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,7 @@ private void setupResponse(final boolean result, final String resourceUuid) {
8787
final HostHAResponse response = new HostHAResponse();
8888
response.setId(resourceUuid);
8989
response.setProvider(getHaProvider().toLowerCase());
90+
response.setStatus(result);
9091
response.setResponseName(getCommandName());
9192
setResponseObject(response);
9293
}

‎core/src/main/java/com/cloud/agent/api/CheckOnHostAnswer.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,8 @@ public CheckOnHostAnswer(CheckOnHostCommand cmd, Boolean alive, String details)
3838

3939
public CheckOnHostAnswer(CheckOnHostCommand cmd, String details) {
4040
super(cmd, false, details);
41+
determined = false;
42+
alive = false;
4143
}
4244

4345
public boolean isDetermined() {
@@ -47,5 +49,4 @@ public boolean isDetermined() {
4749
public boolean isAlive() {
4850
return alive;
4951
}
50-
5152
}

‎core/src/main/java/com/cloud/agent/api/CheckOnHostCommand.java‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424

2525
public class CheckOnHostCommand extends Command {
2626
HostTO host;
27-
boolean reportCheckFailureIfOneStorageIsDown;
27+
boolean reportIfHeartBeatFailedForOneStoragePool;
2828

2929
protected CheckOnHostCommand() {
3030
}
@@ -34,17 +34,17 @@ public CheckOnHostCommand(Host host) {
3434
setWait(20);
3535
}
3636

37-
public CheckOnHostCommand(Host host, boolean reportCheckFailureIfOneStorageIsDown) {
37+
public CheckOnHostCommand(Host host, boolean reportIfHeartBeatFailedForOneStoragePool) {
3838
this(host);
39-
this.reportCheckFailureIfOneStorageIsDown = reportCheckFailureIfOneStorageIsDown;
39+
this.reportIfHeartBeatFailedForOneStoragePool = reportIfHeartBeatFailedForOneStoragePool;
4040
}
4141

4242
public HostTO getHost() {
4343
return host;
4444
}
4545

46-
public boolean isCheckFailedOnOneStorage() {
47-
return reportCheckFailureIfOneStorageIsDown;
46+
public boolean shouldReportIfHeartBeatFailedForOneStoragePool() {
47+
return reportIfHeartBeatFailedForOneStoragePool;
4848
}
4949

5050
@Override

‎engine/components-api/src/main/java/com/cloud/ha/HighAvailabilityManager.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -75,10 +75,10 @@ public interface HighAvailabilityManager extends Manager {
7575
+ " which are registered for the HA event that were successful and are now ready to be purged.",
7676
true, Cluster);
7777

78-
public static final ConfigKey<Boolean> KvmHAFenceHostIfHeartbeatFailsOnStorage = new ConfigKey<>("Advanced", Boolean.class, "kvm.ha.fence.on.storage.heartbeat.failure", "false",
78+
ConfigKey<Boolean> KvmHAFenceHostIfHeartbeatFailsOnStorage = new ConfigKey<>("Advanced", Boolean.class, "kvm.ha.fence.on.storage.heartbeat.failure", "false",
7979
"Proceed fencing the host even the heartbeat failed for only one storage pool", false, ConfigKey.Scope.Zone);
8080

81-
public enum WorkType {
81+
enum WorkType {
8282
Migration, // Migrating VMs off of a host.
8383
Stop, // Stops a VM for storage pool migration purposes. This should be obsolete now.
8484
CheckStop, // Checks if a VM has been stopped.

‎plugins/hypervisors/hyperv/src/main/java/com/cloud/ha/HypervInvestigator.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -41,15 +41,15 @@ public class HypervInvestigator extends AdapterBase implements Investigator {
4141

4242
@Override
4343
public boolean isVmAlive(com.cloud.vm.VirtualMachine vm, Host host) throws UnknownVM {
44-
Status status = isAgentAlive(host);
44+
Status status = getHostAgentStatus(host);
4545
if (status == null) {
4646
throw new UnknownVM();
4747
}
48-
return status == Status.Up ? true : null;
48+
return status == Status.Up;
4949
}
5050

5151
@Override
52-
public Status isAgentAlive(Host agent) {
52+
public Status getHostAgentStatus(Host agent) {
5353
if (agent.getHypervisorType() != Hypervisor.HypervisorType.Hyperv) {
5454
return null;
5555
}

‎plugins/hypervisors/kvm/src/main/java/com/cloud/ha/KVMInvestigator.java‎

Lines changed: 21 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,7 @@
1919
package com.cloud.ha;
2020

2121
import com.cloud.agent.AgentManager;
22-
import com.cloud.agent.api.Answer;
23-
import com.cloud.agent.api.CheckOnHostCommand;
2422
import com.cloud.host.Host;
25-
import com.cloud.host.HostVO;
2623
import com.cloud.host.Status;
2724
import com.cloud.host.dao.HostDao;
2825
import com.cloud.hypervisor.Hypervisor;
@@ -34,11 +31,12 @@
3431
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProviderManager;
3532
import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreDriver;
3633
import org.apache.cloudstack.ha.HAManager;
34+
import org.apache.cloudstack.kvm.ha.KVMHostActivityChecker;
3735
import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
3836
import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
3937

4038
import javax.inject.Inject;
41-
import java.util.Arrays;
39+
import java.util.Collections;
4240
import java.util.List;
4341

4442
public class KVMInvestigator extends AdapterBase implements Investigator {
@@ -54,13 +52,15 @@ public class KVMInvestigator extends AdapterBase implements Investigator {
5452
private HAManager haManager;
5553
@Inject
5654
private DataStoreProviderManager dataStoreProviderMgr;
55+
@Inject
56+
private KVMHostActivityChecker hostActivityChecker;
5757

5858
@Override
5959
public boolean isVmAlive(com.cloud.vm.VirtualMachine vm, Host host) throws UnknownVM {
6060
if (haManager.isHAEligible(host)) {
6161
return haManager.isVMAliveOnHost(host);
6262
}
63-
Status status = isAgentAlive(host);
63+
Status status = getHostAgentStatus(host);
6464
logger.debug("HA: HOST is ineligible legacy state {} for host {}", status, host);
6565
if (status == null) {
6666
throw new UnknownVM();
@@ -73,86 +73,41 @@ public boolean isVmAlive(com.cloud.vm.VirtualMachine vm, Host host) throws Unkno
7373
}
7474

7575
@Override
76-
public Status isAgentAlive(Host agent) {
77-
if (agent.getHypervisorType() != Hypervisor.HypervisorType.KVM && agent.getHypervisorType() != Hypervisor.HypervisorType.LXC) {
76+
public Status getHostAgentStatus(Host host) {
77+
if (host.getHypervisorType() != Hypervisor.HypervisorType.KVM && host.getHypervisorType() != Hypervisor.HypervisorType.LXC) {
7878
return null;
7979
}
8080

81-
if (haManager.isHAEligible(agent)) {
82-
return haManager.getHostStatus(agent);
81+
if (haManager.isHAEligible(host)) {
82+
return haManager.getHostStatusFromHAConfig(host);
8383
}
8484

85-
List<StoragePoolVO> clusterPools = _storagePoolDao.findPoolsInClusters(Arrays.asList(agent.getClusterId()), null);
86-
boolean storageSupportHA = storageSupportHa(clusterPools);
87-
if (!storageSupportHA) {
88-
List<StoragePoolVO> zonePools = _storagePoolDao.findZoneWideStoragePoolsByHypervisor(agent.getDataCenterId(), agent.getHypervisorType());
89-
storageSupportHA = storageSupportHa(zonePools);
85+
List<StoragePoolVO> clusterPools = _storagePoolDao.findPoolsInClusters(Collections.singletonList(host.getClusterId()), null);
86+
boolean storageSupportsHA = storageSupportsHA(clusterPools);
87+
if (!storageSupportsHA) {
88+
List<StoragePoolVO> zonePools = _storagePoolDao.findZoneWideStoragePoolsByHypervisor(host.getDataCenterId(), host.getHypervisorType());
89+
storageSupportsHA = storageSupportsHA(zonePools);
9090
}
91-
if (!storageSupportHA) {
92-
logger.warn("Agent investigation was requested on host {}, but host does not support investigation because it has no NFS storage. Skipping investigation.", agent);
91+
if (!storageSupportsHA) {
92+
logger.warn("Agent investigation was requested on host {}, but host does not support investigation" +
93+
" because it has no HA supported storage. Skipping investigation.", host);
9394
return null;
9495
}
9596

96-
Status hostStatus = null;
97-
Status neighbourStatus = null;
98-
boolean reportFailureIfOneStorageIsDown = HighAvailabilityManager.KvmHAFenceHostIfHeartbeatFailsOnStorage.value();
99-
CheckOnHostCommand cmd = new CheckOnHostCommand(agent, reportFailureIfOneStorageIsDown);
100-
101-
try {
102-
Answer answer = _agentMgr.easySend(agent.getId(), cmd);
103-
if (answer != null) {
104-
hostStatus = answer.getResult() ? Status.Down : Status.Up;
105-
}
106-
} catch (Exception e) {
107-
logger.debug("Failed to send command to host: {}", agent);
108-
}
109-
if (hostStatus == null) {
110-
hostStatus = Status.Disconnected;
111-
}
112-
113-
List<HostVO> neighbors = _resourceMgr.listHostsInClusterByStatus(agent.getClusterId(), Status.Up);
114-
for (HostVO neighbor : neighbors) {
115-
if (neighbor.getId() == agent.getId()
116-
|| (neighbor.getHypervisorType() != Hypervisor.HypervisorType.KVM && neighbor.getHypervisorType() != Hypervisor.HypervisorType.LXC)) {
117-
continue;
118-
}
119-
logger.debug("Investigating host:{} via neighbouring host:{}", agent, neighbor);
120-
try {
121-
Answer answer = _agentMgr.easySend(neighbor.getId(), cmd);
122-
if (answer != null) {
123-
neighbourStatus = answer.getResult() ? Status.Down : Status.Up;
124-
logger.debug("Neighbouring host:{} returned status:{} for the investigated host:{}", neighbor, neighbourStatus, agent);
125-
if (neighbourStatus == Status.Up) {
126-
break;
127-
}
128-
}
129-
} catch (Exception e) {
130-
logger.debug("Failed to send command to host: {}", neighbor);
131-
}
132-
}
133-
if (neighbourStatus == Status.Up && (hostStatus == Status.Disconnected || hostStatus == Status.Down)) {
134-
hostStatus = Status.Disconnected;
135-
}
136-
if (neighbourStatus == Status.Down && (hostStatus == Status.Disconnected || hostStatus == Status.Down)) {
137-
hostStatus = Status.Down;
138-
}
139-
logger.debug("HA: HOST is ineligible legacy state {} for host {}", hostStatus, agent);
140-
return hostStatus;
97+
return hostActivityChecker.getHostAgentStatus(host);
14198
}
14299

143-
private boolean storageSupportHa(List<StoragePoolVO> pools) {
144-
boolean storageSupportHA = false;
100+
private boolean storageSupportsHA(List<StoragePoolVO> pools) {
145101
for (StoragePoolVO pool : pools) {
146102
DataStoreProvider storeProvider = dataStoreProviderMgr.getDataStoreProvider(pool.getStorageProviderName());
147103
DataStoreDriver storeDriver = storeProvider.getDataStoreDriver();
148104
if (storeDriver instanceof PrimaryDataStoreDriver) {
149105
PrimaryDataStoreDriver primaryStoreDriver = (PrimaryDataStoreDriver)storeDriver;
150106
if (primaryStoreDriver.isStorageSupportHA(pool.getPoolType())) {
151-
storageSupportHA = true;
152-
break;
107+
return true;
153108
}
154109
}
155110
}
156-
return storageSupportHA;
111+
return false;
157112
}
158113
}

‎plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHABase.java‎

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -35,10 +35,9 @@
3535
public class KVMHABase {
3636
protected Logger logger = LogManager.getLogger(getClass());
3737
private long _timeout = 60000; /* 1 minutes */
38-
protected long _heartBeatUpdateTimeout = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.HEARTBEAT_UPDATE_TIMEOUT);
39-
protected long _heartBeatUpdateFreq = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_HEARTBEAT_UPDATE_FREQUENCY);
38+
protected long _heartBeatUpdateFreqInMs = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_HEARTBEAT_UPDATE_FREQUENCY);
4039
protected long _heartBeatUpdateMaxTries = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_HEARTBEAT_UPDATE_MAX_TRIES);
41-
protected long _heartBeatUpdateRetrySleep = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_HEARTBEAT_UPDATE_RETRY_SLEEP);
40+
protected long _heartBeatUpdateRetrySleepInMs = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_HEARTBEAT_UPDATE_RETRY_SLEEP);
4241

4342
public static enum PoolType {
4443
PrimaryStorage, SecondaryStorage
@@ -138,7 +137,7 @@ protected String checkingMountPoint(HAStoragePool pool, String poolName) {
138137
/* Can't find the mount point? */
139138
/* we need to mount it under poolName */
140139
if (poolName != null) {
141-
Script mount = new Script("/bin/bash", 60000);
140+
Script mount = new Script("/bin/bash", _timeout);
142141
mount.add("-c");
143142
mount.add("mount " + mountSource + " " + destPath);
144143
String result = mount.execute();
@@ -154,7 +153,6 @@ protected String checkingMountPoint(HAStoragePool pool, String poolName) {
154153
}
155154

156155
protected String getMountPoint(HAStoragePool storagePool) {
157-
158156
StoragePool pool = null;
159157
String poolName = null;
160158
try {
@@ -171,7 +169,6 @@ protected String getMountPoint(HAStoragePool storagePool) {
171169
}
172170
poolName = pool.getName();
173171
}
174-
175172
} catch (LibvirtException e) {
176173
logger.debug("Ignoring libvirt error.", e);
177174
} finally {
@@ -234,7 +231,7 @@ protected String runScriptRetry(String cmdString, OutputInterpreter interpreter)
234231
return result;
235232
}
236233

237-
public Boolean checkingHeartBeat() {
234+
public Boolean hasHeartBeat() {
238235
// TODO Auto-generated method stub
239236
return null;
240237
}

‎plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAChecker.java‎

Lines changed: 16 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -26,44 +26,43 @@
2626
public class KVMHAChecker extends KVMHABase implements Callable<Boolean> {
2727
private List<HAStoragePool> storagePools;
2828
private HostTO host;
29-
private boolean reportFailureIfOneStorageIsDown;
29+
private boolean reportIfHeartBeatFailedForOneStoragePool;
3030

31-
public KVMHAChecker(List<HAStoragePool> pools, HostTO host, boolean reportFailureIfOneStorageIsDown) {
31+
public KVMHAChecker(List<HAStoragePool> pools, HostTO host, boolean reportIfHeartBeatFailedForOneStoragePool) {
3232
this.storagePools = pools;
3333
this.host = host;
34-
this.reportFailureIfOneStorageIsDown = reportFailureIfOneStorageIsDown;
34+
this.reportIfHeartBeatFailedForOneStoragePool = reportIfHeartBeatFailedForOneStoragePool;
3535
}
3636

3737
/*
38-
* True means heartbeaing is on going, or we can't get it's status. False
39-
* means heartbeating is stopped definitely
38+
* True means heart beating is on going, or we can't get it's status.
39+
* False means heart beating is stopped definitely.
4040
*/
4141
@Override
42-
public Boolean checkingHeartBeat() {
43-
boolean validResult = false;
44-
45-
String hostAndPools = String.format("host IP [%s] in pools [%s]", host.getPrivateNetwork().getIp(), storagePools.stream().map(pool -> pool.getPoolUUID()).collect(Collectors.joining(", ")));
46-
47-
logger.debug(String.format("Checking heart beat with KVMHAChecker for %s", hostAndPools));
42+
public Boolean hasHeartBeat() {
43+
String hostAndPools = String.format("host IP [%s] in pools [%s]", host.getPrivateNetwork().getIp(),
44+
storagePools.stream().map(pool -> pool.getPoolUUID()).collect(Collectors.joining(", ")));
45+
logger.debug("Checking heart beat with KVMHAChecker for {}", hostAndPools);
4846

47+
boolean heartBeatCheckResult = false;
4948
for (HAStoragePool pool : storagePools) {
50-
validResult = pool.getPool().checkingHeartBeat(pool, host);
51-
if (reportFailureIfOneStorageIsDown && !validResult) {
49+
heartBeatCheckResult = pool.getPool().hasHeartBeat(pool, host);
50+
if (reportIfHeartBeatFailedForOneStoragePool && !heartBeatCheckResult) {
5251
break;
5352
}
5453
}
5554

56-
if (!validResult) {
57-
logger.warn(String.format("All checks with KVMHAChecker for %s considered it as dead. It may cause a shutdown of the host.", hostAndPools));
55+
if (!heartBeatCheckResult) {
56+
logger.warn("All checks with KVMHAChecker for {} considered it as dead. It may cause a shutdown of the host.", hostAndPools);
5857
}
5958

60-
return validResult;
59+
return heartBeatCheckResult;
6160
}
6261

6362
@Override
6463
public Boolean call() throws Exception {
6564
// logger.addAppender(new org.apache.log4j.ConsoleAppender(new
6665
// org.apache.log4j.PatternLayout(), "System.out"));
67-
return checkingHeartBeat();
66+
return hasHeartBeat();
6867
}
6968
}

0 commit comments

Comments
 (0)