Skip to content

server: double check host capacity when start/migrate a vm #3728

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
yadvr merged 2 commits into from
Jan 28, 2020
Merged

server: double check host capacity when start/migrate a vm #3728

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
30b0fa1
server: update host cpucore capacity when start/migrate a vm
ustcweizhou Nov 28, 2019
a18b1d5
server: double check the capacity of the host when increase host capa…
ustcweizhou Nov 28, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2349,6 +2349,17 @@ protected void migrate(final VMInstanceVO vm, final long srcHostId, final Deploy
_networkMgr.rollbackNicForMigration(vmSrc, profile);
s_logger.info("Migration cancelled because " + e1.getMessage());
throw new ConcurrentOperationException("Migration cancelled because " + e1.getMessage());
} catch (final CloudRuntimeException e2) {
_networkMgr.rollbackNicForMigration(vmSrc, profile);
s_logger.info("Migration cancelled because " + e2.getMessage());
work.setStep(Step.Done);
_workDao.update(work.getId(), work);
try {
stateTransitTo(vm, Event.OperationFailed, srcHostId);
} catch (final NoTransitionException e3) {
s_logger.warn(e3.getMessage());
}
throw new CloudRuntimeException("Migration cancelled because " + e2.getMessage());
Copy link
Copy Markdown
Contributor

@DaanHoogland DaanHoogland Jan 3, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

can we add the cause or log its stack-trace before abandoning it here?

}

boolean migrated = false;
Expand Down
57 changes: 54 additions & 3 deletions server/src/main/java/com/cloud/capacity/CapacityManagerImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,7 @@ public boolean releaseVmCapacity(VirtualMachine vm, final boolean moveFromReserv
final ServiceOfferingVO svo = _offeringsDao.findById(vm.getId(), vm.getServiceOfferingId());
CapacityVO capacityCpu = _capacityDao.findByHostIdType(hostId, Capacity.CAPACITY_TYPE_CPU);
CapacityVO capacityMemory = _capacityDao.findByHostIdType(hostId, Capacity.CAPACITY_TYPE_MEMORY);
CapacityVO capacityCpuCore = _capacityDao.findByHostIdType(hostId, Capacity.CAPACITY_TYPE_CPU_CORE);
Long clusterId = null;
if (hostId != null) {
HostVO host = _hostDao.findById(hostId);
Expand All @@ -182,28 +183,34 @@ public boolean releaseVmCapacity(VirtualMachine vm, final boolean moveFromReserv

clusterId = host.getClusterId();
}
if (capacityCpu == null || capacityMemory == null || svo == null) {
if (capacityCpu == null || capacityMemory == null || svo == null || capacityCpuCore == null) {
return false;
}

try {
final Long clusterIdFinal = clusterId;
final long capacityCpuId = capacityCpu.getId();
final long capacityMemoryId = capacityMemory.getId();
final long capacityCpuCoreId = capacityCpuCore.getId();

Transaction.execute(new TransactionCallbackNoReturn() {
@Override
public void doInTransactionWithoutResult(TransactionStatus status) {
CapacityVO capacityCpu = _capacityDao.lockRow(capacityCpuId, true);
CapacityVO capacityMemory = _capacityDao.lockRow(capacityMemoryId, true);
CapacityVO capacityCpuCore = _capacityDao.lockRow(capacityCpuCoreId, true);

long usedCpu = capacityCpu.getUsedCapacity();
long usedMem = capacityMemory.getUsedCapacity();
long usedCpuCore = capacityCpuCore.getUsedCapacity();
long reservedCpu = capacityCpu.getReservedCapacity();
long reservedMem = capacityMemory.getReservedCapacity();
long reservedCpuCore = capacityCpuCore.getReservedCapacity();
long actualTotalCpu = capacityCpu.getTotalCapacity();
float cpuOvercommitRatio = Float.parseFloat(_clusterDetailsDao.findDetail(clusterIdFinal, "cpuOvercommitRatio").getValue());
float memoryOvercommitRatio = Float.parseFloat(_clusterDetailsDao.findDetail(clusterIdFinal, "memoryOvercommitRatio").getValue());
int vmCPU = svo.getCpu() * svo.getSpeed();
int vmCPUCore = svo.getCpu();
long vmMem = svo.getRamSize() * 1024L * 1024L;
long actualTotalMem = capacityMemory.getTotalCapacity();
long totalMem = (long)(actualTotalMem * memoryOvercommitRatio);
Expand All @@ -221,6 +228,9 @@ public void doInTransactionWithoutResult(TransactionStatus status) {
if (usedMem >= vmMem) {
capacityMemory.setUsedCapacity(usedMem - vmMem);
}
if (usedCpuCore >= vmCPUCore) {
capacityCpuCore.setUsedCapacity(usedCpuCore - vmCPUCore);
}

if (moveToReservered) {
if (reservedCpu + vmCPU <= totalCpu) {
Expand All @@ -229,6 +239,7 @@ public void doInTransactionWithoutResult(TransactionStatus status) {
if (reservedMem + vmMem <= totalMem) {
capacityMemory.setReservedCapacity(reservedMem + vmMem);
}
capacityCpuCore.setReservedCapacity(reservedCpuCore + vmCPUCore);
}
} else {
if (reservedCpu >= vmCPU) {
Expand All @@ -237,6 +248,9 @@ public void doInTransactionWithoutResult(TransactionStatus status) {
if (reservedMem >= vmMem) {
capacityMemory.setReservedCapacity(reservedMem - vmMem);
}
if (reservedCpuCore >= vmCPUCore) {
capacityCpuCore.setReservedCapacity(reservedCpuCore - vmCPUCore);
}
}

s_logger.debug("release cpu from host: " + hostId + ", old used: " + usedCpu + ",reserved: " + reservedCpu + ", actual total: " + actualTotalCpu +
Expand All @@ -249,6 +263,7 @@ public void doInTransactionWithoutResult(TransactionStatus status) {

_capacityDao.update(capacityCpu.getId(), capacityCpu);
_capacityDao.update(capacityMemory.getId(), capacityMemory);
_capacityDao.update(capacityCpuCore.getId(), capacityCpuCore);
}
});

Expand All @@ -263,8 +278,9 @@ public void doInTransactionWithoutResult(TransactionStatus status) {
@Override
public void allocateVmCapacity(VirtualMachine vm, final boolean fromLastHost) {

final long vmId = vm.getId();
final long hostId = vm.getHostId();
HostVO host = _hostDao.findById(hostId);
final HostVO host = _hostDao.findById(hostId);
final long clusterId = host.getClusterId();
final float cpuOvercommitRatio = Float.parseFloat(_clusterDetailsDao.findDetail(clusterId, "cpuOvercommitRatio").getValue());
final float memoryOvercommitRatio = Float.parseFloat(_clusterDetailsDao.findDetail(clusterId, "memoryOvercommitRatio").getValue());
Expand All @@ -273,28 +289,35 @@ public void allocateVmCapacity(VirtualMachine vm, final boolean fromLastHost) {

CapacityVO capacityCpu = _capacityDao.findByHostIdType(hostId, Capacity.CAPACITY_TYPE_CPU);
CapacityVO capacityMem = _capacityDao.findByHostIdType(hostId, Capacity.CAPACITY_TYPE_MEMORY);
CapacityVO capacityCpuCore = _capacityDao.findByHostIdType(hostId, Capacity.CAPACITY_TYPE_CPU_CORE);

if (capacityCpu == null || capacityMem == null || svo == null) {
if (capacityCpu == null || capacityMem == null || svo == null || capacityCpuCore == null) {
return;
}

final int cpu = svo.getCpu() * svo.getSpeed();
final int cpucore = svo.getCpu();
final int cpuspeed = svo.getSpeed();
final long ram = svo.getRamSize() * 1024L * 1024L;

try {
final long capacityCpuId = capacityCpu.getId();
final long capacityMemId = capacityMem.getId();
final long capacityCpuCoreId = capacityCpuCore.getId();

Transaction.execute(new TransactionCallbackNoReturn() {
@Override
public void doInTransactionWithoutResult(TransactionStatus status) {
CapacityVO capacityCpu = _capacityDao.lockRow(capacityCpuId, true);
CapacityVO capacityMem = _capacityDao.lockRow(capacityMemId, true);
CapacityVO capacityCpuCore = _capacityDao.lockRow(capacityCpuCoreId, true);

long usedCpu = capacityCpu.getUsedCapacity();
long usedMem = capacityMem.getUsedCapacity();
long usedCpuCore = capacityCpuCore.getUsedCapacity();
long reservedCpu = capacityCpu.getReservedCapacity();
long reservedMem = capacityMem.getReservedCapacity();
long reservedCpuCore = capacityCpuCore.getReservedCapacity();
long actualTotalCpu = capacityCpu.getTotalCapacity();
long actualTotalMem = capacityMem.getTotalCapacity();
long totalCpu = (long)(actualTotalCpu * cpuOvercommitRatio);
Expand All @@ -313,6 +336,7 @@ public void doInTransactionWithoutResult(TransactionStatus status) {
}
capacityCpu.setUsedCapacity(usedCpu + cpu);
capacityMem.setUsedCapacity(usedMem + ram);
capacityCpuCore.setUsedCapacity(usedCpuCore + cpucore);

if (fromLastHost) {
/* alloc from reserved */
Expand All @@ -324,6 +348,7 @@ public void doInTransactionWithoutResult(TransactionStatus status) {
if (reservedCpu >= cpu && reservedMem >= ram) {
capacityCpu.setReservedCapacity(reservedCpu - cpu);
capacityMem.setReservedCapacity(reservedMem - ram);
capacityCpuCore.setReservedCapacity(reservedCpuCore - cpucore);
}
} else {
/* alloc from free resource */
Expand All @@ -343,12 +368,38 @@ public void doInTransactionWithoutResult(TransactionStatus status) {
totalMem + "; new used: " + capacityMem.getUsedCapacity() + ", reserved: " + capacityMem.getReservedCapacity() + "; requested mem: " + ram +
",alloc_from_last:" + fromLastHost);

long cluster_id = host.getClusterId();
ClusterDetailsVO cluster_detail_cpu = _clusterDetailsDao.findDetail(cluster_id, "cpuOvercommitRatio");
ClusterDetailsVO cluster_detail_ram = _clusterDetailsDao.findDetail(cluster_id, "memoryOvercommitRatio");
Float cpuOvercommitRatio = Float.parseFloat(cluster_detail_cpu.getValue());
Float memoryOvercommitRatio = Float.parseFloat(cluster_detail_ram.getValue());

boolean hostHasCpuCapability, hostHasCapacity = false;
hostHasCpuCapability = checkIfHostHasCpuCapability(host.getId(), cpucore, cpuspeed);

if (hostHasCpuCapability) {
// first check from reserved capacity
hostHasCapacity = checkIfHostHasCapacity(host.getId(), cpu, ram, true, cpuOvercommitRatio, memoryOvercommitRatio, true);

// if not reserved, check the free capacity
if (!hostHasCapacity)
hostHasCapacity = checkIfHostHasCapacity(host.getId(), cpu, ram, false, cpuOvercommitRatio, memoryOvercommitRatio, true);
}

if (!hostHasCapacity || !hostHasCpuCapability) {
throw new CloudRuntimeException("Host does not have enough capacity for vm " + vmId);
}

_capacityDao.update(capacityCpu.getId(), capacityCpu);
_capacityDao.update(capacityMem.getId(), capacityMem);
_capacityDao.update(capacityCpuCore.getId(), capacityCpuCore);
}
});
} catch (Exception e) {
s_logger.error("Exception allocating VM capacity", e);
if (e instanceof CloudRuntimeException) {
throw e;
}
return;
}
}
Expand Down