diff --git a/PendingReleaseNotes b/PendingReleaseNotes index 9670b6e7c13a..8dabde7e2414 100644 --- a/PendingReleaseNotes +++ b/PendingReleaseNotes @@ -39,3 +39,22 @@ example.ver.1 > example.ver.2: which can now be attached to Instances. This is to prevent the Secondary Storage to grow to enormous sizes as Linux Distributions keep growing in size while a stripped down Linux should fit on a 2.88MB floppy. + + * New VM allocation algorithm 'balancedweighted' for vm.allocation.algorithm. Existing + algorithms and the default are unchanged; the new one is opt-in. + + Existing algorithms rank hosts on allocated capacity alone. Under a large overprovisioning + factor that reads badly: allocation is measured against a total that has already been + multiplied by the factor, so a host under real strain can still report a low percentage and + keep attracting new VMs. Concurrent deployments make it worse, since they all read the same + figures before any of them is accounted for. + + 'balancedweighted' ranks hosts on a blend of allocated CPU and memory, measured CPU and + memory utilisation, VM count, and how many VMs started on the host recently. It holds back + hosts that are measurably too busy, and chooses at random from among the best scoring hosts + so that simultaneous deployments do not all pick the same one. + + Tuned with the host.weighted.* settings, most of which are cluster scoped. Measured + utilisation comes from a moving average configured with host.load.sample.interval and + host.load.half.life. Until a management server has collected samples, ranking falls back to + allocation figures alone. diff --git a/api/src/main/java/com/cloud/deploy/DeploymentClusterPlanner.java b/api/src/main/java/com/cloud/deploy/DeploymentClusterPlanner.java index 9471c3d5c84c..5bf7dffbbf9b 100644 --- a/api/src/main/java/com/cloud/deploy/DeploymentClusterPlanner.java +++ b/api/src/main/java/com/cloud/deploy/DeploymentClusterPlanner.java @@ -62,11 +62,14 @@ public interface DeploymentClusterPlanner extends DeploymentPlanner { "vm.allocation.algorithm", "Advanced", "random", - "Order in which hosts within a cluster will be considered for VM allocation. The value can be 'random', 'firstfit', 'userdispersing', or 'firstfitleastconsumed'.", + "Order in which hosts within a cluster will be considered for VM allocation. The value can be 'random', " + + "'firstfit', 'userdispersing', 'firstfitleastconsumed', or 'balancedweighted'. 'balancedweighted' " + + "ranks hosts on a blend of allocated capacity, measured utilisation, VM count and how many VMs " + + "started recently, and is tuned with the host.weighted.* settings.", true, ConfigKey.Scope.Global, null, null, null, null, null, ConfigKey.Kind.Select, - "random,firstfit,userdispersing,firstfitleastconsumed"); + "random,firstfit,userdispersing,firstfitleastconsumed,balancedweighted"); /** * This is called to determine list of possible clusters where a virtual diff --git a/api/src/main/java/com/cloud/deploy/DeploymentPlanner.java b/api/src/main/java/com/cloud/deploy/DeploymentPlanner.java index 22d796d4a775..2d2964916eed 100644 --- a/api/src/main/java/com/cloud/deploy/DeploymentPlanner.java +++ b/api/src/main/java/com/cloud/deploy/DeploymentPlanner.java @@ -70,7 +70,7 @@ public interface DeploymentPlanner extends Adapter { boolean canHandle(VirtualMachineProfile vm, DeploymentPlan plan, ExcludeList avoid); public enum AllocationAlgorithm { - random, firstfit, userdispersing, firstfitleastconsumed; + random, firstfit, userdispersing, firstfitleastconsumed, balancedweighted; } public enum PlannerResourceUsage { diff --git a/api/src/main/java/com/cloud/host/HostScoringWeights.java b/api/src/main/java/com/cloud/host/HostScoringWeights.java new file mode 100644 index 000000000000..886487aeedf6 --- /dev/null +++ b/api/src/main/java/com/cloud/host/HostScoringWeights.java @@ -0,0 +1,57 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.host; + +import org.apache.cloudstack.framework.config.ConfigKey; + +/** + * How much each signal counts when ranking hosts by how loaded they are. + * + * Shared by initial placement and by rebalancing on purpose. If the two weighted these differently + * they would disagree about which host is the better one, and rebalancing could move VMs off hosts + * that placement had just chosen, only for placement to put them back. + * + * Weights are relative to each other; only their ratios matter, and zero disables a term. Terms that + * only make sense for one of the two - how many VMs a host carries, how many started recently - stay + * with whichever uses them. + */ +public interface HostScoringWeights { + + String WEIGHT_DESCRIPTION_SUFFIX = " Relative weight, only meaningful compared with the other " + + "host.weighted.* weights. Zero disables the term. Used by both the 'balancedweighted' " + + "allocation algorithm and the 'weighted' DRS algorithm."; + + ConfigKey CpuAllocatedWeight = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, + Double.class, "host.weighted.cpu.allocated.weight", "1.0", + "How much CPU allocated on a host counts against it." + WEIGHT_DESCRIPTION_SUFFIX, + true, ConfigKey.Scope.Cluster); + + ConfigKey CpuUsedWeight = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, + Double.class, "host.weighted.cpu.used.weight", "2.0", + "How much measured CPU utilisation counts against a host." + WEIGHT_DESCRIPTION_SUFFIX, + true, ConfigKey.Scope.Cluster); + + ConfigKey MemoryAllocatedWeight = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, + Double.class, "host.weighted.memory.allocated.weight", "1.0", + "How much memory allocated on a host counts against it." + WEIGHT_DESCRIPTION_SUFFIX, + true, ConfigKey.Scope.Cluster); + + ConfigKey MemoryUsedWeight = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, + Double.class, "host.weighted.memory.used.weight", "2.0", + "How much measured memory utilisation counts against a host." + WEIGHT_DESCRIPTION_SUFFIX, + true, ConfigKey.Scope.Cluster); +} diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java index 1a5b8cedd9ea..0b5fb4c64f9b 100755 --- a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java @@ -131,6 +131,18 @@ public interface VMInstanceDao extends GenericDao, StateDao< List listHostIdsByVmCount(long dcId, Long podId, Long clusterId, long accountId); + /** + * Counts the VMs occupying each host in a zone, pod or cluster, in one query. + * + * @param changedStateAfter + * cut-off for the second count: VMs whose state last changed after this. Approximates + * the VMs still working through their startup load, which neither allocation nor a + * utilisation average has caught up with yet. + * @return host id to {total VMs, VMs that changed state recently}. Every host in scope appears, + * including hosts with no VMs. + */ + Map> countVmsByHost(long dcId, Long podId, Long clusterId, Date changedStateAfter); + Long countRunningAndStartingByAccount(long accountId); Long countByZoneAndState(long zoneId, State state); diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java index ae1e838649ba..87135c035410 100755 --- a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java @@ -18,6 +18,7 @@ import java.sql.PreparedStatement; import java.sql.ResultSet; +import java.sql.Timestamp; import java.sql.SQLException; import java.util.ArrayList; import java.util.Collection; @@ -154,6 +155,15 @@ public class VMInstanceDaoImpl extends GenericDaoBase implem private static final String COUNT_VMS_BASED_ON_VGPU_TYPES2 = "GROUP BY gpu_card.name, vgpu_profile.name"; + // %s is the "changed state recently" test, or a constant 0 when no cut-off is given. It is not + // a bound parameter because there is no timestamp that reliably means "never" - update_time is + // a TIMESTAMP column, so anything past 2038 is out of range. + private static final String COUNT_VMS_BY_HOST = "SELECT host.id, COUNT(vm.id), SUM(IF(%s, 1, 0)) " + + "FROM `cloud`.`host` host LEFT JOIN `cloud`.`vm_instance` vm " + + "ON vm.host_id = host.id AND vm.state IN ('Running', 'Starting', 'Stopping', 'Migrating') " + + "AND vm.removed IS NULL WHERE host.type = 'Routing' AND host.removed IS NULL AND host.data_center_id = ? "; + private static final String COUNT_VMS_BY_HOST_PART2 = " GROUP BY host.id "; + private static final String UPDATE_SYSTEM_VM_TEMPLATE_ID_FOR_HYPERVISOR = "UPDATE `cloud`.`vm_instance` SET vm_template_id = ? WHERE type <> 'User' AND hypervisor_type = ? AND removed is NULL"; private static final String COUNT_VMS_BY_ZONE_AND_STATE_AND_HOST_TAG = "SELECT COUNT(1) FROM vm_instance vi JOIN service_offering so ON vi.service_offering_id=so.id " + @@ -795,6 +805,44 @@ public Pair, Map> listPodIdsInZoneByVmCount(long dataCe } } + + @Override + public Map> countVmsByHost(long dcId, Long podId, Long clusterId, Date changedStateAfter) { + TransactionLegacy txn = TransactionLegacy.currentTxn(); + Map> result = new HashMap<>(); + String sql = String.format(COUNT_VMS_BY_HOST, changedStateAfter != null ? "vm.update_time > ?" : "0"); + if (podId != null) { + sql = sql + " AND host.pod_id = ? "; + } + if (clusterId != null) { + sql = sql + " AND host.cluster_id = ? "; + } + sql = sql + COUNT_VMS_BY_HOST_PART2; + try { + PreparedStatement pstmt = txn.prepareAutoCloseStatement(sql); + int index = 1; + if (changedStateAfter != null) { + pstmt.setTimestamp(index++, new Timestamp(changedStateAfter.getTime())); + } + pstmt.setLong(index++, dcId); + if (podId != null) { + pstmt.setLong(index++, podId); + } + if (clusterId != null) { + pstmt.setLong(index++, clusterId); + } + ResultSet rs = pstmt.executeQuery(); + while (rs.next()) { + result.put(rs.getLong(1), new Pair<>(rs.getLong(2), rs.getLong(3))); + } + return result; + } catch (SQLException e) { + throw new CloudRuntimeException("DB Exception on: " + sql, e); + } catch (Exception e) { + throw new CloudRuntimeException("Caught: " + sql, e); + } + } + @Override public List listHostIdsByVmCount(long dcId, Long podId, Long clusterId, long accountId) { TransactionLegacy txn = TransactionLegacy.currentTxn(); diff --git a/server/src/main/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java b/server/src/main/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java index 4bc34d8a5c60..6a5669d17975 100644 --- a/server/src/main/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java +++ b/server/src/main/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java @@ -16,6 +16,7 @@ // under the License. package com.cloud.agent.manager.allocator.impl; +import static com.cloud.deploy.DeploymentPlanner.AllocationAlgorithm.balancedweighted; import static com.cloud.deploy.DeploymentPlanner.AllocationAlgorithm.firstfitleastconsumed; import static com.cloud.deploy.DeploymentPlanner.AllocationAlgorithm.random; import static com.cloud.deploy.DeploymentPlanner.AllocationAlgorithm.userdispersing; @@ -95,6 +96,8 @@ public class FirstFitAllocator extends BaseAllocator { CapacityDao _capacityDao; @Inject VMInstanceDetailsDao vmInstanceDetailsDao; + @Inject + WeightedHostScorer weightedHostScorer; boolean _checkHvm = true; @@ -209,6 +212,8 @@ protected List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan hosts = reorderHostsByNumberOfVms(plan, hosts, account); } else if (firstfitleastconsumed.toString().equals(vmAllocationAlgorithm)) { hosts = reorderHostsByCapacity(plan, hosts); + } else if (balancedweighted.toString().equals(vmAllocationAlgorithm)) { + hosts = weightedHostScorer.rank(plan.getDataCenterId(), plan.getPodId(), plan.getClusterId(), hosts); } logger.debug("FirstFitAllocator has {} hosts to check for allocation {}.", hosts.size(), hosts); diff --git a/server/src/main/java/com/cloud/agent/manager/allocator/impl/HostLoad.java b/server/src/main/java/com/cloud/agent/manager/allocator/impl/HostLoad.java new file mode 100644 index 000000000000..2e3d14ba8384 --- /dev/null +++ b/server/src/main/java/com/cloud/agent/manager/allocator/impl/HostLoad.java @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.manager.allocator.impl; + +/** + * Smoothed view of what a host is actually doing, as opposed to what has been allocated on it. + * Fractions are of the host's real capacity and ignore overprovisioning. + */ +public class HostLoad { + + public static final HostLoad UNKNOWN = new HostLoad(0, 0, 0); + + private final double cpuUtilisation; + private final double memoryUtilisation; + private final long samples; + + public HostLoad(double cpuUtilisation, double memoryUtilisation, long samples) { + this.cpuUtilisation = cpuUtilisation; + this.memoryUtilisation = memoryUtilisation; + this.samples = samples; + } + + public double getCpuUtilisation() { + return cpuUtilisation; + } + + public double getMemoryUtilisation() { + return memoryUtilisation; + } + + public long getSamples() { + return samples; + } + + /** + * False until enough has been observed to rank on. Callers fall back to allocation figures. + */ + public boolean isUsable() { + return samples > 0; + } + + @Override + public String toString() { + return String.format("HostLoad[cpu=%.3f, memory=%.3f, samples=%d]", cpuUtilisation, memoryUtilisation, samples); + } +} diff --git a/server/src/main/java/com/cloud/agent/manager/allocator/impl/HostLoadTracker.java b/server/src/main/java/com/cloud/agent/manager/allocator/impl/HostLoadTracker.java new file mode 100644 index 000000000000..4543af9b8b14 --- /dev/null +++ b/server/src/main/java/com/cloud/agent/manager/allocator/impl/HostLoadTracker.java @@ -0,0 +1,260 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.manager.allocator.impl; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import javax.inject.Inject; + +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.Configurable; +import org.apache.cloudstack.managed.context.ManagedContextRunnable; + +import com.cloud.host.HostStats; +import com.cloud.host.HostVO; +import com.cloud.host.Status; +import com.cloud.host.dao.HostDao; +import com.cloud.server.StatsCollector; +import com.cloud.deploy.DeploymentClusterPlanner; +import com.cloud.deploy.DeploymentPlanner.AllocationAlgorithm; +import com.cloud.utils.component.ManagerBase; +import com.cloud.utils.concurrency.NamedThreadFactory; + +/** + * Keeps a smoothed view of how hard each host is actually working. + * + * StatsCollector already polls every host, but it keeps only the newest sample and nothing uses it + * for placement. A single sample is too noisy to rank on: a host can look idle moments before a + * batch of VMs starts work. This folds those samples into an exponentially weighted moving average + * so ranking reflects a trend rather than an instant. + * + * The average is per management server and is not persisted. Every management server polls every + * host, so all of them converge on the same picture, and a restarted server simply reports nothing + * usable until it has sampled - callers then fall back to allocation figures. + * + * What getCpuUtilization means depends on the hypervisor, and only KVM reports what this class + * assumes: + * + *
    + *
  • KVM reports busy time as a percentage of the host's cores, which is what is wanted.
    • + *
  • VMware reports the share of CPU that is reserved rather than the share that is busy, so + * the CPU term becomes a second allocation signal there rather than a load signal.
    • + *
  • XenServer sums per-core averages without dividing by core count, so the value ranges up to + * the number of cores and is under-reported here by roughly that factor.
    • + *
+ * + * Memory is taken as used over total and is sound everywhere. + */ +public class HostLoadTracker extends ManagerBase implements Configurable { + + public static final ConfigKey HostLoadSampleInterval = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, + Integer.class, "host.load.sample.interval", "60", + "Seconds between samples of host CPU and memory utilisation, for placement algorithms that " + + "consider actual load. Should not be shorter than host.stats.interval.", + false, ConfigKey.Scope.Global); + + public static final ConfigKey HostLoadStaleAfter = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, + Integer.class, "host.load.stale.after", "600", + "Seconds after which a host's utilisation average is considered out of date and stops being used " + + "for placement. A host whose agent stops reporting would otherwise keep vouching for itself " + + "with figures that never change.", + true, ConfigKey.Scope.Global); + + public static final ConfigKey HostLoadHalfLife = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, + Integer.class, "host.load.half.life", "300", + "Half life in seconds of the moving average of host utilisation. Larger values react more " + + "slowly and are less affected by short spikes or by guests periodically releasing memory.", + true, ConfigKey.Scope.Global); + + @Inject + private HostDao hostDao; + + @Inject + private StatsCollector statsCollector; + + private final Map samples = new ConcurrentHashMap<>(); + + private ScheduledExecutorService executor; + + @Override + public boolean start() { + int interval = Math.max(1, HostLoadSampleInterval.value()); + executor = Executors.newSingleThreadScheduledExecutor( + new NamedThreadFactory("HostLoadTracker")); + // catch Throwable: an escaping error would cancel all future runs, and the failure would be + // silent - placement would quietly go back to ranking on allocation alone + executor.scheduleWithFixedDelay(new ManagedContextRunnable() { + @Override + protected void runInContext() { + try { + sampleAllHosts(); + } catch (Throwable t) { + logger.warn("Unable to sample host load", t); + } + } + }, interval, interval, TimeUnit.SECONDS); + return true; + } + + @Override + public boolean stop() { + if (executor != null) { + executor.shutdownNow(); + } + return true; + } + + protected void sampleAllHosts() { + if (!isInUse()) { + samples.clear(); + return; + } + for (HostVO host : hostDao.listByType(com.cloud.host.Host.Type.Routing)) { + if (host.getStatus() != Status.Up) { + samples.remove(host.getId()); + continue; + } + record(host.getId(), statsCollector.getHostStats(host.getId())); + } + } + + /** + * Only the placement algorithms that read these figures pay for collecting them. + */ + protected boolean isInUse() { + return AllocationAlgorithm.balancedweighted.toString() + .equals(DeploymentClusterPlanner.VmAllocationAlgorithm.value()); + } + + protected void record(long hostId, HostStats stats) { + record(hostId, stats, System.currentTimeMillis()); + } + + protected void record(long hostId, HostStats stats, long now) { + if (stats == null) { + return; + } + double totalMemory = stats.getTotalMemoryKBs(); + if (totalMemory <= 0) { + return; + } + + Sample previous = samples.get(hostId); + if (previous != null && previous.isSameReadingAs(stats)) { + // StatsCollector keeps the previous entry when a poll fails, so an unchanged object is + // a reading we have already folded, not a fresh measurement + return; + } + + // getCpuUtilization is a percentage of the host's real cores. That holds for KVM; see the + // class javadoc for what it means on other hypervisors. + double cpu = clamp(stats.getCpuUtilization() / 100.0); + double memory = clamp((totalMemory - stats.getFreeMemoryKBs()) / totalMemory); + int halfLife = HostLoadHalfLife.value(); + + samples.compute(hostId, (id, current) -> current == null + ? new Sample(cpu, memory, now, stats) + : current.fold(cpu, memory, now, halfLife, stats)); + } + + public HostLoad getLoad(long hostId) { + return getLoad(hostId, System.currentTimeMillis()); + } + + protected HostLoad getLoad(long hostId, long now) { + Sample sample = samples.get(hostId); + if (sample == null) { + return HostLoad.UNKNOWN; + } + long staleAfter = Math.max(1, HostLoadStaleAfter.value()) * 1000L; + if (now - sample.updatedAt > staleAfter) { + // the host has stopped reporting; stop letting its last known figures speak for it + return HostLoad.UNKNOWN; + } + return sample.toHostLoad(); + } + + protected void clear() { + samples.clear(); + } + + private static double clamp(double value) { + if (Double.isNaN(value) || value < 0) { + return 0; + } + return Math.min(value, 1); + } + + @Override + public String getConfigComponentName() { + return HostLoadTracker.class.getSimpleName(); + } + + @Override + public ConfigKey[] getConfigKeys() { + return new ConfigKey[] {HostLoadSampleInterval, HostLoadHalfLife, HostLoadStaleAfter}; + } + + /** + * One host's running average. Weighting is by elapsed time rather than by sample count, so a + * missed poll decays the old value by the right amount instead of over-weighting it. + */ + private static final class Sample { + private final double cpu; + private final double memory; + private final long updatedAt; + private final long count; + private final HostStats reading; + + private Sample(double cpu, double memory, long updatedAt, HostStats reading) { + this(cpu, memory, updatedAt, 1, reading); + } + + private Sample(double cpu, double memory, long updatedAt, long count, HostStats reading) { + this.cpu = cpu; + this.memory = memory; + this.updatedAt = updatedAt; + this.count = count; + this.reading = reading; + } + + private boolean isSameReadingAs(HostStats stats) { + return reading == stats; + } + + private Sample fold(double newCpu, double newMemory, long now, int halfLifeSeconds, HostStats reading) { + double alpha = alpha(now - updatedAt, halfLifeSeconds); + return new Sample(cpu + alpha * (newCpu - cpu), memory + alpha * (newMemory - memory), now, + count + 1, reading); + } + + private static double alpha(long elapsedMillis, int halfLifeSeconds) { + if (halfLifeSeconds <= 0 || elapsedMillis <= 0) { + return 1; + } + return 1 - Math.exp(-(elapsedMillis / 1000.0) * Math.log(2) / halfLifeSeconds); + } + + private HostLoad toHostLoad() { + return new HostLoad(cpu, memory, count); + } + } +} diff --git a/server/src/main/java/com/cloud/agent/manager/allocator/impl/WeightedHostScorer.java b/server/src/main/java/com/cloud/agent/manager/allocator/impl/WeightedHostScorer.java new file mode 100644 index 000000000000..191f76caa6c5 --- /dev/null +++ b/server/src/main/java/com/cloud/agent/manager/allocator/impl/WeightedHostScorer.java @@ -0,0 +1,417 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.manager.allocator.impl; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.stream.Collectors; + +import javax.inject.Inject; + +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.Configurable; + +import com.cloud.capacity.Capacity; +import com.cloud.capacity.CapacityManager; +import com.cloud.capacity.CapacityVO; +import com.cloud.capacity.dao.CapacityDao; +import com.cloud.dc.ClusterDetailsDao; +import com.cloud.dc.ClusterDetailsVO; +import com.cloud.host.Host; +import com.cloud.host.HostScoringWeights; +import com.cloud.utils.Pair; +import com.cloud.utils.component.AdapterBase; +import com.cloud.vm.VmDetailConstants; +import com.cloud.vm.dao.VMInstanceDao; + +/** + * Ranks hosts on a blend of what has been allocated on them and what they are actually doing. + * + * Ordering purely by allocated capacity misreads a heavily overprovisioned cluster: allocation is + * measured against a total that has been multiplied by the overprovisioning factor, so hosts under + * real strain can still look close to empty and keep attracting new VMs. This blends allocation + * with measured utilisation, VM count, and how many VMs started on the host recently, since a VM + * that has just started is usually working harder than its long run average. + * + * Scores run from 0 (idle) upwards and lower is better. Hosts are then chosen from among the best + * rather than strictly in order - see {@link #applySelectionSpread}. + */ +public class WeightedHostScorer extends AdapterBase implements Configurable { + + private static final String WEIGHT_DESCRIPTION_SUFFIX = + " Relative weight, only meaningful compared with the other host.weighted.* weights. Zero disables the term."; + + public static final ConfigKey VmCountWeight = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, + Double.class, "host.weighted.vm.count.weight", "1.0", + "How much the number of VMs already on a host counts against it, regardless of how busy they are." + + WEIGHT_DESCRIPTION_SUFFIX, + true, ConfigKey.Scope.Cluster); + + public static final ConfigKey RecentStartWeight = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, + Double.class, "host.weighted.recent.start.weight", "2.0", + "How much VMs started recently on a host count against it. Guards against sending a burst of new " + + "VMs to one host, since neither allocation nor utilisation has caught up with them yet." + + WEIGHT_DESCRIPTION_SUFFIX, + true, ConfigKey.Scope.Cluster); + + public static final ConfigKey DominantResourceWeight = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, + Double.class, "host.weighted.dominant.resource.weight", "1.0", + "How much a host's single most stressed resource counts against it, on top of the average across " + + "resources. Keeps a host that is fine on average but nearly out of one resource from ranking well." + + WEIGHT_DESCRIPTION_SUFFIX, + true, ConfigKey.Scope.Cluster); + + public static final ConfigKey RecentStartWindow = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, + Integer.class, "host.weighted.recent.start.window", "300", + "Seconds for which a newly started VM counts as recently started.", + true, ConfigKey.Scope.Global); + + public static final ConfigKey ExpectedVmsPerHost = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, + Integer.class, "host.weighted.expected.vms.per.host", "50", + "Roughly how many VMs a host is expected to carry. Used only to bring VM counts onto the same " + + "0 to 1 scale as the other terms; it is not a limit and is never enforced.", + true, ConfigKey.Scope.Cluster); + + public static final ConfigKey CpuUtilisationThreshold = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, + Double.class, "host.weighted.cpu.utilisation.threshold", "0.85", + "Hosts whose measured CPU utilisation is above this fraction are held back from new VMs. " + + "Ignored if it would leave nowhere to deploy. Set to 1 to disable.", + true, ConfigKey.Scope.Cluster); + + public static final ConfigKey MemoryUtilisationThreshold = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, + Double.class, "host.weighted.memory.utilisation.threshold", "0.90", + "Hosts whose measured memory utilisation is above this fraction are held back from new VMs. " + + "Ignored if it would leave nowhere to deploy. Set to 1 to disable.", + true, ConfigKey.Scope.Cluster); + + public static final ConfigKey SelectionSpread = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, + Integer.class, "host.weighted.selection.spread", "3", + "How many of the best scoring hosts to choose between at random. Ranking strictly by score sends " + + "concurrent deployments to the same host, because they all read the same figures before any " + + "of them is accounted for. 1 restores strict ordering.", + true, ConfigKey.Scope.Cluster); + + private static final Pair NO_VMS = new Pair<>(0L, 0L); + + @Inject + private CapacityDao capacityDao; + + @Inject + private ClusterDetailsDao clusterDetailsDao; + + @Inject + private VMInstanceDao vmInstanceDao; + + @Inject + private HostLoadTracker hostLoadTracker; + + protected Random random = new Random(); + + /** + * Orders hosts best first. Hosts absent from the capacity tables keep their original relative + * order at the end of the list rather than being dropped. + */ + public List rank(long zoneId, Long podId, Long clusterId, List hosts) { + if (hosts == null || hosts.size() <= 1) { + return hosts == null ? new ArrayList<>() : new ArrayList<>(hosts); + } + + Map scores = score(zoneId, podId, clusterId, hosts); + + List unscored = new ArrayList<>(); + List measured = new ArrayList<>(); + List unmeasured = new ArrayList<>(); + for (Host host : hosts) { + if (!scores.containsKey(host.getId())) { + unscored.add(host); + } else if (hostLoadTracker.getLoad(host.getId()).isUsable()) { + measured.add(host); + } else { + unmeasured.add(host); + } + } + + Comparator byScore = Comparator.comparingDouble(h -> scores.get(h.getId())); + measured.sort(byScore); + unmeasured.sort(byScore); + + List healthy = new ArrayList<>(); + List tooBusy = new ArrayList<>(); + partitionByUtilisation(clusterId, measured, healthy, tooBusy); + + List result = new ArrayList<>(); + if (healthy.isEmpty() && unmeasured.isEmpty()) { + logger.warn("Every candidate host is above its utilisation threshold, so the thresholds are being " + + "ignored for this deployment. The cluster is short of capacity."); + result.addAll(tooBusy); + applySelectionSpread(clusterId, result); + } else { + result.addAll(healthy); + // spread only over hosts known to be healthy, before anything else is appended, + // otherwise a busy or unmeasured host can be shuffled into the lead + applySelectionSpread(clusterId, result); + // a host we cannot measure is not assumed to be idle: it ranks behind every host we can + result.addAll(unmeasured); + result.addAll(tooBusy); + } + + if (!tooBusy.isEmpty()) { + logger.debug("Holding back {} host(s) above their utilisation threshold: {}", tooBusy.size(), tooBusy); + } + logger.debug("Weighted host ranking: {}", () -> result.stream() + .filter(h -> scores.containsKey(h.getId())) + .map(h -> String.format("%s=%.4f", h.getName(), scores.get(h.getId()))) + .collect(Collectors.joining(", "))); + + result.addAll(unscored); + return result; + } + + protected Map score(long zoneId, Long podId, Long clusterId, List hosts) { + List capacities = capacityDao.listHostCapacityByCapacityTypes(zoneId, clusterId, + List.of(Capacity.CAPACITY_TYPE_CPU, Capacity.CAPACITY_TYPE_MEMORY)); + Map> vmCounts = vmInstanceDao.countVmsByHost(zoneId, podId, clusterId, + new Date(System.currentTimeMillis() - RecentStartWindow.value() * 1000L)); + + Map allocated = allocatedFractions(capacities); + + Weights weights = new Weights(clusterId); + Map scores = new HashMap<>(); + for (Host host : hosts) { + Double[] alloc = allocated.get(host.getId()); + if (alloc == null) { + continue; + } + Pair counts = vmCounts.getOrDefault(host.getId(), NO_VMS); + scores.put(host.getId(), scoreHost(weights, alloc[0], alloc[1], hostLoadTracker.getLoad(host.getId()), + counts.first(), counts.second())); + } + return scores; + } + + /** + * Allocated CPU and memory as a fraction of what a host can hand out. + * + * op_host_capacity stores totals raw; overprovisioning is applied when they are read, so the + * cluster's ratio has to be applied here too. Without it the fraction reaches 1 at the host's + * physical size and every host on an overcommitted cluster clamps to 1, which is where this + * algorithm is most needed. + * + * Only hosts with both a CPU and a memory row are returned. A host missing one would otherwise + * score as if that resource were untouched, making it the most attractive host in the cluster. + */ + protected Map allocatedFractions(List capacities) { + Map fractions = new HashMap<>(); + Map seen = new HashMap<>(); + for (CapacityVO capacity : capacities) { + long total = capacity.getTotalCapacity(); + if (total <= 0) { + continue; + } + boolean isCpu = capacity.getCapacityType() == Capacity.CAPACITY_TYPE_CPU; + float overcommit = overcommitRatio(capacity.getClusterId(), isCpu); + double allocatable = total * overcommit; + double used = (double) (capacity.getUsedCapacity() + capacity.getReservedCapacity()) / allocatable; + + Double[] entry = fractions.computeIfAbsent(capacity.getHostOrPoolId(), id -> new Double[] {0.0, 0.0}); + entry[isCpu ? 0 : 1] = clamp(used); + seen.merge(capacity.getHostOrPoolId(), isCpu ? 1 : 2, Integer::sum); + } + fractions.keySet().removeIf(hostId -> seen.getOrDefault(hostId, 0) != 3); + return fractions; + } + + /** + * The cluster's overprovisioning factor, defaulting to none if it cannot be read. + */ + protected float overcommitRatio(Long clusterId, boolean forCpu) { + if (clusterId == null) { + return 1f; + } + String key = forCpu ? VmDetailConstants.CPU_OVER_COMMIT_RATIO : VmDetailConstants.MEMORY_OVER_COMMIT_RATIO; + ClusterDetailsVO detail = clusterDetailsDao.findDetail(clusterId, key); + if (detail == null || detail.getValue() == null) { + return 1f; + } + try { + float ratio = Float.parseFloat(detail.getValue()); + return ratio > 0 ? ratio : 1f; + } catch (NumberFormatException e) { + logger.warn("Cluster {} has an unreadable {} of [{}], treating it as 1.", clusterId, key, detail.getValue()); + return 1f; + } + } + + /** + * The blend. Every term is a fraction of the host's capacity for that resource so the weights + * are directly comparable, and the dominant resource term is added on top of the weighted mean + * so that being nearly out of any one resource is penalised even when the average looks fine. + */ + protected double scoreHostIn(Long clusterId, double cpuAllocated, double memoryAllocated, HostLoad load, + long vmCount, long recentStarts) { + return scoreHost(new Weights(clusterId), cpuAllocated, memoryAllocated, load, vmCount, recentStarts); + } + + protected double scoreHost(Weights weights, double cpuAllocated, double memoryAllocated, HostLoad load, + long vmCount, long recentStarts) { + // a host with no usable load figures is ranked on allocation alone, and is placed behind + // every measured host by the caller rather than being assumed idle + double cpuUsedWeight = load.isUsable() ? weights.cpuUsed : 0; + double memoryUsedWeight = load.isUsable() ? weights.memoryUsed : 0; + + double vmCountTerm = clamp(vmCount / weights.vmScale); + double recentStartTerm = clamp(recentStarts / weights.vmScale); + + double weightSum = weights.cpuAllocated + cpuUsedWeight + weights.memoryAllocated + memoryUsedWeight + + weights.vmCount + weights.recentStart; + + double mean = 0; + if (weightSum > 0) { + mean = (weights.cpuAllocated * cpuAllocated + + cpuUsedWeight * load.getCpuUtilisation() + + weights.memoryAllocated * memoryAllocated + + memoryUsedWeight * load.getMemoryUtilisation() + + weights.vmCount * vmCountTerm + + weights.recentStart * recentStartTerm) / weightSum; + } + + if (weights.dominant <= 0) { + return mean; + } + return (mean + weights.dominant * dominantResource(cpuAllocated, memoryAllocated, load)) / (1 + weights.dominant); + } + + /** + * The most stressed resource on the host. Allocation and utilisation are both considered for + * each resource and the larger is taken, because memory that has been reclaimed from idle + * guests can be taken back as soon as those guests get busy. + */ + protected double dominantResource(double cpuAllocated, double memoryAllocated, HostLoad load) { + double cpu = load.isUsable() ? Math.max(cpuAllocated, load.getCpuUtilisation()) : cpuAllocated; + double memory = load.isUsable() ? Math.max(memoryAllocated, load.getMemoryUtilisation()) : memoryAllocated; + return Math.max(cpu, memory); + } + + /** + * Splits measurably busy hosts out from the rest. Only hosts with load samples can be held + * back; a host that cannot be measured is dealt with by the caller. + */ + protected void partitionByUtilisation(Long clusterId, List measured, List healthy, List tooBusy) { + double cpuThreshold = valueIn(CpuUtilisationThreshold, clusterId); + double memoryThreshold = valueIn(MemoryUtilisationThreshold, clusterId); + + for (Host host : measured) { + HostLoad load = hostLoadTracker.getLoad(host.getId()); + if (load.getCpuUtilisation() > cpuThreshold || load.getMemoryUtilisation() > memoryThreshold) { + tooBusy.add(host); + } else { + healthy.add(host); + } + } + } + + /** + * Shuffles the best few hosts so that deployments made at the same moment do not all pick the + * same one. Capacity is only charged once a VM starts, so until then every concurrent decision + * sees the same figures and strict ordering makes them agree. + */ + protected void applySelectionSpread(Long clusterId, List ranked) { + int spread = Math.min((int) valueIn(SelectionSpread, clusterId), ranked.size()); + if (spread > 1) { + Collections.shuffle(ranked.subList(0, spread), random); + } + } + + /** + * The weights for one ranking, read once rather than per host. + * + * A negative weight would invert the ranking and make the most loaded host the best, so they + * are floored at zero and the bad value is reported. + */ + protected final class Weights { + private final double cpuAllocated; + private final double cpuUsed; + private final double memoryAllocated; + private final double memoryUsed; + private final double vmCount; + private final double recentStart; + private final double dominant; + private final double vmScale; + + protected Weights(Long clusterId) { + cpuAllocated = nonNegative(HostScoringWeights.CpuAllocatedWeight, clusterId); + cpuUsed = nonNegative(HostScoringWeights.CpuUsedWeight, clusterId); + memoryAllocated = nonNegative(HostScoringWeights.MemoryAllocatedWeight, clusterId); + memoryUsed = nonNegative(HostScoringWeights.MemoryUsedWeight, clusterId); + vmCount = nonNegative(VmCountWeight, clusterId); + recentStart = nonNegative(RecentStartWeight, clusterId); + dominant = nonNegative(DominantResourceWeight, clusterId); + vmScale = Math.max(1, valueIn(ExpectedVmsPerHost, clusterId)); + } + + private double nonNegative(ConfigKey key, Long clusterId) { + double value = valueIn(key, clusterId); + if (value < 0) { + logger.warn("{} is set to {}, which would rank the most loaded host first. Treating it as 0.", + key.key(), value); + return 0; + } + return value; + } + } + + private double valueIn(ConfigKey key, Long clusterId) { + T value = clusterId == null ? key.value() : key.valueIn(clusterId); + return value == null ? 0 : value.doubleValue(); + } + + private static double clamp(double value) { + if (Double.isNaN(value) || value < 0) { + return 0; + } + return Math.min(value, 1); + } + + @Override + public String getConfigComponentName() { + return CapacityManager.class.getSimpleName(); + } + + @Override + public ConfigKey[] getConfigKeys() { + return new ConfigKey[] { + HostScoringWeights.CpuAllocatedWeight, + HostScoringWeights.CpuUsedWeight, + HostScoringWeights.MemoryAllocatedWeight, + HostScoringWeights.MemoryUsedWeight, + VmCountWeight, + RecentStartWeight, + DominantResourceWeight, + RecentStartWindow, + ExpectedVmsPerHost, + CpuUtilisationThreshold, + MemoryUtilisationThreshold, + SelectionSpread + }; + } +} diff --git a/server/src/main/resources/META-INF/cloudstack/server-allocator/spring-server-allocator-context.xml b/server/src/main/resources/META-INF/cloudstack/server-allocator/spring-server-allocator-context.xml index 28b96b8d194b..69aea2cf8451 100644 --- a/server/src/main/resources/META-INF/cloudstack/server-allocator/spring-server-allocator-context.xml +++ b/server/src/main/resources/META-INF/cloudstack/server-allocator/spring-server-allocator-context.xml @@ -31,6 +31,12 @@ + + + + diff --git a/server/src/test/java/com/cloud/agent/manager/allocator/impl/HostLoadTrackerTest.java b/server/src/test/java/com/cloud/agent/manager/allocator/impl/HostLoadTrackerTest.java new file mode 100644 index 000000000000..0be9b1310f04 --- /dev/null +++ b/server/src/test/java/com/cloud/agent/manager/allocator/impl/HostLoadTrackerTest.java @@ -0,0 +1,182 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.manager.allocator.impl; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.host.HostStats; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +@RunWith(MockitoJUnitRunner.class) +public class HostLoadTrackerTest { + + private static final long HOST_ID = 1L; + private static final long HALF_LIFE_MS = 300 * 1000L; + + @InjectMocks + private HostLoadTracker tracker = new HostLoadTracker(); + + private long now; + + @Before + public void setUp() { + tracker.clear(); + now = 1_000_000L; + } + + private HostStats stats(double cpuPercent, double usedMemoryFraction) { + HostStats stats = Mockito.mock(HostStats.class); + Mockito.lenient().when(stats.getCpuUtilization()).thenReturn(cpuPercent); + Mockito.lenient().when(stats.getTotalMemoryKBs()).thenReturn(1000.0); + Mockito.lenient().when(stats.getFreeMemoryKBs()).thenReturn(1000.0 * (1 - usedMemoryFraction)); + return stats; + } + + private void sample(double cpuPercent, double usedMemoryFraction, long advanceMs) { + now += advanceMs; + tracker.record(HOST_ID, stats(cpuPercent, usedMemoryFraction), now); + } + + private HostLoad load() { + return tracker.getLoad(HOST_ID, now); + } + + @Test + public void testUnknownHostIsNotUsable() { + assertFalse(tracker.getLoad(999L, now).isUsable()); + } + + @Test + public void testFirstSampleIsTakenAsIs() { + sample(40, 0.6, 0); + + HostLoad load = load(); + assertTrue(load.isUsable()); + assertEquals(0.40, load.getCpuUtilisation(), 1e-6); + assertEquals(0.60, load.getMemoryUtilisation(), 1e-6); + } + + @Test + public void testSingleSpikeDoesNotDominateTheAverage() { + sample(10, 0.1, 0); + sample(100, 0.1, 60 * 1000L); + + // one sample a fifth of a half life in should move the average part of the way, not all of it + double cpu = load().getCpuUtilisation(); + assertTrue("a single spike must not take over the average: " + cpu, cpu < 0.30); + assertTrue("but it must move it: " + cpu, cpu > 0.10); + } + + @Test + public void testSustainedLoadConvergesOnTheNewValue() { + sample(10, 0.1, 0); + for (int i = 0; i < 40; i++) { + sample(90, 0.9, 60 * 1000L); + } + + HostLoad load = load(); + assertEquals(0.90, load.getCpuUtilisation(), 0.01); + assertEquals(0.90, load.getMemoryUtilisation(), 0.01); + } + + @Test + public void testHalfLifeMovesAverageHalfWay() { + sample(0, 0, 0); + sample(100, 1.0, HALF_LIFE_MS); + + assertEquals(0.5, load().getCpuUtilisation(), 0.01); + } + + @Test + public void testMissedSamplesDecayByElapsedTimeNotSampleCount() { + sample(0, 0, 0); + sample(100, 1.0, 4 * HALF_LIFE_MS); + + // four half lives of catching up in one sample, so almost all the way there + assertTrue(load().getCpuUtilisation() > 0.9); + } + + @Test + public void testNullStatsAreIgnored() { + tracker.record(HOST_ID, null, now); + assertFalse(load().isUsable()); + } + + @Test + public void testHostReportingNoMemoryIsIgnored() { + HostStats broken = Mockito.mock(HostStats.class); + Mockito.lenient().when(broken.getTotalMemoryKBs()).thenReturn(0.0); + + tracker.record(HOST_ID, broken, now); + + assertFalse(load().isUsable()); + } + + @Test + public void testUnchangedReadingIsNotFoldedAgain() { + // StatsCollector keeps the previous entry when a poll fails, so the same object comes back + HostStats reading = stats(10, 0.1); + tracker.record(HOST_ID, reading, now); + // stay inside the staleness window so this tests folding, not expiry + for (int i = 0; i < 5; i++) { + now += 60 * 1000L; + tracker.record(HOST_ID, reading, now); + } + + assertEquals("re-reading one measurement must not count as six", 1, load().getSamples()); + } + + @Test + public void testAHostThatStopsReportingBecomesUnusable() { + HostStats reading = stats(10, 0.1); + tracker.record(HOST_ID, reading, now); + assertTrue(load().isUsable()); + + // the agent stops updating; StatsCollector keeps handing back the same stale entry + for (int i = 0; i < 20; i++) { + now += 60 * 1000L; + tracker.record(HOST_ID, reading, now); + } + + assertFalse("a host that stopped reporting must not keep vouching for itself", load().isUsable()); + } + + @Test + public void testFreshReadingsKeepAHostUsable() { + for (int i = 0; i < 20; i++) { + sample(10 + i, 0.1, 60 * 1000L); + } + assertTrue(load().isUsable()); + } + + @Test + public void testOutOfRangeValuesAreClamped() { + sample(250, 2.0, 0); + + HostLoad load = load(); + assertEquals(1.0, load.getCpuUtilisation(), 1e-6); + assertEquals(1.0, load.getMemoryUtilisation(), 1e-6); + } +} diff --git a/server/src/test/java/com/cloud/agent/manager/allocator/impl/WeightedHostScorerRankTest.java b/server/src/test/java/com/cloud/agent/manager/allocator/impl/WeightedHostScorerRankTest.java new file mode 100644 index 000000000000..4521d5652e02 --- /dev/null +++ b/server/src/test/java/com/cloud/agent/manager/allocator/impl/WeightedHostScorerRankTest.java @@ -0,0 +1,219 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.manager.allocator.impl; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.Set; +import java.util.stream.Collectors; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.capacity.Capacity; +import com.cloud.capacity.CapacityVO; +import com.cloud.capacity.dao.CapacityDao; +import com.cloud.dc.ClusterDetailsDao; +import com.cloud.dc.ClusterDetailsVO; +import com.cloud.host.Host; +import com.cloud.utils.Pair; +import com.cloud.vm.dao.VMInstanceDao; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * Exercises rank() as the allocator calls it, rather than the scoring function alone, so the + * capacity denominator, the utilisation thresholds and the random spread are all covered. + */ +@RunWith(MockitoJUnitRunner.class) +public class WeightedHostScorerRankTest { + + private static final long ZONE = 1L; + private static final long CLUSTER = 7L; + private static final long CORES = 192L * 2400L; + private static final long MEMORY = 1_132_000L * 1024L * 1024L; + private static final int CPU_OVERCOMMIT = 10; + private static final int MEMORY_OVERCOMMIT = 4; + + @Mock + private CapacityDao capacityDao; + + @Mock + private VMInstanceDao vmInstanceDao; + + @Mock + private ClusterDetailsDao clusterDetailsDao; + + @Mock + private HostLoadTracker hostLoadTracker; + + @InjectMocks + private WeightedHostScorer scorer = new WeightedHostScorer(); + + private final List capacities = new ArrayList<>(); + private final Map> vmCounts = new HashMap<>(); + private final Map hosts = new HashMap<>(); + + @Before + public void setUp() { + scorer.random = new Random(1L); + Mockito.lenient().when(clusterDetailsDao.findDetail(Mockito.eq(CLUSTER), Mockito.contains("cpu"))) + .thenReturn(new ClusterDetailsVO(CLUSTER, "cpuOvercommitRatio", String.valueOf(CPU_OVERCOMMIT))); + Mockito.lenient().when(clusterDetailsDao.findDetail(Mockito.eq(CLUSTER), Mockito.contains("memory"))) + .thenReturn(new ClusterDetailsVO(CLUSTER, "memoryOvercommitRatio", String.valueOf(MEMORY_OVERCOMMIT))); + Mockito.lenient().when(capacityDao.listHostCapacityByCapacityTypes(Mockito.eq(ZONE), Mockito.eq(CLUSTER), Mockito.any())) + .thenReturn(capacities); + Mockito.lenient().when(vmInstanceDao.countVmsByHost(Mockito.eq(ZONE), Mockito.any(), Mockito.eq(CLUSTER), Mockito.any())) + .thenReturn(vmCounts); + } + + private CapacityVO capacity(long hostId, short type, long used, long total) { + CapacityVO capacity = new CapacityVO(hostId, ZONE, 1L, CLUSTER, used, total, type); + capacity.setReservedCapacity(0L); + return capacity; + } + + /** Registers a host with a share of its allocatable CPU and memory already committed. */ + private Host host(long id, String name, double cpuAllocatedFraction, double memoryAllocatedFraction, + HostLoad load, long vms) { + Host host = Mockito.mock(Host.class); + Mockito.lenient().when(host.getId()).thenReturn(id); + Mockito.lenient().when(host.getName()).thenReturn(name); + capacities.add(capacity(id, Capacity.CAPACITY_TYPE_CPU, + (long) (CORES * CPU_OVERCOMMIT * cpuAllocatedFraction), CORES)); + capacities.add(capacity(id, Capacity.CAPACITY_TYPE_MEMORY, + (long) (MEMORY * MEMORY_OVERCOMMIT * memoryAllocatedFraction), MEMORY)); + vmCounts.put(id, new Pair<>(vms, 0L)); + Mockito.lenient().when(hostLoadTracker.getLoad(id)).thenReturn(load); + hosts.put(id, host); + return host; + } + + private List rankedNames(List input) { + return scorer.rank(ZONE, 1L, CLUSTER, input).stream().map(Host::getName).collect(Collectors.toList()); + } + + @Test + public void testAllocationIsMeasuredAgainstTheOvercommittedTotal() { + // both hosts are well past their physical CPU, which is normal at a factor of 10. + // if the denominator ignored the factor both would clamp to 1.0 and rank equal. + Host light = host(1L, "light", 0.20, 0.20, new HostLoad(0.1, 0.1, 5), 20); + Host heavy = host(2L, "heavy", 0.80, 0.20, new HostLoad(0.1, 0.1, 5), 20); + + Map scores = scorer.score(ZONE, 1L, CLUSTER, Arrays.asList(light, heavy)); + + assertTrue("hosts past their physical size must still be distinguishable", + scores.get(heavy.getId()) > scores.get(light.getId())); + } + + @Test + public void testBusyHostRanksBehindQuietOneAtEqualAllocation() { + Host quiet = host(1L, "quiet", 0.30, 0.30, new HostLoad(0.05, 0.05, 10), 30); + Host busy = host(2L, "busy", 0.30, 0.30, new HostLoad(0.70, 0.30, 10), 30); + + assertEquals("quiet", rankedNames(Arrays.asList(busy, quiet)).get(0)); + } + + @Test + public void testHostOverThresholdIsNotChosenWhileAHealthyOneExists() { + // one healthy host and five over threshold: the spread must not shuffle a busy host in front + List input = new ArrayList<>(); + input.add(host(1L, "healthy", 0.30, 0.30, new HostLoad(0.50, 0.50, 10), 30)); + for (long id = 2; id <= 6; id++) { + input.add(host(id, "busy" + id, 0.30, 0.30, new HostLoad(0.95, 0.50, 10), 30)); + } + + for (int attempt = 0; attempt < 50; attempt++) { + assertEquals("the only healthy host must always lead", "healthy", rankedNames(input).get(0)); + } + } + + @Test + public void testUnmeasuredHostRanksBehindEveryMeasuredHost() { + // a host whose stats have stopped must not look idle and collect the deployments + Host measured = host(1L, "measured", 0.60, 0.60, new HostLoad(0.40, 0.40, 10), 60); + Host unmeasured = host(2L, "unmeasured", 0.05, 0.05, HostLoad.UNKNOWN, 2); + + List ranked = rankedNames(Arrays.asList(unmeasured, measured)); + + assertEquals("measured", ranked.get(0)); + assertEquals("unmeasured", ranked.get(1)); + } + + @Test + public void testRankingFallsBackToAllocationWhenNothingIsMeasured() { + Host light = host(1L, "light", 0.10, 0.10, HostLoad.UNKNOWN, 5); + Host heavy = host(2L, "heavy", 0.90, 0.90, HostLoad.UNKNOWN, 90); + + assertEquals("light", rankedNames(Arrays.asList(heavy, light)).get(0)); + } + + @Test + public void testEveryHostIsStillOfferedWhenTheWholeClusterIsBusy() { + List input = Arrays.asList( + host(1L, "a", 0.30, 0.30, new HostLoad(0.95, 0.50, 10), 30), + host(2L, "b", 0.40, 0.30, new HostLoad(0.97, 0.50, 10), 40)); + + List ranked = rankedNames(input); + + assertEquals("deployment must remain possible", 2, ranked.size()); + } + + @Test + public void testSpreadVariesTheLeadAmongHealthyHosts() { + List input = Arrays.asList( + host(1L, "a", 0.30, 0.30, new HostLoad(0.10, 0.10, 10), 30), + host(2L, "b", 0.31, 0.30, new HostLoad(0.10, 0.10, 10), 30), + host(3L, "c", 0.32, 0.30, new HostLoad(0.10, 0.10, 10), 30), + host(4L, "d", 0.90, 0.30, new HostLoad(0.10, 0.10, 10), 90)); + + Set leaders = new HashSet<>(); + for (int attempt = 0; attempt < 200; attempt++) { + leaders.add(rankedNames(input).get(0)); + } + + assertTrue("concurrent deployments must not all pick one host", leaders.size() > 1); + assertFalse("the clearly worst host must never lead", leaders.contains("d")); + } + + @Test + public void testHostMissingACapacityRowIsNotRankedFirst() { + Host complete = host(1L, "complete", 0.60, 0.60, new HostLoad(0.40, 0.40, 10), 60); + Host partial = Mockito.mock(Host.class); + Mockito.lenient().when(partial.getId()).thenReturn(2L); + Mockito.lenient().when(partial.getName()).thenReturn("partial"); + // only a CPU row: memory must not be treated as untouched + capacities.add(capacity(2L, Capacity.CAPACITY_TYPE_CPU, 0L, CORES)); + + List ranked = rankedNames(Arrays.asList(partial, complete)); + + assertEquals("complete", ranked.get(0)); + assertEquals("partial", ranked.get(1)); + } +} diff --git a/server/src/test/java/com/cloud/agent/manager/allocator/impl/WeightedHostScorerTest.java b/server/src/test/java/com/cloud/agent/manager/allocator/impl/WeightedHostScorerTest.java new file mode 100644 index 000000000000..d5097d79d21a --- /dev/null +++ b/server/src/test/java/com/cloud/agent/manager/allocator/impl/WeightedHostScorerTest.java @@ -0,0 +1,175 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.manager.allocator.impl; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.host.Host; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; + +@RunWith(MockitoJUnitRunner.class) +public class WeightedHostScorerTest { + + private static final HostLoad IDLE = new HostLoad(0.0, 0.0, 10); + + @Mock + private HostLoadTracker hostLoadTracker; + + @InjectMocks + private WeightedHostScorer scorer = new WeightedHostScorer(); + + private long nextHostId; + + @Before + public void setUp() { + nextHostId = 1; + } + + private Host host(String name) { + Host host = Mockito.mock(Host.class); + Mockito.lenient().when(host.getId()).thenReturn(nextHostId++); + Mockito.lenient().when(host.getName()).thenReturn(name); + return host; + } + + private double score(double cpuAllocated, double memAllocated, HostLoad load, long vms, long recentStarts) { + return scorer.scoreHostIn(null, cpuAllocated, memAllocated, load, vms, recentStarts); + } + + @Test + public void testIdleHostScoresZero() { + assertEquals(0.0, score(0, 0, IDLE, 0, 0), 1e-9); + } + + @Test + public void testMoreAllocationScoresHigher() { + assertTrue(score(0.5, 0.5, IDLE, 0, 0) > score(0.1, 0.1, IDLE, 0, 0)); + } + + @Test + public void testBusyHostScoresHigherThanIdleHostWithSameAllocation() { + HostLoad busy = new HostLoad(0.9, 0.5, 10); + assertTrue("measured load must separate hosts that look identical by allocation", + score(0.05, 0.05, busy, 20, 0) > score(0.05, 0.05, IDLE, 20, 0)); + } + + @Test + public void testUtilisationIgnoredUntilThereAreSamples() { + // a host with no samples must not be treated as idle, it must rank on allocation alone + double noSamples = score(0.4, 0.4, HostLoad.UNKNOWN, 10, 0); + double idleSamples = score(0.4, 0.4, IDLE, 10, 0); + assertTrue("a host with no load samples should not outrank a measurably idle one", + noSamples >= idleSamples); + } + + @Test + public void testDominantResourcePenalisesLopsidedHost() { + // same mean across resources, but one host is nearly out of memory + double balanced = score(0.5, 0.5, IDLE, 0, 0); + double lopsided = score(0.05, 0.95, IDLE, 0, 0); + assertTrue("a host nearly out of one resource must not rank as well as an evenly loaded one", + lopsided > balanced); + } + + @Test + public void testDominantResourceUsesUtilisationWhenHigherThanAllocation() { + HostLoad reclaimedButBusy = new HostLoad(0.95, 0.95, 10); + assertTrue(score(0.05, 0.05, reclaimedButBusy, 0, 0) > score(0.05, 0.05, IDLE, 0, 0)); + } + + @Test + public void testVmCountPenalisesHost() { + assertTrue(score(0.1, 0.1, IDLE, 120, 0) > score(0.1, 0.1, IDLE, 5, 0)); + } + + @Test + public void testRecentStartsPenaliseHost() { + assertTrue("VMs that just started are not yet visible in allocation or utilisation", + score(0.1, 0.1, IDLE, 20, 20) > score(0.1, 0.1, IDLE, 20, 0)); + } + + @Test + public void testScoreStaysWithinUnitRange() { + assertTrue(score(1, 1, new HostLoad(1, 1, 10), 1000, 1000) <= 1.0); + assertTrue(score(0, 0, IDLE, 0, 0) >= 0.0); + } + + @Test + public void testBusyHostIsHeldBackByThreshold() { + Host quiet = host("quiet"); + Host busy = host("busy"); + Mockito.when(hostLoadTracker.getLoad(quiet.getId())).thenReturn(new HostLoad(0.10, 0.10, 10)); + Mockito.when(hostLoadTracker.getLoad(busy.getId())).thenReturn(new HostLoad(0.99, 0.10, 10)); + + List healthy = new ArrayList<>(); + List tooBusy = new ArrayList<>(); + scorer.partitionByUtilisation(null, new ArrayList<>(List.of(quiet, busy)), healthy, tooBusy); + + assertEquals(1, healthy.size()); + assertSame(quiet, healthy.get(0)); + assertSame("host over the CPU threshold must be held back", busy, tooBusy.get(0)); + } + + @Test + public void testThresholdIsIgnoredWhenEveryHostIsBusy() { + Host a = host("a"); + Host b = host("b"); + Mockito.when(hostLoadTracker.getLoad(Mockito.anyLong())).thenReturn(new HostLoad(0.99, 0.99, 10)); + + List healthy = new ArrayList<>(); + List tooBusy = new ArrayList<>(); + scorer.partitionByUtilisation(null, new ArrayList<>(List.of(a, b)), healthy, tooBusy); + + assertEquals("both hosts are over threshold", 2, tooBusy.size()); + assertTrue(healthy.isEmpty()); + } + + @Test + public void testSelectionSpreadVariesTheChosenHost() { + Set chosen = new HashSet<>(); + List hosts = List.of(host("a"), host("b"), host("c"), host("d"), host("e")); + for (int i = 0; i < 200; i++) { + List ranked = new ArrayList<>(hosts); + scorer.applySelectionSpread(null, ranked); + chosen.add(ranked.get(0).getName()); + } + assertNotEquals("strict ordering sends every concurrent deployment to the same host", 1, chosen.size()); + assertTrue("only the best scoring hosts should be candidates", chosen.size() <= 3); + } + + @Test + public void testSelectionSpreadLeavesShortListsAlone() { + List ranked = new ArrayList<>(List.of(host("only"))); + scorer.applySelectionSpread(null, ranked); + assertEquals(1, ranked.size()); + } +} diff --git a/server/src/test/java/com/cloud/agent/manager/allocator/impl/WeightedPlacementDistributionTest.java b/server/src/test/java/com/cloud/agent/manager/allocator/impl/WeightedPlacementDistributionTest.java new file mode 100644 index 000000000000..22fa6c565839 --- /dev/null +++ b/server/src/test/java/com/cloud/agent/manager/allocator/impl/WeightedPlacementDistributionTest.java @@ -0,0 +1,319 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.manager.allocator.impl; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; +import java.util.Random; + +import org.junit.Test; + +import static org.junit.Assert.assertTrue; + +/** + * Simulates placement over a churning, heavily overprovisioned fleet. + * + * This is the regression fixture for the failure the weighted scoring exists to prevent. Ranking + * hosts on allocated capacity alone is blind to what a host is really doing, and under a large + * overprovisioning factor the gap between the two can be enormous: a host can report a few percent + * allocated while its cores are saturated. Anything the scheduler is not told about - VMs it has + * lost track of, guests using far more than their share - is invisible, so the emptiest looking + * host keeps being chosen no matter how hard it is working. + * + * Three things are modelled that a single-shot unit test cannot show: + * + * - deployments arrive in concurrent batches and every decision in a batch reads the same figures, + * because capacity is only charged once a VM starts + * - real load that allocation cannot account for + * - VMs are short lived and their lifetimes vary, so hosts empty unevenly + * + * Deterministic: fixed seeds, no wall clock. + */ +public class WeightedPlacementDistributionTest { + + private static final int HOSTS = 9; + private static final int CORES_PER_HOST = 192; + private static final int MEMORY_MB_PER_HOST = 1_132_000; + private static final int CPU_OVERCOMMIT = 10; + private static final int MEMORY_OVERCOMMIT = 4; + + private static final int VM_CORES = 4; + private static final int VM_MEMORY_MB = 8_192; + + private static final int BATCHES = 300; + private static final int VMS_PER_BATCH = 8; + private static final int SPREAD = 3; + + /** Share of VMs that peg their cores for their whole life, as build runners do. */ + private static final double BUSY_FRACTION = 0.35; + + /** + * Cores in use on some hosts that allocation knows nothing about. Stands in for anything the + * scheduler cannot see - VMs it believes are gone, or guests far exceeding their request. + */ + private static final int UNACCOUNTED_CORES = 60; + + private static final class SimHost { + final long id; + final double unaccountedCores; + int vms; + int recentStarts; + double busyCores; + double memoryMb; + + SimHost(long id, double unaccountedCores) { + this.id = id; + this.unaccountedCores = unaccountedCores; + } + + /** What the capacity tables would report: uniform per VM, against an inflated total. */ + double cpuAllocated() { + return (double) vms * VM_CORES / (CORES_PER_HOST * CPU_OVERCOMMIT); + } + + double memoryAllocated() { + return (double) vms * VM_MEMORY_MB / ((double) MEMORY_MB_PER_HOST * MEMORY_OVERCOMMIT); + } + + /** What the host is really doing, including what allocation cannot see. */ + double realCores() { + return busyCores + unaccountedCores; + } + + HostLoad load() { + return new HostLoad(Math.min(1, realCores() / CORES_PER_HOST), + Math.min(1, memoryMb / MEMORY_MB_PER_HOST), 10); + } + } + + /** Frozen figures for one batch, so every decision in the batch sees the same thing. */ + private static final class HostView { + final SimHost host; + final double cpuAllocated; + final double memoryAllocated; + final HostLoad load; + final int vms; + final int recentStarts; + + HostView(SimHost host) { + this.host = host; + this.cpuAllocated = host.cpuAllocated(); + this.memoryAllocated = host.memoryAllocated(); + this.load = host.load(); + this.vms = host.vms; + this.recentStarts = host.recentStarts; + } + } + + private interface Placement { + SimHost choose(List snapshot, Random random); + } + + /** What firstfitleastconsumed does today: strictly the lowest allocated fraction. */ + private static final Placement LEAST_ALLOCATED = (snapshot, random) -> + snapshot.stream().min(Comparator.comparingDouble(v -> v.cpuAllocated)).orElseThrow().host; + + /** + * Allocation-only ranking with the same random spread as the weighted arm. Isolates what the + * scoring contributes from what the spread alone contributes. + */ + private static final Placement LEAST_ALLOCATED_WITH_SPREAD = (snapshot, random) -> { + List ranked = new ArrayList<>(snapshot); + ranked.sort(Comparator.comparingDouble(v -> v.cpuAllocated)); + return ranked.get(random.nextInt(Math.min(SPREAD, ranked.size()))).host; + }; + + private Placement weighted() { + WeightedHostScorer scorer = new WeightedHostScorer(); + return (snapshot, random) -> { + List ranked = new ArrayList<>(snapshot); + ranked.sort(Comparator.comparingDouble(v -> + scorer.scoreHostIn(null, v.cpuAllocated, v.memoryAllocated, v.load, v.vms, v.recentStarts))); + return ranked.get(random.nextInt(Math.min(SPREAD, ranked.size()))).host; + }; + } + + private static final class Result { + final double[] vmCounts; + final double[] realCores; + + Result(double[] vmCounts, double[] realCores) { + this.vmCounts = vmCounts; + this.realCores = realCores; + } + + double vmSkew() { + return max(vmCounts) / mean(vmCounts); + } + + double loadSkew() { + return max(realCores) / mean(realCores); + } + } + + /** The VMs to be placed, fixed before any arm runs so all arms see the same workload. */ + private List workload(long seed) { + Random random = new Random(seed); + List vms = new ArrayList<>(); + for (int i = 0; i < BATCHES * VMS_PER_BATCH; i++) { + vms.add(new int[] {random.nextDouble() < BUSY_FRACTION ? 1 : 0, 10 + random.nextInt(50)}); + } + return vms; + } + + private Result run(Placement placement, long seed) { + List workload = workload(seed); + // a separate stream for placement decisions, so arms that consult it differently still see + // the same workload + Random random = new Random(seed ^ 0x5DEECE66DL); + int next = 0; + List hosts = new ArrayList<>(); + for (int i = 0; i < HOSTS; i++) { + // a third of the fleet carries load the scheduler cannot account for + hosts.add(new SimHost(i + 1, i % 3 == 0 ? UNACCOUNTED_CORES : 0)); + } + List live = new ArrayList<>(); // {hostIndex, busy, batchesLeft} + + for (int batch = 0; batch < BATCHES; batch++) { + live.removeIf(vm -> { + if (--vm[2] > 0) { + return false; + } + SimHost host = hosts.get(vm[0]); + host.vms--; + host.memoryMb -= VM_MEMORY_MB; + if (vm[1] == 1) { + host.busyCores -= VM_CORES; + } + return true; + }); + + hosts.forEach(h -> h.recentStarts = 0); + + List snapshot = new ArrayList<>(); + for (SimHost host : hosts) { + snapshot.add(new HostView(host)); + } + + for (int i = 0; i < VMS_PER_BATCH; i++) { + SimHost chosen = placement.choose(snapshot, random); + int[] vm = workload.get(next++); + boolean busy = vm[0] == 1; + chosen.vms++; + chosen.recentStarts++; + chosen.memoryMb += VM_MEMORY_MB; + if (busy) { + chosen.busyCores += VM_CORES; + } + // lifetimes vary, so hosts do not empty in the order they filled + live.add(new int[] {hosts.indexOf(chosen), vm[0], vm[1]}); + } + } + + return new Result(hosts.stream().mapToDouble(h -> h.vms).toArray(), + hosts.stream().mapToDouble(SimHost::realCores).toArray()); + } + + private static double max(double[] values) { + return Arrays.stream(values).max().orElse(0); + } + + private static double mean(double[] values) { + return Arrays.stream(values).average().orElse(0); + } + + @Test + public void testAllocationOnlyOrderingPilesRealLoadOntoTheBusiestHosts() { + Result result = run(LEAST_ALLOCATED, 42L); + + assertTrue(String.format("expected allocation-only ranking to be blind to real load, " + + "got cores %s (max/mean %.2f)", Arrays.toString(result.realCores), result.loadSkew()), + result.loadSkew() > 1.4); + } + + @Test + public void testWeightedScoringKeepsRealLoadEven() { + Result result = run(weighted(), 42L); + + // a third of the fleet carries a fixed handicap the scheduler can only stop adding to, not + // remove, so some residual skew is expected. Measured across seeds: allocation-only ranking + // lands at 1.84 to 2.01, weighted at 1.27 to 1.40. + assertTrue(String.format("real load should be spread, got cores %s (max/mean %.2f)", + Arrays.toString(result.realCores), result.loadSkew()), + result.loadSkew() < 1.5); + } + + @Test + public void testWeightedScoringBeatsAllocationOnlyOnEverySeed() { + for (long seed : new long[] {1L, 7L, 42L, 99L, 12345L}) { + double baseline = run(LEAST_ALLOCATED, seed).loadSkew(); + double improved = run(weighted(), seed).loadSkew(); + assertTrue(String.format("seed %d: weighted %.2f should beat allocation-only %.2f", + seed, improved, baseline), + improved < baseline); + } + } + + @Test + public void testTheScoringNotJustTheSpreadIsWhatEvensOutRealLoad() { + // the control: same random spread, ranking still blind to real load. If the spread alone + // were doing the work, this arm would do as well as the weighted one. + for (long seed : new long[] {1L, 42L, 12345L}) { + double spreadOnly = run(LEAST_ALLOCATED_WITH_SPREAD, seed).loadSkew(); + double weighted = run(weighted(), seed).loadSkew(); + assertTrue(String.format("seed %d: weighted %.2f should beat spread-only %.2f", + seed, weighted, spreadOnly), + weighted < spreadOnly); + } + } + + @Test + public void testWeightedScoringGivesFewerVmsToHostsCarryingHiddenLoad() { + Result result = run(weighted(), 42L); + + // hosts 0, 3 and 6 carry load that allocation cannot see, so they should get fewer VMs. + // uneven VM counts are the right answer here - it is real load that should come out even. + double withHiddenLoad = mean(new double[] {result.vmCounts[0], result.vmCounts[3], result.vmCounts[6]}); + double withoutHiddenLoad = mean(new double[] {result.vmCounts[1], result.vmCounts[2], result.vmCounts[4], + result.vmCounts[5], result.vmCounts[7], result.vmCounts[8]}); + + assertTrue(String.format("hosts with hidden load should take fewer VMs: %.1f vs %.1f (counts %s)", + withHiddenLoad, withoutHiddenLoad, Arrays.toString(result.vmCounts)), + withHiddenLoad < withoutHiddenLoad * 0.75); + assertTrue("no host should be left completely unused: " + Arrays.toString(result.vmCounts), + min(result.vmCounts) > 0); + } + + @Test + public void testAllocationOnlyOrderingIgnoresHiddenLoadEntirely() { + Result result = run(LEAST_ALLOCATED, 42L); + + double withHiddenLoad = mean(new double[] {result.vmCounts[0], result.vmCounts[3], result.vmCounts[6]}); + double withoutHiddenLoad = mean(new double[] {result.vmCounts[1], result.vmCounts[2], result.vmCounts[4], + result.vmCounts[5], result.vmCounts[7], result.vmCounts[8]}); + + assertTrue(String.format("allocation-only ranking should treat loaded and idle hosts alike, got %.1f vs %.1f", + withHiddenLoad, withoutHiddenLoad), + withHiddenLoad > withoutHiddenLoad * 0.9); + } + + private static double min(double[] values) { + return Arrays.stream(values).min().orElse(0); + } +}