Skip to content

Commit 8af021c

Browse files
DaanHooglandDaanHoogland
committed
Merge branch '4.20'
2 parents f47bd61 + 5f93ce7 commit 8af021c

34 files changed

Lines changed: 589 additions & 227 deletions

File tree

‎agent/bindir/cloud-setup-agent.in‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,19 @@ import os
2020
import logging
2121
import sys
2222
import socket
23+
24+
# ---- This snippet of code adds the sources path and the waf configured PYTHONDIR to the Python path ----
25+
# ---- We do this so cloud_utils can be looked up in the following order:
26+
# ---- 1) Sources directory
27+
# ---- 2) waf configured PYTHONDIR
28+
# ---- 3) System Python path
29+
for pythonpath in (
30+
"@PYTHONDIR@",
31+
os.path.join(os.path.dirname(__file__),os.path.pardir,os.path.pardir,"python","lib"),
32+
):
33+
if os.path.isdir(pythonpath): sys.path.insert(0,pythonpath)
34+
# ---- End snippet of code ----
35+
2336
from cloudutils.cloudException import CloudRuntimeException, CloudInternalException
2437
from cloudutils.utilities import initLoging, bash
2538
from cloudutils.configFileOps import configFileOps

‎agent/bindir/libvirtqemuhook.in‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,19 @@ import sys
2020
import os
2121
import subprocess
2222
from threading import Timer
23+
24+
# ---- This snippet of code adds the sources path and the waf configured PYTHONDIR to the Python path ----
25+
# ---- We do this so cloud_utils can be looked up in the following order:
26+
# ---- 1) Sources directory
27+
# ---- 2) waf configured PYTHONDIR
28+
# ---- 3) System Python path
29+
for pythonpath in (
30+
"@PYTHONDIR@",
31+
os.path.join(os.path.dirname(__file__),os.path.pardir,os.path.pardir,"python","lib"),
32+
):
33+
if os.path.isdir(pythonpath): sys.path.insert(0,pythonpath)
34+
# ---- End snippet of code ----
35+
2336
from xml.dom.minidom import parse
2437
from cloudutils.configFileOps import configFileOps
2538
from cloudutils.networkConfig import networkConfig

‎api/src/main/java/org/apache/cloudstack/api/response/NetworkResponse.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,12 +28,11 @@
2828
import org.apache.cloudstack.api.EntityReference;
2929

3030
import com.cloud.network.Network;
31-
import com.cloud.projects.ProjectAccount;
3231
import com.cloud.serializer.Param;
3332
import com.google.gson.annotations.SerializedName;
3433

3534
@SuppressWarnings("unused")
36-
@EntityReference(value = {Network.class, ProjectAccount.class})
35+
@EntityReference(value = {Network.class})
3736
public class NetworkResponse extends BaseResponseWithAssociatedNetwork implements ControlledEntityResponse, SetResourceIconResponse {
3837

3938
@SerializedName(ApiConstants.ID)

‎client/bindir/cloud-setup-management.in‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,13 +16,27 @@
1616
# specific language governing permissions and limitations
1717
# under the License.
1818

19+
import os
1920
import sys
21+
# ---- This snippet of code adds the sources path and the waf configured PYTHONDIR to the Python path ----
22+
# ---- We do this so cloud_utils can be looked up in the following order:
23+
# ---- 1) Sources directory
24+
# ---- 2) waf configured PYTHONDIR
25+
# ---- 3) System Python path
26+
for pythonpath in (
27+
"@PYTHONDIR@",
28+
os.path.join(os.path.dirname(__file__),os.path.pardir,os.path.pardir,"python","lib"),
29+
):
30+
if os.path.isdir(pythonpath): sys.path.insert(0,pythonpath)
31+
# ---- End snippet of code ----
32+
2033
from cloudutils.syscfg import sysConfigFactory
2134
from cloudutils.utilities import initLoging, UnknownSystemException
2235
from cloudutils.cloudException import CloudRuntimeException, CloudInternalException
2336
from cloudutils.globalEnv import globalEnv
2437
from cloudutils.serviceConfigServer import cloudManagementConfig
2538
from optparse import OptionParser
39+
2640
if __name__ == '__main__':
2741
initLoging("@MSLOGDIR@/setupManagement.log")
2842
glbEnv = globalEnv()

‎engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/BasePrimaryDataStoreLifeCycleImpl.java‎

Lines changed: 51 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,23 +22,29 @@
2222

2323
import javax.inject.Inject;
2424

25-
import com.cloud.dc.dao.DataCenterDao;
2625
import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope;
2726
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
27+
import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
28+
import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
2829
import org.apache.cloudstack.storage.volume.datastore.PrimaryDataStoreHelper;
2930

3031
import com.cloud.agent.AgentManager;
3132
import com.cloud.agent.api.Answer;
3233
import com.cloud.agent.api.DeleteStoragePoolCommand;
34+
import com.cloud.dc.dao.DataCenterDao;
3335
import com.cloud.host.HostVO;
3436
import com.cloud.host.dao.HostDao;
3537
import com.cloud.hypervisor.Hypervisor.HypervisorType;
3638
import com.cloud.resource.ResourceManager;
3739
import com.cloud.storage.StorageManager;
3840
import com.cloud.storage.StoragePool;
3941
import com.cloud.storage.StoragePoolHostVO;
42+
import com.cloud.storage.VMTemplateStoragePoolVO;
43+
import com.cloud.storage.VMTemplateStorageResourceAssoc;
4044
import com.cloud.storage.dao.StoragePoolHostDao;
45+
import com.cloud.template.TemplateManager;
4146
import com.cloud.utils.Pair;
47+
4248
import org.apache.logging.log4j.LogManager;
4349
import org.apache.logging.log4j.Logger;
4450

@@ -59,6 +65,10 @@ public class BasePrimaryDataStoreLifeCycleImpl {
5965
protected DataCenterDao zoneDao;
6066
@Inject
6167
protected StoragePoolHostDao storagePoolHostDao;
68+
@Inject
69+
private PrimaryDataStoreDao primaryDataStoreDao;
70+
@Inject
71+
private TemplateManager templateMgr;
6272

6373
private List<HostVO> getPoolHostsList(ClusterScope clusterScope, HypervisorType hypervisorType) {
6474
List<HostVO> hosts;
@@ -81,7 +91,7 @@ public void changeStoragePoolScopeToZone(DataStore store, ClusterScope clusterSc
8191
try {
8292
storageMgr.connectHostToSharedPool(host, store.getId());
8393
} catch (Exception e) {
84-
logger.warn("Unable to establish a connection between " + host + " and " + store, e);
94+
logger.warn("Unable to establish a connection between {} and {}", host, store, e);
8595
}
8696
}
8797
}
@@ -99,7 +109,7 @@ public void changeStoragePoolScopeToCluster(DataStore store, ClusterScope cluste
99109

100110
if (answer != null) {
101111
if (!answer.getResult()) {
102-
logger.debug("Failed to delete storage pool: " + answer.getResult());
112+
logger.debug("Failed to delete storage pool: {}", answer.getResult());
103113
} else if (HypervisorType.KVM != hypervisorType) {
104114
break;
105115
}
@@ -108,4 +118,42 @@ public void changeStoragePoolScopeToCluster(DataStore store, ClusterScope cluste
108118
}
109119
dataStoreHelper.switchToCluster(store, clusterScope);
110120
}
121+
122+
private void evictTemplates(StoragePoolVO storagePoolVO) {
123+
List<VMTemplateStoragePoolVO> unusedTemplatesInPool = templateMgr.getUnusedTemplatesInPool(storagePoolVO);
124+
for (VMTemplateStoragePoolVO templatePoolVO : unusedTemplatesInPool) {
125+
if (templatePoolVO.getDownloadState() == VMTemplateStorageResourceAssoc.Status.DOWNLOADED) {
126+
templateMgr.evictTemplateFromStoragePool(templatePoolVO);
127+
}
128+
}
129+
}
130+
131+
private void deleteAgentStoragePools(StoragePool storagePool) {
132+
List<StoragePoolHostVO> poolHostVOs = storagePoolHostDao.listByPoolId(storagePool.getId());
133+
for (StoragePoolHostVO poolHostVO : poolHostVOs) {
134+
DeleteStoragePoolCommand deleteStoragePoolCommand = new DeleteStoragePoolCommand(storagePool);
135+
final Answer answer = agentMgr.easySend(poolHostVO.getHostId(), deleteStoragePoolCommand);
136+
if (answer != null && answer.getResult()) {
137+
logger.info("Successfully deleted storage pool: {} from host: {}", storagePool.getId(), poolHostVO.getHostId());
138+
} else {
139+
if (answer != null) {
140+
logger.error("Failed to delete storage pool: {} from host: {} , result: {}", storagePool.getId(), poolHostVO.getHostId(), answer.getResult());
141+
} else {
142+
logger.error("Failed to delete storage pool: {} from host: {}", storagePool.getId(), poolHostVO.getHostId());
143+
}
144+
}
145+
}
146+
}
147+
148+
protected boolean cleanupDatastore(DataStore store) {
149+
StoragePool storagePool = (StoragePool)store;
150+
StoragePoolVO storagePoolVO = primaryDataStoreDao.findById(storagePool.getId());
151+
if (storagePoolVO == null) {
152+
return false;
153+
}
154+
155+
evictTemplates(storagePoolVO);
156+
deleteAgentStoragePools(storagePool);
157+
return true;
158+
}
111159
}

‎framework/config/src/main/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImpl.java‎

Lines changed: 7 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,7 @@ public class ConfigDepotImpl implements ConfigDepot, ConfigDepotAdmin {
8585
List<ScopedConfigStorage> _scopedStorages;
8686
Set<Configurable> _configured = Collections.synchronizedSet(new HashSet<Configurable>());
8787
Set<String> newConfigs = Collections.synchronizedSet(new HashSet<>());
88-
LazyCache<String, String> configCache;
88+
LazyCache<Ternary<String, ConfigKey.Scope, Long>, String> configCache;
8989

9090
private HashMap<String, Pair<String, ConfigKey<?>>> _allKeys = new HashMap<String, Pair<String, ConfigKey<?>>>(1007);
9191

@@ -275,15 +275,10 @@ public ConfigurationDao global() {
275275
return _configDao;
276276
}
277277

278-
protected String getConfigStringValueInternal(String cacheKey) {
279-
String[] parts = cacheKey.split("-");
280-
String key = parts[0];
281-
ConfigKey.Scope scope = ConfigKey.Scope.Global;
282-
Long scopeId = null;
283-
try {
284-
scope = ConfigKey.Scope.valueOf(parts[1]);
285-
scopeId = Long.valueOf(parts[2]);
286-
} catch (IllegalArgumentException ignored) {}
278+
protected String getConfigStringValueInternal(Ternary<String, ConfigKey.Scope, Long> cacheKey) {
279+
String key = cacheKey.first();
280+
ConfigKey.Scope scope = cacheKey.second();
281+
Long scopeId = cacheKey.third();
287282
if (!ConfigKey.Scope.Global.equals(scope) && scopeId != null) {
288283
ScopedConfigStorage scopedConfigStorage = getScopedStorage(scope);
289284
if (scopedConfigStorage == null) {
@@ -298,8 +293,8 @@ protected String getConfigStringValueInternal(String cacheKey) {
298293
return null;
299294
}
300295

301-
private String getConfigCacheKey(String key, ConfigKey.Scope scope, Long scopeId) {
302-
return String.format("%s-%s-%d", key, scope, (scopeId == null ? 0 : scopeId));
296+
protected Ternary<String, ConfigKey.Scope, Long> getConfigCacheKey(String key, ConfigKey.Scope scope, Long scopeId) {
297+
return new Ternary<>(key, scope, scopeId);
303298
}
304299

305300
@Override

‎framework/config/src/test/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImplTest.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,12 @@ public void testGetConfigStringValue() {
8989
runTestGetConfigStringValue("test", "value");
9090
}
9191

92+
@Test
93+
public void testGetConfigStringValue_nameWithCharacters() {
94+
runTestGetConfigStringValue("test.1-1", "value");
95+
runTestGetConfigStringValue("test_1#2", "value");
96+
}
97+
9298
private void runTestGetConfigStringValueExpiry(long wait, int configDBRetrieval) {
9399
String key = "test1";
94100
String value = "expiry";

‎framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/GenericPresetVariable.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,8 +49,12 @@ public void setName(String name) {
4949
fieldNamesToIncludeInToString.add("name");
5050
}
5151

52+
/***
53+
* Converts the preset variable into a valid JSON object that will be injected into the JS interpreter.
54+
* This method should not be overridden or changed.
55+
*/
5256
@Override
53-
public String toString() {
57+
public final String toString() {
5458
return ReflectionToStringBuilderUtils.reflectOnlySelectedFields(this, fieldNamesToIncludeInToString.toArray(new String[0]));
5559
}
5660
}

‎framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Resource.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,8 +40,12 @@ public void setDomainId(String domainId) {
4040
this.domainId = domainId;
4141
}
4242

43+
/***
44+
* Converts the preset variable into a valid JSON object that will be injected into the JS interpreter.
45+
* This method should not be overridden or changed.
46+
*/
4347
@Override
44-
public String toString() {
48+
public final String toString() {
4549
return ToStringBuilder.reflectionToString(this, ToStringStyle.JSON_STYLE);
4650
}
4751

‎plugins/metrics/src/main/java/org/apache/cloudstack/metrics/MetricsServiceImpl.java‎

Lines changed: 6 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -189,13 +189,6 @@ protected MetricsServiceImpl() {
189189
super();
190190
}
191191

192-
private Double findRatioValue(final String value) {
193-
if (value != null) {
194-
return Double.valueOf(value);
195-
}
196-
return 1.0;
197-
}
198-
199192
private void updateHostMetrics(final HostMetrics hostMetrics, final HostJoinVO host) {
200193
hostMetrics.addCpuAllocated(host.getCpuReservedCapacity() + host.getCpuUsedCapacity());
201194
hostMetrics.addMemoryAllocated(host.getMemReservedCapacity() + host.getMemUsedCapacity());
@@ -767,14 +760,10 @@ public List<ClusterMetricsResponse> listClusterMetrics(Pair<List<ClusterResponse
767760
if (AllowListMetricsComputation.value()) {
768761
List<Ternary<Long, Long, Long>> cpuList = new ArrayList<>();
769762
List<Ternary<Long, Long, Long>> memoryList = new ArrayList<>();
770-
for (final Host host : hostDao.findByClusterId(clusterId)) {
771-
if (host == null || host.getType() != Host.Type.Routing) {
772-
continue;
773-
}
774-
updateHostMetrics(hostMetrics, hostJoinDao.findById(host.getId()));
775-
HostJoinVO hostJoin = hostJoinDao.findById(host.getId());
776-
cpuList.add(new Ternary<>(hostJoin.getCpuUsedCapacity(), hostJoin.getCpuReservedCapacity(), hostJoin.getCpus() * hostJoin.getSpeed()));
777-
memoryList.add(new Ternary<>(hostJoin.getMemUsedCapacity(), hostJoin.getMemReservedCapacity(), hostJoin.getTotalMemory()));
763+
for (final HostJoinVO host : hostJoinDao.findByClusterId(clusterId, Host.Type.Routing)) {
764+
updateHostMetrics(hostMetrics, host);
765+
cpuList.add(new Ternary<>(host.getCpuUsedCapacity(), host.getCpuReservedCapacity(), host.getCpus() * host.getSpeed()));
766+
memoryList.add(new Ternary<>(host.getMemUsedCapacity(), host.getMemReservedCapacity(), host.getTotalMemory()));
778767
}
779768
try {
780769
Double imbalance = ClusterDrsAlgorithm.getClusterImbalance(clusterId, cpuList, memoryList, null);
@@ -955,11 +944,8 @@ public List<ZoneMetricsResponse> listZoneMetrics(List<ZoneResponse> zoneResponse
955944
if (cluster == null) {
956945
continue;
957946
}
958-
for (final Host host: hostDao.findByClusterId(cluster.getId())) {
959-
if (host == null || host.getType() != Host.Type.Routing) {
960-
continue;
961-
}
962-
updateHostMetrics(hostMetrics, hostJoinDao.findById(host.getId()));
947+
for (final HostJoinVO host: hostJoinDao.findByClusterId(cluster.getId(), Host.Type.Routing)) {
948+
updateHostMetrics(hostMetrics, host);
963949
}
964950
}
965951
} else {

0 commit comments

Comments
 (0)