Skip to content

Commit d722415

Browse files
abh1sarDaan Hoogland
abh1sar
authored and
Daan Hoogland
committed
[22.0] secondary storage resource limit for upload
1 parent 5d5ee7b commit d722415

4 files changed

Lines changed: 182 additions & 25 deletions

File tree

‎core/src/main/java/org/apache/cloudstack/storage/command/UploadStatusCommand.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ public enum EntityType {
2828
}
2929
private String entityUuid;
3030
private EntityType entityType;
31+
private Boolean abort;
3132

3233
protected UploadStatusCommand() {
3334
}
@@ -37,6 +38,11 @@ public UploadStatusCommand(String entityUuid, EntityType entityType) {
3738
this.entityType = entityType;
3839
}
3940

41+
public UploadStatusCommand(String entityUuid, EntityType entityType, Boolean abort) {
42+
this(entityUuid, entityType);
43+
this.abort = abort;
44+
}
45+
4046
public String getEntityUuid() {
4147
return entityUuid;
4248
}
@@ -45,6 +51,10 @@ public EntityType getEntityType() {
4551
return entityType;
4652
}
4753

54+
public Boolean getAbort() {
55+
return abort;
56+
}
57+
4858
@Override
4959
public boolean executeInSequence() {
5060
return false;

‎server/src/main/java/com/cloud/storage/ImageStoreUploadMonitorImpl.java‎

Lines changed: 138 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,6 @@
2525
import javax.inject.Inject;
2626
import javax.naming.ConfigurationException;
2727

28-
import com.cloud.agent.api.to.OVFInformationTO;
2928
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
3029
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
3130
import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint;
@@ -37,6 +36,7 @@
3736
import org.apache.cloudstack.framework.config.ConfigKey;
3837
import org.apache.cloudstack.framework.config.Configurable;
3938
import org.apache.cloudstack.managed.context.ManagedContextRunnable;
39+
import org.apache.cloudstack.reservation.dao.ReservationDao;
4040
import org.apache.cloudstack.storage.command.UploadStatusAnswer;
4141
import org.apache.cloudstack.storage.command.UploadStatusAnswer.UploadStatus;
4242
import org.apache.cloudstack.storage.command.UploadStatusCommand;
@@ -55,6 +55,7 @@
5555
import com.cloud.agent.api.Answer;
5656
import com.cloud.agent.api.Command;
5757
import com.cloud.agent.api.StartupCommand;
58+
import com.cloud.agent.api.to.OVFInformationTO;
5859
import com.cloud.alert.AlertManager;
5960
import com.cloud.api.query.dao.TemplateJoinDao;
6061
import com.cloud.api.query.vo.TemplateJoinVO;
@@ -65,12 +66,16 @@
6566
import com.cloud.host.Host;
6667
import com.cloud.host.Status;
6768
import com.cloud.host.dao.HostDao;
69+
import com.cloud.resourcelimit.CheckedReservation;
6870
import com.cloud.storage.Volume.Event;
6971
import com.cloud.storage.dao.VMTemplateDao;
7072
import com.cloud.storage.dao.VMTemplateZoneDao;
7173
import com.cloud.storage.dao.VolumeDao;
7274
import com.cloud.template.VirtualMachineTemplate;
75+
import com.cloud.user.Account;
76+
import com.cloud.user.AccountManager;
7377
import com.cloud.user.ResourceLimitService;
78+
import com.cloud.user.dao.AccountDao;
7479
import com.cloud.utils.component.ManagerBase;
7580
import com.cloud.utils.concurrency.NamedThreadFactory;
7681
import com.cloud.utils.db.Transaction;
@@ -117,6 +122,12 @@ public class ImageStoreUploadMonitorImpl extends ManagerBase implements ImageSto
117122
private TemplateJoinDao templateJoinDao;
118123
@Inject
119124
private DeployAsIsHelper deployAsIsHelper;
125+
@Inject
126+
private AccountDao accountDao;
127+
@Inject
128+
private AccountManager _accountMgr;
129+
@Inject
130+
private ReservationDao reservationDao;
120131

121132
private long _nodeId;
122133
private ScheduledExecutorService _executor = null;
@@ -205,6 +216,36 @@ protected class UploadStatusCheck extends ManagedContextRunnable {
205216
public UploadStatusCheck() {
206217
}
207218

219+
private Answer sendUploadStatusCommandForVolume(EndPoint ep, UploadStatusCommand cmd, VolumeVO volume) {
220+
Answer answer = null;
221+
try {
222+
answer = ep.sendMessage(cmd);
223+
} catch (CloudRuntimeException e) {
224+
logger.warn("Unable to get upload status for volume {}. Error details: {}", volume, e.getMessage());
225+
answer = new UploadStatusAnswer(cmd, UploadStatus.UNKNOWN, e.getMessage());
226+
}
227+
if (answer == null || !(answer instanceof UploadStatusAnswer)) {
228+
logger.warn("No or invalid answer corresponding to UploadStatusCommand for volume {}", volume);
229+
return null;
230+
}
231+
return answer;
232+
}
233+
234+
private Answer sendUploadStatusCommandForTemplate(EndPoint ep, UploadStatusCommand cmd, VMTemplateVO template) {
235+
Answer answer = null;
236+
try {
237+
answer = ep.sendMessage(cmd);
238+
} catch (CloudRuntimeException e) {
239+
logger.warn("Unable to get upload status for template {}. Error details: {}", template, e.getMessage());
240+
answer = new UploadStatusAnswer(cmd, UploadStatus.UNKNOWN, e.getMessage());
241+
}
242+
if (answer == null || !(answer instanceof UploadStatusAnswer)) {
243+
logger.warn("No or invalid answer corresponding to UploadStatusCommand for template {}", template);
244+
return null;
245+
}
246+
return answer;
247+
}
248+
208249
@Override
209250
protected void runInContext() {
210251
// 1. Select all entries with download_state = Not_Downloaded or Download_In_Progress
@@ -231,18 +272,17 @@ protected void runInContext() {
231272
UploadStatusCommand cmd = new UploadStatusCommand(volume.getUuid(), EntityType.Volume);
232273
if (host != null && host.getManagementServerId() != null) {
233274
if (_nodeId == host.getManagementServerId().longValue()) {
234-
Answer answer = null;
235-
try {
236-
answer = ep.sendMessage(cmd);
237-
} catch (CloudRuntimeException e) {
238-
logger.warn("Unable to get upload status for volume {}. Error details: {}", volume, e.getMessage());
239-
answer = new UploadStatusAnswer(cmd, UploadStatus.UNKNOWN, e.getMessage());
240-
}
241-
if (answer == null || !(answer instanceof UploadStatusAnswer)) {
242-
logger.warn("No or invalid answer corresponding to UploadStatusCommand for volume {}", volume);
275+
Answer answer = sendUploadStatusCommandForVolume(ep, cmd, volume);
276+
if (answer == null) {
243277
continue;
244278
}
245-
handleVolumeStatusResponse((UploadStatusAnswer)answer, volume, volumeDataStore);
279+
if (!handleVolumeStatusResponse((UploadStatusAnswer)answer, volume, volumeDataStore)) {
280+
cmd = new UploadStatusCommand(volume.getUuid(), EntityType.Volume, true);
281+
answer = sendUploadStatusCommandForVolume(ep, cmd, volume);
282+
if (answer == null) {
283+
logger.warn("Unable to abort upload for volume {}", volume);
284+
}
285+
}
246286
}
247287
} else {
248288
String error = "Volume " + volume.getUuid() + " failed to upload as SSVM is either destroyed or SSVM agent not in 'Up' state";
@@ -275,18 +315,17 @@ protected void runInContext() {
275315
UploadStatusCommand cmd = new UploadStatusCommand(template.getUuid(), EntityType.Template);
276316
if (host != null && host.getManagementServerId() != null) {
277317
if (_nodeId == host.getManagementServerId().longValue()) {
278-
Answer answer = null;
279-
try {
280-
answer = ep.sendMessage(cmd);
281-
} catch (CloudRuntimeException e) {
282-
logger.warn("Unable to get upload status for template {}. Error details: {}", template, e.getMessage());
283-
answer = new UploadStatusAnswer(cmd, UploadStatus.UNKNOWN, e.getMessage());
284-
}
285-
if (answer == null || !(answer instanceof UploadStatusAnswer)) {
286-
logger.warn("No or invalid answer corresponding to UploadStatusCommand for template {}", template);
318+
Answer answer = sendUploadStatusCommandForTemplate(ep, cmd, template);
319+
if (answer == null) {
287320
continue;
288321
}
289-
handleTemplateStatusResponse((UploadStatusAnswer)answer, template, templateDataStore);
322+
if (!handleTemplateStatusResponse((UploadStatusAnswer) answer, template, templateDataStore)) {
323+
cmd = new UploadStatusCommand(template.getUuid(), EntityType.Template, true);
324+
answer = sendUploadStatusCommandForTemplate(ep, cmd, template);
325+
if (answer == null) {
326+
logger.warn("Unable to abort upload for template {}", template);
327+
}
328+
}
290329
}
291330
} else {
292331
String error = String.format(
@@ -303,7 +342,41 @@ protected void runInContext() {
303342
}
304343
}
305344

306-
private void handleVolumeStatusResponse(final UploadStatusAnswer answer, final VolumeVO volume, final VolumeDataStoreVO volumeDataStore) {
345+
private Boolean checkAndUpdateSecondaryStorageResourceLimit(Long accountId, Long lastSize, Long currentSize) {
346+
if (lastSize >= currentSize) {
347+
return true;
348+
}
349+
Long usage = currentSize - lastSize;
350+
try (CheckedReservation secStorageReservation = new CheckedReservation(_accountMgr.getAccount(accountId), Resource.ResourceType.secondary_storage, null, null, usage, reservationDao, _resourceLimitMgr)) {
351+
_resourceLimitMgr.incrementResourceCount(accountId, Resource.ResourceType.secondary_storage, usage);
352+
return true;
353+
} catch (Exception e) {
354+
_resourceLimitMgr.decrementResourceCount(accountId, Resource.ResourceType.secondary_storage, lastSize);
355+
return false;
356+
}
357+
}
358+
359+
private Boolean checkAndUpdateVolumeResourceLimit(VolumeVO volume, VolumeDataStoreVO volumeDataStore, UploadStatusAnswer answer) {
360+
boolean success = true;
361+
Long currentSize = answer.getVirtualSize() != 0 ? answer.getVirtualSize() : answer.getPhysicalSize();
362+
Long lastSize = volume.getSize() != null ? volume.getSize() : 0L;
363+
if (!checkAndUpdateSecondaryStorageResourceLimit(volume.getAccountId(), volume.getSize(), currentSize)) {
364+
volumeDataStore.setDownloadState(VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR);
365+
volumeDataStore.setState(State.Failed);
366+
volumeDataStore.setErrorString("Storage Limit Reached");
367+
Account owner = accountDao.findById(volume.getAccountId());
368+
String msg = String.format("Upload of volume [%s] failed because its owner [%s] does not have enough secondary storage space available.", volume.getUuid(), owner.getUuid());
369+
logger.error(msg);
370+
success = false;
371+
}
372+
VolumeVO volumeUpdate = _volumeDao.findById(volume.getId());
373+
volumeUpdate.setSize(currentSize);
374+
_volumeDao.update(volumeUpdate.getId(), volumeUpdate);
375+
return success;
376+
}
377+
378+
private boolean handleVolumeStatusResponse(final UploadStatusAnswer answer, final VolumeVO volume, final VolumeDataStoreVO volumeDataStore) {
379+
final boolean[] needAbort = new boolean[]{false};
307380
final StateMachine2<Volume.State, Event, Volume> stateMachine = Volume.State.getStateMachine();
308381
Transaction.execute(new TransactionCallbackNoReturn() {
309382
@Override
@@ -315,6 +388,11 @@ public void doInTransactionWithoutResult(TransactionStatus status) {
315388
try {
316389
switch (answer.getStatus()) {
317390
case COMPLETED:
391+
if (!checkAndUpdateVolumeResourceLimit(tmpVolume, tmpVolumeDataStore, answer)) {
392+
stateMachine.transitTo(tmpVolume, Event.OperationFailed, null, _volumeDao);
393+
sendAlert = true;
394+
break;
395+
}
318396
tmpVolumeDataStore.setDownloadState(VMTemplateStorageResourceAssoc.Status.DOWNLOADED);
319397
tmpVolumeDataStore.setState(State.Ready);
320398
tmpVolumeDataStore.setInstallPath(answer.getInstallPath());
@@ -326,7 +404,6 @@ public void doInTransactionWithoutResult(TransactionStatus status) {
326404
volumeUpdate.setSize(answer.getVirtualSize());
327405
_volumeDao.update(tmpVolume.getId(), volumeUpdate);
328406
stateMachine.transitTo(tmpVolume, Event.OperationSucceeded, null, _volumeDao);
329-
_resourceLimitMgr.incrementResourceCount(volume.getAccountId(), Resource.ResourceType.secondary_storage, answer.getVirtualSize());
330407

331408
// publish usage events
332409
UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_UPLOAD, tmpVolume.getAccountId(),
@@ -339,6 +416,12 @@ public void doInTransactionWithoutResult(TransactionStatus status) {
339416
}
340417
break;
341418
case IN_PROGRESS:
419+
if (!checkAndUpdateVolumeResourceLimit(tmpVolume, tmpVolumeDataStore, answer)) {
420+
stateMachine.transitTo(tmpVolume, Event.OperationFailed, null, _volumeDao);
421+
sendAlert = true;
422+
needAbort[0] = true;
423+
break;
424+
}
342425
if (tmpVolume.getState() == Volume.State.NotUploaded) {
343426
tmpVolumeDataStore.setDownloadState(VMTemplateStorageResourceAssoc.Status.DOWNLOAD_IN_PROGRESS);
344427
tmpVolumeDataStore.setDownloadPercent(answer.getDownloadPercent());
@@ -387,10 +470,29 @@ public void doInTransactionWithoutResult(TransactionStatus status) {
387470
}
388471
}
389472
});
473+
return !needAbort[0];
474+
}
475+
476+
private Boolean checkAndUpdateTemplateResourceLimit(VMTemplateVO template, TemplateDataStoreVO templateDataStore, UploadStatusAnswer answer) {
477+
boolean success = true;
478+
Long currentSize = answer.getVirtualSize() != 0 ? answer.getVirtualSize() : answer.getPhysicalSize();
479+
Long lastSize = template.getSize() != null ? template.getSize() : 0L;
480+
if (!checkAndUpdateSecondaryStorageResourceLimit(template.getAccountId(), lastSize, currentSize)) {
481+
templateDataStore.setDownloadState(VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR);
482+
templateDataStore.setErrorString("Storage Limit Reached");
483+
templateDataStore.setState(State.Failed);
484+
Account owner = accountDao.findById(template.getAccountId());
485+
String msg = String.format("Upload of template [%s] failed because its owner [%s] does not have enough secondary storage space available.", template.getUuid(), owner.getUuid());
486+
logger.error(msg);
487+
success = false;
488+
}
489+
templateDataStore.setSize(currentSize);
490+
return success;
390491
}
391492

392-
private void handleTemplateStatusResponse(final UploadStatusAnswer answer, final VMTemplateVO template, final TemplateDataStoreVO templateDataStore) {
493+
private boolean handleTemplateStatusResponse(final UploadStatusAnswer answer, final VMTemplateVO template, final TemplateDataStoreVO templateDataStore) {
393494
final StateMachine2<VirtualMachineTemplate.State, VirtualMachineTemplate.Event, VirtualMachineTemplate> stateMachine = VirtualMachineTemplate.State.getStateMachine();
495+
final boolean[] needAbort = new boolean[]{false};
394496
Transaction.execute(new TransactionCallbackNoReturn() {
395497
@Override
396498
public void doInTransactionWithoutResult(TransactionStatus status) {
@@ -401,6 +503,11 @@ public void doInTransactionWithoutResult(TransactionStatus status) {
401503
try {
402504
switch (answer.getStatus()) {
403505
case COMPLETED:
506+
if (!checkAndUpdateTemplateResourceLimit(tmpTemplate, tmpTemplateDataStore, answer)) {
507+
stateMachine.transitTo(tmpTemplate, VirtualMachineTemplate.Event.OperationFailed, null, _templateDao);
508+
sendAlert = true;
509+
break;
510+
}
404511
tmpTemplateDataStore.setDownloadState(VMTemplateStorageResourceAssoc.Status.DOWNLOADED);
405512
tmpTemplateDataStore.setState(State.Ready);
406513
tmpTemplateDataStore.setInstallPath(answer.getInstallPath());
@@ -453,6 +560,12 @@ public void doInTransactionWithoutResult(TransactionStatus status) {
453560
}
454561
break;
455562
case IN_PROGRESS:
563+
if (!checkAndUpdateTemplateResourceLimit(tmpTemplate, tmpTemplateDataStore, answer)) {
564+
stateMachine.transitTo(tmpTemplate, VirtualMachineTemplate.Event.OperationFailed, null, _templateDao);
565+
sendAlert = true;
566+
needAbort[0] = true;
567+
break;
568+
}
456569
if (tmpTemplate.getState() == VirtualMachineTemplate.State.NotUploaded) {
457570
tmpTemplateDataStore.setDownloadState(VMTemplateStorageResourceAssoc.Status.DOWNLOAD_IN_PROGRESS);
458571
stateMachine.transitTo(tmpTemplate, VirtualMachineTemplate.Event.UploadRequested, null, _templateDao);
@@ -502,6 +615,7 @@ public void doInTransactionWithoutResult(TransactionStatus status) {
502615
}
503616
}
504617
});
618+
return !needAbort[0];
505619
}
506620

507621
}

‎services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/HttpUploadServerHandler.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,7 @@ public void channelUnregistered(ChannelHandlerContext ctx) throws Exception {
130130
if (decoder != null) {
131131
decoder.cleanFiles();
132132
}
133+
storageResource.deregisterUploadChannel(uuid);
133134
requestProcessed = false;
134135
}
135136

@@ -182,6 +183,7 @@ public void channelRead0(ChannelHandlerContext ctx, HttpObject msg) throws Excep
182183
requestProcessed = true;
183184
return;
184185
}
186+
storageResource.registerUploadChannel(uuid, ctx.channel());
185187
//set the base directory to download the file
186188
DiskFileUpload.baseDirectory = uploadEntity.getInstallPathPrefix();
187189
this.processTimeout = uploadEntity.getProcessTimeout();

0 commit comments

Comments
 (0)