Skip to content

Commit 81a8ac8

Browse files
abh1sarDaan Hoogland
abh1sar
authored and
Daan Hoogland
committed
secondary storage resource limit for upload
1 parent d0f6730 commit 81a8ac8

4 files changed

Lines changed: 173 additions & 41 deletions

File tree

‎core/src/main/java/org/apache/cloudstack/storage/command/UploadStatusCommand.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ public enum EntityType {
2828
}
2929
private String entityUuid;
3030
private EntityType entityType;
31+
private Boolean abort;
3132

3233
protected UploadStatusCommand() {
3334
}
@@ -37,6 +38,11 @@ public UploadStatusCommand(String entityUuid, EntityType entityType) {
3738
this.entityType = entityType;
3839
}
3940

41+
public UploadStatusCommand(String entityUuid, EntityType entityType, Boolean abort) {
42+
this(entityUuid, entityType);
43+
this.abort = abort;
44+
}
45+
4046
public String getEntityUuid() {
4147
return entityUuid;
4248
}
@@ -45,6 +51,10 @@ public EntityType getEntityType() {
4551
return entityType;
4652
}
4753

54+
public Boolean getAbort() {
55+
return abort;
56+
}
57+
4858
@Override
4959
public boolean executeInSequence() {
5060
return false;

‎server/src/main/java/com/cloud/storage/ImageStoreUploadMonitorImpl.java‎

Lines changed: 129 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -26,10 +26,10 @@
2626
import javax.naming.ConfigurationException;
2727

2828
import com.cloud.agent.api.to.OVFInformationTO;
29-
import com.cloud.exception.ResourceAllocationException;
3029
import com.cloud.resourcelimit.CheckedReservation;
3130
import com.cloud.user.Account;
3231
import com.cloud.user.dao.AccountDao;
32+
import com.cloud.user.AccountManager;
3333
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
3434
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
3535
import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint;
@@ -126,6 +126,8 @@ public class ImageStoreUploadMonitorImpl extends ManagerBase implements ImageSto
126126
private ReservationDao reservationDao;
127127
@Inject
128128
private AccountDao accountDao;
129+
@Inject
130+
private AccountManager _accountMgr;
129131

130132
private long _nodeId;
131133
private ScheduledExecutorService _executor = null;
@@ -214,6 +216,36 @@ protected class UploadStatusCheck extends ManagedContextRunnable {
214216
public UploadStatusCheck() {
215217
}
216218

219+
private Answer sendUploadStatusCommandForVolume(EndPoint ep, UploadStatusCommand cmd, VolumeVO volume) {
220+
Answer answer = null;
221+
try {
222+
answer = ep.sendMessage(cmd);
223+
} catch (CloudRuntimeException e) {
224+
logger.warn("Unable to get upload status for volume {}. Error details: {}", volume, e.getMessage());
225+
answer = new UploadStatusAnswer(cmd, UploadStatus.UNKNOWN, e.getMessage());
226+
}
227+
if (answer == null || !(answer instanceof UploadStatusAnswer)) {
228+
logger.warn("No or invalid answer corresponding to UploadStatusCommand for volume {}", volume);
229+
return null;
230+
}
231+
return answer;
232+
}
233+
234+
private Answer sendUploadStatusCommandForTemplate(EndPoint ep, UploadStatusCommand cmd, VMTemplateVO template) {
235+
Answer answer = null;
236+
try {
237+
answer = ep.sendMessage(cmd);
238+
} catch (CloudRuntimeException e) {
239+
logger.warn("Unable to get upload status for template {}. Error details: {}", template, e.getMessage());
240+
answer = new UploadStatusAnswer(cmd, UploadStatus.UNKNOWN, e.getMessage());
241+
}
242+
if (answer == null || !(answer instanceof UploadStatusAnswer)) {
243+
logger.warn("No or invalid answer corresponding to UploadStatusCommand for template {}", template);
244+
return null;
245+
}
246+
return answer;
247+
}
248+
217249
@Override
218250
protected void runInContext() {
219251
// 1. Select all entries with download_state = Not_Downloaded or Download_In_Progress
@@ -240,18 +272,17 @@ protected void runInContext() {
240272
UploadStatusCommand cmd = new UploadStatusCommand(volume.getUuid(), EntityType.Volume);
241273
if (host != null && host.getManagementServerId() != null) {
242274
if (_nodeId == host.getManagementServerId().longValue()) {
243-
Answer answer = null;
244-
try {
245-
answer = ep.sendMessage(cmd);
246-
} catch (CloudRuntimeException e) {
247-
logger.warn("Unable to get upload status for volume {}. Error details: {}", volume, e.getMessage());
248-
answer = new UploadStatusAnswer(cmd, UploadStatus.UNKNOWN, e.getMessage());
249-
}
250-
if (answer == null || !(answer instanceof UploadStatusAnswer)) {
251-
logger.warn("No or invalid answer corresponding to UploadStatusCommand for volume {}", volume);
275+
Answer answer = sendUploadStatusCommandForVolume(ep, cmd, volume);
276+
if (answer == null) {
252277
continue;
253278
}
254-
handleVolumeStatusResponse((UploadStatusAnswer)answer, volume, volumeDataStore);
279+
if (!handleVolumeStatusResponse((UploadStatusAnswer)answer, volume, volumeDataStore)) {
280+
cmd = new UploadStatusCommand(volume.getUuid(), EntityType.Volume, true);
281+
answer = sendUploadStatusCommandForVolume(ep, cmd, volume);
282+
if (answer == null) {
283+
logger.warn("Unable to abort upload for volume {}", volume);
284+
}
285+
}
255286
}
256287
} else {
257288
String error = "Volume " + volume.getUuid() + " failed to upload as SSVM is either destroyed or SSVM agent not in 'Up' state";
@@ -284,18 +315,17 @@ protected void runInContext() {
284315
UploadStatusCommand cmd = new UploadStatusCommand(template.getUuid(), EntityType.Template);
285316
if (host != null && host.getManagementServerId() != null) {
286317
if (_nodeId == host.getManagementServerId().longValue()) {
287-
Answer answer = null;
288-
try {
289-
answer = ep.sendMessage(cmd);
290-
} catch (CloudRuntimeException e) {
291-
logger.warn("Unable to get upload status for template {}. Error details: {}", template, e.getMessage());
292-
answer = new UploadStatusAnswer(cmd, UploadStatus.UNKNOWN, e.getMessage());
293-
}
294-
if (answer == null || !(answer instanceof UploadStatusAnswer)) {
295-
logger.warn("No or invalid answer corresponding to UploadStatusCommand for template {}", template);
318+
Answer answer = sendUploadStatusCommandForTemplate(ep, cmd, template);
319+
if (answer == null) {
296320
continue;
297321
}
298-
handleTemplateStatusResponse((UploadStatusAnswer)answer, template, templateDataStore);
322+
if (!handleTemplateStatusResponse((UploadStatusAnswer) answer, template, templateDataStore)) {
323+
cmd = new UploadStatusCommand(template.getUuid(), EntityType.Template, true);
324+
answer = sendUploadStatusCommandForTemplate(ep, cmd, template);
325+
if (answer == null) {
326+
logger.warn("Unable to abort upload for template {}", template);
327+
}
328+
}
299329
}
300330
} else {
301331
String error = String.format(
@@ -312,7 +342,41 @@ protected void runInContext() {
312342
}
313343
}
314344

315-
private void handleVolumeStatusResponse(final UploadStatusAnswer answer, final VolumeVO volume, final VolumeDataStoreVO volumeDataStore) {
345+
private Boolean checkAndUpdateSecondaryStorageResourceLimit(Long accountId, Long lastSize, Long currentSize) {
346+
if (lastSize >= currentSize) {
347+
return true;
348+
}
349+
Long usage = currentSize - lastSize;
350+
try (CheckedReservation secStorageReservation = new CheckedReservation(_accountMgr.getAccount(accountId), Resource.ResourceType.secondary_storage, null, null, usage, reservationDao, _resourceLimitMgr)) {
351+
_resourceLimitMgr.incrementResourceCount(accountId, Resource.ResourceType.secondary_storage, usage);
352+
return true;
353+
} catch (Exception e) {
354+
_resourceLimitMgr.decrementResourceCount(accountId, Resource.ResourceType.secondary_storage, lastSize);
355+
return false;
356+
}
357+
}
358+
359+
private Boolean checkAndUpdateVolumeResourceLimit(VolumeVO volume, VolumeDataStoreVO volumeDataStore, UploadStatusAnswer answer) {
360+
boolean success = true;
361+
Long currentSize = answer.getVirtualSize() != 0 ? answer.getVirtualSize() : answer.getPhysicalSize();
362+
Long lastSize = volume.getSize() != null ? volume.getSize() : 0L;
363+
if (!checkAndUpdateSecondaryStorageResourceLimit(volume.getAccountId(), volume.getSize(), currentSize)) {
364+
volumeDataStore.setDownloadState(VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR);
365+
volumeDataStore.setState(State.Failed);
366+
volumeDataStore.setErrorString("Storage Limit Reached");
367+
Account owner = accountDao.findById(volume.getAccountId());
368+
String msg = String.format("Upload of volume [%s] failed because its owner [%s] does not have enough secondary storage space available.", volume.getUuid(), owner.getUuid());
369+
logger.error(msg);
370+
success = false;
371+
}
372+
VolumeVO volumeUpdate = _volumeDao.findById(volume.getId());
373+
volumeUpdate.setSize(currentSize);
374+
_volumeDao.update(volumeUpdate.getId(), volumeUpdate);
375+
return success;
376+
}
377+
378+
private boolean handleVolumeStatusResponse(final UploadStatusAnswer answer, final VolumeVO volume, final VolumeDataStoreVO volumeDataStore) {
379+
final boolean[] needAbort = new boolean[]{false};
316380
final StateMachine2<Volume.State, Event, Volume> stateMachine = Volume.State.getStateMachine();
317381
Transaction.execute(new TransactionCallbackNoReturn() {
318382
@Override
@@ -324,6 +388,11 @@ public void doInTransactionWithoutResult(TransactionStatus status) {
324388
try {
325389
switch (answer.getStatus()) {
326390
case COMPLETED:
391+
if (!checkAndUpdateVolumeResourceLimit(tmpVolume, tmpVolumeDataStore, answer)) {
392+
stateMachine.transitTo(tmpVolume, Event.OperationFailed, null, _volumeDao);
393+
sendAlert = true;
394+
break;
395+
}
327396
tmpVolumeDataStore.setDownloadState(VMTemplateStorageResourceAssoc.Status.DOWNLOADED);
328397
tmpVolumeDataStore.setState(State.Ready);
329398
tmpVolumeDataStore.setInstallPath(answer.getInstallPath());
@@ -335,7 +404,6 @@ public void doInTransactionWithoutResult(TransactionStatus status) {
335404
volumeUpdate.setSize(answer.getVirtualSize());
336405
_volumeDao.update(tmpVolume.getId(), volumeUpdate);
337406
stateMachine.transitTo(tmpVolume, Event.OperationSucceeded, null, _volumeDao);
338-
_resourceLimitMgr.incrementResourceCount(volume.getAccountId(), Resource.ResourceType.secondary_storage, answer.getVirtualSize());
339407

340408
// publish usage events
341409
UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_UPLOAD, tmpVolume.getAccountId(),
@@ -348,6 +416,12 @@ public void doInTransactionWithoutResult(TransactionStatus status) {
348416
}
349417
break;
350418
case IN_PROGRESS:
419+
if (!checkAndUpdateVolumeResourceLimit(tmpVolume, tmpVolumeDataStore, answer)) {
420+
stateMachine.transitTo(tmpVolume, Event.OperationFailed, null, _volumeDao);
421+
sendAlert = true;
422+
needAbort[0] = true;
423+
break;
424+
}
351425
if (tmpVolume.getState() == Volume.State.NotUploaded) {
352426
tmpVolumeDataStore.setDownloadState(VMTemplateStorageResourceAssoc.Status.DOWNLOAD_IN_PROGRESS);
353427
tmpVolumeDataStore.setDownloadPercent(answer.getDownloadPercent());
@@ -396,10 +470,29 @@ public void doInTransactionWithoutResult(TransactionStatus status) {
396470
}
397471
}
398472
});
473+
return !needAbort[0];
474+
}
475+
476+
private Boolean checkAndUpdateTemplateResourceLimit(VMTemplateVO template, TemplateDataStoreVO templateDataStore, UploadStatusAnswer answer) {
477+
boolean success = true;
478+
Long currentSize = answer.getVirtualSize() != 0 ? answer.getVirtualSize() : answer.getPhysicalSize();
479+
Long lastSize = template.getSize() != null ? template.getSize() : 0L;
480+
if (!checkAndUpdateSecondaryStorageResourceLimit(template.getAccountId(), lastSize, currentSize)) {
481+
templateDataStore.setDownloadState(VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR);
482+
templateDataStore.setErrorString("Storage Limit Reached");
483+
templateDataStore.setState(State.Failed);
484+
Account owner = accountDao.findById(template.getAccountId());
485+
String msg = String.format("Upload of template [%s] failed because its owner [%s] does not have enough secondary storage space available.", template.getUuid(), owner.getUuid());
486+
logger.error(msg);
487+
success = false;
488+
}
489+
templateDataStore.setSize(currentSize);
490+
return success;
399491
}
400492

401-
private void handleTemplateStatusResponse(final UploadStatusAnswer answer, final VMTemplateVO template, final TemplateDataStoreVO templateDataStore) {
493+
private boolean handleTemplateStatusResponse(final UploadStatusAnswer answer, final VMTemplateVO template, final TemplateDataStoreVO templateDataStore) {
402494
final StateMachine2<VirtualMachineTemplate.State, VirtualMachineTemplate.Event, VirtualMachineTemplate> stateMachine = VirtualMachineTemplate.State.getStateMachine();
495+
final boolean[] needAbort = new boolean[]{false};
403496
Transaction.execute(new TransactionCallbackNoReturn() {
404497
@Override
405498
public void doInTransactionWithoutResult(TransactionStatus status) {
@@ -410,6 +503,11 @@ public void doInTransactionWithoutResult(TransactionStatus status) {
410503
try {
411504
switch (answer.getStatus()) {
412505
case COMPLETED:
506+
if (!checkAndUpdateTemplateResourceLimit(tmpTemplate, tmpTemplateDataStore, answer)) {
507+
stateMachine.transitTo(tmpTemplate, VirtualMachineTemplate.Event.OperationFailed, null, _templateDao);
508+
sendAlert = true;
509+
break;
510+
}
413511
tmpTemplateDataStore.setDownloadState(VMTemplateStorageResourceAssoc.Status.DOWNLOADED);
414512
tmpTemplateDataStore.setState(State.Ready);
415513
tmpTemplateDataStore.setInstallPath(answer.getInstallPath());
@@ -445,22 +543,6 @@ public void doInTransactionWithoutResult(TransactionStatus status) {
445543
break;
446544
}
447545
}
448-
449-
Account owner = accountDao.findById(template.getAccountId());
450-
long templateSize = answer.getVirtualSize();
451-
452-
try (CheckedReservation secondaryStorageReservation = new CheckedReservation(owner, Resource.ResourceType.secondary_storage, null, null, templateSize, reservationDao, _resourceLimitMgr)) {
453-
_resourceLimitMgr.incrementResourceCount(owner.getId(), Resource.ResourceType.secondary_storage, templateSize);
454-
} catch (ResourceAllocationException e) {
455-
tmpTemplateDataStore.setDownloadState(VMTemplateStorageResourceAssoc.Status.UPLOAD_ERROR);
456-
tmpTemplateDataStore.setState(State.Failed);
457-
stateMachine.transitTo(tmpTemplate, VirtualMachineTemplate.Event.OperationFailed, null, _templateDao);
458-
msg = String.format("Upload of template [%s] failed because its owner [%s] does not have enough secondary storage space available.", template.getUuid(), owner.getUuid());
459-
logger.warn(msg);
460-
sendAlert = true;
461-
break;
462-
}
463-
464546
stateMachine.transitTo(tmpTemplate, VirtualMachineTemplate.Event.OperationSucceeded, null, _templateDao);
465547
//publish usage event
466548
String etype = EventTypes.EVENT_TEMPLATE_CREATE;
@@ -477,6 +559,12 @@ public void doInTransactionWithoutResult(TransactionStatus status) {
477559
}
478560
break;
479561
case IN_PROGRESS:
562+
if (!checkAndUpdateTemplateResourceLimit(tmpTemplate, tmpTemplateDataStore, answer)) {
563+
stateMachine.transitTo(tmpTemplate, VirtualMachineTemplate.Event.OperationFailed, null, _templateDao);
564+
sendAlert = true;
565+
needAbort[0] = true;
566+
break;
567+
}
480568
if (tmpTemplate.getState() == VirtualMachineTemplate.State.NotUploaded) {
481569
tmpTemplateDataStore.setDownloadState(VMTemplateStorageResourceAssoc.Status.DOWNLOAD_IN_PROGRESS);
482570
stateMachine.transitTo(tmpTemplate, VirtualMachineTemplate.Event.UploadRequested, null, _templateDao);
@@ -526,6 +614,7 @@ public void doInTransactionWithoutResult(TransactionStatus status) {
526614
}
527615
}
528616
});
617+
return !needAbort[0];
529618
}
530619

531620
}

‎services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/HttpUploadServerHandler.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,7 @@ public void channelUnregistered(ChannelHandlerContext ctx) throws Exception {
130130
if (decoder != null) {
131131
decoder.cleanFiles();
132132
}
133+
storageResource.deregisterUploadChannel(uuid);
133134
requestProcessed = false;
134135
}
135136

@@ -182,6 +183,7 @@ public void channelRead0(ChannelHandlerContext ctx, HttpObject msg) throws Excep
182183
requestProcessed = true;
183184
return;
184185
}
186+
storageResource.registerUploadChannel(uuid, ctx.channel());
185187
//set the base directory to download the file
186188
DiskFileUpload.baseDirectory = uploadEntity.getInstallPathPrefix();
187189
this.processTimeout = uploadEntity.getProcessTimeout();

0 commit comments

Comments
 (0)