Skip to content

Commit f1c6817

Browse files
author
zhijian.liu
committed
<fix>[kms]: add delete secret for vm
Resolves: ZSV-11630
1 parent 52600df commit f1c6817

8 files changed

Lines changed: 329 additions & 7 deletions

File tree

‎conf/springConfigXml/Kvm.xml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -256,6 +256,7 @@
256256
<zstack:plugin>
257257
<zstack:extension interface="org.zstack.kvm.KVMStartVmExtensionPoint" />
258258
<zstack:extension interface="org.zstack.header.vm.PreVmInstantiateResourceExtensionPoint" />
259+
<zstack:extension interface="org.zstack.header.vm.VmAfterExpungeExtensionPoint" />
259260
</zstack:plugin>
260261
</bean>
261262

‎header/src/main/java/org/zstack/header/secret/SecretHostDeleteMsg.java‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
package org.zstack.header.secret;
2+
3+
import org.zstack.header.host.HostMessage;
4+
import org.zstack.header.message.NeedReplyMessage;
5+
6+
/**
7+
* Request to delete existing secret on KVM host by vmUuid/purpose/keyVersion.
8+
* keyVersion can be null, then agent may remove all matched versions for the vm+purpose.
9+
*/
10+
public class SecretHostDeleteMsg extends NeedReplyMessage implements HostMessage {
11+
private String hostUuid;
12+
private String vmUuid;
13+
private String purpose;
14+
private Integer keyVersion;
15+
16+
@Override
17+
public String getHostUuid() {
18+
return hostUuid;
19+
}
20+
21+
public void setHostUuid(String hostUuid) {
22+
this.hostUuid = hostUuid;
23+
}
24+
25+
public String getVmUuid() {
26+
return vmUuid;
27+
}
28+
29+
public void setVmUuid(String vmUuid) {
30+
this.vmUuid = vmUuid;
31+
}
32+
33+
public String getPurpose() {
34+
return purpose;
35+
}
36+
37+
public void setPurpose(String purpose) {
38+
this.purpose = purpose;
39+
}
40+
41+
public Integer getKeyVersion() {
42+
return keyVersion;
43+
}
44+
45+
public void setKeyVersion(Integer keyVersion) {
46+
this.keyVersion = keyVersion;
47+
}
48+
}

‎header/src/main/java/org/zstack/header/secret/SecretHostDeleteReply.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
package org.zstack.header.secret;
2+
3+
import org.zstack.header.message.MessageReply;
4+
5+
/** Reply for SecretHostDeleteMsg. */
6+
public class SecretHostDeleteReply extends MessageReply {
7+
}

‎plugin/kvm/src/main/java/org/zstack/kvm/KVMAgentCommands.java‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -508,6 +508,39 @@ public void setSecretUuid(String secretUuid) {
508508
}
509509
}
510510

511+
public static class SecretHostDeleteCmd extends AgentCommand {
512+
private String vmUuid;
513+
private String purpose;
514+
private Integer keyVersion;
515+
516+
public String getVmUuid() {
517+
return vmUuid;
518+
}
519+
520+
public void setVmUuid(String vmUuid) {
521+
this.vmUuid = vmUuid;
522+
}
523+
524+
public String getPurpose() {
525+
return purpose;
526+
}
527+
528+
public void setPurpose(String purpose) {
529+
this.purpose = purpose;
530+
}
531+
532+
public Integer getKeyVersion() {
533+
return keyVersion;
534+
}
535+
536+
public void setKeyVersion(Integer keyVersion) {
537+
this.keyVersion = keyVersion;
538+
}
539+
}
540+
541+
public static class SecretHostDeleteResponse extends AgentResponse {
542+
}
543+
511544
public static class PingCmd extends AgentCommand {
512545
public String hostUuid;
513546
public Map<String, Object> configs;

‎plugin/kvm/src/main/java/org/zstack/kvm/KVMConstant.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,7 @@ public interface KVMConstant {
131131
String KVM_VERIFY_ENVELOPE_KEY_PATH = "/host/key/envelope/checkEnvelopeKey";
132132
String KVM_GET_SECRET_PATH = "/host/key/envelope/getSecret";
133133
String KVM_ENSURE_SECRET_PATH = "/host/key/envelope/ensureSecret";
134+
String KVM_DELETE_SECRET_PATH = "/host/key/envelope/deleteSecret";
134135

135136
/** HTTP timeout in seconds for envelope key sync (verify/create/rotate/get) to agent. */
136137
long ENVELOPE_KEY_HTTP_TIMEOUT_SEC = 5L;

‎plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,8 @@
5757
import org.zstack.header.host.MigrateVmOnHypervisorMsg.StorageMigrationPolicy;
5858
import org.zstack.header.secret.SecretHostDefineMsg;
5959
import org.zstack.header.secret.SecretHostDefineReply;
60+
import org.zstack.header.secret.SecretHostDeleteMsg;
61+
import org.zstack.header.secret.SecretHostDeleteReply;
6062
import org.zstack.header.secret.SecretHostGetMsg;
6163
import org.zstack.header.secret.SecretHostGetReply;
6264
import org.zstack.header.message.APIMessage;
@@ -756,6 +758,8 @@ protected void handleLocalMessage(Message msg) {
756758
handle((SecretHostGetMsg) msg);
757759
} else if (msg instanceof SecretHostDefineMsg) {
758760
handle((SecretHostDefineMsg) msg);
761+
} else if (msg instanceof SecretHostDeleteMsg) {
762+
handle((SecretHostDeleteMsg) msg);
759763
} else {
760764
super.handleLocalMessage(msg);
761765
}
@@ -5465,6 +5469,42 @@ public Class<KVMAgentCommands.SecretHostDefineResponse> getReturnClass() {
54655469
}, TimeUnit.SECONDS, KVMConstant.ENVELOPE_KEY_HTTP_TIMEOUT_SEC);
54665470
}
54675471

5472+
private void handle(SecretHostDeleteMsg msg) {
5473+
SecretHostDeleteReply reply = new SecretHostDeleteReply();
5474+
if (StringUtils.isBlank(msg.getVmUuid()) || StringUtils.isBlank(msg.getPurpose())) {
5475+
reply.setError(operr("vmUuid and purpose are required for delete secret"));
5476+
bus.reply(msg, reply);
5477+
return;
5478+
}
5479+
5480+
String url = buildUrl(KVMConstant.KVM_DELETE_SECRET_PATH);
5481+
KVMAgentCommands.SecretHostDeleteCmd cmd = new KVMAgentCommands.SecretHostDeleteCmd();
5482+
cmd.setVmUuid(msg.getVmUuid());
5483+
cmd.setPurpose(msg.getPurpose());
5484+
cmd.setKeyVersion(msg.getKeyVersion());
5485+
5486+
restf.asyncJsonPost(url, cmd, new JsonAsyncRESTCallback<KVMAgentCommands.SecretHostDeleteResponse>(msg, reply) {
5487+
@Override
5488+
public void fail(ErrorCode err) {
5489+
reply.setError(err != null ? err : operr("delete secret on agent failed"));
5490+
bus.reply(msg, reply);
5491+
}
5492+
5493+
@Override
5494+
public void success(KVMAgentCommands.SecretHostDeleteResponse rsp) {
5495+
if (rsp == null || !rsp.isSuccess()) {
5496+
reply.setError(buildSecretAgentError(rsp, "delete secret failed"));
5497+
}
5498+
bus.reply(msg, reply);
5499+
}
5500+
5501+
@Override
5502+
public Class<KVMAgentCommands.SecretHostDeleteResponse> getReturnClass() {
5503+
return KVMAgentCommands.SecretHostDeleteResponse.class;
5504+
}
5505+
}, TimeUnit.SECONDS, KVMConstant.ENVELOPE_KEY_HTTP_TIMEOUT_SEC);
5506+
}
5507+
54685508
private ErrorCode buildSecretAgentError(KVMAgentCommands.AgentResponse rsp, String defaultMessage) {
54695509
if (rsp != null && rsp.getError() != null) {
54705510
ErrorCode err = new ErrorCode();

‎plugin/kvm/src/main/java/org/zstack/kvm/tpm/KvmTpmExtensions.java‎

Lines changed: 112 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@
22

33
import org.apache.commons.lang.StringUtils;
44
import org.springframework.beans.factory.annotation.Autowired;
5-
import org.zstack.compute.vm.VmGlobalConfig;
65
import org.zstack.compute.vm.devices.TpmEncryptedResourceKeyBackend;
76
import org.zstack.compute.vm.devices.TpmEncryptedResourceKeyBackend.CloneEncryptedResourceKeyContext;
87
import org.zstack.core.Platform;
@@ -21,11 +20,13 @@
2120
import org.zstack.header.core.workflow.FlowTrigger;
2221
import org.zstack.header.core.workflow.NoRollbackFlow;
2322
import org.zstack.header.errorcode.ErrorCode;
23+
import org.zstack.header.core.NoErrorCompletion;
2424
import org.zstack.header.host.HostConstant;
2525
import org.zstack.header.message.MessageReply;
2626
import org.zstack.header.keyprovider.EncryptedResourceKeyManager;
2727
import org.zstack.header.keyprovider.EncryptedResourceKeyManager.GetOrCreateResourceKeyContext;
2828
import org.zstack.header.keyprovider.EncryptedResourceKeyManager.ResourceKeyResult;
29+
import org.zstack.header.secret.SecretHostDeleteMsg;
2930
import org.zstack.header.secret.SecretHostDefineMsg;
3031
import org.zstack.header.storage.snapshot.group.VolumeSnapshotGroupVO;
3132
import org.zstack.header.storage.snapshot.group.VolumeSnapshotGroupVO_;
@@ -35,11 +36,17 @@
3536
import org.zstack.header.tpm.entity.TpmSpec;
3637
import org.zstack.header.tpm.entity.TpmVO;
3738
import org.zstack.header.tpm.entity.TpmVO_;
39+
import org.zstack.header.vm.HaStartVmInstanceMsg;
3840
import org.zstack.header.vm.PreVmInstantiateResourceExtensionPoint;
41+
import org.zstack.header.vm.VmAfterExpungeExtensionPoint;
42+
import org.zstack.header.vm.VmInstanceInventory;
43+
import org.zstack.header.vm.VmInstanceMigrateExtensionPoint;
3944
import org.zstack.header.vm.VmInstanceSpec;
4045
import org.zstack.header.vm.VmInstantiateResourceException;
4146
import org.zstack.header.vm.additions.VmHostBackupFileVO;
4247
import org.zstack.header.vm.additions.VmHostBackupFileVO_;
48+
import org.zstack.header.vm.VmInstanceVO;
49+
import org.zstack.header.vm.VmInstanceVO_;
4350
import org.zstack.header.vm.additions.VmHostFileType;
4451
import org.zstack.header.vm.additions.VmHostFileVO;
4552
import org.zstack.header.vm.additions.VmHostFileVO_;
@@ -60,7 +67,9 @@
6067
import static org.zstack.core.Platform.operr;
6168

6269
public class KvmTpmExtensions implements KVMStartVmExtensionPoint,
63-
PreVmInstantiateResourceExtensionPoint {
70+
PreVmInstantiateResourceExtensionPoint,
71+
VmInstanceMigrateExtensionPoint,
72+
VmAfterExpungeExtensionPoint {
6473
private static final CLogger logger = Utils.getLogger(KvmTpmExtensions.class);
6574

6675
@Autowired
@@ -122,6 +131,16 @@ public void beforeStartVmOnKvm(KVMHostInventory host, VmInstanceSpec spec, KVMAg
122131

123132
@Override
124133
public void startVmOnKvmSuccess(KVMHostInventory host, VmInstanceSpec spec) {
134+
if (spec.getMessage() instanceof HaStartVmInstanceMsg) {
135+
String vmUuid = spec.getVmInventory() == null ? null : spec.getVmInventory().getUuid();
136+
String srcHostUuid = spec.getVmInventory() == null ? null : spec.getVmInventory().getLastHostUuid();
137+
Integer keyVersion = findTpmKeyVersionByVmUuid(vmUuid);
138+
boolean vmIsOnDestHost = isVmCurrentlyOnExpectedHost(vmUuid, host.getUuid());
139+
if (vmIsOnDestHost && StringUtils.isNotBlank(srcHostUuid) && !host.getUuid().equals(srcHostUuid)) {
140+
deleteHostSecretBestEffort(srcHostUuid, vmUuid, keyVersion,
141+
"ha-start-success");
142+
}
143+
}
125144
clearRollbackInfo(spec);
126145
}
127146

@@ -229,10 +248,11 @@ public void rollback(FlowRollback trigger, Map data) {
229248

230249
@Override
231250
public boolean skip(Map data) {
232-
boolean shouldSkip = VmGlobalConfig.ALLOWED_TPM_VM_WITHOUT_KMS.value(Boolean.class)
233-
&& StringUtils.isBlank(context.providerUuid);
251+
boolean shouldSkip = StringUtils.isBlank(context.providerUuid);
234252
if (shouldSkip) {
235-
logger.info("skip create-dek: allowed.tpm.vm.without.kms is enabled and no KMS provider bound");
253+
logger.info(String.format(
254+
"skip ensure-resource-key-ref for tpm[uuid:%s] due to missing key provider binding, try host secret first",
255+
context.tpmUuid));
236256
}
237257
return shouldSkip;
238258
}
@@ -253,7 +273,6 @@ public void run(FlowTrigger trigger, Map data) {
253273
keyCtx.setResourceUuid(context.tpmUuid);
254274
keyCtx.setResourceType(TpmVO.class.getSimpleName());
255275
keyCtx.setKeyProviderUuid(context.providerUuid);
256-
keyCtx.setKeyProviderName(context.providerName);
257276
keyCtx.setPurpose("vtpm");
258277

259278
resourceKeyManager.getOrCreateKey(keyCtx, new ReturnValueCompletion<ResourceKeyResult>(trigger) {
@@ -488,4 +507,91 @@ private String findSourceTpmUuidFromSnapshotTpmBackupFile(String tpmBackupFileUu
488507
.select(TpmVO_.uuid)
489508
.findValue();
490509
}
510+
511+
@Override
512+
public void afterMigrateVm(VmInstanceInventory inv, String srcHostUuid, NoErrorCompletion completion) {
513+
String vmUuid = inv == null ? null : inv.getUuid();
514+
String destHostUuid = inv == null ? null : inv.getHostUuid();
515+
if (StringUtils.isBlank(vmUuid) || StringUtils.isBlank(srcHostUuid) || srcHostUuid.equals(destHostUuid)) {
516+
completion.done();
517+
return;
518+
}
519+
520+
if (!isVmCurrentlyOnExpectedHost(vmUuid, destHostUuid)) {
521+
completion.done();
522+
return;
523+
}
524+
525+
Integer keyVersion = findTpmKeyVersionByVmUuid(vmUuid);
526+
deleteHostSecretBestEffort(srcHostUuid, vmUuid, keyVersion, "after-migrate");
527+
completion.done();
528+
}
529+
530+
@Override
531+
public void vmAfterExpunge(VmInstanceInventory vm) {
532+
String vmUuid = vm.getUuid();
533+
Integer keyVersion = findTpmKeyVersionByVmUuid(vmUuid);
534+
535+
java.util.Set<String> hostUuids = new java.util.HashSet<>();
536+
if (StringUtils.isNotBlank(vm.getHostUuid())) {
537+
hostUuids.add(vm.getHostUuid());
538+
}
539+
if (StringUtils.isNotBlank(vm.getLastHostUuid())) {
540+
hostUuids.add(vm.getLastHostUuid());
541+
}
542+
543+
if (hostUuids.isEmpty()) {
544+
return;
545+
}
546+
547+
for (String hostUuid : hostUuids) {
548+
deleteHostSecretBestEffort(hostUuid, vmUuid, keyVersion, "expunge");
549+
}
550+
}
551+
552+
private Integer findTpmKeyVersionByVmUuid(String vmUuid) {
553+
if (StringUtils.isBlank(vmUuid)) {
554+
return null;
555+
}
556+
String tpmUuid = Q.New(TpmVO.class)
557+
.eq(TpmVO_.vmInstanceUuid, vmUuid)
558+
.select(TpmVO_.uuid)
559+
.findValue();
560+
return tpmUuid == null ? null : resourceKeyBackend.findKeyVersionByTpm(tpmUuid);
561+
}
562+
563+
private boolean isVmCurrentlyOnExpectedHost(String vmUuid, String expectedHostUuid) {
564+
if (StringUtils.isBlank(vmUuid) || StringUtils.isBlank(expectedHostUuid)) {
565+
return false;
566+
}
567+
568+
String currentHostUuid = Q.New(VmInstanceVO.class)
569+
.select(VmInstanceVO_.hostUuid)
570+
.eq(VmInstanceVO_.uuid, vmUuid)
571+
.findValue();
572+
return expectedHostUuid.equals(currentHostUuid);
573+
}
574+
575+
private void deleteHostSecretBestEffort(String hostUuid, String vmUuid, Integer keyVersion, String reason) {
576+
if (StringUtils.isBlank(hostUuid) || StringUtils.isBlank(vmUuid)) {
577+
return;
578+
}
579+
580+
SecretHostDeleteMsg dmsg = new SecretHostDeleteMsg();
581+
dmsg.setHostUuid(hostUuid);
582+
dmsg.setVmUuid(vmUuid);
583+
dmsg.setPurpose("vtpm");
584+
dmsg.setKeyVersion(keyVersion);
585+
bus.makeTargetServiceIdByResourceUuid(dmsg, HostConstant.SERVICE_ID, hostUuid);
586+
bus.send(dmsg, new CloudBusCallBack(null) {
587+
@Override
588+
public void run(MessageReply reply) {
589+
if (!reply.isSuccess()) {
590+
logger.warn(String.format(
591+
"best-effort delete host secret failed on %s for vm[uuid:%s], host[uuid:%s]: %s",
592+
reason, vmUuid, hostUuid, reply.getError().getDetails()));
593+
}
594+
}
595+
});
596+
}
491597
}

0 commit comments

Comments
 (0)