Skip to content

Commit 6013cb5

Browse files
author
zhong.zhou
committed
<fix>[kvm]: define secret on migrate destination
Resolves: ZSV-11729 Change-Id: I616a776e78666179637976616c776162626c6533
1 parent 1751228 commit 6013cb5

5 files changed

Lines changed: 264 additions & 4 deletions

File tree

‎header/src/main/java/org/zstack/header/secret/SecretHostDefineMsg.java‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,9 @@ public class SecretHostDefineMsg extends NeedReplyMessage implements HostMessage
1717
private String purpose;
1818
private String providerName;
1919
private String description;
20+
private Integer keyVersion;
21+
private String usageInstance;
22+
private String secretUuid;
2023

2124
@Override
2225
public String getHostUuid() {
@@ -66,4 +69,28 @@ public String getDescription() {
6669
public void setDescription(String description) {
6770
this.description = description;
6871
}
72+
73+
public Integer getKeyVersion() {
74+
return keyVersion;
75+
}
76+
77+
public void setKeyVersion(Integer keyVersion) {
78+
this.keyVersion = keyVersion;
79+
}
80+
81+
public String getUsageInstance() {
82+
return usageInstance;
83+
}
84+
85+
public void setUsageInstance(String usageInstance) {
86+
this.usageInstance = usageInstance;
87+
}
88+
89+
public String getSecretUuid() {
90+
return secretUuid;
91+
}
92+
93+
public void setSecretUuid(String secretUuid) {
94+
this.secretUuid = secretUuid;
95+
}
6996
}

‎plugin/kvm/src/main/java/org/zstack/kvm/KVMAgentCommands.java‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -412,6 +412,9 @@ public static class SecretHostDefineCmd extends AgentCommand {
412412
private String purpose;
413413
private String providerName;
414414
private String description;
415+
private Integer keyVersion;
416+
private String usageInstance;
417+
private String secretUuid;
415418

416419
public String getEncryptedDek() {
417420
return encryptedDek;
@@ -452,6 +455,30 @@ public String getDescription() {
452455
public void setDescription(String description) {
453456
this.description = description;
454457
}
458+
459+
public Integer getKeyVersion() {
460+
return keyVersion;
461+
}
462+
463+
public void setKeyVersion(Integer keyVersion) {
464+
this.keyVersion = keyVersion;
465+
}
466+
467+
public String getUsageInstance() {
468+
return usageInstance;
469+
}
470+
471+
public void setUsageInstance(String usageInstance) {
472+
this.usageInstance = usageInstance;
473+
}
474+
475+
public String getSecretUuid() {
476+
return secretUuid;
477+
}
478+
479+
public void setSecretUuid(String secretUuid) {
480+
this.secretUuid = secretUuid;
481+
}
455482
}
456483

457484
public static class SecretHostDefineResponse extends AgentResponse {
@@ -466,6 +493,57 @@ public void setSecretUuid(String secretUuid) {
466493
}
467494
}
468495

496+
public static class SecretHostGetCmd extends AgentCommand {
497+
private String vmUuid;
498+
private String purpose;
499+
private Integer keyVersion;
500+
private String usageInstance;
501+
502+
public String getVmUuid() {
503+
return vmUuid;
504+
}
505+
506+
public void setVmUuid(String vmUuid) {
507+
this.vmUuid = vmUuid;
508+
}
509+
510+
public String getPurpose() {
511+
return purpose;
512+
}
513+
514+
public void setPurpose(String purpose) {
515+
this.purpose = purpose;
516+
}
517+
518+
public Integer getKeyVersion() {
519+
return keyVersion;
520+
}
521+
522+
public void setKeyVersion(Integer keyVersion) {
523+
this.keyVersion = keyVersion;
524+
}
525+
526+
public String getUsageInstance() {
527+
return usageInstance;
528+
}
529+
530+
public void setUsageInstance(String usageInstance) {
531+
this.usageInstance = usageInstance;
532+
}
533+
}
534+
535+
public static class SecretHostGetResponse extends AgentResponse {
536+
private String secretUuid;
537+
538+
public String getSecretUuid() {
539+
return secretUuid;
540+
}
541+
542+
public void setSecretUuid(String secretUuid) {
543+
this.secretUuid = secretUuid;
544+
}
545+
}
546+
469547
public static class PingCmd extends AgentCommand {
470548
public String hostUuid;
471549
public Map<String, Object> configs;

‎plugin/kvm/src/main/java/org/zstack/kvm/KVMConstant.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,7 @@ public interface KVMConstant {
130130
String KVM_ROTATE_ENVELOPE_KEY_PATH = "/host/key/envelope/rotateEnvelopeKey";
131131
String KVM_VERIFY_ENVELOPE_KEY_PATH = "/host/key/envelope/checkEnvelopeKey";
132132
String KVM_ENSURE_SECRET_PATH = "/host/key/envelope/ensureSecret";
133+
String KVM_GET_SECRET_PATH = "/host/key/envelope/getSecret";
133134

134135
/** HTTP timeout in seconds for envelope key sync (verify/create/rotate/get) to agent. */
135136
long ENVELOPE_KEY_HTTP_TIMEOUT_SEC = 5L;

‎plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java‎

Lines changed: 156 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@
1212
import org.zstack.compute.cluster.arch.ClusterResourceConfigInitializer;
1313
import org.zstack.compute.host.*;
1414
import org.zstack.compute.vm.*;
15+
import org.zstack.compute.vm.devices.TpmEncryptedResourceKeyBackend;
16+
import org.zstack.compute.vm.devices.VmTpmManager;
1517
import org.zstack.core.asyncbatch.While;
1618
import org.zstack.core.cloudbus.ResourceDestinationMaker;
1719
import org.zstack.core.timeout.TimeHelper;
@@ -55,6 +57,9 @@
5557
import org.zstack.header.exception.CloudRuntimeException;
5658
import org.zstack.header.host.*;
5759
import org.zstack.header.host.MigrateVmOnHypervisorMsg.StorageMigrationPolicy;
60+
import org.zstack.header.keyprovider.EncryptedResourceKeyManager;
61+
import org.zstack.header.keyprovider.EncryptedResourceKeyManager.GetOrCreateResourceKeyContext;
62+
import org.zstack.header.keyprovider.EncryptedResourceKeyManager.ResourceKeyResult;
5863
import org.zstack.header.secret.SecretHostDefineMsg;
5964
import org.zstack.header.secret.SecretHostDefineReply;
6065
import org.zstack.header.message.APIMessage;
@@ -68,6 +73,7 @@
6873
import org.zstack.header.rest.RESTFacade;
6974
import org.zstack.header.storage.primary.*;
7075
import org.zstack.header.tag.SystemTagInventory;
76+
import org.zstack.header.tpm.entity.TpmVO;
7177
import org.zstack.header.vm.*;
7278
import org.zstack.header.vm.devices.DeviceAddress;
7379
import org.zstack.header.volume.*;
@@ -164,6 +170,10 @@ public class KVMHost extends HostBase implements Host {
164170
private AccountManager accountMgr;
165171
@Autowired
166172
private ResourceDestinationMaker destMaker;
173+
@Autowired
174+
private TpmEncryptedResourceKeyBackend tpmKeyBackend;
175+
@Autowired
176+
private EncryptedResourceKeyManager resourceKeyManager;
167177

168178
private KVMHostContext context;
169179

@@ -3162,6 +3172,146 @@ public void fail(ErrorCode errorCode) {
31623172
}
31633173
});
31643174

3175+
flow(new NoRollbackFlow() {
3176+
String __name__ = "ensure-secret-on-dst-before-migrate";
3177+
3178+
@Override
3179+
public void run(FlowTrigger trigger, Map data) {
3180+
String tpmUuid = VmTpmManager.findTpmUuidForVmOrNull(vmUuid);
3181+
if (StringUtils.isBlank(tpmUuid)) {
3182+
trigger.next();
3183+
return;
3184+
}
3185+
String providerUuid = tpmKeyBackend.findKeyProviderUuidByTpm(tpmUuid);
3186+
String providerName = tpmKeyBackend.findKeyProviderNameByTpm(tpmUuid);
3187+
if (StringUtils.isBlank(providerUuid) && StringUtils.isBlank(providerName)) {
3188+
trigger.fail(operr("missing TPM resource key binding for tpm[uuid:%s] before migrate", tpmUuid));
3189+
return;
3190+
}
3191+
3192+
GetOrCreateResourceKeyContext keyCtx = new GetOrCreateResourceKeyContext();
3193+
keyCtx.setResourceUuid(tpmUuid);
3194+
keyCtx.setResourceType(TpmVO.class.getSimpleName());
3195+
keyCtx.setKeyProviderUuid(providerUuid);
3196+
keyCtx.setKeyProviderName(providerName);
3197+
keyCtx.setPurpose("vtpm");
3198+
resourceKeyManager.getOrCreateKey(keyCtx, new ReturnValueCompletion<ResourceKeyResult>(trigger) {
3199+
@Override
3200+
public void success(ResourceKeyResult result) {
3201+
String effectiveProviderName = result.getKeyProviderName();
3202+
if (StringUtils.isBlank(effectiveProviderName)) {
3203+
effectiveProviderName = providerName;
3204+
}
3205+
if (StringUtils.isBlank(effectiveProviderName)) {
3206+
trigger.fail(operr("missing effective key provider name for tpm[uuid:%s] before migrate", tpmUuid));
3207+
return;
3208+
}
3209+
3210+
final String usageInstance = "tpm0";
3211+
final Integer[] keyVersionHolder = new Integer[1];
3212+
final String[] sourceSecretUuidHolder = new String[1];
3213+
3214+
FlowChain chain = FlowChainBuilder.newSimpleFlowChain();
3215+
chain.setName(String.format("get-and-ensure-vtpm-secret-vm-%s", vmUuid));
3216+
chain.then(new NoRollbackFlow() {
3217+
String __name__ = "get-source-secret-uuid";
3218+
3219+
@Override
3220+
public void run(FlowTrigger innerTrigger, Map data) {
3221+
Object keyVersionObj = SQL.New("select keyVersion from EncryptedResourceKeyRefVO " +
3222+
"where resourceType = :resourceType and resourceUuid = :resourceUuid")
3223+
.param("resourceType", TpmVO.class.getSimpleName())
3224+
.param("resourceUuid", tpmUuid)
3225+
.find();
3226+
if (keyVersionObj == null) {
3227+
innerTrigger.fail(operr("cannot find keyVersion in EncryptedResourceKeyRefVO for tpm[uuid:%s] before migrate", tpmUuid));
3228+
return;
3229+
}
3230+
if (!(keyVersionObj instanceof Number)) {
3231+
innerTrigger.fail(operr("invalid keyVersion type[%s] in EncryptedResourceKeyRefVO for tpm[uuid:%s] before migrate",
3232+
keyVersionObj.getClass().getName(), tpmUuid));
3233+
return;
3234+
}
3235+
3236+
keyVersionHolder[0] = ((Number) keyVersionObj).intValue();
3237+
KVMAgentCommands.SecretHostGetCmd getCmd = new KVMAgentCommands.SecretHostGetCmd();
3238+
getCmd.setVmUuid(vmUuid);
3239+
getCmd.setPurpose("vtpm");
3240+
getCmd.setKeyVersion(0);
3241+
getCmd.setUsageInstance(usageInstance);
3242+
restf.asyncJsonPost(buildUrl(KVMConstant.KVM_GET_SECRET_PATH), getCmd,
3243+
new JsonAsyncRESTCallback<KVMAgentCommands.SecretHostGetResponse>(innerTrigger) {
3244+
@Override
3245+
public void fail(ErrorCode err) {
3246+
innerTrigger.fail(err != null ? err : operr("get secret on source host failed before migrate"));
3247+
}
3248+
3249+
@Override
3250+
public void success(KVMAgentCommands.SecretHostGetResponse getRsp) {
3251+
if (getRsp == null || !getRsp.isSuccess() || StringUtils.isBlank(getRsp.getSecretUuid())) {
3252+
String details = getRsp == null ? "empty response" : getRsp.getError();
3253+
innerTrigger.fail(operr("failed to get source secret uuid before migrate: %s", details));
3254+
return;
3255+
}
3256+
3257+
sourceSecretUuidHolder[0] = getRsp.getSecretUuid();
3258+
innerTrigger.next();
3259+
}
3260+
3261+
@Override
3262+
public Class<KVMAgentCommands.SecretHostGetResponse> getReturnClass() {
3263+
return KVMAgentCommands.SecretHostGetResponse.class;
3264+
}
3265+
}, TimeUnit.SECONDS, KVMConstant.ENVELOPE_KEY_HTTP_TIMEOUT_SEC);
3266+
}
3267+
}).then(new NoRollbackFlow() {
3268+
String __name__ = "ensure-secret-on-dst";
3269+
3270+
@Override
3271+
public void run(FlowTrigger innerTrigger, Map data) {
3272+
SecretHostDefineMsg innerMsg = new SecretHostDefineMsg();
3273+
innerMsg.setHostUuid(dstHostUuid);
3274+
innerMsg.setVmUuid(vmUuid);
3275+
innerMsg.setDekBase64(result.getDekBase64());
3276+
innerMsg.setPurpose("vtpm");
3277+
innerMsg.setKeyVersion(0);
3278+
innerMsg.setUsageInstance(usageInstance);
3279+
innerMsg.setSecretUuid(sourceSecretUuidHolder[0]);
3280+
innerMsg.setDescription(String.format("Define secret for VM %s before live migration", vmUuid));
3281+
bus.makeTargetServiceIdByResourceUuid(innerMsg, HostConstant.SERVICE_ID, innerMsg.getHostUuid());
3282+
bus.send(innerMsg, new CloudBusCallBack(innerTrigger) {
3283+
@Override
3284+
public void run(MessageReply reply) {
3285+
if (reply.isSuccess()) {
3286+
innerTrigger.next();
3287+
} else {
3288+
innerTrigger.fail(reply.getError());
3289+
}
3290+
}
3291+
});
3292+
}
3293+
}).error(new FlowErrorHandler(trigger) {
3294+
@Override
3295+
public void handle(ErrorCode errCode, Map data) {
3296+
trigger.fail(errCode);
3297+
}
3298+
}).done(new FlowDoneHandler(trigger) {
3299+
@Override
3300+
public void handle(Map data) {
3301+
trigger.next();
3302+
}
3303+
}).start();
3304+
}
3305+
3306+
@Override
3307+
public void fail(ErrorCode errorCode) {
3308+
trigger.fail(errorCode);
3309+
}
3310+
});
3311+
}
3312+
3313+
});
3314+
31653315
flow(new NoRollbackFlow() {
31663316
String __name__ = "migrate-vm";
31673317

@@ -5317,8 +5467,9 @@ private void handle(SecretHostDefineMsg msg) {
53175467
bus.reply(msg, reply);
53185468
return;
53195469
}
5320-
if (StringUtils.isBlank(msg.getVmUuid()) || StringUtils.isBlank(msg.getPurpose()) || StringUtils.isBlank(msg.getProviderName())) {
5321-
reply.setError(operr("vmUuid, purpose and providerName are required for ensure secret"));
5470+
if (StringUtils.isBlank(msg.getVmUuid()) || StringUtils.isBlank(msg.getPurpose())
5471+
|| msg.getKeyVersion() == null || StringUtils.isBlank(msg.getUsageInstance())) {
5472+
reply.setError(operr("vmUuid, purpose, keyVersion and usageInstance are required for ensure secret"));
53225473
bus.reply(msg, reply);
53235474
return;
53245475
}
@@ -5396,7 +5547,9 @@ private void handle(SecretHostDefineMsg msg) {
53965547
cmd.setEncryptedDek(envelopeDekBase64);
53975548
cmd.setVmUuid(msg.getVmUuid());
53985549
cmd.setPurpose(msg.getPurpose());
5399-
cmd.setProviderName(msg.getProviderName());
5550+
cmd.setKeyVersion(msg.getKeyVersion());
5551+
cmd.setUsageInstance(msg.getUsageInstance());
5552+
cmd.setSecretUuid(msg.getSecretUuid());
54005553
cmd.setDescription(msg.getDescription() != null ? msg.getDescription() : "");
54015554
restf.asyncJsonPost(url, cmd, new JsonAsyncRESTCallback<KVMAgentCommands.SecretHostDefineResponse>(msg, reply) {
54025555
@Override

‎plugin/kvm/src/main/java/org/zstack/kvm/tpm/KvmTpmExtensions.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -231,7 +231,8 @@ public void run(FlowTrigger trigger, Map data) {
231231
innerMsg.setVmUuid(spec.getVmInventory().getUuid());
232232
innerMsg.setDekBase64(context.dekBase64);
233233
innerMsg.setPurpose("vtpm");
234-
innerMsg.setProviderName(context.providerName);
234+
innerMsg.setKeyVersion(0);
235+
innerMsg.setUsageInstance("tpm0");
235236
innerMsg.setDescription("Define secret for VM " + spec.getVmInventory().getUuid());
236237
bus.makeTargetServiceIdByResourceUuid(innerMsg, HostConstant.SERVICE_ID, innerMsg.getHostUuid());
237238
bus.send(innerMsg, new CloudBusCallBack(trigger) {

0 commit comments

Comments
 (0)