Skip to content

Commit 74de51b

Browse files
author
zhong.zhou
committed
<fix>[kvm]: define secret on migrate destination
Resolves: ZSV-11729 Change-Id: I616a776e78666179637976616c776162626c6533
1 parent 78c679d commit 74de51b

6 files changed

Lines changed: 188 additions & 74 deletions

File tree

‎compute/src/main/java/org/zstack/compute/vm/VmInstanceBase.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7947,6 +7947,7 @@ public void handle(final ErrorCode errCode, Map data) {
79477947
// the resource uuid is set
79487948
try {
79497949
dbf.eoCleanup(VmInstanceVO.class, self.getUuid());
7950+
// detach resource ref
79507951
} catch (Exception e) {
79517952
logger.warn(e.getMessage());
79527953
}

‎header/src/main/java/org/zstack/header/keyprovider/EncryptedResourceKeyManager.java‎

Lines changed: 4 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,10 @@ void getOrCreateKey(GetOrCreateResourceKeyContext ctx,
2929
/**
3030
* Roll back a newly created resource key during upper-layer workflow rollback.
3131
* <p>
32-
* If the key record already existed before creation, implementation should restore it
33-
* to its previous empty-placeholder state instead of deleting the relationship.
32+
* When {@link ResourceKeyResult#isCreatedNewKey()} is true, the implementation deletes the
33+
* key-tool secret if one was materialized, then removes the {@code EncryptedResourceKeyRef} row
34+
* for the resource (same storage effect as detaching the key provider from the resource).
35+
* When {@code createdNewKey} is false (existing secret was reused), this is a no-op.
3436
*/
3537
void rollbackCreatedKey(ResourceKeyResult result, Completion completion);
3638

@@ -91,7 +93,6 @@ class ResourceKeyResult {
9193
private String dekBase64;
9294
private String secretRef;
9395
private boolean createdNewKey;
94-
private boolean refExistedBeforeCreate;
9596

9697
public String getResourceUuid() {
9798
return resourceUuid;
@@ -156,13 +157,5 @@ public boolean isCreatedNewKey() {
156157
public void setCreatedNewKey(boolean createdNewKey) {
157158
this.createdNewKey = createdNewKey;
158159
}
159-
160-
public boolean isRefExistedBeforeCreate() {
161-
return refExistedBeforeCreate;
162-
}
163-
164-
public void setRefExistedBeforeCreate(boolean refExistedBeforeCreate) {
165-
this.refExistedBeforeCreate = refExistedBeforeCreate;
166-
}
167160
}
168161
}

‎header/src/main/java/org/zstack/header/secret/SecretHostDefineMsg.java‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@ public class SecretHostDefineMsg extends NeedReplyMessage implements HostMessage
1717
private String purpose;
1818
private Integer keyVersion;
1919
private String description;
20+
private String usageInstance;
21+
private String secretUuid;
2022

2123
@Override
2224
public String getHostUuid() {
@@ -66,4 +68,20 @@ public String getDescription() {
6668
public void setDescription(String description) {
6769
this.description = description;
6870
}
71+
72+
public String getUsageInstance() {
73+
return usageInstance;
74+
}
75+
76+
public void setUsageInstance(String usageInstance) {
77+
this.usageInstance = usageInstance;
78+
}
79+
80+
public String getSecretUuid() {
81+
return secretUuid;
82+
}
83+
84+
public void setSecretUuid(String secretUuid) {
85+
this.secretUuid = secretUuid;
86+
}
6987
}

‎plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java‎

Lines changed: 154 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@
1212
import org.zstack.compute.cluster.arch.ClusterResourceConfigInitializer;
1313
import org.zstack.compute.host.*;
1414
import org.zstack.compute.vm.*;
15+
import org.zstack.compute.vm.devices.TpmEncryptedResourceKeyBackend;
16+
import org.zstack.compute.vm.devices.VmTpmManager;
1517
import org.zstack.core.asyncbatch.While;
1618
import org.zstack.core.cloudbus.ResourceDestinationMaker;
1719
import org.zstack.core.timeout.TimeHelper;
@@ -55,6 +57,9 @@
5557
import org.zstack.header.exception.CloudRuntimeException;
5658
import org.zstack.header.host.*;
5759
import org.zstack.header.host.MigrateVmOnHypervisorMsg.StorageMigrationPolicy;
60+
import org.zstack.header.keyprovider.EncryptedResourceKeyManager;
61+
import org.zstack.header.keyprovider.EncryptedResourceKeyManager.GetOrCreateResourceKeyContext;
62+
import org.zstack.header.keyprovider.EncryptedResourceKeyManager.ResourceKeyResult;
5863
import org.zstack.header.secret.SecretHostDefineMsg;
5964
import org.zstack.header.secret.SecretHostDefineReply;
6065
import org.zstack.header.secret.SecretHostDeleteMsg;
@@ -67,6 +72,7 @@
6772
import org.zstack.header.message.NeedReplyMessage;
6873
import org.zstack.header.network.l2.*;
6974
import org.zstack.header.network.l3.L3NetworkInventory;
75+
import org.zstack.header.tpm.entity.TpmVO;
7076
import org.zstack.header.network.l3.L3NetworkVO;
7177
import org.zstack.header.rest.JsonAsyncRESTCallback;
7278
import org.zstack.header.rest.RESTFacade;
@@ -168,6 +174,10 @@ public class KVMHost extends HostBase implements Host {
168174
private AccountManager accountMgr;
169175
@Autowired
170176
private ResourceDestinationMaker destMaker;
177+
@Autowired
178+
private TpmEncryptedResourceKeyBackend tpmKeyBackend;
179+
@Autowired
180+
private EncryptedResourceKeyManager resourceKeyManager;
171181

172182
private KVMHostContext context;
173183

@@ -3170,6 +3180,146 @@ public void fail(ErrorCode errorCode) {
31703180
}
31713181
});
31723182

3183+
flow(new NoRollbackFlow() {
3184+
String __name__ = "ensure-secret-on-dst-before-migrate";
3185+
3186+
@Override
3187+
public void run(FlowTrigger trigger, Map data) {
3188+
String tpmUuid = VmTpmManager.findTpmUuidForVmOrNull(vmUuid);
3189+
if (StringUtils.isBlank(tpmUuid)) {
3190+
trigger.next();
3191+
return;
3192+
}
3193+
String providerUuid = tpmKeyBackend.findKeyProviderUuidByTpm(tpmUuid);
3194+
String providerName = tpmKeyBackend.findKeyProviderNameByTpm(tpmUuid);
3195+
if (StringUtils.isBlank(providerUuid) && StringUtils.isBlank(providerName)) {
3196+
trigger.fail(operr("missing TPM resource key binding for tpm[uuid:%s] before migrate", tpmUuid));
3197+
return;
3198+
}
3199+
3200+
GetOrCreateResourceKeyContext keyCtx = new GetOrCreateResourceKeyContext();
3201+
keyCtx.setResourceUuid(tpmUuid);
3202+
keyCtx.setResourceType(TpmVO.class.getSimpleName());
3203+
keyCtx.setKeyProviderUuid(providerUuid);
3204+
keyCtx.setKeyProviderName(providerName);
3205+
keyCtx.setPurpose("vtpm");
3206+
resourceKeyManager.getOrCreateKey(keyCtx, new ReturnValueCompletion<ResourceKeyResult>(trigger) {
3207+
@Override
3208+
public void success(ResourceKeyResult result) {
3209+
String effectiveProviderName = result.getKeyProviderName();
3210+
if (StringUtils.isBlank(effectiveProviderName)) {
3211+
effectiveProviderName = providerName;
3212+
}
3213+
if (StringUtils.isBlank(effectiveProviderName)) {
3214+
trigger.fail(operr("missing effective key provider name for tpm[uuid:%s] before migrate", tpmUuid));
3215+
return;
3216+
}
3217+
3218+
final String usageInstance = "tpm0";
3219+
final Integer[] keyVersionHolder = new Integer[1];
3220+
final String[] sourceSecretUuidHolder = new String[1];
3221+
3222+
FlowChain chain = FlowChainBuilder.newSimpleFlowChain();
3223+
chain.setName(String.format("get-and-ensure-vtpm-secret-vm-%s", vmUuid));
3224+
chain.then(new NoRollbackFlow() {
3225+
String __name__ = "get-source-secret-uuid";
3226+
3227+
@Override
3228+
public void run(FlowTrigger innerTrigger, Map data) {
3229+
Object keyVersionObj = SQL.New("select keyVersion from EncryptedResourceKeyRefVO " +
3230+
"where resourceType = :resourceType and resourceUuid = :resourceUuid")
3231+
.param("resourceType", TpmVO.class.getSimpleName())
3232+
.param("resourceUuid", tpmUuid)
3233+
.find();
3234+
if (keyVersionObj == null) {
3235+
innerTrigger.fail(operr("cannot find keyVersion in EncryptedResourceKeyRefVO for tpm[uuid:%s] before migrate", tpmUuid));
3236+
return;
3237+
}
3238+
if (!(keyVersionObj instanceof Number)) {
3239+
innerTrigger.fail(operr("invalid keyVersion type[%s] in EncryptedResourceKeyRefVO for tpm[uuid:%s] before migrate",
3240+
keyVersionObj.getClass().getName(), tpmUuid));
3241+
return;
3242+
}
3243+
3244+
keyVersionHolder[0] = ((Number) keyVersionObj).intValue();
3245+
KVMAgentCommands.SecretHostGetCmd getCmd = new KVMAgentCommands.SecretHostGetCmd();
3246+
getCmd.setVmUuid(vmUuid);
3247+
getCmd.setPurpose("vtpm");
3248+
getCmd.setKeyVersion(0);
3249+
getCmd.setUsageInstance(usageInstance);
3250+
restf.asyncJsonPost(buildUrl(KVMConstant.KVM_GET_SECRET_PATH), getCmd,
3251+
new JsonAsyncRESTCallback<KVMAgentCommands.SecretHostGetResponse>(innerTrigger) {
3252+
@Override
3253+
public void fail(ErrorCode err) {
3254+
innerTrigger.fail(err != null ? err : operr("get secret on source host failed before migrate"));
3255+
}
3256+
3257+
@Override
3258+
public void success(KVMAgentCommands.SecretHostGetResponse getRsp) {
3259+
if (getRsp == null || !getRsp.isSuccess() || StringUtils.isBlank(getRsp.getSecretUuid())) {
3260+
String details = getRsp == null ? "empty response" : getRsp.getError();
3261+
innerTrigger.fail(operr("failed to get source secret uuid before migrate: %s", details));
3262+
return;
3263+
}
3264+
3265+
sourceSecretUuidHolder[0] = getRsp.getSecretUuid();
3266+
innerTrigger.next();
3267+
}
3268+
3269+
@Override
3270+
public Class<KVMAgentCommands.SecretHostGetResponse> getReturnClass() {
3271+
return KVMAgentCommands.SecretHostGetResponse.class;
3272+
}
3273+
}, TimeUnit.SECONDS, KVMConstant.ENVELOPE_KEY_HTTP_TIMEOUT_SEC);
3274+
}
3275+
}).then(new NoRollbackFlow() {
3276+
String __name__ = "ensure-secret-on-dst";
3277+
3278+
@Override
3279+
public void run(FlowTrigger innerTrigger, Map data) {
3280+
SecretHostDefineMsg innerMsg = new SecretHostDefineMsg();
3281+
innerMsg.setHostUuid(dstHostUuid);
3282+
innerMsg.setVmUuid(vmUuid);
3283+
innerMsg.setDekBase64(result.getDekBase64());
3284+
innerMsg.setPurpose("vtpm");
3285+
innerMsg.setKeyVersion(0);
3286+
innerMsg.setUsageInstance(usageInstance);
3287+
innerMsg.setSecretUuid(sourceSecretUuidHolder[0]);
3288+
innerMsg.setDescription(String.format("Define secret for VM %s before live migration", vmUuid));
3289+
bus.makeTargetServiceIdByResourceUuid(innerMsg, HostConstant.SERVICE_ID, innerMsg.getHostUuid());
3290+
bus.send(innerMsg, new CloudBusCallBack(innerTrigger) {
3291+
@Override
3292+
public void run(MessageReply reply) {
3293+
if (reply.isSuccess()) {
3294+
innerTrigger.next();
3295+
} else {
3296+
innerTrigger.fail(reply.getError());
3297+
}
3298+
}
3299+
});
3300+
}
3301+
}).error(new FlowErrorHandler(trigger) {
3302+
@Override
3303+
public void handle(ErrorCode errCode, Map data) {
3304+
trigger.fail(errCode);
3305+
}
3306+
}).done(new FlowDoneHandler(trigger) {
3307+
@Override
3308+
public void handle(Map data) {
3309+
trigger.next();
3310+
}
3311+
}).start();
3312+
}
3313+
3314+
@Override
3315+
public void fail(ErrorCode errorCode) {
3316+
trigger.fail(errorCode);
3317+
}
3318+
});
3319+
}
3320+
3321+
});
3322+
31733323
flow(new NoRollbackFlow() {
31743324
String __name__ = "migrate-vm";
31753325

@@ -5372,8 +5522,9 @@ private void handle(SecretHostDefineMsg msg) {
53725522
bus.reply(msg, reply);
53735523
return;
53745524
}
5375-
if (StringUtils.isBlank(msg.getVmUuid()) || StringUtils.isBlank(msg.getPurpose()) || msg.getKeyVersion() == null) {
5376-
reply.setError(operr("vmUuid, purpose and keyVersion are required for ensure secret"));
5525+
if (StringUtils.isBlank(msg.getVmUuid()) || StringUtils.isBlank(msg.getPurpose())
5526+
|| msg.getKeyVersion() == null || StringUtils.isBlank(msg.getUsageInstance())) {
5527+
reply.setError(operr("vmUuid, purpose, keyVersion and usageInstance are required for ensure secret")); reply.setError(operr("vmUuid, purpose and keyVersion are required for ensure secret"));
53775528
bus.reply(msg, reply);
53785529
return;
53795530
}
@@ -5452,6 +5603,7 @@ private void handle(SecretHostDefineMsg msg) {
54525603
cmd.setVmUuid(msg.getVmUuid());
54535604
cmd.setPurpose(msg.getPurpose());
54545605
cmd.setKeyVersion(msg.getKeyVersion());
5606+
cmd.setSecretUuid(msg.getSecretUuid());
54555607
cmd.setDescription(msg.getDescription() != null ? msg.getDescription() : "");
54565608
cmd.setUsageInstance(KVMConstant.HOST_SECRET_USAGE_INSTANCE_VTPM);
54575609
Map<String, String> headers = new HashMap<>();

0 commit comments

Comments
 (0)