Skip to content

Commit 6600b98

Browse files
yuerong996yuerong996
committed
Merge branch 'fb-reg-1-ZSV-11559@@3' into 'feature-zsv-5.0.0-vm-support-vtpm-and-secuceboot'
<feature>[vm]: add MetadataImpact See merge request zstackio/zstack!9523
2 parents 7619fd7 + ab1b9a4 commit 6600b98

150 files changed

Lines changed: 5003 additions & 13 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎compute/src/main/java/org/zstack/compute/vm/CleanupVmInstanceMetadataOnPrimaryStorageGC.java‎

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
package org.zstack.compute.vm;
2+
3+
import org.springframework.beans.factory.annotation.Autowired;
4+
import org.zstack.core.cloudbus.CloudBusCallBack;
5+
import org.zstack.core.componentloader.PluginRegistry;
6+
import org.zstack.core.db.Q;
7+
import org.zstack.core.gc.GC;
8+
import org.zstack.core.gc.GCCompletion;
9+
import org.zstack.core.gc.TimeBasedGarbageCollector;
10+
import org.zstack.header.host.HostVO;
11+
import org.zstack.header.message.MessageReply;
12+
import org.zstack.header.storage.primary.CleanupVmInstanceMetadataOnPrimaryStorageMsg;
13+
import org.zstack.header.storage.primary.PrimaryStorageConstant;
14+
import org.zstack.header.storage.primary.PrimaryStorageVO;
15+
import org.zstack.header.storage.primary.PrimaryStorageVO_;
16+
import org.zstack.header.vm.metadata.VmMetadataPathBuildExtensionPoint;
17+
import org.zstack.utils.Utils;
18+
import org.zstack.utils.logging.CLogger;
19+
20+
public class CleanupVmInstanceMetadataOnPrimaryStorageGC extends TimeBasedGarbageCollector {
21+
private static final CLogger logger = Utils.getLogger(CleanupVmInstanceMetadataOnPrimaryStorageGC.class);
22+
23+
@Autowired
24+
private PluginRegistry pluginRgty;
25+
26+
@GC
27+
public String primaryStorageUuid;
28+
@GC
29+
public String vmUuid;
30+
@GC
31+
public String rootVolumeUuid;
32+
@GC
33+
public String metadataPath;
34+
@GC
35+
public String hostUuid;
36+
37+
public static String getGCName(String vmUuid) {
38+
return String.format("gc-cleanup-vm-metadata-%s", vmUuid);
39+
}
40+
41+
@Override
42+
protected void triggerNow(GCCompletion completion) {
43+
if (!dbf.isExist(primaryStorageUuid, PrimaryStorageVO.class)) {
44+
logger.debug(String.format("[MetadataCleanupGC] primary storage[uuid:%s] no longer exists, " +
45+
"cancel gc for vm[uuid:%s]", primaryStorageUuid, vmUuid));
46+
completion.cancel();
47+
return;
48+
}
49+
50+
String psType = Q.New(PrimaryStorageVO.class).select(PrimaryStorageVO_.type).eq(PrimaryStorageVO_.uuid, primaryStorageUuid).findValue();
51+
if (psType == null) {
52+
logger.debug(String.format("[MetadataCleanupGC] primary storage[uuid:%s] type not found, " +
53+
"cancel gc for vm[uuid:%s]", primaryStorageUuid, vmUuid));
54+
completion.cancel();
55+
return;
56+
}
57+
58+
VmMetadataPathBuildExtensionPoint ext = pluginRgty.getExtensionFromMap(psType, VmMetadataPathBuildExtensionPoint.class);
59+
boolean requireHost = ext != null && ext.requireHostForCleanup();
60+
61+
// Determine effective hostUuid based on whether the PS type requires a host for cleanup.
62+
String effectiveHostUuid = hostUuid;
63+
if (!requireHost) {
64+
effectiveHostUuid = null;
65+
} else {
66+
if (effectiveHostUuid == null) {
67+
logger.debug(String.format("[MetadataCleanupGC] hostUuid is null and ps[uuid:%s, type:%s] " +
68+
"requires host for cleanup, cancel gc for vm[uuid:%s]",
69+
primaryStorageUuid, psType, vmUuid));
70+
completion.cancel();
71+
return;
72+
}
73+
if (!dbf.isExist(effectiveHostUuid, HostVO.class)) {
74+
logger.debug(String.format("[MetadataCleanupGC] host[uuid:%s] no longer exists " +
75+
"and ps[uuid:%s, type:%s] requires host for cleanup, " +
76+
"metadata is unreachable, cancel gc for vm[uuid:%s]",
77+
effectiveHostUuid, primaryStorageUuid, psType, vmUuid));
78+
completion.cancel();
79+
return;
80+
}
81+
}
82+
83+
CleanupVmInstanceMetadataOnPrimaryStorageMsg msg = new CleanupVmInstanceMetadataOnPrimaryStorageMsg();
84+
msg.setPrimaryStorageUuid(primaryStorageUuid);
85+
msg.setVmInstanceUuid(vmUuid);
86+
msg.setRootVolumeUuid(rootVolumeUuid);
87+
msg.setMetadataPath(metadataPath);
88+
msg.setHostUuid(effectiveHostUuid);
89+
90+
bus.makeTargetServiceIdByResourceUuid(msg, PrimaryStorageConstant.SERVICE_ID, primaryStorageUuid);
91+
bus.send(msg, new CloudBusCallBack(completion) {
92+
@Override
93+
public void run(MessageReply reply) {
94+
if (reply.isSuccess()) {
95+
logger.info(String.format("[MetadataCleanupGC] successfully cleaned up metadata " +
96+
"for vm[uuid:%s] on ps[uuid:%s]", vmUuid, primaryStorageUuid));
97+
completion.success();
98+
} else {
99+
logger.warn(String.format("[MetadataCleanupGC] failed to clean up metadata " +
100+
"for vm[uuid:%s] on ps[uuid:%s]: %s", vmUuid, primaryStorageUuid, reply.getError()));
101+
completion.fail(reply.getError());
102+
}
103+
}
104+
});
105+
}
106+
}

‎compute/src/main/java/org/zstack/compute/vm/VmExpungeMetadataFlow.java‎

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
package org.zstack.compute.vm;
2+
3+
import org.springframework.beans.factory.annotation.Autowire;
4+
import org.springframework.beans.factory.annotation.Autowired;
5+
import org.springframework.beans.factory.annotation.Configurable;
6+
import org.zstack.core.cloudbus.CloudBus;
7+
import org.zstack.core.cloudbus.CloudBusCallBack;
8+
import org.zstack.core.componentloader.PluginRegistry;
9+
import org.zstack.core.db.Q;
10+
import org.zstack.header.core.workflow.FlowTrigger;
11+
import org.zstack.header.core.workflow.NoRollbackFlow;
12+
import org.zstack.header.message.MessageReply;
13+
import org.zstack.header.storage.primary.CleanupVmInstanceMetadataOnPrimaryStorageMsg;
14+
import org.zstack.header.storage.primary.PrimaryStorageConstant;
15+
import org.zstack.header.storage.primary.PrimaryStorageVO;
16+
import org.zstack.header.storage.primary.PrimaryStorageVO_;
17+
import org.zstack.header.vm.VmInstanceConstant;
18+
import org.zstack.header.vm.VmInstanceSpec;
19+
import org.zstack.header.vm.metadata.VmMetadataPathBuildExtensionPoint;
20+
import org.zstack.header.volume.VolumeInventory;
21+
import org.zstack.utils.Utils;
22+
import org.zstack.utils.logging.CLogger;
23+
24+
import java.util.Map;
25+
import java.util.concurrent.TimeUnit;
26+
27+
@Configurable(preConstruction = true, autowire = Autowire.BY_TYPE)
28+
public class VmExpungeMetadataFlow extends NoRollbackFlow {
29+
private static final CLogger logger = Utils.getLogger(VmExpungeMetadataFlow.class);
30+
31+
@Autowired
32+
private CloudBus bus;
33+
@Autowired
34+
private PluginRegistry pluginRgty;
35+
36+
@Override
37+
public void run(FlowTrigger trigger, Map data) {
38+
if (!VmGlobalConfig.VM_METADATA_ENABLED.value(Boolean.class)) {
39+
trigger.next();
40+
return;
41+
}
42+
43+
final VmInstanceSpec spec = (VmInstanceSpec) data.get(VmInstanceConstant.Params.VmInstanceSpec.toString());
44+
if (spec == null || spec.getVmInventory() == null) {
45+
logger.warn("[MetadataExpunge] missing VmInstanceSpec or VmInventory, skip metadata cleanup");
46+
trigger.next();
47+
return;
48+
}
49+
50+
final String vmUuid = spec.getVmInventory().getUuid();
51+
52+
VolumeInventory rootVolume = spec.getVmInventory().getRootVolume();
53+
String psUuid = rootVolume != null ? rootVolume.getPrimaryStorageUuid() : null;
54+
if (psUuid == null) {
55+
logger.debug(String.format("[MetadataExpunge] vm[uuid:%s] root volume has no primaryStorageUuid, " +
56+
"skipping metadata cleanup", vmUuid));
57+
trigger.next();
58+
return;
59+
}
60+
61+
62+
String psType = Q.New(PrimaryStorageVO.class).select(PrimaryStorageVO_.type).eq(PrimaryStorageVO_.uuid, psUuid).findValue();
63+
if (psType == null) {
64+
logger.warn(String.format("[MetadataExpunge] primary storage[uuid:%s] not found for vm[uuid:%s], " +
65+
"skip metadata cleanup", psUuid, vmUuid));
66+
trigger.next();
67+
return;
68+
}
69+
70+
VmMetadataPathBuildExtensionPoint ext = pluginRgty.getExtensionFromMap(psType, VmMetadataPathBuildExtensionPoint.class);
71+
if (ext == null) {
72+
logger.warn(String.format("[MetadataExpunge] no VmMetadataPathBuildExtensionPoint found for ps[uuid:%s, type:%s], " +
73+
"skip metadata cleanup", psUuid, psType));
74+
trigger.next();
75+
return;
76+
}
77+
final String metadataPath;
78+
try {
79+
metadataPath = ext.buildVmMetadataPath(psUuid, vmUuid);
80+
} catch (Exception e) {
81+
logger.warn(String.format("[MetadataExpunge] failed to build metadata path for vm[uuid:%s] on ps[uuid:%s], " +
82+
"skip metadata cleanup: %s", vmUuid, psUuid, e.getMessage()));
83+
trigger.next();
84+
return;
85+
}
86+
87+
String hostUuid = null;
88+
if (ext.requireHostForCleanup()) {
89+
hostUuid = spec.getVmInventory().getHostUuid();
90+
if (hostUuid == null) {
91+
hostUuid = spec.getVmInventory().getLastHostUuid();
92+
}
93+
94+
if (hostUuid == null) {
95+
logger.warn(String.format("[MetadataExpunge] vm[uuid:%s] hostUuid is null, " +
96+
"ps[uuid:%s, type:%s] requires host for cleanup, skip without submitting GC",
97+
vmUuid, psUuid, psType));
98+
trigger.next();
99+
return;
100+
}
101+
}
102+
103+
String rootVolumeUuid = rootVolume.getUuid();
104+
CleanupVmInstanceMetadataOnPrimaryStorageMsg cmsg = new CleanupVmInstanceMetadataOnPrimaryStorageMsg();
105+
cmsg.setPrimaryStorageUuid(psUuid);
106+
cmsg.setVmInstanceUuid(vmUuid);
107+
cmsg.setMetadataPath(metadataPath);
108+
cmsg.setRootVolumeUuid(rootVolumeUuid);
109+
cmsg.setHostUuid(hostUuid);
110+
final String finalPsUuid = psUuid;
111+
final String finalHostUuid = hostUuid;
112+
113+
bus.makeTargetServiceIdByResourceUuid(cmsg, PrimaryStorageConstant.SERVICE_ID, psUuid);
114+
bus.send(cmsg, new CloudBusCallBack(trigger) {
115+
@Override
116+
public void run(MessageReply reply) {
117+
if (reply.isSuccess()) {
118+
logger.info(String.format("[MetadataExpunge] successfully deleted metadata for vm[uuid:%s] on ps[uuid:%s]",
119+
vmUuid, finalPsUuid));
120+
} else {
121+
logger.warn(String.format("[MetadataExpunge] failed to delete metadata for vm[uuid:%s] on ps[uuid:%s]: %s, " +
122+
"submitting GC job for retry", vmUuid, finalPsUuid, reply.getError()));
123+
submitGC(finalPsUuid, vmUuid, rootVolumeUuid, metadataPath, finalHostUuid);
124+
}
125+
trigger.next();
126+
}
127+
});
128+
}
129+
130+
private void submitGC(String psUuid, String vmUuid, String rootVolumeUuid, String metadataPath, String hostUuid) {
131+
CleanupVmInstanceMetadataOnPrimaryStorageGC gc = new CleanupVmInstanceMetadataOnPrimaryStorageGC();
132+
gc.NAME = CleanupVmInstanceMetadataOnPrimaryStorageGC.getGCName(vmUuid);
133+
gc.primaryStorageUuid = psUuid;
134+
gc.vmUuid = vmUuid;
135+
gc.rootVolumeUuid = rootVolumeUuid;
136+
gc.metadataPath = metadataPath;
137+
gc.hostUuid = hostUuid;
138+
long gcIntervalSec = TimeUnit.HOURS.toSeconds(VmGlobalConfig.VM_METADATA_CLEANUP_GC_INTERVAL.value(Long.class));
139+
gc.deduplicateSubmit(gcIntervalSec, TimeUnit.SECONDS);
140+
141+
logger.info(String.format("[MetadataExpunge] submitted GC job [%s] for vm[uuid:%s] on ps[uuid:%s]", gc.NAME, vmUuid, psUuid));
142+
}
143+
}

‎compute/src/main/java/org/zstack/compute/vm/VmGlobalConfig.java‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -143,4 +143,52 @@ public class VmGlobalConfig {
143143
@GlobalConfigDef(defaultValue = "false", type = Boolean.class, description = "allowed TPM VM start without KMS")
144144
@GlobalConfigValidation(validValues = {"true", "false"})
145145
public static GlobalConfig ALLOWED_TPM_VM_WITHOUT_KMS = new GlobalConfig(CATEGORY, "allowed.tpm.vm.without.kms");
146+
147+
@GlobalConfigValidation(validValues = {"true", "false"})
148+
public static GlobalConfig VM_METADATA_ENABLED = new GlobalConfig(CATEGORY, "vm.metadata.enabled");
149+
150+
@GlobalConfigValidation()
151+
public static GlobalConfig VM_METADATA_LAST_REFRESH_VERSION = new GlobalConfig(CATEGORY, "vm.metadata.lastRefreshVersion");
152+
153+
@GlobalConfigValidation(numberGreaterThan = 0, numberLessThan = 100)
154+
public static GlobalConfig VM_METADATA_FLUSH_CONCURRENCY = new GlobalConfig(CATEGORY, "vm.metadata.flush.concurrency");
155+
156+
@GlobalConfigValidation(numberGreaterThan = 0, numberLessThan = 300)
157+
public static GlobalConfig VM_METADATA_FLUSH_POLL_INTERVAL = new GlobalConfig(CATEGORY, "vm.metadata.flush.pollInterval");
158+
159+
@GlobalConfigValidation(numberGreaterThan = 0, numberLessThan = 1000)
160+
public static GlobalConfig VM_METADATA_FLUSH_BATCH_SIZE = new GlobalConfig(CATEGORY, "vm.metadata.flush.batchSize");
161+
162+
@GlobalConfigValidation(numberGreaterThan = 0, numberLessThan = 168)
163+
public static GlobalConfig VM_METADATA_CLEANUP_GC_INTERVAL = new GlobalConfig(CATEGORY, "vm.metadata.cleanup.gc.interval");
164+
165+
@GlobalConfigValidation(numberGreaterThan = 0, numberLessThan = 10)
166+
public static GlobalConfig VM_METADATA_FLUSH_MAX_RETRY = new GlobalConfig(CATEGORY, "vm.metadata.flush.maxRetry");
167+
168+
@GlobalConfigValidation(numberGreaterThan = 0, numberLessThan = 120)
169+
public static GlobalConfig VM_METADATA_FLUSH_ZOMBIE_CLAIM_THRESHOLD = new GlobalConfig(CATEGORY, "vm.metadata.flush.zombieClaimThreshold");
170+
171+
@GlobalConfigValidation(numberGreaterThan = 21599, numberLessThan = 172801)
172+
public static GlobalConfig VM_METADATA_MAINTENANCE_CONTENT_DRIFT_INTERVAL = new GlobalConfig(CATEGORY, "vm.metadata.maintenance.contentDriftInterval");
173+
174+
@GlobalConfigValidation(numberGreaterThan = 0, numberLessThan = 86400)
175+
public static GlobalConfig VM_METADATA_MAINTENANCE_STALE_RECOVERY_INTERVAL = new GlobalConfig(CATEGORY, "vm.metadata.maintenance.staleRecoveryInterval");
176+
177+
@GlobalConfigValidation(numberGreaterThan = 0, numberLessThan = 1000)
178+
public static GlobalConfig VM_METADATA_MAINTENANCE_STALE_RECOVERY_MAX_CYCLES = new GlobalConfig(CATEGORY, "vm.metadata.maintenance.staleRecoveryMaxCycles");
179+
180+
@GlobalConfigValidation(numberGreaterThan = 0)
181+
public static GlobalConfig VM_METADATA_PAYLOAD_REJECT_THRESHOLD = new GlobalConfig(CATEGORY, "vm.metadata.payload.rejectThreshold");
182+
183+
@GlobalConfigValidation(numberGreaterThan = 0, numberLessThan = 86400)
184+
public static GlobalConfig VM_METADATA_MAINTENANCE_ORPHAN_CHECK_INTERVAL = new GlobalConfig(CATEGORY, "vm.metadata.maintenance.orphanCheckInterval");
185+
186+
@GlobalConfigValidation(numberGreaterThan = 0, numberLessThan = 20)
187+
public static GlobalConfig VM_METADATA_MAINTENANCE_STALE_RECOVERY_BATCH_SIZE = new GlobalConfig(CATEGORY, "vm.metadata.maintenance.staleRecoveryBatchSize");
188+
189+
@GlobalConfigValidation(numberGreaterThan = 9, numberLessThan = 201)
190+
public static GlobalConfig VM_METADATA_MAINTENANCE_CONTENT_DRIFT_BATCH_SIZE = new GlobalConfig(CATEGORY, "vm.metadata.maintenance.contentDriftBatchSize");
191+
192+
@GlobalConfigValidation(numberGreaterThan = 0, numberLessThan = 31)
193+
public static GlobalConfig VM_METADATA_MAINTENANCE_CONTENT_DRIFT_BATCH_SLEEP_SEC = new GlobalConfig(CATEGORY, "vm.metadata.maintenance.contentDriftBatchSleepSec");
146194
}

0 commit comments

Comments
 (0)