refactor(project): Refactor project cache validation

Author: RandomByte

packages/project/lib/build/ProjectBuilder.js

@@ -180,7 +180,7 @@ class ProjectBuilder {
 			}
 		}
 
-		const projectBuildContexts = await this._createRequiredBuildContexts(requestedProjects);
+		const projectBuildContexts = await this._buildContext.createRequiredProjectContexts(requestedProjects);
 		let fsTarget;
 		if (destPath) {
 			fsTarget = resourceFactory.createAdapter({
@@ -236,7 +236,16 @@ class ProjectBuilder {
 		}));
 
 		const alreadyBuilt = [];
+		const changedDependencyResources = [];
 		for (const projectBuildContext of queue) {
+			if (changedDependencyResources.length) {
+				// Notify build cache of changed resources from dependencies
+				projectBuildContext.dependencyResourcesChanged(changedDependencyResources);
+			}
+			const changedResources = await projectBuildContext.determineChangedResources();
+			for (const resourcePath of changedResources) {
+				changedDependencyResources.push(resourcePath);
+			}
 			if (!await projectBuildContext.requiresBuild()) {
 				const projectName = projectBuildContext.getProject().getName();
 				alreadyBuilt.push(projectName);
@@ -275,19 +284,24 @@ class ProjectBuilder {
 		try {
 			const startTime = process.hrtime();
 			const pWrites = [];
-			for (const projectBuildContext of queue) {
+			while (queue.length) {
+				const projectBuildContext = queue.shift();
 				const project = projectBuildContext.getProject();
 				const projectName = project.getName();
 				const projectType = project.getType();
 				this.#log.verbose(`Processing project ${projectName}...`);
 
 				// Only build projects that are not already build (i.e. provide a matching build manifest)
-				if (alreadyBuilt.includes(projectName)) {
+				if (alreadyBuilt.includes(projectName) || !(await projectBuildContext.requiresBuild())) {
 					this.#log.skipProjectBuild(projectName, projectType);
 				} else {
 					this.#log.startProjectBuild(projectName, projectType);
-					await projectBuildContext.getTaskRunner().runTasks();
+					const changedResources = await projectBuildContext.runTasks();
 					this.#log.endProjectBuild(projectName, projectType);
+					for (const pbc of queue) {
+						// Propagate resource changes to following projects
+						pbc.getBuildCache().dependencyResourcesChanged(changedResources);
+					}
 				}
 				if (!requestedProjects.includes(projectName)) {
 					// Project has not been requested
@@ -306,7 +320,7 @@ class ProjectBuilder {
 						project,
 						this._graph, this._buildContext.getBuildConfig(), this._buildContext.getTaskRepository(),
 						projectBuildContext.getBuildSignature());
-					pWrites.push(projectBuildContext.getBuildCache().storeCache(buildManifest));
+					pWrites.push(projectBuildContext.getBuildCache().writeCache(buildManifest));
 				}
 			}
 			await Promise.all(pWrites);
@@ -337,6 +351,7 @@ class ProjectBuilder {
 
 	async #update(projectBuildContexts, requestedProjects, fsTarget) {
 		const queue = [];
+		const changedDependencyResources = [];
 		await this._graph.traverseDepthFirst(async ({project}) => {
 			const projectName = project.getName();
 			const projectBuildContext = projectBuildContexts.get(projectName);
@@ -345,6 +360,15 @@ class ProjectBuilder {
 				//	=> This project needs to be built or, in case it has already
 				//		been built, it's build result needs to be written out (if requested)
 				queue.push(projectBuildContext);
+
+				if (changedDependencyResources.length) {
+					// Notify build cache of changed resources from dependencies
+					await projectBuildContext.dependencyResourcesChanged(changedDependencyResources);
+				}
+				const changedResources = await projectBuildContext.determineChangedResources();
+				for (const resourcePath of changedResources) {
+					changedDependencyResources.push(resourcePath);
+				}
 			}
 		});
 
@@ -353,7 +377,8 @@ class ProjectBuilder {
 		}));
 
 		const pWrites = [];
-		for (const projectBuildContext of queue) {
+		while (queue.length) {
+			const projectBuildContext = queue.shift();
 			const project = projectBuildContext.getProject();
 			const projectName = project.getName();
 			const projectType = project.getType();
@@ -365,8 +390,12 @@ class ProjectBuilder {
 			}
 
 			this.#log.startProjectBuild(projectName, projectType);
-			await projectBuildContext.runTasks();
+			const changedResources = await projectBuildContext.runTasks();
 			this.#log.endProjectBuild(projectName, projectType);
+			for (const pbc of queue) {
+				// Propagate resource changes to following projects
+				pbc.getBuildCache().dependencyResourcesChanged(changedResources);
+			}
 			if (!requestedProjects.includes(projectName)) {
 				// Project has not been requested
 				//	=> Its resources shall not be part of the build result
@@ -386,52 +415,11 @@ class ProjectBuilder {
 				project,
 				this._graph, this._buildContext.getBuildConfig(), this._buildContext.getTaskRepository(),
 				projectBuildContext.getBuildSignature());
-			pWrites.push(projectBuildContext.getBuildCache().storeCache(buildManifest));
+			pWrites.push(projectBuildContext.getBuildCache().writeCache(buildManifest));
 		}
 		await Promise.all(pWrites);
 	}
 
-	async _createRequiredBuildContexts(requestedProjects) {
-		const requiredProjects = new Set(this._graph.getProjectNames().filter((projectName) => {
-			return requestedProjects.includes(projectName);
-		}));
-
-		const projectBuildContexts = new Map();
-
-		for (const projectName of requiredProjects) {
-			this.#log.verbose(`Creating build context for project ${projectName}...`);
-			const projectBuildContext = await this._buildContext.createProjectContext({
-				project: this._graph.getProject(projectName)
-			});
-
-			projectBuildContexts.set(projectName, projectBuildContext);
-
-			if (await projectBuildContext.requiresBuild()) {
-				const taskRunner = projectBuildContext.getTaskRunner();
-				const requiredDependencies = await taskRunner.getRequiredDependencies();
-
-				if (requiredDependencies.size === 0) {
-					continue;
-				}
-				// This project needs to be built and required dependencies to be built as well
-				this._graph.getDependencies(projectName).forEach((depName) => {
-					if (projectBuildContexts.has(depName)) {
-						// Build context already exists
-						//	=> Dependency will be built
-						return;
-					}
-					if (!requiredDependencies.has(depName)) {
-						return;
-					}
-					// Add dependency to list of projects to build
-					requiredProjects.add(depName);
-				});
-			}
-		}
-
-		return projectBuildContexts;
-	}
-
 	async _getProjectFilter({
 		dependencyIncludes,
 		explicitIncludes,

packages/project/lib/build/TaskRunner.js

@@ -131,7 +131,7 @@ class TaskRunner {
 				await this._executeTask(taskName, taskFunction);
 			}
 		}
-		this._buildCache.allTasksCompleted();
+		return await this._buildCache.allTasksCompleted();
 	}
 
 	/**
@@ -485,13 +485,13 @@ class TaskRunner {
 	 * @returns {Promise} Resolves when task has finished
 	 */
 	async _executeTask(taskName, taskFunction, taskParams) {
-		if (this._buildCache.isTaskCacheValid(taskName)) {
-			// Immediately skip task if cache is valid
-			// Continue if cache is (potentially) invalid, in which case taskFunction will
-			// validate the cache thoroughly
-			this._log.skipTask(taskName);
-			return;
-		}
+		// if (this._buildCache.isTaskCacheValid(taskName)) {
+		// 	// Immediately skip task if cache is valid
+		// 	// Continue if cache is (potentially) invalid, in which case taskFunction will
+		// 	// validate the cache thoroughly
+		// 	this._log.skipTask(taskName);
+		// 	return;
+		// }
 		this._taskStart = performance.now();
 		await taskFunction(taskParams, this._log);
 		if (this._log.isLevelEnabled("perf")) {

packages/project/lib/build/cache/BuildTaskCache.js

@@ -43,7 +43,7 @@ export default class BuildTaskCache {
 	#readTaskMetadataCache;
 	#treeRegistries = [];
 	#useDifferentialUpdate = true;
-	#isNewOrModified;
+	#hasNewOrModifiedCacheEntries = true;
 
 	// ===== LIFECYCLE =====
 
@@ -67,7 +67,7 @@ export default class BuildTaskCache {
 		if (!this.#readTaskMetadataCache) {
 			// No cache reader provided, start with empty graph
 			this.#resourceRequests = new ResourceRequestGraph();
-			this.#isNewOrModified = true;
+			this.#hasNewOrModifiedCacheEntries = true;
 			return;
 		}
 
@@ -78,7 +78,7 @@ export default class BuildTaskCache {
 				`of project '${this.#projectName}'`);
 		}
 		this.#resourceRequests = this.#restoreGraphFromCache(taskMetadata);
-		this.#isNewOrModified = false;
+		this.#hasNewOrModifiedCacheEntries = false; // Using cache
 	}
 
 	// ===== METADATA ACCESS =====
@@ -92,8 +92,8 @@ export default class BuildTaskCache {
 		return this.#taskName;
 	}
 
-	isNewOrModified() {
-		return this.#isNewOrModified;
+	hasNewOrModifiedCacheEntries() {
+		return this.#hasNewOrModifiedCacheEntries;
 	}
 
 	/**
@@ -405,7 +405,7 @@ export default class BuildTaskCache {
 		if (!relevantTree) {
 			return;
 		}
-		this.#isNewOrModified = true;
+		this.#hasNewOrModifiedCacheEntries = true;
 
 		// Update signatures for affected request sets
 		const {requestSetId, signature: originalSignature} = trees.get(relevantTree);
@@ -525,6 +525,32 @@ export default class BuildTaskCache {
 		});
 	}
 
+	async isAffectedByProjectChanges(changedPaths) {
+		await this.#initResourceRequests();
+		const resourceRequests = this.#resourceRequests.getAllRequests();
+		return resourceRequests.some(({type, value}) => {
+			if (type === "path") {
+				return changedPaths.includes(value);
+			}
+			if (type === "patterns") {
+				return micromatch(changedPaths, value).length > 0;
+			}
+		});
+	}
+
+	async isAffectedByDependencyChanges(changedPaths) {
+		await this.#initResourceRequests();
+		const resourceRequests = this.#resourceRequests.getAllRequests();
+		return resourceRequests.some(({type, value}) => {
+			if (type === "dep-path") {
+				return changedPaths.includes(value);
+			}
+			if (type === "dep-patterns") {
+				return micromatch(changedPaths, value).length > 0;
+			}
+		});
+	}
+
 	/**
 	 * Serializes the task cache to a plain object for persistence
 	 *

packages/project/lib/build/cache/CacheManager.js

@@ -161,11 +161,12 @@ export default class CacheManager {
 	 * @private
 	 * @param {string} packageName - Package/project identifier
 	 * @param {string} buildSignature - Build signature hash
+	 * @param {string} kind "source" or "result"
 	 * @returns {string} Absolute path to the index metadata file
 	 */
-	#getIndexCachePath(packageName, buildSignature) {
+	#getIndexCachePath(packageName, buildSignature, kind) {
 		const pkgDir = getPathFromPackageName(packageName);
-		return path.join(this.#indexDir, pkgDir, `${buildSignature}.json`);
+		return path.join(this.#indexDir, pkgDir, `${kind}-${buildSignature}.json`);
 	}
 
 	/**
@@ -176,12 +177,13 @@ export default class CacheManager {
 	 *
 	 * @param {string} projectId - Project identifier (typically package name)
 	 * @param {string} buildSignature - Build signature hash
+	 * @param {string} kind "source" or "result"
 	 * @returns {Promise<object|null>} Parsed index cache object or null if not found
 	 * @throws {Error} If file read fails for reasons other than file not existing
 	 */
-	async readIndexCache(projectId, buildSignature) {
+	async readIndexCache(projectId, buildSignature, kind) {
 		try {
-			const metadata = await readFile(this.#getIndexCachePath(projectId, buildSignature), "utf8");
+			const metadata = await readFile(this.#getIndexCachePath(projectId, buildSignature, kind), "utf8");
 			return JSON.parse(metadata);
 		} catch (err) {
 			if (err.code === "ENOENT") {
@@ -203,11 +205,12 @@ export default class CacheManager {
 	 *
 	 * @param {string} projectId - Project identifier (typically package name)
 	 * @param {string} buildSignature - Build signature hash
+	 * @param {string} kind "source" or "result"
 	 * @param {object} index - Index object containing resource tree and task metadata
 	 * @returns {Promise<void>}
 	 */
-	async writeIndexCache(projectId, buildSignature, index) {
-		const indexPath = this.#getIndexCachePath(projectId, buildSignature);
+	async writeIndexCache(projectId, buildSignature, kind, index) {
+		const indexPath = this.#getIndexCachePath(projectId, buildSignature, kind);
 		await mkdir(path.dirname(indexPath), {recursive: true});
 		await writeFile(indexPath, JSON.stringify(index, null, 2), "utf8");
 	}