From 8123eed0186145fb19f90efea1ff88dfb2760c4b Mon Sep 17 00:00:00 2001 From: David Nguyen <87228593+davidnguyen-tech@users.noreply.github.com> Date: Tue, 31 Mar 2026 17:13:41 +0200 Subject: [PATCH 01/19] Add AndroidInnerLoopParser for build/deploy binlog analysis Add a new C# parser that extracts build and deploy metrics from MSBuild binary logs (.binlog) for MAUI Android inner loop measurements. The parser captures: - Overall build duration (Publish Time) - Build task timings: Csc, XamlC, GenerateJavaStubs, D8, Javac, etc. - Build target timings: CoreCompile, _GenerateJavaStubs, _CompileToDalvik, etc. - Deploy task timings: FastDeploy, AndroidSignPackage, Aapt2Link - Deploy target timings: _Sign, _Upload, _DeployApk, _BuildApkFastDev Register the AndroidInnerLoop MetricType in Startup.cs so the parser can be selected via the measurement framework. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../ScenarioMeasurement/Startup/Startup.cs | 2 + .../Util/Parsers/AndroidInnerLoopParser.cs | 205 ++++++++++++++++++ 2 files changed, 207 insertions(+) create mode 100644 src/tools/ScenarioMeasurement/Util/Parsers/AndroidInnerLoopParser.cs diff --git a/src/tools/ScenarioMeasurement/Startup/Startup.cs b/src/tools/ScenarioMeasurement/Startup/Startup.cs index 3c8a180561b..1e6f6a73bd7 100644 --- a/src/tools/ScenarioMeasurement/Startup/Startup.cs +++ b/src/tools/ScenarioMeasurement/Startup/Startup.cs @@ -27,6 +27,7 @@ enum MetricType WinUIBlazor, TimeToMain2, BuildTime, + AndroidInnerLoop, } public class InnerLoopMarkerEventSource : EventSource @@ -291,6 +292,7 @@ static void checkArg(string arg, string name) MetricType.WinUIBlazor => new WinUIBlazorParser(), MetricType.TimeToMain2 => new TimeToMain2Parser(AddTestProcessEnvironmentVariable), MetricType.BuildTime => new BuildTimeParser(), + MetricType.AndroidInnerLoop => new AndroidInnerLoopParser(), _ => throw new ArgumentOutOfRangeException(), }; diff --git a/src/tools/ScenarioMeasurement/Util/Parsers/AndroidInnerLoopParser.cs b/src/tools/ScenarioMeasurement/Util/Parsers/AndroidInnerLoopParser.cs new file mode 100644 index 00000000000..f5d55d00021 --- /dev/null +++ b/src/tools/ScenarioMeasurement/Util/Parsers/AndroidInnerLoopParser.cs @@ -0,0 +1,205 @@ +using System; +using System.Collections.Generic; +using System.IO; +using Microsoft.Build.Logging.StructuredLogger; +using StructuredLogViewer; +using Microsoft.Diagnostics.Tracing; +using Reporting; + +namespace ScenarioMeasurement; + +/// +/// Parses Android inner loop (build+deploy) target and task durations from a binary log file. +/// +public class AndroidInnerLoopParser : IParser +{ + public void EnableKernelProvider(ITraceSession kernel) { throw new NotImplementedException(); } + public void EnableUserProviders(ITraceSession user) { throw new NotImplementedException(); } + + public IEnumerable Parse(string binlogFile, string processName, IList pids, string commandLine) + { + var publishTimes = new List(); + + // Build tasks (compilation) + var cscTimes = new List(); + var xamlCTimes = new List(); + var generateJavaStubsTimes = new List(); + var linkAssembliesNoShrinkTimes = new List(); + var d8Times = new List(); + var javacTimes = new List(); + var generateTypeMappingsTimes = new List(); + var processAssembliesTimes = new List(); + var generateJavaCallableWrappersTimes = new List(); + var filterAssembliesTimes = new List(); + var waitForAppDetectionTimes = new List(); + var generateMainAndroidManifestTimes = new List(); + var resolveSdksTimes = new List(); + var generateNativeApplicationConfigSourcesTimes = new List(); + + // Deploy tasks + var fastDeployTimes = new List(); + var androidSignPackageTimes = new List(); + var androidApkSignerTimes = new List(); + var aapt2LinkTimes = new List(); + + // Build targets + var coreCompileTargetTimes = new List(); + var xamlCTargetTimes = new List(); + var generateJavaStubsTargetTimes = new List(); + var linkAssembliesNoShrinkTargetTimes = new List(); + var compileToDalvikTargetTimes = new List(); + var compileJavaTargetTimes = new List(); + + // Deploy targets + var signTargetTimes = new List(); + var uploadTargetTimes = new List(); + var deployApkTargetTimes = new List(); + var buildApkFastDevTargetTimes = new List(); + + if (File.Exists(binlogFile)) + { + var build = BinaryLog.ReadBuild(binlogFile); + BuildAnalyzer.AnalyzeBuild(build); + + foreach (var task in build.FindChildrenRecursive()) + { + var name = task.Name; + var s = task.Duration.TotalMilliseconds / 1000.0; + + if (name.Equals("Csc", StringComparison.OrdinalIgnoreCase)) + cscTimes.Add(s); + else if (name.Equals("XamlCTask", StringComparison.OrdinalIgnoreCase)) + xamlCTimes.Add(s); + else if (name.Equals("GenerateJavaStubs", StringComparison.OrdinalIgnoreCase)) + generateJavaStubsTimes.Add(s); + else if (name.Equals("LinkAssembliesNoShrink", StringComparison.OrdinalIgnoreCase)) + linkAssembliesNoShrinkTimes.Add(s); + else if (name.Equals("D8", StringComparison.OrdinalIgnoreCase)) + d8Times.Add(s); + else if (name.Equals("Javac", StringComparison.OrdinalIgnoreCase)) + javacTimes.Add(s); + else if (name.Equals("GenerateTypeMappings", StringComparison.OrdinalIgnoreCase)) + generateTypeMappingsTimes.Add(s); + else if (name.Equals("ProcessAssemblies", StringComparison.OrdinalIgnoreCase)) + processAssembliesTimes.Add(s); + else if (name.Equals("GenerateJavaCallableWrappers", StringComparison.OrdinalIgnoreCase)) + generateJavaCallableWrappersTimes.Add(s); + else if (name.Equals("FilterAssemblies", StringComparison.OrdinalIgnoreCase)) + filterAssembliesTimes.Add(s); + else if (name.Equals("WaitForAppDetection", StringComparison.OrdinalIgnoreCase)) + waitForAppDetectionTimes.Add(s); + else if (name.Equals("GenerateMainAndroidManifest", StringComparison.OrdinalIgnoreCase)) + generateMainAndroidManifestTimes.Add(s); + else if (name.Equals("ResolveSdks", StringComparison.OrdinalIgnoreCase)) + resolveSdksTimes.Add(s); + else if (name.Equals("GenerateNativeApplicationConfigSources", StringComparison.OrdinalIgnoreCase)) + generateNativeApplicationConfigSourcesTimes.Add(s); + else if (name.Equals("FastDeploy", StringComparison.OrdinalIgnoreCase)) + fastDeployTimes.Add(s); + else if (name.Equals("AndroidSignPackage", StringComparison.OrdinalIgnoreCase)) + androidSignPackageTimes.Add(s); + else if (name.Equals("AndroidApkSigner", StringComparison.OrdinalIgnoreCase)) + androidApkSignerTimes.Add(s); + else if (name.Equals("Aapt2Link", StringComparison.OrdinalIgnoreCase)) + aapt2LinkTimes.Add(s); + } + + foreach (var target in build.FindChildrenRecursive()) + { + var name = target.Name; + var s = target.Duration.TotalMilliseconds / 1000.0; + + if (name.Equals("CoreCompile", StringComparison.Ordinal)) + coreCompileTargetTimes.Add(s); + else if (name.Equals("XamlC", StringComparison.Ordinal)) + xamlCTargetTimes.Add(s); + else if (name.Equals("_GenerateJavaStubs", StringComparison.Ordinal)) + generateJavaStubsTargetTimes.Add(s); + else if (name.Equals("_LinkAssembliesNoShrink", StringComparison.Ordinal)) + linkAssembliesNoShrinkTargetTimes.Add(s); + else if (name.Equals("_CompileToDalvik", StringComparison.Ordinal)) + compileToDalvikTargetTimes.Add(s); + else if (name.Equals("_CompileJava", StringComparison.Ordinal)) + compileJavaTargetTimes.Add(s); + else if (name.Equals("_Sign", StringComparison.Ordinal)) + signTargetTimes.Add(s); + else if (name.Equals("_Upload", StringComparison.Ordinal)) + uploadTargetTimes.Add(s); + else if (name.Equals("_DeployApk", StringComparison.Ordinal)) + deployApkTargetTimes.Add(s); + else if (name.Equals("_BuildApkFastDev", StringComparison.Ordinal)) + buildApkFastDevTargetTimes.Add(s); + } + + publishTimes.Add(build.Duration.TotalMilliseconds / 1000.0); + } + + // Overall duration + if (publishTimes.Count > 0) + yield return new Counter { Name = "Publish Time", MetricName = "s", DefaultCounter = true, TopCounter = true, Results = publishTimes.ToArray() }; + + // Build task counters + if (cscTimes.Count > 0) + yield return new Counter { Name = "Csc Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = cscTimes.ToArray() }; + if (xamlCTimes.Count > 0) + yield return new Counter { Name = "XamlC Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = xamlCTimes.ToArray() }; + if (generateJavaStubsTimes.Count > 0) + yield return new Counter { Name = "GenerateJavaStubs Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = generateJavaStubsTimes.ToArray() }; + if (linkAssembliesNoShrinkTimes.Count > 0) + yield return new Counter { Name = "LinkAssembliesNoShrink Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = linkAssembliesNoShrinkTimes.ToArray() }; + if (d8Times.Count > 0) + yield return new Counter { Name = "D8 Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = d8Times.ToArray() }; + if (javacTimes.Count > 0) + yield return new Counter { Name = "Javac Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = javacTimes.ToArray() }; + if (generateTypeMappingsTimes.Count > 0) + yield return new Counter { Name = "GenerateTypeMappings Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = generateTypeMappingsTimes.ToArray() }; + if (processAssembliesTimes.Count > 0) + yield return new Counter { Name = "ProcessAssemblies Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = processAssembliesTimes.ToArray() }; + if (generateJavaCallableWrappersTimes.Count > 0) + yield return new Counter { Name = "GenerateJavaCallableWrappers Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = generateJavaCallableWrappersTimes.ToArray() }; + if (filterAssembliesTimes.Count > 0) + yield return new Counter { Name = "FilterAssemblies Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = filterAssembliesTimes.ToArray() }; + if (waitForAppDetectionTimes.Count > 0) + yield return new Counter { Name = "WaitForAppDetection Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = waitForAppDetectionTimes.ToArray() }; + if (generateMainAndroidManifestTimes.Count > 0) + yield return new Counter { Name = "GenerateMainAndroidManifest Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = generateMainAndroidManifestTimes.ToArray() }; + if (resolveSdksTimes.Count > 0) + yield return new Counter { Name = "ResolveSdks Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = resolveSdksTimes.ToArray() }; + if (generateNativeApplicationConfigSourcesTimes.Count > 0) + yield return new Counter { Name = "GenerateNativeApplicationConfigSources Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = generateNativeApplicationConfigSourcesTimes.ToArray() }; + + // Build target counters + if (coreCompileTargetTimes.Count > 0) + yield return new Counter { Name = "CoreCompile Target Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = coreCompileTargetTimes.ToArray() }; + if (xamlCTargetTimes.Count > 0) + yield return new Counter { Name = "XamlC Target Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = xamlCTargetTimes.ToArray() }; + if (generateJavaStubsTargetTimes.Count > 0) + yield return new Counter { Name = "_GenerateJavaStubs Target Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = generateJavaStubsTargetTimes.ToArray() }; + if (linkAssembliesNoShrinkTargetTimes.Count > 0) + yield return new Counter { Name = "_LinkAssembliesNoShrink Target Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = linkAssembliesNoShrinkTargetTimes.ToArray() }; + if (compileToDalvikTargetTimes.Count > 0) + yield return new Counter { Name = "_CompileToDalvik Target Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = compileToDalvikTargetTimes.ToArray() }; + if (compileJavaTargetTimes.Count > 0) + yield return new Counter { Name = "_CompileJava Target Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = compileJavaTargetTimes.ToArray() }; + + // Deploy target counters + if (signTargetTimes.Count > 0) + yield return new Counter { Name = "_Sign Target Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = signTargetTimes.ToArray() }; + if (uploadTargetTimes.Count > 0) + yield return new Counter { Name = "_Upload Target Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = uploadTargetTimes.ToArray() }; + if (deployApkTargetTimes.Count > 0) + yield return new Counter { Name = "_DeployApk Target Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = deployApkTargetTimes.ToArray() }; + if (buildApkFastDevTargetTimes.Count > 0) + yield return new Counter { Name = "_BuildApkFastDev Target Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = buildApkFastDevTargetTimes.ToArray() }; + + // Task-level counters (granular) + if (fastDeployTimes.Count > 0) + yield return new Counter { Name = "FastDeploy Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = fastDeployTimes.ToArray() }; + if (androidSignPackageTimes.Count > 0) + yield return new Counter { Name = "AndroidSignPackage Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = androidSignPackageTimes.ToArray() }; + if (androidApkSignerTimes.Count > 0) + yield return new Counter { Name = "AndroidApkSigner Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = androidApkSignerTimes.ToArray() }; + if (aapt2LinkTimes.Count > 0) + yield return new Counter { Name = "Aapt2Link Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = aapt2LinkTimes.ToArray() }; + } +} From f6c5869f05d30f713f7690dcf053276b5caf615f Mon Sep 17 00:00:00 2001 From: David Nguyen <87228593+davidnguyen-tech@users.noreply.github.com> Date: Tue, 31 Mar 2026 17:15:48 +0200 Subject: [PATCH 02/19] Add ANDROIDINNERLOOP test type to Python test framework Extend the shared test runner with a new ANDROIDINNERLOOP scenario type that orchestrates first deploy and incremental build+deploy+startup measurements for MAUI Android apps. Changes: - const.py: Add ANDROIDINNERLOOP constant and scenario name mapping - runner.py: Add argument parser and full ANDROIDINNERLOOP handler that performs first build+deploy, then N incremental iterations with source file toggling, binlog capture, startup time measurement via am start, and result upload to perflab - androidhelper.py: Add skip_install, screen_timeout_ms, skip_uninstall, and other parameters to AndroidHelper for inner loop reuse - startup.py: Fix copytree FileExistsError with dirs_exist_ok=True Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/scenarios/shared/androidhelper.py | 269 +++++++++++++++---------- src/scenarios/shared/const.py | 4 +- src/scenarios/shared/runner.py | 277 +++++++++++++++++++++++++- src/scenarios/shared/startup.py | 4 +- 4 files changed, 448 insertions(+), 106 deletions(-) diff --git a/src/scenarios/shared/androidhelper.py b/src/scenarios/shared/androidhelper.py index df3e0ef4d52..297b807b93d 100644 --- a/src/scenarios/shared/androidhelper.py +++ b/src/scenarios/shared/androidhelper.py @@ -21,56 +21,63 @@ def __init__(self): self.startverifierverifyadbinstalls = None self.startpackageverifierenable = None - def setup_device(self, packagename: str, packagepath: str, animationsdisabled: bool, forcewaitstart: bool = True): + def setup_device(self, packagename: str, packagepath: str, animationsdisabled: bool, forcewaitstart: bool = True, skip_install: bool = False, skip_xharness_warmup: bool = False, skip_package_verifier: bool = False, skip_test_launch: bool = False, screen_timeout_ms: int = 2 * 60 * 1000): + if not skip_install and packagepath is None: + raise Exception("packagepath is required when skip_install is False") + run_split_regex = r":\s(.+)" self.screenwasoff = False self.packagename = packagename - # Try calling xharness with stdout=None and stderr=None to hopefully bypass the hang - getLogger().info("Clearing xharness stdout and stderr to avoid hang") - cmdline = xharness_adb() + [ - 'shell', - 'echo', 'Hello World' - ] - RunCommand(cmdline, verbose=False).run() - getLogger().info("Ran echo command to clear stdout and stderr") - - cmdline = xharness_adb() + [ - 'shell', - 'wm', - 'size' - ] - RunCommand(cmdline, verbose=True).run() - - # Capture and disable Android package verifier settings to avoid prompts/overhead during installs - try: - getLogger().info("Capturing current package verifier settings") + if not skip_xharness_warmup: + # Try calling xharness with stdout=None and stderr=None to hopefully bypass the hang + getLogger().info("Clearing xharness stdout and stderr to avoid hang") cmdline = xharness_adb() + [ - 'shell', 'settings', 'get', 'global', 'verifier_verify_adb_installs' + 'shell', + 'echo', 'Hello World' ] - get_verifier_adb_cmd = RunCommand(cmdline, verbose=True) - get_verifier_adb_cmd.run() - self.startverifierverifyadbinstalls = get_verifier_adb_cmd.stdout.strip() + RunCommand(cmdline, verbose=False).run() + getLogger().info("Ran echo command to clear stdout and stderr") cmdline = xharness_adb() + [ - 'shell', 'settings', 'get', 'global', 'package_verifier_enable' - ] - get_pkg_verifier_cmd = RunCommand(cmdline, verbose=True) - get_pkg_verifier_cmd.run() - self.startpackageverifierenable = get_pkg_verifier_cmd.stdout.strip() - - getLogger().info("Disabling package verifier settings for the run") - cmdline = xharness_adb() + [ - 'shell', 'settings', 'put', 'global', 'verifier_verify_adb_installs', '0' - ] - RunCommand(cmdline, verbose=True).run() - cmdline = xharness_adb() + [ - 'shell', 'settings', 'put', 'global', 'package_verifier_enable', '0' + 'shell', + 'wm', + 'size' ] RunCommand(cmdline, verbose=True).run() - except CalledProcessError: - # Best-effort: don't fail setup if device doesn't support these keys; proceed with the run - getLogger().warning("Failed to update package verifier settings; continuing without changes", exc_info=True) + + # Capture and disable Android package verifier settings to avoid prompts/overhead during installs + if skip_package_verifier: + getLogger().info("Skipping package verifier changes") + else: + try: + getLogger().info("Capturing current package verifier settings") + cmdline = xharness_adb() + [ + 'shell', 'settings', 'get', 'global', 'verifier_verify_adb_installs' + ] + get_verifier_adb_cmd = RunCommand(cmdline, verbose=True) + get_verifier_adb_cmd.run() + self.startverifierverifyadbinstalls = get_verifier_adb_cmd.stdout.strip() + + cmdline = xharness_adb() + [ + 'shell', 'settings', 'get', 'global', 'package_verifier_enable' + ] + get_pkg_verifier_cmd = RunCommand(cmdline, verbose=True) + get_pkg_verifier_cmd.run() + self.startpackageverifierenable = get_pkg_verifier_cmd.stdout.strip() + + getLogger().info("Disabling package verifier settings for the run") + cmdline = xharness_adb() + [ + 'shell', 'settings', 'put', 'global', 'verifier_verify_adb_installs', '0' + ] + RunCommand(cmdline, verbose=True).run() + cmdline = xharness_adb() + [ + 'shell', 'settings', 'put', 'global', 'package_verifier_enable', '0' + ] + RunCommand(cmdline, verbose=True).run() + except CalledProcessError: + # Best-effort: don't fail setup if device doesn't support these keys; proceed with the run + getLogger().warning("Failed to update package verifier settings; continuing without changes", exc_info=True) # Get animation values getLogger().info("Getting Values we will need set specifically") @@ -106,7 +113,7 @@ def setup_device(self, packagename: str, packagepath: str, animationsdisabled: b animationValue = 0 else: animationValue = 1 - minimumTimeoutValue = 2 * 60 * 1000 # milliseconds + minimumTimeoutValue = screen_timeout_ms cmdline = xharness_adb() + [ 'shell', 'settings', 'put', 'global', 'window_animation_scale', str(animationValue) ] @@ -157,19 +164,21 @@ def setup_device(self, packagename: str, packagepath: str, animationsdisabled: b self.packagename ] - installCmd = xharnesscommand() + [ - 'android', - 'install', - '--app', packagepath, - '--package-name', - self.packagename, - '-o', - const.TRACEDIR, - '-v' - ] - RunCommand(installCmd, verbose=True).run() + if not skip_install: + installCmd = xharnesscommand() + [ + 'android', + 'install', + '--app', packagepath, + '--package-name', + self.packagename, + '-o', + const.TRACEDIR, + '-v' + ] + RunCommand(installCmd, verbose=True).run() + getLogger().info("Completed install.") - getLogger().info("Completed install, running shell.") + getLogger().info("Resolving launchable activity.") cmdline = xharness_adb() + [ 'shell', f'cmd package resolve-activity --brief {self.packagename} | tail -n 1' @@ -205,50 +214,52 @@ def setup_device(self, packagename: str, packagepath: str, animationsdisabled: b getLogger().exception("Failed to make screen interactive.") raise Exception("Failed to make screen interactive.") - # Actual testing some run stuff - getLogger().info("Test run to check if permissions are needed") self.activityname = getActivity.stdout.strip() - # -W in the start command waits for the app to finish initial draw. - self.startappcommand = xharness_adb() + [ - 'shell', - 'am', - 'start-activity', - '-W', - '-n', - self.activityname - ] + if not skip_test_launch: + # Test run to check if permissions are needed + getLogger().info("Test run to check if permissions are needed") + + # -W in the start command waits for the app to finish initial draw. + self.startappcommand = xharness_adb() + [ + 'shell', + 'am', + 'start-activity', + '-W', + '-n', + self.activityname + ] - testRun = RunCommand(self.startappcommand, verbose=True) - testRun.run() - testRunStats = re.findall(run_split_regex, testRun.stdout) # Split results saving value (List: Starting, Status, LaunchState, Activity, TotalTime, WaitTime) - getLogger().info(f"Test run activity: {testRunStats[3]}") - time.sleep(10) # Add delay to ensure app is fully installed and give it some time to settle - - RunCommand(self.stopappcommand, verbose=True).run() - if "com.google.android.permissioncontroller" in testRunStats[3]: - # On perm screen, use the buttons to close it. it will stay away until the app is reinstalled - RunCommand(keyInputCmd + ['22'], verbose=True).run() # Select next button - time.sleep(1) - RunCommand(keyInputCmd + ['22'], verbose=True).run() # Select next button - time.sleep(1) - RunCommand(keyInputCmd + ['66'], verbose=True).run() # Press enter to close main perm screen - time.sleep(1) - RunCommand(keyInputCmd + ['22'], verbose=True).run() # Select next button - time.sleep(1) - RunCommand(keyInputCmd + ['66'], verbose=True).run() # Press enter to close out of second screen - time.sleep(1) - - # Check to make sure it worked testRun = RunCommand(self.startappcommand, verbose=True) testRun.run() - testRunStats = re.findall(run_split_regex, testRun.stdout) + testRunStats = re.findall(run_split_regex, testRun.stdout) # Split results saving value (List: Starting, Status, LaunchState, Activity, TotalTime, WaitTime) getLogger().info(f"Test run activity: {testRunStats[3]}") - RunCommand(self.stopappcommand, verbose=True).run() + time.sleep(10) # Add delay to ensure app is fully installed and give it some time to settle + RunCommand(self.stopappcommand, verbose=True).run() if "com.google.android.permissioncontroller" in testRunStats[3]: - getLogger().exception("Failed to get past permission screen, run locally to see if enough next button presses were used.") - raise Exception("Failed to get past permission screen, run locally to see if enough next button presses were used.") + # On perm screen, use the buttons to close it. it will stay away until the app is reinstalled + RunCommand(keyInputCmd + ['22'], verbose=True).run() # Select next button + time.sleep(1) + RunCommand(keyInputCmd + ['22'], verbose=True).run() # Select next button + time.sleep(1) + RunCommand(keyInputCmd + ['66'], verbose=True).run() # Press enter to close main perm screen + time.sleep(1) + RunCommand(keyInputCmd + ['22'], verbose=True).run() # Select next button + time.sleep(1) + RunCommand(keyInputCmd + ['66'], verbose=True).run() # Press enter to close out of second screen + time.sleep(1) + + # Check to make sure it worked + testRun = RunCommand(self.startappcommand, verbose=True) + testRun.run() + testRunStats = re.findall(run_split_regex, testRun.stdout) + getLogger().info(f"Test run activity: {testRunStats[3]}") + RunCommand(self.stopappcommand, verbose=True).run() + + if "com.google.android.permissioncontroller" in testRunStats[3]: + getLogger().exception("Failed to get past permission screen, run locally to see if enough next button presses were used.") + raise Exception("Failed to get past permission screen, run locally to see if enough next button presses were used.") self.startappcommand = xharness_adb() + [ 'shell', @@ -263,24 +274,80 @@ def setup_device(self, packagename: str, packagepath: str, animationsdisabled: b self.activityname ] - def close_device(self): + def measure_cold_startup(self, packagename: str, activityname: str) -> int: + """Measure app cold startup time in milliseconds. + + Uses am start-activity -W -S to force-stop and cold-start the app. + Primary: parses TotalTime from am start stdout. + Fallback: parses WaitTime from am start stdout. + Last resort: parses 'Displayed' time from logcat. + + Returns startup time in milliseconds. + """ + stop_app_cmd = xharness_adb() + ['shell', 'am', 'force-stop', packagename] + start_app_cmd = xharness_adb() + ['shell', 'am', 'start-activity', '-W', '-S', '-n', activityname] + clear_logs_cmd = xharness_adb() + ['logcat', '-c'] + retrieve_time_cmd = xharness_adb() + [ + 'shell', + f"logcat -d | grep -E 'ActivityManager|ActivityTaskManager' | grep ': Displayed {activityname}'" + ] + + RunCommand(clear_logs_cmd, verbose=True).run() + start_result = RunCommand(start_app_cmd, verbose=True) + start_result.run() + + startup_ms = None + + # Primary: parse TotalTime or WaitTime from am start -W output + total_match = re.search(r"TotalTime:\s*(\d+)", start_result.stdout) + wait_match = re.search(r"WaitTime:\s*(\d+)", start_result.stdout) + if total_match: + startup_ms = int(total_match.group(1)) + getLogger().info("Startup time (TotalTime): %d ms" % startup_ms) + elif wait_match: + startup_ms = int(wait_match.group(1)) + getLogger().info("Startup time (WaitTime): %d ms" % startup_ms) + + # Fallback: parse 'Displayed' time from logcat + if startup_ms is None: + RunCommand(stop_app_cmd, verbose=True).run() + retrieve_result = RunCommand(retrieve_time_cmd, verbose=True) + retrieve_result.run() + dirty_capture = re.search(r"\+(\d*s?\d+)ms", retrieve_result.stdout) + if not dirty_capture: + raise Exception("Failed to capture startup time from am start output or logcat!") + capture_list = dirty_capture.group(1).split('s') + if len(capture_list) == 1: + startup_ms = int(capture_list[0]) + elif len(capture_list) == 2: + startup_ms = int(capture_list[0]) * 1000 + int(capture_list[1].zfill(3)) + else: + raise Exception("Android time capture failed! Unexpected format: %s" % dirty_capture.group(0)) + getLogger().info("Startup time (logcat): %d ms" % startup_ms) + else: + RunCommand(stop_app_cmd, verbose=True).run() + + return startup_ms + + def close_device(self, skip_uninstall: bool = False): keyInputCmd = xharness_adb() + [ 'shell', 'input', 'keyevent' ] - getLogger().info("Stopping App for uninstall") - RunCommand(self.stopappcommand, verbose=True).run() - - getLogger().info("Uninstalling app") - uninstallAppCmd = xharnesscommand() + [ - 'android', - 'uninstall', - '--package-name', - self.packagename - ] - RunCommand(uninstallAppCmd, verbose=True).run() + if not skip_uninstall: + getLogger().info("Stopping App for uninstall") + RunCommand(self.stopappcommand, verbose=True).run() + + getLogger().info("Uninstalling app") + uninstallAppCmd = xharnesscommand() + [ + 'android', + 'uninstall', + '--package-name', + self.packagename + ] + RunCommand(uninstallAppCmd, verbose=True).run() keyInputCmd = xharness_adb() + [ diff --git a/src/scenarios/shared/const.py b/src/scenarios/shared/const.py index 074fa7fdf89..3492887ff13 100644 --- a/src/scenarios/shared/const.py +++ b/src/scenarios/shared/const.py @@ -19,6 +19,7 @@ ANDROIDINSTRUMENTATION = "androidinstrumentation" DEVICEPOWERCONSUMPTION = "devicepowerconsumption" BUILDTIME = "buildtime" +ANDROIDINNERLOOP = "androidinnerloop" SCENARIO_NAMES = {STARTUP: 'Startup', SDK: 'SDK', @@ -27,7 +28,8 @@ INNERLOOP: 'Innerloop', INNERLOOPMSBUILD: 'InnerLoopMsBuild', DOTNETWATCH: 'DotnetWatch', - BUILDTIME: 'BuildTime'} + BUILDTIME: 'BuildTime', + ANDROIDINNERLOOP: 'AndroidInnerLoop'} BINDIR = 'bin' PUBDIR = 'pub' diff --git a/src/scenarios/shared/runner.py b/src/scenarios/shared/runner.py index 165f0e882ba..1678d6cb40e 100644 --- a/src/scenarios/shared/runner.py +++ b/src/scenarios/shared/runner.py @@ -174,6 +174,18 @@ def parseargs(self): buildtimeparser.add_argument('--binlog-path', help='Location of binlog', dest='binlogpath') self.add_common_arguments(buildtimeparser) + androidinnerloopparser = subparsers.add_parser(const.ANDROIDINNERLOOP, + description='measure first and incremental deploy time via binlogs') + androidinnerloopparser.add_argument('--csproj-path', help='Path to .csproj file to build', dest='csprojpath') + androidinnerloopparser.add_argument('--edit-src', help='Path to modified source file (copied before incremental deploy)', dest='editsrc') + androidinnerloopparser.add_argument('--edit-dest', help='Destination path for the modified file', dest='editdest') + androidinnerloopparser.add_argument('--framework', '-f', help='Target framework (e.g., net10.0-android)', dest='framework') + androidinnerloopparser.add_argument('--configuration', '-c', help='Build configuration', dest='configuration', default='Debug') + androidinnerloopparser.add_argument('--msbuild-args', help='Additional MSBuild arguments', dest='msbuildargs', default='') + androidinnerloopparser.add_argument('--package-name', help='Android package name for startup measurement (e.g. com.companyname.mauiandroidinnerloop)', dest='packagename') + androidinnerloopparser.add_argument('--inner-loop-iterations', help='Number of incremental build+deploy+startup iterations (1+)', type=int, default=10, dest='innerloopiterations') + self.add_common_arguments(androidinnerloopparser) + args = parser.parse_args() if not args.testtype: @@ -196,7 +208,17 @@ def parseargs(self): if self.testtype == const.BUILDTIME: self.binlogpath = args.binlogpath - + + if self.testtype == const.ANDROIDINNERLOOP: + self.csprojpath = args.csprojpath + self.editsrcs = args.editsrc.split(';') if args.editsrc else [] + self.editdests = args.editdest.split(';') if args.editdest else [] + self.framework = args.framework + self.configuration = args.configuration + self.msbuildargs = args.msbuildargs or os.environ.get('PERFLAB_MSBUILD_ARGS', '') + self.packagename = args.packagename + self.innerloopiterations = args.innerloopiterations + if self.testtype == const.DEVICESTARTUP: self.packagepath = args.packagepath self.packagename = args.packagename @@ -974,4 +996,255 @@ def run(self): if not (self.binlogpath and os.path.exists(os.path.join(const.TRACEDIR, self.binlogpath))): raise Exception("For build time measurements a valid binlog path must be provided.") self.traits.add_traits(overwrite=True, apptorun="app", startupmetric=const.BUILDTIME, tracename=self.binlogpath, scenarioname=self.scenarioname) - startup.parsetraces(self.traits) \ No newline at end of file + startup.parsetraces(self.traits) + + elif self.testtype == const.ANDROIDINNERLOOP: + import hashlib + import subprocess + from shutil import copytree + from performance.common import runninginlab + from performance.constants import UPLOAD_CONTAINER, UPLOAD_STORAGE_URI, UPLOAD_QUEUE + from shared.util import helixuploaddir + import upload + + def merge_build_and_startup(build_report_path, startup_results, final_report_path): + """Load the build metrics report, append a startup time counter, write to final path.""" + with open(build_report_path, 'r') as f: + report = json.load(f) + startup_counter = { + "name": "Time to Main", + "topCounter": True, + "defaultCounter": False, + "higherIsBetter": False, + "metricName": "ms", + "results": startup_results + } + # Report structure: { "tests": [ { "counters": [...] } ] } + report["tests"][0]["counters"].append(startup_counter) + with open(final_report_path, 'w') as f: + json.dump(report, f, indent=2) + getLogger().info("Merged report written to: %s" % final_report_path) + + def run_incremental_iteration(iteration, num_iterations, base_cmd, edit_pairs, + packagename, activityname, + scenarioprefix, startup, traits, measure_startup_fn): + """Run one incremental build+deploy+startup iteration. + + edit_pairs is a list of (dest_path, original_content, modified_content) tuples. + Returns (startup_ms, counters_list, binlog_path, test_metadata). + """ + import subprocess + + getLogger().info("=== Incremental iteration %d/%d ===" % (iteration, num_iterations)) + + # Toggle source files + for dest, original, modified in edit_pairs: + if iteration % 2 == 1: + with open(dest, 'w') as f: + f.write(modified) + content_hash = hashlib.md5(modified.encode()).hexdigest()[:8] + getLogger().info("Applied modified source: %s (hash=%s, len=%d)" % (dest, content_hash, len(modified))) + else: + with open(dest, 'w') as f: + f.write(original) + content_hash = hashlib.md5(original.encode()).hexdigest()[:8] + getLogger().info("Restored original source: %s (hash=%s, len=%d)" % (dest, content_hash, len(original))) + + # Incremental build+deploy with per-iteration binlog + iter_binlog_name = 'incremental-build-and-deploy-%d.binlog' % iteration + iter_binlog = os.path.join(const.TRACEDIR, iter_binlog_name) + incremental_cmd = base_cmd + [f'-bl:{iter_binlog}'] + getLogger().info("Incremental build+deploy: %s" % ' '.join(incremental_cmd)) + subprocess.run(incremental_cmd, check=True) + + # Measure startup + ms = measure_startup_fn(packagename, activityname) + getLogger().info("Incremental iteration %d/%d: build+deploy done, startup: %d ms" % (iteration, num_iterations, ms)) + + # Parse this iteration's binlog → temp build report + iter_report_name = 'incremental-build-report-%d.json' % iteration + iter_report = os.path.join(const.TRACEDIR, iter_report_name) + startup.reportjson = iter_report + traits.add_traits(overwrite=True, apptorun="app", startupmetric=const.ANDROIDINNERLOOP, + tracename=iter_binlog_name, + scenarioname=scenarioprefix + " - Incremental Build and Deploy", + upload_to_perflab_container=False) + startup.parsetraces(traits) + + # Extract build counters and test metadata from temp report + with open(iter_report, 'r') as f: + iter_data = json.load(f) + test_obj = iter_data["tests"][0] + counters = test_obj["counters"] + # Return test metadata (without counters) for building the final report + test_metadata = test_obj.copy() + test_metadata["counters"] = [] + + # Clean up temp report (leave binlog for later cleanup) + if os.path.exists(iter_report): + os.remove(iter_report) + getLogger().info("Removed temp report: %s" % iter_report) + + return ms, counters, iter_binlog, test_metadata + + # --- Validate inputs --- + if not self.csprojpath: + raise Exception("For Android inner loop measurements, --csproj-path must be provided.") + if not self.packagename: + raise Exception("For Android inner loop measurements, --package-name must be provided.") + scenarioprefix = self.scenarioname or "MAUI Android Build and Deploy" + + os.makedirs(const.TRACEDIR, exist_ok=True) + first_binlog = os.path.join(const.TRACEDIR, 'first-build-and-deploy.binlog') + + # Build the base MSBuild command + base_cmd = ['dotnet', 'build', self.csprojpath, '-t:Install'] + if self.configuration: + base_cmd.extend(['-c', self.configuration]) + if self.framework: + base_cmd.extend(['-f', self.framework]) + if self.msbuildargs: + for arg in re.split(r'[;\s]+', self.msbuildargs): + if arg.strip(): + base_cmd.append(arg.strip()) + + # --- First build + deploy --- + first_cmd = base_cmd + [f'-bl:{first_binlog}'] + getLogger().info("First build+deploy: %s" % ' '.join(first_cmd)) + subprocess.run(first_cmd, check=True) + + # --- Device setup for physical devices (screen wake, animations, timeout) --- + # Physical Helix CI devices have screens OFF. When the screen is off, + # am start-activity -W reports LaunchState: UNKNOWN, omits TotalTime, and + # ActivityTaskManager never logs the "Displayed" line — making startup + # measurement impossible. + androidHelper = AndroidHelper() + try: + androidHelper.setup_device(self.packagename, packagepath=None, animationsdisabled=True, skip_install=True, skip_xharness_warmup=True, skip_package_verifier=True, skip_test_launch=True, screen_timeout_ms=30 * 60 * 1000) + + activityname = androidHelper.activityname + getLogger().info("Using resolved activity: %s" % activityname) + + # --- First startup measurement --- + first_startup_ms = androidHelper.measure_cold_startup(self.packagename, activityname) + getLogger().info("First deploy startup: %d ms" % first_startup_ms) + + # --- Parse first build report --- + startup = StartupWrapper() + first_build_report = os.path.join(const.TRACEDIR, 'first-build-and-deploy-perf-lab-report.json') + startup.reportjson = first_build_report + saved_upload = self.traits.upload_to_perflab_container + self.traits.add_traits(overwrite=True, apptorun="app", startupmetric=const.ANDROIDINNERLOOP, + tracename='first-build-and-deploy.binlog', + scenarioname=scenarioprefix + " - First Build and Deploy", + upload_to_perflab_container=False) + startup.parsetraces(self.traits) + + # Merge first build metrics + startup → first e2e report + first_e2e_report = os.path.join(const.TRACEDIR, 'first-debug-e2e-perf-lab-report.json') + merge_build_and_startup(first_build_report, [first_startup_ms], first_e2e_report) + + # --- Incremental loop --- + num_iterations = self.innerloopiterations + getLogger().info("Starting incremental loop: %d iterations" % num_iterations) + + # Build list of (dest, original_content, modified_content) tuples for toggling + edit_pairs = [] + if self.editsrcs and self.editdests: + if len(self.editsrcs) != len(self.editdests): + raise Exception("--edit-src and --edit-dest must have the same number of semicolon-separated paths") + for src, dest in zip(self.editsrcs, self.editdests): + original = None + modified = None + with open(dest, 'r') as f: + original = f.read() + with open(src, 'r') as f: + modified = f.read() + edit_pairs.append((dest, original, modified)) + getLogger().info("Edit pair: %s <-> %s" % (src, dest)) + else: + raise Exception("No edit-src/edit-dest specified; incremental builds require file pairs to toggle") + + incremental_startup_results = [] + aggregated_counters = {} # counter_name -> aggregated counter dict + report_template = None # test metadata from first parsed report + intermediate_files = [] # files to clean up + + for iteration in range(1, num_iterations + 1): + ms, counters, iter_binlog, test_metadata = run_incremental_iteration( + iteration, num_iterations, base_cmd, + edit_pairs, + self.packagename, activityname, scenarioprefix, startup, self.traits, + androidHelper.measure_cold_startup) + + incremental_startup_results.append(ms) + intermediate_files.append(iter_binlog) + + # Save test metadata from the first iteration + if report_template is None: + report_template = test_metadata + + for counter in counters: + name = counter["name"] + if name not in aggregated_counters: + aggregated_counters[name] = { + "name": name, + "topCounter": counter.get("topCounter", False), + "defaultCounter": counter.get("defaultCounter", False), + "higherIsBetter": counter.get("higherIsBetter", False), + "metricName": counter.get("metricName", "ms"), + "results": [] + } + aggregated_counters[name]["results"].extend(counter.get("results", [])) + + # --- Aggregate incremental results --- + incremental_e2e_report = os.path.join(const.TRACEDIR, 'incremental-debug-e2e-perf-lab-report.json') + final_counters = list(aggregated_counters.values()) + final_counters.append({ + "name": "Time to Main", + "topCounter": True, + "defaultCounter": False, + "higherIsBetter": False, + "metricName": "ms", + "results": incremental_startup_results + }) + if report_template is not None: + report_template["counters"] = final_counters + final_report_data = {"tests": [report_template]} + else: + # Fallback: should not happen if at least 1 iteration ran + final_report_data = {"tests": [{"counters": final_counters}]} + with open(incremental_e2e_report, 'w') as f: + json.dump(final_report_data, f, indent=2) + getLogger().info("Final incremental E2E report written to: %s" % incremental_e2e_report) + + # --- Cleanup and upload --- + # Clean up intermediates from TRACEDIR + for f_path in intermediate_files + [first_build_report]: + if f_path.endswith('.binlog'): + getLogger().info("Keeping binlog for upload: %s" % f_path) + continue + if os.path.exists(f_path): + os.remove(f_path) + getLogger().info("Removed intermediate: %s" % f_path) + + # Wipe helix upload traces dir so copytree repopulates it cleanly + if runninginlab(): + traces_upload = os.path.join(helixuploaddir() or '', 'traces') + if os.path.exists(traces_upload): + rmtree(traces_upload) + + # Final upload + self.traits.add_traits(overwrite=True, upload_to_perflab_container=saved_upload) + helix_upload_dir = helixuploaddir() + if runninginlab() and helix_upload_dir is not None: + copytree(const.TRACEDIR, os.path.join(helix_upload_dir, 'traces'), dirs_exist_ok=True) + if self.traits.upload_to_perflab_container: + for report_path in [first_e2e_report, incremental_e2e_report]: + upload_code = upload.upload(report_path, UPLOAD_CONTAINER, UPLOAD_QUEUE, UPLOAD_STORAGE_URI) + getLogger().info("Upload code for %s: %s" % (os.path.basename(report_path), upload_code)) + if upload_code != 0: + sys.exit(upload_code) + + finally: + androidHelper.close_device(skip_uninstall=True) diff --git a/src/scenarios/shared/startup.py b/src/scenarios/shared/startup.py index da0fa548680..e2f51133d0b 100644 --- a/src/scenarios/shared/startup.py +++ b/src/scenarios/shared/startup.py @@ -85,7 +85,7 @@ def parsetraces(self, traits: TestTraits): helix_upload_dir = helixuploaddir() if runninginlab() and helix_upload_dir is not None: - copytree(TRACEDIR, os.path.join(helix_upload_dir, 'traces')) + copytree(TRACEDIR, os.path.join(helix_upload_dir, 'traces'), dirs_exist_ok=True) if traits.upload_to_perflab_container: import upload upload_code = upload.upload(self.reportjson, upload_container, UPLOAD_QUEUE, UPLOAD_STORAGE_URI) @@ -163,7 +163,7 @@ def runtests(self, traits: TestTraits): helix_upload_dir = helixuploaddir() if runninginlab() and helix_upload_dir is not None: - copytree(TRACEDIR, os.path.join(helix_upload_dir, 'traces')) + copytree(TRACEDIR, os.path.join(helix_upload_dir, 'traces'), dirs_exist_ok=True) if traits.upload_to_perflab_container: import upload upload_code = upload.upload(self.reportjson, upload_container, UPLOAD_QUEUE, UPLOAD_STORAGE_URI) From 77405688c81521374d90723cb5171012cd8d5bc2 Mon Sep 17 00:00:00 2001 From: David Nguyen <87228593+davidnguyen-tech@users.noreply.github.com> Date: Tue, 31 Mar 2026 17:16:03 +0200 Subject: [PATCH 03/19] Add MAUI Android inner loop scenario scripts Add the scenario-specific scripts for MAUI Android inner loop measurements: - pre.py: Bootstraps the .NET SDK, installs maui-android workload, restores NuGet packages, installs Android SDK dependencies (build tools, platform SDK, Java), and creates the MAUI test app with modified source files for incremental measurement - setup_helix.py: Helix-specific environment setup that discovers dotnet/SDK/Android/Java paths, installs workloads, and prepares the build environment on Helix agents - test.py: Entry point that invokes the shared test runner with ANDROIDINNERLOOP test type - post.py: Cleanup script that disables device animations, restores screen settings, and uninstalls the test APK using ADB Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/scenarios/mauiandroidinnerloop/post.py | 36 ++ src/scenarios/mauiandroidinnerloop/pre.py | 210 ++++++++++++ .../mauiandroidinnerloop/setup_helix.py | 315 ++++++++++++++++++ src/scenarios/mauiandroidinnerloop/test.py | 14 + 4 files changed, 575 insertions(+) create mode 100644 src/scenarios/mauiandroidinnerloop/post.py create mode 100644 src/scenarios/mauiandroidinnerloop/pre.py create mode 100644 src/scenarios/mauiandroidinnerloop/setup_helix.py create mode 100644 src/scenarios/mauiandroidinnerloop/test.py diff --git a/src/scenarios/mauiandroidinnerloop/post.py b/src/scenarios/mauiandroidinnerloop/post.py new file mode 100644 index 00000000000..e63eb8a009c --- /dev/null +++ b/src/scenarios/mauiandroidinnerloop/post.py @@ -0,0 +1,36 @@ +''' +post cleanup script +''' + +import os +import subprocess +import sys +import traceback +from performance.logger import setup_loggers, getLogger +from shared.postcommands import clean_directories +from shared.util import xharness_adb +from test import EXENAME + +setup_loggers(True) +logger = getLogger(__name__) + +# Pin to the emulator device to avoid "more than one device/emulator" errors +# 'emulator-5554' is the standard ADB serial for the first Android emulator instance +# and is stable across all Android SDK versions. +if 'ANDROID_SERIAL' not in os.environ: + os.environ['ANDROID_SERIAL'] = 'emulator-5554' + logger.info("Set ANDROID_SERIAL=emulator-5554") + +try: + # Uninstall the app from the connected device so re-runs start from a clean state + package_name = f'com.companyname.{EXENAME.lower()}' + logger.info(f"Uninstalling {package_name} from device") + subprocess.run(xharness_adb() + ['uninstall', package_name], check=False) + + logger.info("Shutting down dotnet build servers") + subprocess.run(['dotnet', 'build-server', 'shutdown'], check=False) + + clean_directories() +except Exception as e: + logger.error(f"Post cleanup failed: {e}\n{traceback.format_exc()}") + sys.exit(1) diff --git a/src/scenarios/mauiandroidinnerloop/pre.py b/src/scenarios/mauiandroidinnerloop/pre.py new file mode 100644 index 00000000000..a8f8b4b3a7c --- /dev/null +++ b/src/scenarios/mauiandroidinnerloop/pre.py @@ -0,0 +1,210 @@ +''' +pre-command: Set up a MAUI Android app for deploy measurement. +Creates the template (without restore) and prepares the modified file for incremental deploy. +NuGet packages are restored on the Helix machine, not shipped in the payload. +''' +import json +import os +import re +import shutil +import sys +from performance.common import get_repo_root_path +from performance.logger import setup_loggers, getLogger +from shared import const +from shared.mauisharedpython import extract_latest_dotnet_feed_from_nuget_config, MauiNuGetConfigContext +from shared.precommands import PreCommands +from test import EXENAME + +def install_maui_android_workload(precommands: PreCommands): + ''' + Install the maui-android workload (not the full 'maui' workload). + The full 'maui' workload includes iOS/macOS/Windows components that aren't + available on Linux. Since this scenario only needs Android, 'maui-android' + is sufficient and works on both Windows and Linux. + ''' + # Why this is complex: we can't simply run `dotnet workload install maui-android` + # because that would install the latest public version, which may not match the + # SDK version being tested. Instead, we resolve the exact workload manifest version + # from the NuGet feed that matches our SDK, create a rollback file pinning that + # version, and install using --from-rollback-file. + logger.info("########## Installing maui-android workload ##########") + + if precommands.has_workload: + logger.info("Skipping maui-android installation due to --has-workload=true") + return + + feed = extract_latest_dotnet_feed_from_nuget_config( + path=os.path.join(get_repo_root_path(), "NuGet.config") + ) + logger.info(f"Installing the latest maui-android workload from feed {feed}") + + workload = "microsoft.net.sdk.android" + try: + packages = precommands.get_packages_for_sdk_from_feed(workload, feed) + except Exception as e: + logger.warning(f"Failed to get packages for {workload} from latest feed: {e}") + logger.info("Trying second latest feed as fallback") + fallback_feed = extract_latest_dotnet_feed_from_nuget_config( + path=os.path.join(get_repo_root_path(), "NuGet.config"), + offset=1 + ) + logger.info(f"Using fallback feed: {fallback_feed}") + packages = precommands.get_packages_for_sdk_from_feed(workload, fallback_feed) + + # Filter to manifest packages only + pattern = r'Microsoft\.NET\.Sdk\..*\.Manifest\-\d+\.\d+\.\d+(\-(preview|rc|alpha)\.\d+)?$' + packages = [pkg for pkg in packages if re.match(pattern, pkg['id'])] + logger.info(f"After manifest pattern filtering, found {len(packages)} packages for {workload}") + + # Extract SDK and .NET versions from package IDs + for package in packages: + match = re.search(r'Manifest-(.+)$', package["id"]) + if not match: + raise Exception(f"Unable to find .NET SDK version in package ID: {package['id']}") + sdk_version = match.group(1) + package['sdk_version'] = sdk_version + + ver_match = re.search(r'^\d+\.\d+', sdk_version) + if not ver_match: + raise Exception(f"Unable to find .NET version in SDK version '{sdk_version}'") + package['dotnet_version'] = ver_match.group(0) + + # Keep only packages targeting the highest .NET version + dotnet_versions = [float(pkg['dotnet_version']) for pkg in packages] + highest = max(dotnet_versions) + packages = [pkg for pkg in packages if float(pkg['dotnet_version']) == highest] + logger.info(f"After .NET version filtering for {workload}: {len(packages)} packages (highest={highest})") + + # Prefer non-preview packages + preview_pattern = r'\-(preview|rc|alpha)\.\d+$' + non_preview = [pkg for pkg in packages if not re.search(preview_pattern, pkg['id'])] + if non_preview: + packages = non_preview + + # Sort by SDK version descending and take the latest + packages.sort(key=lambda x: x['sdk_version'], reverse=True) + if not packages: + raise Exception(f"No packages available for {workload} after filtering") + + latest = packages[0] + logger.info(f"Latest package: ID={latest['id']}, Version={latest['latestVersion']}, SDK={latest['sdk_version']}") + + # Create rollback file with only the android workload + rollback_value = f"{latest['latestVersion']}/{latest['sdk_version']}" + rollback_dict = {workload: rollback_value} + logger.info(f"Rollback dictionary: {rollback_dict}") + with open("rollback_maui.json", "w", encoding="utf-8") as f: + f.write(json.dumps(rollback_dict, indent=4)) + logger.info("Created rollback_maui.json file") + + # Install maui-android (not 'maui') — works on both Windows and Linux + precommands.install_workload('maui-android', ['--from-rollback-file', 'rollback_maui.json']) + logger.info("########## Finished installing maui-android workload ##########") + +setup_loggers(True) +logger = getLogger(__name__) +logger.info("Starting pre-command for MAUI Android deploy measurement") + +precommands = PreCommands() + +with MauiNuGetConfigContext(precommands.framework): + install_maui_android_workload(precommands) + precommands.print_dotnet_info() + + # Create template without restoring packages — packages will be restored + # on the Helix machine to avoid shipping ~1-2GB in the workitem payload. + precommands.new(template='maui', + output_dir=const.APPDIR, + bin_dir=const.BINDIR, + exename=EXENAME, + working_directory=sys.path[0], + no_restore=True, + extra_args=['-sc']) + + # Copy the merged NuGet.config into the app directory. This file contains + # MAUI NuGet feed URLs added by MauiNuGetConfigContext. The Helix machine + # needs these feeds during restore, and we must copy before the context + # manager restores the original NuGet.config. + repo_root = os.path.normpath(os.path.join(sys.path[0], '..', '..', '..')) + repo_nuget_config = os.path.join(repo_root, 'NuGet.config') + app_nuget_config = os.path.join(const.APPDIR, 'NuGet.config') + shutil.copy2(repo_nuget_config, app_nuget_config) + logger.info(f"Copied merged NuGet.config from {repo_nuget_config} to {app_nuget_config}") + + # Inject properties into the csproj so they apply to every command that + # targets this project (restore, build, install). + csproj_path = os.path.join(const.APPDIR, f'{EXENAME}.csproj') + with open(csproj_path, 'r') as f: + csproj_content = f.read() + + logger.info(f"Original .csproj content:\n{csproj_content}") + + injected_props = { + # Preview SDKs may lack prune-package-data files, causing NETSDK1226. + 'AllowMissingPrunePackageData': 'true', + # The perf repo globally disables the Roslyn compiler server to avoid + # BenchmarkDotNet file-locking issues. Re-enable it here to match real + # MAUI developer inner loop experience. + 'UseSharedCompilation': 'true', + } + csproj_modified = csproj_content + if '' not in csproj_modified: + raise Exception( + f"Cannot inject properties into {csproj_path}: " + f"no found in the generated template." + ) + for prop_name, prop_value in injected_props.items(): + if prop_name not in csproj_modified: + csproj_modified = csproj_modified.replace( + '', + f' <{prop_name}>{prop_value}\n ', + 1 # only the first PropertyGroup + ) + + with open(csproj_path, 'w') as f: + f.write(csproj_modified) + + logger.info(f"Updated {csproj_path} with injected properties") + logger.info(f"Modified .csproj content:\n{csproj_modified}") + + # Create modified source files in src/ for the incremental deploy simulation. + # The runner toggles between original and modified versions each iteration, + # exercising both the C# compiler (Csc) and XAML compiler (XamlC) paths. + src_dir = os.path.join(sys.path[0], const.SRCDIR) + os.makedirs(src_dir, exist_ok=True) + + # --- Modified MainPage.xaml.cs: add a debug line in the constructor --- + cs_original = os.path.join(const.APPDIR, 'Pages', 'MainPage.xaml.cs') + cs_modified = os.path.join(src_dir, 'MainPage.xaml.cs') + + with open(cs_original, 'r') as f: + cs_content = f.read() + + cs_modified_content = cs_content.replace( + 'InitializeComponent();', + 'InitializeComponent();\n\t\tSystem.Diagnostics.Debug.WriteLine("incremental-touch");' + ) + if cs_modified_content == cs_content: + raise Exception("Could not find 'InitializeComponent();' in %s — template may have changed" % cs_original) + + with open(cs_modified, 'w') as f: + f.write(cs_modified_content) + logger.info(f"Modified MainPage.xaml.cs written to {cs_modified}") + + # --- Modified MainPage.xaml: change a label's text --- + xaml_original = os.path.join(const.APPDIR, 'Pages', 'MainPage.xaml') + xaml_modified = os.path.join(src_dir, 'MainPage.xaml') + + with open(xaml_original, 'r') as f: + xaml_content = f.read() + + xaml_modified_content = xaml_content.replace( + 'Text="Task Categories"', + 'Text="Task Categories (updated)"' + ) + if xaml_modified_content == xaml_content: + raise Exception("Could not find 'Text=\"Task Categories\"' in %s — template may have changed" % xaml_original) + + with open(xaml_modified, 'w') as f: + f.write(xaml_modified_content) + logger.info(f"Modified MainPage.xaml written to {xaml_modified}") diff --git a/src/scenarios/mauiandroidinnerloop/setup_helix.py b/src/scenarios/mauiandroidinnerloop/setup_helix.py new file mode 100644 index 00000000000..369c7d9ae03 --- /dev/null +++ b/src/scenarios/mauiandroidinnerloop/setup_helix.py @@ -0,0 +1,315 @@ +#!/usr/bin/env python3 +"""setup_helix.py — Helix machine setup for MAUI Android inner loop (Windows + Linux).""" + +import os +import platform +import stat +import subprocess +import sys +import time +from datetime import datetime + +# --- Constants --- +IS_WINDOWS = platform.system() == "Windows" +EXE = ".exe" if IS_WINDOWS else "" + +# --- Logging --- +_logfile = None + + +def log(msg, tee=False): + """Write *msg* with a timestamp to the log file.""" + line = f"[{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] {msg}" + if _logfile: + _logfile.write(line + "\n") + _logfile.flush() + if tee: + print(line, flush=True) + + +def log_raw(msg, tee=False): + """Write *msg* verbatim (no timestamp) to the log file.""" + if _logfile: + _logfile.write(msg + "\n") + _logfile.flush() + if tee: + print(msg, flush=True) + + +def run_cmd(args, check=True, **kwargs): + """Run a command, logging stdout/stderr. Returns CompletedProcess.""" + log(f"Running: {args}") + result = subprocess.run( + args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, + **kwargs, + ) + if result.stdout: + for line in result.stdout.splitlines(): + log_raw(line) + if check and result.returncode != 0: + raise subprocess.CalledProcessError(result.returncode, args, result.stdout) + return result + + +def _chmod_exec(path): + if not IS_WINDOWS and os.path.isfile(path): + os.chmod(path, os.stat(path).st_mode | stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH) + + +# --- ADB device setup --- +def _count_adb_devices(): + result = subprocess.run(["adb", "devices"], capture_output=True, text=True) + count = 0 + for line in result.stdout.splitlines()[1:]: + parts = line.split() + if len(parts) >= 2 and parts[1] == "device": + count += 1 + return count + + +def _setup_adb_windows(android_home): + """Windows: wait for a physical device, then verify.""" + log("Waiting for device (timeout 30s)...") + try: + run_cmd(["adb", "wait-for-device"], check=False, timeout=30) + except subprocess.TimeoutExpired: + log("WARNING: adb wait-for-device timed out") + device_count = _count_adb_devices() + log(f"Device count: {device_count}") + if device_count == 0: + log("WARNING: No devices detected — build -t:Install will likely fail") + + +def _setup_adb_linux(): + """Linux: target emulator-5554, wait for device and boot.""" + log("ANDROID_SERIAL=emulator-5554 (set by PreCommands)") + log("Waiting for device (timeout 30s)...") + try: + run_cmd(["adb", "wait-for-device"], check=False, timeout=30) + except subprocess.TimeoutExpired: + log("WARNING: adb wait-for-device timed out") + device_count = _count_adb_devices() + log(f"Device count: {device_count}") + if device_count == 0: + log("WARNING: No devices detected — build -t:Install will likely fail") + return + # Wait for emulator boot + log("Waiting for emulator to fully boot (up to 60s)...") + boot_wait = 0 + boot_completed = "" + while boot_wait < 60: + result = subprocess.run( + ["adb", "shell", "getprop", "sys.boot_completed"], + capture_output=True, text=True, + ) + boot_completed = result.stdout.strip() + if boot_completed == "1": + log(f"Emulator fully booted after {boot_wait}s") + break + time.sleep(5) + boot_wait += 5 + log(f" sys.boot_completed={boot_completed} (waited {boot_wait}s)") + if boot_completed != "1": + log(f"WARNING: Emulator did not report boot_completed=1 after 60s") + + +def _dump_log(): + if not _logfile: + return + _logfile.flush() + try: + with open(_logfile.name, "r") as f: + print(f.read()) + except Exception: + pass + + +# --- Orchestration --- +def print_diagnostics(): + log_raw("=== DIAGNOSTICS ===", tee=True) + log_raw(f"DOTNET_ROOT={os.environ.get('DOTNET_ROOT', '')}") + + +def setup_dotnet(correlation_payload): + """Return the path to the dotnet executable in the correlation payload.""" + dotnet_root = os.path.join(correlation_payload, "dotnet") + dotnet_exe = os.path.join(dotnet_root, f"dotnet{EXE}") + log(f"DOTNET_ROOT={dotnet_root}") + run_cmd([dotnet_exe, "--version"], check=False) + return dotnet_exe + + +def install_workload(ctx): + """Step 1: Install workloads via restore (dependencies) then install (pinned). + + First, ``workload restore`` installs whatever workloads the project + needs (including iOS/MacCatalyst if MAUI requires them) at whatever + version is available in the feeds. This satisfies dependency packages + that may not exist at the pinned version. + + Then, ``workload install maui-android --from-rollback-file`` pins the + android workload to the exact version from the rollback file we want + to test against. + """ + log_raw("=== STEP 1: Workload Install ===", tee=True) + + # 1a. Workload restore — satisfy all project dependencies first. + log("Step 1a: workload restore (satisfy project dependencies)", tee=True) + result = run_cmd( + [ctx["dotnet_exe"], "workload", "restore", ctx["csproj"], + "--configfile", ctx["nuget_config"], + f"-p:TargetFrameworks={ctx['framework']}"], + check=False, + ) + if result.returncode != 0: + log(f"WARNING: workload restore exited with code {result.returncode} " + "— continuing with pinned install", tee=True) + else: + log("Workload restore succeeded") + + # 1b. Workload install — pin maui-android to the rollback version. + rollback_file = os.path.join(ctx["workitem_root"], "rollback_maui.json") + log(f"Step 1b: workload install maui-android " + f"(pinned via {rollback_file})", tee=True) + result = run_cmd( + [ctx["dotnet_exe"], "workload", "install", "maui-android", + "--from-rollback-file", rollback_file, + "--configfile", ctx["nuget_config"]], + check=False, + ) + if result.returncode != 0: + log(f"STEP 1 FAILED: workload install exited with code " + f"{result.returncode}", tee=True) + _dump_log() + sys.exit(1) + log("Workload install (pinned maui-android) succeeded") + + +def install_android_dependencies(ctx): + """Restore the real MAUI csproj and run InstallAndroidDependencies. + + Uses the real project at ``ctx["csproj"]`` directly. + ``AllowMissingPrunePackageData=true`` bypasses NETSDK1226 on preview + SDKs that lack prune package data. + """ + log_raw("=== Installing Android SDK & Java Dependencies ===", tee=True) + + android_home = os.path.join(ctx["workitem_root"], "android-sdk") + java_home = os.path.join(ctx["workitem_root"], "jdk") + os.makedirs(android_home, exist_ok=True) + os.makedirs(java_home, exist_ok=True) + + csproj = ctx["csproj"] + log(f"Using project: {csproj}") + + # Restore the project. Override TargetFrameworks to android-only. + result = run_cmd( + [ctx["dotnet_exe"], "restore", csproj, + "--configfile", ctx["nuget_config"], + f"-p:TargetFrameworks={ctx['framework']}"], + check=False, + ) + if result.returncode != 0: + log("WARNING: restore failed — " + "InstallAndroidDependencies may not work") + + # Use dotnet msbuild (not dotnet build) to run only this target + # without the full build pipeline. + result = run_cmd( + [ctx["dotnet_exe"], "msbuild", csproj, + "-t:InstallAndroidDependencies", + f"/p:AndroidSdkDirectory={android_home}", + f"/p:JavaSdkDirectory={java_home}", + "/p:AcceptAndroidSdkLicenses=True", + f"/p:TargetFramework={ctx['framework']}"], + check=False, + ) + if result.returncode != 0: + log("InstallAndroidDependencies FAILED", tee=True) + _dump_log() + sys.exit(1) + log("Android SDK and Java dependencies installed successfully") + + # Set up paths for adb and log the values (env vars set by PreCommands) + platform_tools = os.path.join(android_home, "platform-tools") + log(f"ANDROID_HOME={android_home}") + log(f"JAVA_HOME={java_home}") + + _chmod_exec(os.path.join(platform_tools, "adb")) + ctx["android_home"] = android_home + + +def setup_android_sdk(ctx): + """Install Android SDK and Java via the built-in MSBuild target.""" + log_raw("=== Setting up Android SDK ===") + install_android_dependencies(ctx) + + +def setup_adb_device(ctx): + """Restart ADB server and run platform-specific device setup.""" + log_raw("=== ADB DEVICE SETUP ===") + run_cmd(["adb", "kill-server"], check=False) + run_cmd(["adb", "start-server"], check=False) + run_cmd(["adb", "devices", "-l"], check=False) + if IS_WINDOWS: + _setup_adb_windows(ctx["android_home"]) + else: + _setup_adb_linux() + + +def restore_packages(ctx): + """Step 2: Restore NuGet packages.""" + log_raw("=== STEP 2: Restore ===", tee=True) + restore_args = [ + ctx["dotnet_exe"], "restore", ctx["csproj"], + "--configfile", ctx["nuget_config"], + f"/p:TargetFrameworks={ctx['framework']}", + ] + if ctx["msbuild_args"]: + restore_args.extend(ctx["msbuild_args"].split()) + result = run_cmd(restore_args, check=False) + if result.returncode != 0: + log(f"STEP 2 FAILED with exit code {result.returncode}", tee=True) + _dump_log() + sys.exit(2) + log("Restore succeeded") + + +# --- Main --- +def main(): + if len(sys.argv) < 3: + print(f"Usage: {sys.argv[0]} FRAMEWORK MSBUILD_ARGS", + file=sys.stderr) + sys.exit(1) + + global _logfile + upload_root = os.environ.get("HELIX_WORKITEM_UPLOAD_ROOT") + if upload_root: + _logfile = open(os.path.join(upload_root, "output.log"), "a") + + workitem_root = os.environ.get("HELIX_WORKITEM_ROOT", ".") + correlation_payload = os.environ.get("HELIX_CORRELATION_PAYLOAD", ".") + ctx = { + "framework": sys.argv[1], + "msbuild_args": sys.argv[2], + "workitem_root": workitem_root, + "correlation_payload": correlation_payload, + "nuget_config": os.path.join(workitem_root, "app", "NuGet.config"), + "csproj": os.path.join(workitem_root, "app", "MauiAndroidInnerLoop.csproj"), + } + + ctx["dotnet_exe"] = setup_dotnet(correlation_payload) + print_diagnostics() + + install_workload(ctx) + setup_android_sdk(ctx) + setup_adb_device(ctx) + restore_packages(ctx) + + log_raw("=== SETUP SUCCEEDED ===", tee=True) + _dump_log() + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/scenarios/mauiandroidinnerloop/test.py b/src/scenarios/mauiandroidinnerloop/test.py new file mode 100644 index 00000000000..ddaf0bda63f --- /dev/null +++ b/src/scenarios/mauiandroidinnerloop/test.py @@ -0,0 +1,14 @@ +''' +MAUI Android Inner Loop (Debug End-2-End) Time Measurement +Orchestrates first build-deploy-startup → file edit → incremental build-deploy-startup → parse binlogs and startup times. +''' +import os +from shared.runner import TestTraits, Runner + +EXENAME = 'MauiAndroidInnerLoop' + +if __name__ == "__main__": + traits = TestTraits(exename=EXENAME, + guiapp='false', + ) + Runner(traits).run() From 3f5e643656074f14723b132fbc91d41fd20bcf94 Mon Sep 17 00:00:00 2001 From: David Nguyen <87228593+davidnguyen-tech@users.noreply.github.com> Date: Tue, 31 Mar 2026 17:16:21 +0200 Subject: [PATCH 04/19] Wire MAUI Android inner loop into CI pipeline Add CI infrastructure to run inner loop measurements on Helix: - maui_scenarios_android_innerloop.proj: MSBuild project that defines Helix work items with per-platform PreCommands for environment setup, SDK discovery, workload installation, and test invocation - sdk-perf-jobs.yml: Add 6 inner loop job definitions covering Pixel 8, Galaxy A16, and Android 36 emulator queues, each with Mono and CoreCLR runtime configurations - build-machine-matrix.yml: Add ubuntu-x64-android-emulator build machine mapping to Ubuntu.2204.Amd64.Android.36 queue - run-performance-job.yml: Support androidinnerloop runtime flavor - run_performance_job.py: Extend maui_scenarios_android run_kind matching to include innerloop variant; copy binlogs to artifacts Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../maui_scenarios_android_innerloop.proj | 98 +++++++++++++++ eng/pipelines/sdk-perf-jobs.yml | 114 ++++++++++++++++++ .../templates/build-machine-matrix.yml | 13 ++ .../templates/run-performance-job.yml | 2 +- scripts/run_performance_job.py | 6 +- 5 files changed, 229 insertions(+), 4 deletions(-) create mode 100644 eng/performance/maui_scenarios_android_innerloop.proj diff --git a/eng/performance/maui_scenarios_android_innerloop.proj b/eng/performance/maui_scenarios_android_innerloop.proj new file mode 100644 index 00000000000..bcff5027140 --- /dev/null +++ b/eng/performance/maui_scenarios_android_innerloop.proj @@ -0,0 +1,98 @@ + + + + + + + true + + + + <_MSBuildArgs Condition="'$(RuntimeFlavor)' == 'mono'">/p:UseMonoRuntime=true + <_MSBuildArgs Condition="'$(RuntimeFlavor)' == 'coreclr'">/p:UseMonoRuntime=false + + + <_MSBuildArgs Condition="'$(RuntimeFlavor)' == 'coreclr' and '$(CodegenType)' == 'JIT'">$(_MSBuildArgs) /p:PublishReadyToRun=false /p:PublishReadyToRunComposite=false + + + android-arm64 + android-x64 + <_MSBuildArgs>$(_MSBuildArgs) /p:RuntimeIdentifier=$(AndroidRid) + + + <_MSBuildArgs>$(_MSBuildArgs) /p:SupportedOSPlatformVersion=23 + + <_MSBuildArgs Condition="!$(_MSBuildArgs.Contains('/p:TargetFrameworks='))">$(_MSBuildArgs) /p:TargetFrameworks=$(PERFLAB_Framework)-android + + $(RuntimeFlavor)_$(CodegenType) + 10 + + + + + + + + + + + + 01:00 + + + + + + mauiandroidinnerloop + $(ScenariosDir)%(ScenarioDirectoryName) + + + + + + + $(Python) pre.py default -f $(PERFLAB_Framework) + %(PreparePayloadWorkItem.PayloadDirectory) + + + + + + <_WindowsEnvVars>set DOTNET_ROOT=%HELIX_CORRELATION_PAYLOAD%\dotnet;set DOTNET_CLI_TELEMETRY_OPTOUT=1;set DOTNET_MULTILEVEL_LOOKUP=0;set NUGET_PACKAGES=%HELIX_WORKITEM_ROOT%\.packages;set ANDROID_HOME=%HELIX_WORKITEM_ROOT%\android-sdk;set ANDROID_SDK_ROOT=%HELIX_WORKITEM_ROOT%\android-sdk;set JAVA_HOME=%HELIX_WORKITEM_ROOT%\jdk;set PATH=%HELIX_CORRELATION_PAYLOAD%\dotnet;;%HELIX_WORKITEM_ROOT%\android-sdk\platform-tools;;%HELIX_WORKITEM_ROOT%\jdk\bin;;%PATH% + <_LinuxEnvVars>export DOTNET_ROOT=$HELIX_CORRELATION_PAYLOAD/dotnet;export DOTNET_CLI_TELEMETRY_OPTOUT=1;export DOTNET_MULTILEVEL_LOOKUP=0;export NUGET_PACKAGES=$HELIX_WORKITEM_ROOT/.packages;export ANDROID_HOME=$HELIX_WORKITEM_ROOT/android-sdk;export ANDROID_SDK_ROOT=$HELIX_WORKITEM_ROOT/android-sdk;export JAVA_HOME=$HELIX_WORKITEM_ROOT/jdk;export ANDROID_SERIAL=emulator-5554;export PATH=$HELIX_CORRELATION_PAYLOAD/dotnet:$HELIX_WORKITEM_ROOT/android-sdk/platform-tools:$HELIX_WORKITEM_ROOT/jdk/bin:$PATH + + + + + + $(_WindowsEnvVars);$(Python) setup_helix.py $(PERFLAB_Framework)-android "$(_MSBuildArgs)" + $(Python) test.py androidinnerloop --csproj-path app/MauiAndroidInnerLoop.csproj --edit-src "src/MainPage.xaml.cs;src/MainPage.xaml" --edit-dest "app/Pages/MainPage.xaml.cs;app/Pages/MainPage.xaml" --package-name com.companyname.mauiandroidinnerloop -f $(PERFLAB_Framework)-android -c Debug --msbuild-args "$(_MSBuildArgs)" --scenario-name "%(Identity)" --inner-loop-iterations $(InnerLoopIterations) $(ScenarioArgs) + $(Python) post.py + output.log + + + + + + + $(_LinuxEnvVars);$(Python) setup_helix.py $(PERFLAB_Framework)-android "$(_MSBuildArgs)" + $(Python) test.py androidinnerloop --csproj-path app/MauiAndroidInnerLoop.csproj --edit-src "src/MainPage.xaml.cs;src/MainPage.xaml" --edit-dest "app/Pages/MainPage.xaml.cs;app/Pages/MainPage.xaml" --package-name com.companyname.mauiandroidinnerloop -f $(PERFLAB_Framework)-android -c Debug --msbuild-args "$(_MSBuildArgs)" --scenario-name "%(Identity)" --inner-loop-iterations $(InnerLoopIterations) $(ScenarioArgs) + $(Python) post.py + output.log + + + + + + diff --git a/eng/pipelines/sdk-perf-jobs.yml b/eng/pipelines/sdk-perf-jobs.yml index 4c3a62d1a72..f089963da10 100644 --- a/eng/pipelines/sdk-perf-jobs.yml +++ b/eng/pipelines/sdk-perf-jobs.yml @@ -527,6 +527,120 @@ jobs: ${{ each parameter in parameters.jobParameters }}: ${{ parameter.key }}: ${{ parameter.value }} + # Maui Android inner loop benchmarks on Pixel (Mono Default) - Debug + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64-android-arm64-pixel + isPublic: false + jobParameters: + runKind: maui_scenarios_android_innerloop + projectFileName: maui_scenarios_android_innerloop.proj + channels: + - main + runtimeFlavor: mono + codeGenType: Default + buildConfig: Debug + additionalJobIdentifier: Mono_Debug_InnerLoop + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # Maui Android inner loop benchmarks on Pixel (CoreCLR Default) - Debug + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64-android-arm64-pixel + isPublic: false + jobParameters: + runKind: maui_scenarios_android_innerloop + projectFileName: maui_scenarios_android_innerloop.proj + channels: + - main + runtimeFlavor: coreclr + codeGenType: Default + buildConfig: Debug + additionalJobIdentifier: CoreCLR_Debug_InnerLoop + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # Maui Android inner loop benchmarks on Galaxy (Mono Default) - Debug + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64-android-arm64-galaxy + isPublic: false + jobParameters: + runKind: maui_scenarios_android_innerloop + projectFileName: maui_scenarios_android_innerloop.proj + channels: + - main + runtimeFlavor: mono + codeGenType: Default + buildConfig: Debug + additionalJobIdentifier: Mono_Debug_InnerLoop_Galaxy + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # Maui Android inner loop benchmarks on Galaxy (CoreCLR Default) - Debug + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64-android-arm64-galaxy + isPublic: false + jobParameters: + runKind: maui_scenarios_android_innerloop + projectFileName: maui_scenarios_android_innerloop.proj + channels: + - main + runtimeFlavor: coreclr + codeGenType: Default + buildConfig: Debug + additionalJobIdentifier: CoreCLR_Debug_InnerLoop_Galaxy + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # Maui Android inner loop benchmarks on emulator (Mono Default) - Debug + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - ubuntu-x64-android-emulator + isPublic: false + jobParameters: + runKind: maui_scenarios_android_innerloop + projectFileName: maui_scenarios_android_innerloop.proj + channels: + - main + runtimeFlavor: mono + codeGenType: Default + buildConfig: Debug + additionalJobIdentifier: Mono_Debug_InnerLoop_Emulator + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # Maui Android inner loop benchmarks on emulator (CoreCLR Default) - Debug + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - ubuntu-x64-android-emulator + isPublic: false + jobParameters: + runKind: maui_scenarios_android_innerloop + projectFileName: maui_scenarios_android_innerloop.proj + channels: + - main + runtimeFlavor: coreclr + codeGenType: Default + buildConfig: Debug + additionalJobIdentifier: CoreCLR_Debug_InnerLoop_Emulator + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + # Maui iOS scenario benchmarks (Mono - Default) - Debug - template: /eng/pipelines/templates/build-machine-matrix.yml parameters: diff --git a/eng/pipelines/templates/build-machine-matrix.yml b/eng/pipelines/templates/build-machine-matrix.yml index 71ffc558754..f558c755adb 100644 --- a/eng/pipelines/templates/build-machine-matrix.yml +++ b/eng/pipelines/templates/build-machine-matrix.yml @@ -146,6 +146,19 @@ jobs: machinePool: GalaxyA16 ${{ insert }}: ${{ parameters.jobParameters }} +- ${{ if and(containsValue(parameters.buildMachines, 'ubuntu-x64-android-emulator'), not(eq(parameters.isPublic, true))) }}: # Ubuntu x64 Android emulator only used in private builds currently + - template: ${{ parameters.jobTemplate }} + parameters: + osGroup: ubuntu + archType: x64 + osVersion: 2204 + pool: + vmImage: ubuntu-latest + container: ubuntu_x64_build_container + queue: Ubuntu.2204.Amd64.Android.36 + machinePool: AndroidEmulator + ${{ insert }}: ${{ parameters.jobParameters }} + - ${{ if and(containsValue(parameters.buildMachines, 'osx-x64-ios-arm64'), not(eq(parameters.isPublic, true))) }}: # iPhone ARM64 17 only used in private builds currently - template: ${{ parameters.jobTemplate }} parameters: diff --git a/eng/pipelines/templates/run-performance-job.yml b/eng/pipelines/templates/run-performance-job.yml index 4fe69d8c312..0045cdb5a03 100644 --- a/eng/pipelines/templates/run-performance-job.yml +++ b/eng/pipelines/templates/run-performance-job.yml @@ -190,7 +190,7 @@ jobs: - '--is-scenario' - ${{ if ne(length(parameters.runEnvVars), 0) }}: - "--run-env-vars ${{ join(' ', parameters.runEnvVars)}}" - - ${{ if and(in(parameters.runKind, 'maui_scenarios_ios', 'maui_scenarios_android'), ne(parameters.runtimeFlavor, '')) }}: + - ${{ if and(in(parameters.runKind, 'maui_scenarios_ios', 'maui_scenarios_android', 'maui_scenarios_android_innerloop'), ne(parameters.runtimeFlavor, '')) }}: - '--runtime-flavor ${{ parameters.runtimeFlavor }}' - ${{ if ne(parameters.osVersion, '') }}: - '--os-version ${{ parameters.osVersion }}' diff --git a/scripts/run_performance_job.py b/scripts/run_performance_job.py index 7e7e48c8c44..ad1c51908a8 100644 --- a/scripts/run_performance_job.py +++ b/scripts/run_performance_job.py @@ -570,9 +570,9 @@ def get_run_configurations( configurations["iOSLlvmBuild"] = str(ios_llvm_build) # .NET Android and .NET MAUI Android sample app scenarios - if run_kind == "maui_scenarios_android": + if run_kind in ["maui_scenarios_android", "maui_scenarios_android_innerloop"]: if not runtime_flavor in ("mono", "coreclr"): - raise Exception("Runtime flavor must be specified for maui_scenarios_android") + raise Exception(f"Runtime flavor must be specified for {run_kind}") configurations["CodegenType"] = str(codegen_type) configurations["RuntimeType"] = str(runtime_flavor) if build_config is not None and build_config != DEFAULT_BUILD_CONFIG: @@ -1190,7 +1190,7 @@ def publish_dotnet_app_to_payload(payload_dir_name: str, csproj_path: str, self_ verbose=True).run() # Search for additional binlogs generated by the maui scenarios prepare payload work items to copy to the artifacts log dir - if args.run_kind in ["maui_scenarios_android", "maui_scenarios_ios"]: + if args.run_kind in ["maui_scenarios_android", "maui_scenarios_ios", "maui_scenarios_android_innerloop"]: for binlog_path in glob(os.path.join(payload_dir, "scenarios_out", "**", "*.binlog"), recursive=True): shutil.copy(binlog_path, ci_artifacts_log_dir) From 746709af59818f7a8f62ffc8d11f01a746773499 Mon Sep 17 00:00:00 2001 From: David Nguyen <87228593+davidnguyen-tech@users.noreply.github.com> Date: Wed, 1 Apr 2026 11:28:54 +0200 Subject: [PATCH 05/19] Make screen timeout configurable and validate cold startup LaunchState Add --screen-timeout-ms CLI argument (default 1800000 = 30 min) and ScreenTimeoutMs MSBuild property so the screen timeout can be tuned from the .proj file without code changes. Add LaunchState validation to AndroidHelper.measure_cold_startup(): if am start reports anything other than COLD (e.g. UNKNOWN when the screen is off), the method now throws with a clear diagnostic message suggesting to increase --screen-timeout-ms. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/performance/maui_scenarios_android_innerloop.proj | 8 ++++++-- src/scenarios/shared/androidhelper.py | 10 ++++++++++ src/scenarios/shared/runner.py | 4 +++- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/eng/performance/maui_scenarios_android_innerloop.proj b/eng/performance/maui_scenarios_android_innerloop.proj index bcff5027140..f733bbe160f 100644 --- a/eng/performance/maui_scenarios_android_innerloop.proj +++ b/eng/performance/maui_scenarios_android_innerloop.proj @@ -33,6 +33,10 @@ $(RuntimeFlavor)_$(CodegenType) 10 + + 1800000 + $(_LinuxEnvVars);$(Python) setup_helix.py $(PERFLAB_Framework)-android "$(_MSBuildArgs)" diff --git a/src/scenarios/mauiandroidinnerloop/post.py b/src/scenarios/mauiandroidinnerloop/post.py index e63eb8a009c..de98a0c507b 100644 --- a/src/scenarios/mauiandroidinnerloop/post.py +++ b/src/scenarios/mauiandroidinnerloop/post.py @@ -2,7 +2,6 @@ post cleanup script ''' -import os import subprocess import sys import traceback @@ -14,13 +13,6 @@ setup_loggers(True) logger = getLogger(__name__) -# Pin to the emulator device to avoid "more than one device/emulator" errors -# 'emulator-5554' is the standard ADB serial for the first Android emulator instance -# and is stable across all Android SDK versions. -if 'ANDROID_SERIAL' not in os.environ: - os.environ['ANDROID_SERIAL'] = 'emulator-5554' - logger.info("Set ANDROID_SERIAL=emulator-5554") - try: # Uninstall the app from the connected device so re-runs start from a clean state package_name = f'com.companyname.{EXENAME.lower()}' diff --git a/src/scenarios/shared/runner.py b/src/scenarios/shared/runner.py index c2045e60f8f..d93169dd8ea 100644 --- a/src/scenarios/shared/runner.py +++ b/src/scenarios/shared/runner.py @@ -1100,7 +1100,7 @@ def run_incremental_iteration(iteration, num_iterations, base_cmd, edit_pairs, first_binlog = os.path.join(const.TRACEDIR, 'first-build-and-deploy.binlog') # Build the base MSBuild command - base_cmd = ['dotnet', 'build', self.csprojpath, '-t:Install'] + base_cmd = ['dotnet', 'build', self.csprojpath, '-t:Install', '--no-restore'] if self.configuration: base_cmd.extend(['-c', self.configuration]) if self.framework: diff --git a/src/tools/ScenarioMeasurement/Util/Parsers/AndroidInnerLoopParser.cs b/src/tools/ScenarioMeasurement/Util/Parsers/AndroidInnerLoopParser.cs index f5d55d00021..d112a2bd451 100644 --- a/src/tools/ScenarioMeasurement/Util/Parsers/AndroidInnerLoopParser.cs +++ b/src/tools/ScenarioMeasurement/Util/Parsers/AndroidInnerLoopParser.cs @@ -136,7 +136,7 @@ public IEnumerable Parse(string binlogFile, string processName, IList 0) - yield return new Counter { Name = "Publish Time", MetricName = "s", DefaultCounter = true, TopCounter = true, Results = publishTimes.ToArray() }; + yield return new Counter { Name = "Build+Deploy Time", MetricName = "s", DefaultCounter = true, TopCounter = true, Results = publishTimes.ToArray() }; // Build task counters if (cscTimes.Count > 0) From 26ea3ea95e8e8be13fbc2c5a58c7653e7a1f60be Mon Sep 17 00:00:00 2001 From: David Nguyen <87228593+davidnguyen-tech@users.noreply.github.com> Date: Tue, 7 Apr 2026 15:08:30 +0200 Subject: [PATCH 07/19] Address review notes 2 --- .../maui_scenarios_android_innerloop.proj | 2 - eng/pipelines/sdk-perf-jobs.yml | 44 +-------- src/scenarios/mauiandroidinnerloop/pre.py | 97 ++----------------- src/scenarios/shared/mauisharedpython.py | 58 +++++++---- 4 files changed, 49 insertions(+), 152 deletions(-) diff --git a/eng/performance/maui_scenarios_android_innerloop.proj b/eng/performance/maui_scenarios_android_innerloop.proj index 981c1baa8d6..40a74270d37 100644 --- a/eng/performance/maui_scenarios_android_innerloop.proj +++ b/eng/performance/maui_scenarios_android_innerloop.proj @@ -77,7 +77,6 @@ <_LinuxEnvVars>export DOTNET_ROOT=$HELIX_CORRELATION_PAYLOAD/dotnet;export DOTNET_CLI_TELEMETRY_OPTOUT=1;export DOTNET_MULTILEVEL_LOOKUP=0;export NUGET_PACKAGES=$HELIX_WORKITEM_ROOT/.packages;export ANDROID_HOME=$HELIX_WORKITEM_ROOT/android-sdk;export ANDROID_SDK_ROOT=$HELIX_WORKITEM_ROOT/android-sdk;export JAVA_HOME=$HELIX_WORKITEM_ROOT/jdk;export ANDROID_SERIAL=emulator-5554;export PATH=$HELIX_CORRELATION_PAYLOAD/dotnet:$HELIX_WORKITEM_ROOT/android-sdk/platform-tools:$HELIX_WORKITEM_ROOT/jdk/bin:$PATH - $(_WindowsEnvVars);$(Python) setup_helix.py $(PERFLAB_Framework)-android "$(_MSBuildArgs)" @@ -87,7 +86,6 @@ - $(_LinuxEnvVars);$(Python) setup_helix.py $(PERFLAB_Framework)-android "$(_MSBuildArgs)" diff --git a/eng/pipelines/sdk-perf-jobs.yml b/eng/pipelines/sdk-perf-jobs.yml index f089963da10..dda52b3e49b 100644 --- a/eng/pipelines/sdk-perf-jobs.yml +++ b/eng/pipelines/sdk-perf-jobs.yml @@ -527,12 +527,13 @@ jobs: ${{ each parameter in parameters.jobParameters }}: ${{ parameter.key }}: ${{ parameter.value }} - # Maui Android inner loop benchmarks on Pixel (Mono Default) - Debug + # Maui Android inner loop benchmarks on devices (Mono Default) - Debug - template: /eng/pipelines/templates/build-machine-matrix.yml parameters: jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml buildMachines: - win-x64-android-arm64-pixel + - win-x64-android-arm64-galaxy isPublic: false jobParameters: runKind: maui_scenarios_android_innerloop @@ -546,49 +547,12 @@ jobs: ${{ each parameter in parameters.jobParameters }}: ${{ parameter.key }}: ${{ parameter.value }} - # Maui Android inner loop benchmarks on Pixel (CoreCLR Default) - Debug + # Maui Android inner loop benchmarks on devices (CoreCLR Default) - Debug - template: /eng/pipelines/templates/build-machine-matrix.yml parameters: jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml buildMachines: - win-x64-android-arm64-pixel - isPublic: false - jobParameters: - runKind: maui_scenarios_android_innerloop - projectFileName: maui_scenarios_android_innerloop.proj - channels: - - main - runtimeFlavor: coreclr - codeGenType: Default - buildConfig: Debug - additionalJobIdentifier: CoreCLR_Debug_InnerLoop - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} - - # Maui Android inner loop benchmarks on Galaxy (Mono Default) - Debug - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml - buildMachines: - - win-x64-android-arm64-galaxy - isPublic: false - jobParameters: - runKind: maui_scenarios_android_innerloop - projectFileName: maui_scenarios_android_innerloop.proj - channels: - - main - runtimeFlavor: mono - codeGenType: Default - buildConfig: Debug - additionalJobIdentifier: Mono_Debug_InnerLoop_Galaxy - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} - - # Maui Android inner loop benchmarks on Galaxy (CoreCLR Default) - Debug - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml - buildMachines: - win-x64-android-arm64-galaxy isPublic: false jobParameters: @@ -599,7 +563,7 @@ jobs: runtimeFlavor: coreclr codeGenType: Default buildConfig: Debug - additionalJobIdentifier: CoreCLR_Debug_InnerLoop_Galaxy + additionalJobIdentifier: CoreCLR_Debug_InnerLoop ${{ each parameter in parameters.jobParameters }}: ${{ parameter.key }}: ${{ parameter.value }} diff --git a/src/scenarios/mauiandroidinnerloop/pre.py b/src/scenarios/mauiandroidinnerloop/pre.py index a8f8b4b3a7c..39011bd4210 100644 --- a/src/scenarios/mauiandroidinnerloop/pre.py +++ b/src/scenarios/mauiandroidinnerloop/pre.py @@ -3,104 +3,15 @@ Creates the template (without restore) and prepares the modified file for incremental deploy. NuGet packages are restored on the Helix machine, not shipped in the payload. ''' -import json import os -import re import shutil import sys -from performance.common import get_repo_root_path from performance.logger import setup_loggers, getLogger from shared import const -from shared.mauisharedpython import extract_latest_dotnet_feed_from_nuget_config, MauiNuGetConfigContext +from shared.mauisharedpython import install_latest_maui, MauiNuGetConfigContext from shared.precommands import PreCommands from test import EXENAME -def install_maui_android_workload(precommands: PreCommands): - ''' - Install the maui-android workload (not the full 'maui' workload). - The full 'maui' workload includes iOS/macOS/Windows components that aren't - available on Linux. Since this scenario only needs Android, 'maui-android' - is sufficient and works on both Windows and Linux. - ''' - # Why this is complex: we can't simply run `dotnet workload install maui-android` - # because that would install the latest public version, which may not match the - # SDK version being tested. Instead, we resolve the exact workload manifest version - # from the NuGet feed that matches our SDK, create a rollback file pinning that - # version, and install using --from-rollback-file. - logger.info("########## Installing maui-android workload ##########") - - if precommands.has_workload: - logger.info("Skipping maui-android installation due to --has-workload=true") - return - - feed = extract_latest_dotnet_feed_from_nuget_config( - path=os.path.join(get_repo_root_path(), "NuGet.config") - ) - logger.info(f"Installing the latest maui-android workload from feed {feed}") - - workload = "microsoft.net.sdk.android" - try: - packages = precommands.get_packages_for_sdk_from_feed(workload, feed) - except Exception as e: - logger.warning(f"Failed to get packages for {workload} from latest feed: {e}") - logger.info("Trying second latest feed as fallback") - fallback_feed = extract_latest_dotnet_feed_from_nuget_config( - path=os.path.join(get_repo_root_path(), "NuGet.config"), - offset=1 - ) - logger.info(f"Using fallback feed: {fallback_feed}") - packages = precommands.get_packages_for_sdk_from_feed(workload, fallback_feed) - - # Filter to manifest packages only - pattern = r'Microsoft\.NET\.Sdk\..*\.Manifest\-\d+\.\d+\.\d+(\-(preview|rc|alpha)\.\d+)?$' - packages = [pkg for pkg in packages if re.match(pattern, pkg['id'])] - logger.info(f"After manifest pattern filtering, found {len(packages)} packages for {workload}") - - # Extract SDK and .NET versions from package IDs - for package in packages: - match = re.search(r'Manifest-(.+)$', package["id"]) - if not match: - raise Exception(f"Unable to find .NET SDK version in package ID: {package['id']}") - sdk_version = match.group(1) - package['sdk_version'] = sdk_version - - ver_match = re.search(r'^\d+\.\d+', sdk_version) - if not ver_match: - raise Exception(f"Unable to find .NET version in SDK version '{sdk_version}'") - package['dotnet_version'] = ver_match.group(0) - - # Keep only packages targeting the highest .NET version - dotnet_versions = [float(pkg['dotnet_version']) for pkg in packages] - highest = max(dotnet_versions) - packages = [pkg for pkg in packages if float(pkg['dotnet_version']) == highest] - logger.info(f"After .NET version filtering for {workload}: {len(packages)} packages (highest={highest})") - - # Prefer non-preview packages - preview_pattern = r'\-(preview|rc|alpha)\.\d+$' - non_preview = [pkg for pkg in packages if not re.search(preview_pattern, pkg['id'])] - if non_preview: - packages = non_preview - - # Sort by SDK version descending and take the latest - packages.sort(key=lambda x: x['sdk_version'], reverse=True) - if not packages: - raise Exception(f"No packages available for {workload} after filtering") - - latest = packages[0] - logger.info(f"Latest package: ID={latest['id']}, Version={latest['latestVersion']}, SDK={latest['sdk_version']}") - - # Create rollback file with only the android workload - rollback_value = f"{latest['latestVersion']}/{latest['sdk_version']}" - rollback_dict = {workload: rollback_value} - logger.info(f"Rollback dictionary: {rollback_dict}") - with open("rollback_maui.json", "w", encoding="utf-8") as f: - f.write(json.dumps(rollback_dict, indent=4)) - logger.info("Created rollback_maui.json file") - - # Install maui-android (not 'maui') — works on both Windows and Linux - precommands.install_workload('maui-android', ['--from-rollback-file', 'rollback_maui.json']) - logger.info("########## Finished installing maui-android workload ##########") - setup_loggers(True) logger = getLogger(__name__) logger.info("Starting pre-command for MAUI Android deploy measurement") @@ -108,7 +19,11 @@ def install_maui_android_workload(precommands: PreCommands): precommands = PreCommands() with MauiNuGetConfigContext(precommands.framework): - install_maui_android_workload(precommands) + install_latest_maui( + precommands, + workloads=["microsoft.net.sdk.android"], + workload_name='maui-android', + ) precommands.print_dotnet_info() # Create template without restoring packages — packages will be restored diff --git a/src/scenarios/shared/mauisharedpython.py b/src/scenarios/shared/mauisharedpython.py index 3b80dcb91cb..e91c113c08d 100644 --- a/src/scenarios/shared/mauisharedpython.py +++ b/src/scenarios/shared/mauisharedpython.py @@ -388,29 +388,49 @@ def __exit__(self, exc_type, exc_val, exc_tb): def install_latest_maui( precommands: PreCommands, - feed=extract_latest_dotnet_feed_from_nuget_config(path=os.path.join(get_repo_root_path(), "NuGet.config")) + feed: str | None = None, + workloads: list[str] | None = None, + workload_name: str = 'maui' ): ''' - Install the latest maui workload using the provided feed. - This function will create a rollback file and install the maui workload using that file. + Install the latest MAUI workload using the provided feed. + Creates a rollback file and installs the workload using that file. + + Args: + precommands: PreCommands instance for running dotnet commands. + feed: NuGet feed URL to resolve workload packages from. + If None, resolves from NuGet.config at call time. + workloads: NuGet SDK package names to resolve (e.g., + ["microsoft.net.sdk.android"]). + If None, defaults to the full set of 6 MAUI workloads. + workload_name: The dotnet CLI workload name to install + (e.g., 'maui', 'maui-android'). ''' - getLogger().info("########## Installing latest MAUI workload ##########") + getLogger().info(f"########## Installing latest {workload_name} workload ##########") if precommands.has_workload: - getLogger().info("Skipping maui installation due to --has-workload=true") + getLogger().info(f"Skipping {workload_name} installation due to --has-workload=true") return - maui_rollback_dict: dict[str, str] = { - "microsoft.net.sdk.android" : "", - "microsoft.net.sdk.ios" : "", - "microsoft.net.sdk.maccatalyst" : "", - "microsoft.net.sdk.macos" : "", - "microsoft.net.sdk.maui" : "", - "microsoft.net.sdk.tvos" : "" - } + if feed is None: + feed = extract_latest_dotnet_feed_from_nuget_config( + path=os.path.join(get_repo_root_path(), "NuGet.config") + ) + + if workloads is None: + workloads = [ + "microsoft.net.sdk.android", + "microsoft.net.sdk.ios", + "microsoft.net.sdk.maccatalyst", + "microsoft.net.sdk.macos", + "microsoft.net.sdk.maui", + "microsoft.net.sdk.tvos", + ] + + maui_rollback_dict = {name: "" for name in workloads} - getLogger().info(f"Installing the latest maui workload from feed {feed}") + getLogger().info(f"Installing the latest {workload_name} workload from feed {feed}") # Get the latest published version of the maui workloads for workload in maui_rollback_dict.keys(): @@ -453,10 +473,10 @@ def install_latest_maui( getLogger().debug(f"Extracted .NET version '{dotnet_version}' from SDK version '{sdk_version}'") else: getLogger().error(f"Unable to find .NET version in SDK version '{sdk_version}' for package {package['id']}") - raise Exception("Unable to find .NET version in SDK version") + raise Exception(f"Unable to find .NET version in SDK version '{sdk_version}' for package {package['id']}") else: getLogger().error(f"Unable to find .NET SDK version in package ID: {package['id']}") - raise Exception("Unable to find .NET SDK version in package ID") + raise Exception(f"Unable to find .NET SDK version in package ID: {package['id']}") # Filter out packages that have lower 'dotnet_version' than the rest of the packages # Sometimes feed can contain packages from previous release versions, so we need to filter them out @@ -514,6 +534,6 @@ def install_latest_maui( getLogger().info("Created rollback_maui.json file") # Install the workload using the rollback file - getLogger().info("Installing maui workload with rollback file") - precommands.install_workload('maui', ['--from-rollback-file', 'rollback_maui.json']) - getLogger().info("########## Finished installing latest MAUI workload ##########") + getLogger().info(f"Installing {workload_name} workload with rollback file") + precommands.install_workload(workload_name, ['--from-rollback-file', 'rollback_maui.json']) + getLogger().info(f"########## Finished installing latest {workload_name} workload ##########") From 59d3d405d65e06ff5c77185d006d123b3c28e866 Mon Sep 17 00:00:00 2001 From: David Nguyen <87228593+davidnguyen-tech@users.noreply.github.com> Date: Tue, 21 Apr 2026 18:51:59 +0200 Subject: [PATCH 08/19] Add SDK version capture + diagnostics to mauiandroidinnerloop Hooks the scenario into the canonical versions.json pipeline used by the other MAUI/.NET Android scenarios so SDK versions land in the Startup tool's Reporter (build.AdditionalData), and adds a single 'dotnet --info' call on Helix for triage of SDK/runtime mismatches between the build machine and the Helix machine. - pre.py: log rollback_maui.json contents after install_latest_maui so the workload version pinned for the build is visible in build logs. - setup_helix.py: run 'dotnet --info' on the Helix machine before the measured build so triage doesn't have to dig through binlogs to know what SDK was actually used. - shared/runner.py (ANDROIDINNERLOOP branch): require -c/--configuration and -f/--framework (the .proj already passes both). After the first Helix build, read SDK versions from the linked/ or apk-staging assets folder via get_sdk_versions, write them with versions_write_json, then versions_read_json_file_save_env so PERFLAB_DATA_* env vars feed Reporter.cs. Wrapped in try/except so version capture never regresses the measurement. Unlike the other scenarios this capture has to live in runner.py rather than pre.py because the scenario's first build is itself the measured build and runs on Helix, not on the build machine. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/scenarios/mauiandroidinnerloop/pre.py | 6 ++++ .../mauiandroidinnerloop/setup_helix.py | 1 + src/scenarios/shared/runner.py | 34 +++++++++++++++---- 3 files changed, 34 insertions(+), 7 deletions(-) diff --git a/src/scenarios/mauiandroidinnerloop/pre.py b/src/scenarios/mauiandroidinnerloop/pre.py index 39011bd4210..e37d3a44b70 100644 --- a/src/scenarios/mauiandroidinnerloop/pre.py +++ b/src/scenarios/mauiandroidinnerloop/pre.py @@ -24,6 +24,12 @@ workloads=["microsoft.net.sdk.android"], workload_name='maui-android', ) + + # Log the generated rollback file for diagnostics. install_latest_maui + # writes this; if it's missing the install failed and we want to crash. + with open("rollback_maui.json", "r") as f: + logger.info(f"Generated rollback_maui.json contents:\n{f.read()}") + precommands.print_dotnet_info() # Create template without restoring packages — packages will be restored diff --git a/src/scenarios/mauiandroidinnerloop/setup_helix.py b/src/scenarios/mauiandroidinnerloop/setup_helix.py index 369c7d9ae03..48a4a640cd3 100644 --- a/src/scenarios/mauiandroidinnerloop/setup_helix.py +++ b/src/scenarios/mauiandroidinnerloop/setup_helix.py @@ -303,6 +303,7 @@ def main(): install_workload(ctx) setup_android_sdk(ctx) + run_cmd([ctx["dotnet_exe"], "--info"], check=False) setup_adb_device(ctx) restore_packages(ctx) diff --git a/src/scenarios/shared/runner.py b/src/scenarios/shared/runner.py index d93169dd8ea..0663eb07eb7 100644 --- a/src/scenarios/shared/runner.py +++ b/src/scenarios/shared/runner.py @@ -28,6 +28,7 @@ from performance.common import RunCommand, iswin, extension, helixworkitemroot from performance.logger import setup_loggers from shared.testtraits import TestTraits, testtypes +from shared.versionmanager import versions_write_json, versions_read_json_file_save_env, get_sdk_versions from subprocess import CalledProcessError @@ -179,8 +180,8 @@ def parseargs(self): androidinnerloopparser.add_argument('--csproj-path', help='Path to .csproj file to build', dest='csprojpath') androidinnerloopparser.add_argument('--edit-src', help='Path to modified source file (copied before incremental deploy)', dest='editsrc') androidinnerloopparser.add_argument('--edit-dest', help='Destination path for the modified file', dest='editdest') - androidinnerloopparser.add_argument('--framework', '-f', help='Target framework (e.g., net10.0-android)', dest='framework') - androidinnerloopparser.add_argument('--configuration', '-c', help='Build configuration', dest='configuration', default='Debug') + androidinnerloopparser.add_argument('--framework', '-f', help='Target framework (e.g., net10.0-android)', dest='framework', required=True) + androidinnerloopparser.add_argument('--configuration', '-c', help='Build configuration', dest='configuration', required=True) androidinnerloopparser.add_argument('--msbuild-args', help='Additional MSBuild arguments', dest='msbuildargs', default='') androidinnerloopparser.add_argument('--package-name', help='Android package name for startup measurement (e.g. com.companyname.mauiandroidinnerloop)', dest='packagename') androidinnerloopparser.add_argument('--inner-loop-iterations', help='Number of incremental build+deploy+startup iterations (1+)', type=int, default=10, dest='innerloopiterations') @@ -1100,11 +1101,7 @@ def run_incremental_iteration(iteration, num_iterations, base_cmd, edit_pairs, first_binlog = os.path.join(const.TRACEDIR, 'first-build-and-deploy.binlog') # Build the base MSBuild command - base_cmd = ['dotnet', 'build', self.csprojpath, '-t:Install', '--no-restore'] - if self.configuration: - base_cmd.extend(['-c', self.configuration]) - if self.framework: - base_cmd.extend(['-f', self.framework]) + base_cmd = ['dotnet', 'build', self.csprojpath, '-t:Install', '--no-restore', '-c', self.configuration, '-f', self.framework] if self.msbuildargs: for arg in re.split(r'[;\s]+', self.msbuildargs): if arg.strip(): @@ -1115,6 +1112,29 @@ def run_incremental_iteration(iteration, num_iterations, base_cmd, edit_pairs, getLogger().info("First build+deploy: %s" % ' '.join(first_cmd)) subprocess.run(first_cmd, check=True) + # Capture SDK versions from the just-built output. Other Android + # scenarios do this in pre.py on the build machine, but our first + # build runs on Helix and IS the measured build, so we have to + # do it here. + try: + fw_obj_dir = os.path.join(os.path.dirname(self.csprojpath), 'obj', self.configuration, self.framework, 'android-arm64') + dll_folder = os.path.join(fw_obj_dir, 'linked') + if not os.path.isdir(dll_folder): + dll_folder = os.path.join(fw_obj_dir, 'android', 'assets', 'arm64-v8a') + + getLogger().info("Capturing SDK versions from: %s" % dll_folder) + version_dict = get_sdk_versions(dll_folder) + + os.makedirs(const.TRACEDIR, exist_ok=True) + versions_path = os.path.join(const.TRACEDIR, 'versions.json') + versions_write_json(version_dict, versions_path) + # Surface PERFLAB_DATA_* env vars so the Startup tool's Reporter + # picks them up into build.AdditionalData. + versions_read_json_file_save_env(versions_path) + except Exception as ex: + # Never let version capture regress the measurement pipeline. + getLogger().warning("Version capture failed, continuing without versions.json: %s" % ex) + # --- Device setup for physical devices (screen wake, animations, timeout) --- # Physical Helix CI devices have screens OFF. When the screen is off, # am start-activity -W reports LaunchState: UNKNOWN, omits TotalTime, and From efd221768e786c9a8a437bfac8a7162dda27db05 Mon Sep 17 00:00:00 2001 From: David Nguyen <87228593+davidnguyen-tech@users.noreply.github.com> Date: Tue, 21 Apr 2026 19:52:36 +0200 Subject: [PATCH 09/19] Address copilot review notes - runner.py: mark csprojpath, edit-src, edit-dest, package-name as required=True so argparse surfaces missing args with a clear usage error instead of a late runtime exception. Drop the now-dead input validation block and the dead else-raise under editsrcs/editdests. - runner.py: discover the android-* RID folder under the build's obj tree instead of hardcoding android-arm64, so the emulator track (android-x64) captures SDK versions too. - startup.py: add copy_traces parameter (default True) to StartupWrapper.parsetraces. When False, skip the per-call TRACEDIR -> Helix upload-dir copy and perflab container upload. - runner.py (inner loop): pass copy_traces=False for the per-iteration parse calls. The final copy at end-of-run still runs, so trace uploads land once rather than O(N^2). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/scenarios/shared/runner.py | 67 ++++++++++++++++++--------------- src/scenarios/shared/startup.py | 9 ++++- 2 files changed, 45 insertions(+), 31 deletions(-) diff --git a/src/scenarios/shared/runner.py b/src/scenarios/shared/runner.py index 0663eb07eb7..999b9555dca 100644 --- a/src/scenarios/shared/runner.py +++ b/src/scenarios/shared/runner.py @@ -177,13 +177,13 @@ def parseargs(self): androidinnerloopparser = subparsers.add_parser(const.ANDROIDINNERLOOP, description='measure first and incremental deploy time via binlogs') - androidinnerloopparser.add_argument('--csproj-path', help='Path to .csproj file to build', dest='csprojpath') - androidinnerloopparser.add_argument('--edit-src', help='Path to modified source file (copied before incremental deploy)', dest='editsrc') - androidinnerloopparser.add_argument('--edit-dest', help='Destination path for the modified file', dest='editdest') + androidinnerloopparser.add_argument('--csproj-path', help='Path to .csproj file to build', dest='csprojpath', required=True) + androidinnerloopparser.add_argument('--edit-src', help='Path to modified source file (copied before incremental deploy)', dest='editsrc', required=True) + androidinnerloopparser.add_argument('--edit-dest', help='Destination path for the modified file', dest='editdest', required=True) androidinnerloopparser.add_argument('--framework', '-f', help='Target framework (e.g., net10.0-android)', dest='framework', required=True) androidinnerloopparser.add_argument('--configuration', '-c', help='Build configuration', dest='configuration', required=True) androidinnerloopparser.add_argument('--msbuild-args', help='Additional MSBuild arguments', dest='msbuildargs', default='') - androidinnerloopparser.add_argument('--package-name', help='Android package name for startup measurement (e.g. com.companyname.mauiandroidinnerloop)', dest='packagename') + androidinnerloopparser.add_argument('--package-name', help='Android package name for startup measurement (e.g. com.companyname.mauiandroidinnerloop)', dest='packagename', required=True) androidinnerloopparser.add_argument('--inner-loop-iterations', help='Number of incremental build+deploy+startup iterations (1+)', type=int, default=10, dest='innerloopiterations') androidinnerloopparser.add_argument('--screen-timeout-ms', help='Screen timeout in milliseconds. Must be large enough so the display stays on for the entire scenario; a screen-off mid-run breaks cold startup measurement (am start reports LaunchState: UNKNOWN).', type=int, default=30 * 60 * 1000, dest='screentimeoutms') self.add_common_arguments(androidinnerloopparser) @@ -1072,7 +1072,7 @@ def run_incremental_iteration(iteration, num_iterations, base_cmd, edit_pairs, tracename=iter_binlog_name, scenarioname=scenarioprefix + " - Incremental Build and Deploy", upload_to_perflab_container=False) - startup.parsetraces(traits) + startup.parsetraces(traits, copy_traces=False) # Extract build counters and test metadata from temp report with open(iter_report, 'r') as f: @@ -1090,11 +1090,6 @@ def run_incremental_iteration(iteration, num_iterations, base_cmd, edit_pairs, return ms, counters, iter_binlog, test_metadata - # --- Validate inputs --- - if not self.csprojpath: - raise Exception("For Android inner loop measurements, --csproj-path must be provided.") - if not self.packagename: - raise Exception("For Android inner loop measurements, --package-name must be provided.") scenarioprefix = self.scenarioname or "MAUI Android Build and Deploy" os.makedirs(const.TRACEDIR, exist_ok=True) @@ -1115,12 +1110,27 @@ def run_incremental_iteration(iteration, num_iterations, base_cmd, edit_pairs, # Capture SDK versions from the just-built output. Other Android # scenarios do this in pre.py on the build machine, but our first # build runs on Helix and IS the measured build, so we have to - # do it here. + # do it here. Discover the RID folder the build actually produced + # (android-arm64 on physical devices, android-x64 on emulator). try: - fw_obj_dir = os.path.join(os.path.dirname(self.csprojpath), 'obj', self.configuration, self.framework, 'android-arm64') - dll_folder = os.path.join(fw_obj_dir, 'linked') - if not os.path.isdir(dll_folder): - dll_folder = os.path.join(fw_obj_dir, 'android', 'assets', 'arm64-v8a') + framework_obj_root = os.path.join(os.path.dirname(self.csprojpath), 'obj', self.configuration, self.framework) + abi_by_rid = {'android-arm': 'armeabi-v7a', 'android-arm64': 'arm64-v8a', 'android-x86': 'x86', 'android-x64': 'x86_64'} + + dll_folder = None + for rid_dir in sorted(glob.glob(os.path.join(framework_obj_root, 'android-*'))): + rid = os.path.basename(rid_dir) + linked = os.path.join(rid_dir, 'linked') + if os.path.isdir(linked): + dll_folder = linked + break + abi = abi_by_rid.get(rid) + if abi: + staging = os.path.join(rid_dir, 'android', 'assets', abi) + if os.path.isdir(staging): + dll_folder = staging + break + if dll_folder is None: + raise FileNotFoundError("No android-* RID folder with DLLs found under %s" % framework_obj_root) getLogger().info("Capturing SDK versions from: %s" % dll_folder) version_dict = get_sdk_versions(dll_folder) @@ -1160,7 +1170,7 @@ def run_incremental_iteration(iteration, num_iterations, base_cmd, edit_pairs, tracename='first-build-and-deploy.binlog', scenarioname=scenarioprefix + " - First Build and Deploy", upload_to_perflab_container=False) - startup.parsetraces(self.traits) + startup.parsetraces(self.traits, copy_traces=False) # Merge first build metrics + startup → first e2e report first_e2e_report = os.path.join(const.TRACEDIR, 'first-debug-e2e-perf-lab-report.json') @@ -1171,21 +1181,18 @@ def run_incremental_iteration(iteration, num_iterations, base_cmd, edit_pairs, getLogger().info("Starting incremental loop: %d iterations" % num_iterations) # Build list of (dest, original_content, modified_content) tuples for toggling + if len(self.editsrcs) != len(self.editdests): + raise Exception("--edit-src and --edit-dest must have the same number of semicolon-separated paths") edit_pairs = [] - if self.editsrcs and self.editdests: - if len(self.editsrcs) != len(self.editdests): - raise Exception("--edit-src and --edit-dest must have the same number of semicolon-separated paths") - for src, dest in zip(self.editsrcs, self.editdests): - original = None - modified = None - with open(dest, 'r') as f: - original = f.read() - with open(src, 'r') as f: - modified = f.read() - edit_pairs.append((dest, original, modified)) - getLogger().info("Edit pair: %s <-> %s" % (src, dest)) - else: - raise Exception("No edit-src/edit-dest specified; incremental builds require file pairs to toggle") + for src, dest in zip(self.editsrcs, self.editdests): + original = None + modified = None + with open(dest, 'r') as f: + original = f.read() + with open(src, 'r') as f: + modified = f.read() + edit_pairs.append((dest, original, modified)) + getLogger().info("Edit pair: %s <-> %s" % (src, dest)) incremental_startup_results = [] aggregated_counters = {} # counter_name -> aggregated counter dict diff --git a/src/scenarios/shared/startup.py b/src/scenarios/shared/startup.py index e2f51133d0b..331b90aadde 100644 --- a/src/scenarios/shared/startup.py +++ b/src/scenarios/shared/startup.py @@ -56,7 +56,7 @@ def __init__(self): def _setstartuppath(self, path: str): self.startuppath = os.path.join(path, "Startup%s" % extension()) - def parsetraces(self, traits: TestTraits): + def parsetraces(self, traits: TestTraits, copy_traces: bool = True): directory = TRACEDIR if traits.tracefolder: directory = TRACEDIR + '/' + traits.tracefolder @@ -83,6 +83,13 @@ def parsetraces(self, traits: TestTraits): # rethrow the original exception raise + # copy_traces=False skips the per-call TRACEDIR -> Helix upload-dir copy and + # the perflab container upload. Callers that parse repeatedly (e.g. the + # inner loop scenario) should pass False and do one final copy at the end + # to avoid O(N^2) I/O over accumulating binlogs. + if not copy_traces: + return + helix_upload_dir = helixuploaddir() if runninginlab() and helix_upload_dir is not None: copytree(TRACEDIR, os.path.join(helix_upload_dir, 'traces'), dirs_exist_ok=True) From d73f5fbe8f421334428672165d5db2c5fe4be131 Mon Sep 17 00:00:00 2001 From: David Nguyen <87228593+davidnguyen-tech@users.noreply.github.com> Date: Wed, 22 Apr 2026 12:16:28 +0200 Subject: [PATCH 10/19] Log rollback_maui.json contents on Helix machine Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/scenarios/mauiandroidinnerloop/setup_helix.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/scenarios/mauiandroidinnerloop/setup_helix.py b/src/scenarios/mauiandroidinnerloop/setup_helix.py index 48a4a640cd3..f0864c2860a 100644 --- a/src/scenarios/mauiandroidinnerloop/setup_helix.py +++ b/src/scenarios/mauiandroidinnerloop/setup_helix.py @@ -171,6 +171,8 @@ def install_workload(ctx): rollback_file = os.path.join(ctx["workitem_root"], "rollback_maui.json") log(f"Step 1b: workload install maui-android " f"(pinned via {rollback_file})", tee=True) + with open(rollback_file, "r", encoding="utf-8") as f: + log_raw(f"rollback_maui.json contents:\n{f.read()}", tee=True) result = run_cmd( [ctx["dotnet_exe"], "workload", "install", "maui-android", "--from-rollback-file", rollback_file, From d5da6c11915062b9f58c81c2995f6dfaf092ccae Mon Sep 17 00:00:00 2001 From: David Nguyen <87228593+davidnguyen-tech@users.noreply.github.com> Date: Wed, 22 Apr 2026 18:30:07 +0200 Subject: [PATCH 11/19] Switch MAUI Android Inner Loop to 'dotnet run -p:WaitForExit=false' Replace the 'dotnet build -t:Install' invocation with 'dotnet run --project -p:WaitForExit=false' so the first-build/incremental measurements chain Build -> Install -> _Run in a single MSBuild invocation, mirroring the VS Code F5 path documented in dotnet/android build-properties.md:1827-1843. WaitForExit=false issues 'am start -S' and returns immediately instead of blocking on logcat. The existing AndroidHelper.measure_cold_startup call still force-stops via 'am start -W -S' to obtain TotalTime; the user-visible double-launch is intentional and metrics-neutral. Counter names ('First Build and Deploy', 'Incremental Build and Deploy') and binlog filenames are unchanged so PerfLab dashboard continuity is preserved. Validated locally with 3 iterations on Mono and CoreCLR against an Android device (PR #5165). --- src/scenarios/shared/runner.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/scenarios/shared/runner.py b/src/scenarios/shared/runner.py index 999b9555dca..8c18ec9a217 100644 --- a/src/scenarios/shared/runner.py +++ b/src/scenarios/shared/runner.py @@ -1095,8 +1095,12 @@ def run_incremental_iteration(iteration, num_iterations, base_cmd, edit_pairs, os.makedirs(const.TRACEDIR, exist_ok=True) first_binlog = os.path.join(const.TRACEDIR, 'first-build-and-deploy.binlog') - # Build the base MSBuild command - base_cmd = ['dotnet', 'build', self.csprojpath, '-t:Install', '--no-restore', '-c', self.configuration, '-f', self.framework] + # Build the base MSBuild command. `dotnet run -p:WaitForExit=false` + # chains Build → Install → _Run in a single MSBuild invocation, + # mirroring the VS Code F5 path (see dotnet/android + # build-properties.md:1827-1843). WaitForExit=false returns once the + # activity is launched instead of blocking on logcat. + base_cmd = ['dotnet', 'run', '--project', self.csprojpath, '-p:WaitForExit=false', '--no-restore', '-c', self.configuration, '-f', self.framework] if self.msbuildargs: for arg in re.split(r'[;\s]+', self.msbuildargs): if arg.strip(): @@ -1158,6 +1162,10 @@ def run_incremental_iteration(iteration, num_iterations, base_cmd, edit_pairs, getLogger().info("Using resolved activity: %s" % activityname) # --- First startup measurement --- + # `dotnet run -p:WaitForExit=false` already issued `am start -S` + # after deploy. measure_cold_startup below force-stops with -W -S + # and re-launches to obtain TotalTime. The user-visible + # double-launch is intentional and metrics-neutral. first_startup_ms = androidHelper.measure_cold_startup(self.packagename, activityname) getLogger().info("First deploy startup: %d ms" % first_startup_ms) From c47d731979babe9e22121507f4367d705ad4bab7 Mon Sep 17 00:00:00 2001 From: David Nguyen <87228593+davidnguyen-tech@users.noreply.github.com> Date: Thu, 23 Apr 2026 13:15:54 +0200 Subject: [PATCH 12/19] Migrate MAUI Android Inner Loop startup measurement to logcat polling Replace the double-launch am start -W -S approach in measure_cold_startup with logcat-based polling of ActivityTaskManager's 'Displayed /: +NNNms' line, which measures the SAME first-frame-drawn event without re-launching. Changes: - androidhelper.py: Extract screen-wake block from setup_device into ensure_screen_on() so it can be called before the first dotnet run. Add clear_logcat() to reset the logcat buffer before each launch. Add measure_startup_from_logcat() which polls 'adb shell logcat -d' using subprocess.run (not RunCommand, to avoid CalledProcessError on grep's exit-code-1 no-match result), parses '+NNNms'/'Ns NNNms' with the same regex as the existing fallback, and raises with a logcat tail on timeout. - runner.py: Move AndroidHelper() instantiation and try/finally block to before the first 'dotnet run' so ensure_screen_on() + clear_logcat() can fire before am start is issued. Physical Helix CI devices start with screens off; ActivityTaskManager never emits Displayed with screen off. Add pre_launch_fn=None to run_incremental_iteration signature, called before each dotnet run (wired to androidHelper.clear_logcat). Replace all measure_cold_startup calls with measure_startup_from_logcat. Metric name 'Time to Main', units ms, and JSON structure are unchanged. measure_cold_startup is preserved for non-ANDROIDINNERLOOP callers. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/scenarios/shared/androidhelper.py | 139 +++++++++++++++++++++----- src/scenarios/shared/runner.py | 120 ++++++++++++---------- 2 files changed, 178 insertions(+), 81 deletions(-) diff --git a/src/scenarios/shared/androidhelper.py b/src/scenarios/shared/androidhelper.py index ac27cd9de3b..e2a469473e9 100644 --- a/src/scenarios/shared/androidhelper.py +++ b/src/scenarios/shared/androidhelper.py @@ -1,4 +1,5 @@ import re +import subprocess import time from performance.common import RunCommand from logging import getLogger @@ -187,32 +188,7 @@ def setup_device(self, packagename: str, packagepath: str, animationsdisabled: b getActivity.run() getLogger().info(f"Target Activity {getActivity.stdout}") - # More setup stuff - checkScreenOnCmd = xharness_adb() + [ - 'shell', - 'dumpsys input_method | grep mInteractive' - ] - checkScreenOn = RunCommand(checkScreenOnCmd, verbose=True) - checkScreenOn.run() - - keyInputCmd = xharness_adb() + [ - 'shell', - 'input', - 'keyevent' - ] - - if "mInteractive=false" in checkScreenOn.stdout: - # Turn on the screen to make interactive and see if it worked - getLogger().info("Screen was off, turning on.") - self.screenwasoff = True - RunCommand(keyInputCmd + ['26'], verbose=True).run() # Press the power key - RunCommand(keyInputCmd + ['82'], verbose=True).run() # Unlock the screen with menu key (only works if it is not a password lock) - - checkScreenOn = RunCommand(checkScreenOnCmd, verbose=True) - checkScreenOn.run() - if "mInteractive=false" in checkScreenOn.stdout: - getLogger().exception("Failed to make screen interactive.") - raise Exception("Failed to make screen interactive.") + self.ensure_screen_on() self.activityname = getActivity.stdout.strip() @@ -274,6 +250,117 @@ def setup_device(self, packagename: str, packagepath: str, animationsdisabled: b self.activityname ] + def ensure_screen_on(self) -> None: + """Wake the device screen and unlock it if it is off. + + Must be called before any launch we want ActivityTaskManager to log a + 'Displayed' line for — Android never emits that line when the screen is + off. Physical Helix CI devices start with screens off. + + Sets self.screenwasoff = True the FIRST time the screen is found off so + close_device() can restore it. A second call when the screen is already + on leaves the flag unchanged. + """ + checkScreenOnCmd = xharness_adb() + [ + 'shell', + 'dumpsys input_method | grep mInteractive' + ] + checkScreenOn = RunCommand(checkScreenOnCmd, verbose=True) + checkScreenOn.run() + + keyInputCmd = xharness_adb() + [ + 'shell', + 'input', + 'keyevent' + ] + + if "mInteractive=false" in checkScreenOn.stdout: + # Turn on the screen to make interactive and see if it worked + getLogger().info("Screen was off, turning on.") + # Guard so a second call doesn't overwrite a True we already set. + if not self.screenwasoff: + self.screenwasoff = True + RunCommand(keyInputCmd + ['26'], verbose=True).run() # Press the power key + RunCommand(keyInputCmd + ['82'], verbose=True).run() # Unlock with menu key (no-op on password lock) + + checkScreenOn = RunCommand(checkScreenOnCmd, verbose=True) + checkScreenOn.run() + if "mInteractive=false" in checkScreenOn.stdout: + getLogger().exception("Failed to make screen interactive.") + raise Exception("Failed to make screen interactive.") + + def clear_logcat(self) -> None: + """Clear the logcat ring buffer before a measured launch. + + Called before every dotnet-run invocation so stale 'Displayed' lines + from previous iterations don't contaminate the next measurement. + """ + RunCommand(xharness_adb() + ['logcat', '-c'], verbose=True).run() + + def measure_startup_from_logcat(self, packagename: str, activityname: str, timeout_s: int = 30) -> int: + """Measure app startup time from logcat's 'Displayed' line. + + Polls 'adb shell logcat -d' for the 'Displayed /: +NNNms' + line emitted by ActivityTaskManager (API 29+) / ActivityManager (API + 23-28) after the first frame is drawn. + + Prerequisites (caller's responsibility): + - ensure_screen_on() called before the launch — Android never logs + 'Displayed' when the screen is off. + - clear_logcat() called immediately before the launch — otherwise a + stale line from a previous iteration is returned instantly. + + activityname is accepted for API symmetry with measure_cold_startup but + is not used in the grep; packagename alone uniquely identifies the app + and is robust to activity-name format variations. + + Returns startup time in milliseconds. + Raises Exception with a logcat tail on timeout. + """ + poll_cmd = xharness_adb() + [ + 'shell', + # Use packagename (not activityname) to match regardless of whether + # the activity is reported as '.MainActivity' or the fully-qualified + # 'com.company.app.MainActivity'. + f"logcat -d | grep -E 'ActivityTaskManager|ActivityManager' | grep 'Displayed {packagename}/'" + ] + + deadline = time.time() + timeout_s + result = None + while time.time() < deadline: + # Use subprocess.run directly instead of RunCommand because grep + # returns exit code 1 when no lines match, which RunCommand treats + # as an error and raises CalledProcessError. + result = subprocess.run(poll_cmd, capture_output=True, text=True) + if result.stdout.strip(): + break + time.sleep(1) + else: + # Dump recent logcat to help diagnose screen-off or timing issues + debug_cmd = xharness_adb() + ['shell', 'logcat -d -t 40'] + debug_result = subprocess.run(debug_cmd, capture_output=True, text=True) + raise Exception( + "Timed out waiting for 'Displayed %s/' in logcat after %d seconds.\n" + "Last 40 logcat lines:\n%s" % (packagename, timeout_s, debug_result.stdout) + ) + + # Parse '+NNNms' or '+Ns NNNms' — same regex as measure_cold_startup's fallback + dirty_capture = re.search(r"\+(\d*s?\d+)ms", result.stdout) + if dirty_capture: + capture_list = dirty_capture.group(1).split('s') + if len(capture_list) == 1: + startup_ms = int(capture_list[0]) + elif len(capture_list) == 2: + startup_ms = int(capture_list[0]) * 1000 + int(capture_list[1].zfill(3)) + else: + raise Exception("Android time capture failed! Unexpected format: %s" % dirty_capture.group(0)) + getLogger().info("Startup time (logcat Displayed): %d ms" % startup_ms) + return startup_ms + + raise Exception( + "Logcat returned output but no '+NNNms' pattern found.\nOutput: %s" % result.stdout + ) + def measure_cold_startup(self, packagename: str, activityname: str) -> int: """Measure app cold startup time in milliseconds. diff --git a/src/scenarios/shared/runner.py b/src/scenarios/shared/runner.py index 8c18ec9a217..88fb767d6c0 100644 --- a/src/scenarios/shared/runner.py +++ b/src/scenarios/shared/runner.py @@ -1030,10 +1030,13 @@ def merge_build_and_startup(build_report_path, startup_results, final_report_pat def run_incremental_iteration(iteration, num_iterations, base_cmd, edit_pairs, packagename, activityname, - scenarioprefix, startup, traits, measure_startup_fn): + scenarioprefix, startup, traits, measure_startup_fn, + pre_launch_fn=None): """Run one incremental build+deploy+startup iteration. edit_pairs is a list of (dest_path, original_content, modified_content) tuples. + pre_launch_fn, if provided, is called before each dotnet run (e.g. to clear + the logcat buffer so stale 'Displayed' lines don't pollute the measurement). Returns (startup_ms, counters_list, binlog_path, test_metadata). """ import subprocess @@ -1058,6 +1061,8 @@ def run_incremental_iteration(iteration, num_iterations, base_cmd, edit_pairs, iter_binlog = os.path.join(const.TRACEDIR, iter_binlog_name) incremental_cmd = base_cmd + [f'-bl:{iter_binlog}'] getLogger().info("Incremental build+deploy: %s" % ' '.join(incremental_cmd)) + if pre_launch_fn is not None: + pre_launch_fn() subprocess.run(incremental_cmd, check=True) # Measure startup @@ -1107,66 +1112,70 @@ def run_incremental_iteration(iteration, num_iterations, base_cmd, edit_pairs, base_cmd.append(arg.strip()) # --- First build + deploy --- - first_cmd = base_cmd + [f'-bl:{first_binlog}'] - getLogger().info("First build+deploy: %s" % ' '.join(first_cmd)) - subprocess.run(first_cmd, check=True) - - # Capture SDK versions from the just-built output. Other Android - # scenarios do this in pre.py on the build machine, but our first - # build runs on Helix and IS the measured build, so we have to - # do it here. Discover the RID folder the build actually produced - # (android-arm64 on physical devices, android-x64 on emulator). - try: - framework_obj_root = os.path.join(os.path.dirname(self.csprojpath), 'obj', self.configuration, self.framework) - abi_by_rid = {'android-arm': 'armeabi-v7a', 'android-arm64': 'arm64-v8a', 'android-x86': 'x86', 'android-x64': 'x86_64'} - - dll_folder = None - for rid_dir in sorted(glob.glob(os.path.join(framework_obj_root, 'android-*'))): - rid = os.path.basename(rid_dir) - linked = os.path.join(rid_dir, 'linked') - if os.path.isdir(linked): - dll_folder = linked - break - abi = abi_by_rid.get(rid) - if abi: - staging = os.path.join(rid_dir, 'android', 'assets', abi) - if os.path.isdir(staging): - dll_folder = staging - break - if dll_folder is None: - raise FileNotFoundError("No android-* RID folder with DLLs found under %s" % framework_obj_root) - - getLogger().info("Capturing SDK versions from: %s" % dll_folder) - version_dict = get_sdk_versions(dll_folder) - - os.makedirs(const.TRACEDIR, exist_ok=True) - versions_path = os.path.join(const.TRACEDIR, 'versions.json') - versions_write_json(version_dict, versions_path) - # Surface PERFLAB_DATA_* env vars so the Startup tool's Reporter - # picks them up into build.AdditionalData. - versions_read_json_file_save_env(versions_path) - except Exception as ex: - # Never let version capture regress the measurement pipeline. - getLogger().warning("Version capture failed, continuing without versions.json: %s" % ex) - - # --- Device setup for physical devices (screen wake, animations, timeout) --- - # Physical Helix CI devices have screens OFF. When the screen is off, - # am start-activity -W reports LaunchState: UNKNOWN, omits TotalTime, and - # ActivityTaskManager never logs the "Displayed" line — making startup - # measurement impossible. + # AndroidHelper is instantiated here (before first_cmd) so we can + # wake the screen and clear logcat BEFORE dotnet run fires am start. + # Physical Helix CI devices have screens OFF; ActivityTaskManager + # only emits 'Displayed' when the screen is on. androidHelper = AndroidHelper() try: + first_cmd = base_cmd + [f'-bl:{first_binlog}'] + getLogger().info("First build+deploy: %s" % ' '.join(first_cmd)) + # Wake screen before first launch so ActivityTaskManager logs the 'Displayed' line. + # Physical Helix CI devices have screens OFF; Displayed is never logged with screen off. + androidHelper.ensure_screen_on() + androidHelper.clear_logcat() + subprocess.run(first_cmd, check=True) + + # Capture SDK versions from the just-built output. Other Android + # scenarios do this in pre.py on the build machine, but our first + # build runs on Helix and IS the measured build, so we have to + # do it here. Discover the RID folder the build actually produced + # (android-arm64 on physical devices, android-x64 on emulator). + try: + framework_obj_root = os.path.join(os.path.dirname(self.csprojpath), 'obj', self.configuration, self.framework) + abi_by_rid = {'android-arm': 'armeabi-v7a', 'android-arm64': 'arm64-v8a', 'android-x86': 'x86', 'android-x64': 'x86_64'} + + dll_folder = None + for rid_dir in sorted(glob.glob(os.path.join(framework_obj_root, 'android-*'))): + rid = os.path.basename(rid_dir) + linked = os.path.join(rid_dir, 'linked') + if os.path.isdir(linked): + dll_folder = linked + break + abi = abi_by_rid.get(rid) + if abi: + staging = os.path.join(rid_dir, 'android', 'assets', abi) + if os.path.isdir(staging): + dll_folder = staging + break + if dll_folder is None: + raise FileNotFoundError("No android-* RID folder with DLLs found under %s" % framework_obj_root) + + getLogger().info("Capturing SDK versions from: %s" % dll_folder) + version_dict = get_sdk_versions(dll_folder) + + os.makedirs(const.TRACEDIR, exist_ok=True) + versions_path = os.path.join(const.TRACEDIR, 'versions.json') + versions_write_json(version_dict, versions_path) + # Surface PERFLAB_DATA_* env vars so the Startup tool's Reporter + # picks them up into build.AdditionalData. + versions_read_json_file_save_env(versions_path) + except Exception as ex: + # Never let version capture regress the measurement pipeline. + getLogger().warning("Version capture failed, continuing without versions.json: %s" % ex) + + # --- Device setup for physical devices (animations, screen timeout, activity resolution) --- + # Note: ensure_screen_on() was already called above before the first deploy. androidHelper.setup_device(self.packagename, packagepath=None, animationsdisabled=True, skip_install=True, skip_xharness_warmup=True, skip_package_verifier=True, skip_test_launch=True, screen_timeout_ms=self.screentimeoutms) activityname = androidHelper.activityname getLogger().info("Using resolved activity: %s" % activityname) - # --- First startup measurement --- - # `dotnet run -p:WaitForExit=false` already issued `am start -S` - # after deploy. measure_cold_startup below force-stops with -W -S - # and re-launches to obtain TotalTime. The user-visible - # double-launch is intentional and metrics-neutral. - first_startup_ms = androidHelper.measure_cold_startup(self.packagename, activityname) + # --- First startup measurement (logcat-based) --- + # `dotnet run -p:WaitForExit=false` issued `am start -S` (no -W). + # Poll logcat for ActivityTaskManager's 'Displayed /: +NNNms' line. + # Screen was guaranteed on by ensure_screen_on() above. + first_startup_ms = androidHelper.measure_startup_from_logcat(self.packagename, activityname) getLogger().info("First deploy startup: %d ms" % first_startup_ms) # --- Parse first build report --- @@ -1212,7 +1221,8 @@ def run_incremental_iteration(iteration, num_iterations, base_cmd, edit_pairs, iteration, num_iterations, base_cmd, edit_pairs, self.packagename, activityname, scenarioprefix, startup, self.traits, - androidHelper.measure_cold_startup) + androidHelper.measure_startup_from_logcat, + pre_launch_fn=androidHelper.clear_logcat) incremental_startup_results.append(ms) intermediate_files.append(iter_binlog) From 7c1e3bf0f4e7b730826b9340ad49d50ea102b4a4 Mon Sep 17 00:00:00 2001 From: David Nguyen <87228593+davidnguyen-tech@users.noreply.github.com> Date: Mon, 27 Apr 2026 14:37:44 +0200 Subject: [PATCH 13/19] Remove stale Android cold startup helper Delete the unused cold-start helper now that Android inner loop startup uses the logcat Displayed path, and update the remaining logcat parser comments to describe the active behavior without referencing the removed helper. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/scenarios/shared/androidhelper.py | 75 ++------------------------- 1 file changed, 5 insertions(+), 70 deletions(-) diff --git a/src/scenarios/shared/androidhelper.py b/src/scenarios/shared/androidhelper.py index e2a469473e9..1c2ffda54ce 100644 --- a/src/scenarios/shared/androidhelper.py +++ b/src/scenarios/shared/androidhelper.py @@ -310,9 +310,9 @@ def measure_startup_from_logcat(self, packagename: str, activityname: str, timeo - clear_logcat() called immediately before the launch — otherwise a stale line from a previous iteration is returned instantly. - activityname is accepted for API symmetry with measure_cold_startup but - is not used in the grep; packagename alone uniquely identifies the app - and is robust to activity-name format variations. + activityname is accepted to preserve runner call compatibility. It is + not used in the grep; packagename alone uniquely identifies the app and + is robust to activity-name format variations. Returns startup time in milliseconds. Raises Exception with a logcat tail on timeout. @@ -344,7 +344,7 @@ def measure_startup_from_logcat(self, packagename: str, activityname: str, timeo "Last 40 logcat lines:\n%s" % (packagename, timeout_s, debug_result.stdout) ) - # Parse '+NNNms' or '+Ns NNNms' — same regex as measure_cold_startup's fallback + # Parse '+NNNms' or '+Ns NNNms' from ActivityTaskManager/ActivityManager. dirty_capture = re.search(r"\+(\d*s?\d+)ms", result.stdout) if dirty_capture: capture_list = dirty_capture.group(1).split('s') @@ -361,71 +361,6 @@ def measure_startup_from_logcat(self, packagename: str, activityname: str, timeo "Logcat returned output but no '+NNNms' pattern found.\nOutput: %s" % result.stdout ) - def measure_cold_startup(self, packagename: str, activityname: str) -> int: - """Measure app cold startup time in milliseconds. - - Uses am start-activity -W -S to force-stop and cold-start the app. - Primary: parses TotalTime from am start stdout. - Fallback: parses WaitTime from am start stdout. - Last resort: parses 'Displayed' time from logcat. - - Returns startup time in milliseconds. - """ - stop_app_cmd = xharness_adb() + ['shell', 'am', 'force-stop', packagename] - start_app_cmd = xharness_adb() + ['shell', 'am', 'start-activity', '-W', '-S', '-n', activityname] - clear_logs_cmd = xharness_adb() + ['logcat', '-c'] - retrieve_time_cmd = xharness_adb() + [ - 'shell', - f"logcat -d | grep -E 'ActivityManager|ActivityTaskManager' | grep ': Displayed {activityname}'" - ] - - RunCommand(clear_logs_cmd, verbose=True).run() - start_result = RunCommand(start_app_cmd, verbose=True) - start_result.run() - - startup_ms = None - - # Primary: parse TotalTime or WaitTime from am start -W output - total_match = re.search(r"TotalTime:\s*(\d+)", start_result.stdout) - wait_match = re.search(r"WaitTime:\s*(\d+)", start_result.stdout) - launch_state_match = re.search(r"LaunchState:\s*(\w+)", start_result.stdout) - if launch_state_match: - launch_state = launch_state_match.group(1) - getLogger().info("LaunchState: %s" % launch_state) - if launch_state != "COLD": - raise Exception( - "Expected LaunchState: COLD but got LaunchState: %s. " - "If UNKNOWN, the device screen may have turned off mid-run — " - "increase --screen-timeout-ms." % launch_state - ) - if total_match: - startup_ms = int(total_match.group(1)) - getLogger().info("Startup time (TotalTime): %d ms" % startup_ms) - elif wait_match: - startup_ms = int(wait_match.group(1)) - getLogger().info("Startup time (WaitTime): %d ms" % startup_ms) - - # Fallback: parse 'Displayed' time from logcat - if startup_ms is None: - RunCommand(stop_app_cmd, verbose=True).run() - retrieve_result = RunCommand(retrieve_time_cmd, verbose=True) - retrieve_result.run() - dirty_capture = re.search(r"\+(\d*s?\d+)ms", retrieve_result.stdout) - if not dirty_capture: - raise Exception("Failed to capture startup time from am start output or logcat!") - capture_list = dirty_capture.group(1).split('s') - if len(capture_list) == 1: - startup_ms = int(capture_list[0]) - elif len(capture_list) == 2: - startup_ms = int(capture_list[0]) * 1000 + int(capture_list[1].zfill(3)) - else: - raise Exception("Android time capture failed! Unexpected format: %s" % dirty_capture.group(0)) - getLogger().info("Startup time (logcat): %d ms" % startup_ms) - else: - RunCommand(stop_app_cmd, verbose=True).run() - - return startup_ms - def close_device(self, skip_uninstall: bool = False): keyInputCmd = xharness_adb() + [ 'shell', @@ -507,4 +442,4 @@ def close_device(self, skip_uninstall: bool = False): RunCommand(cmdline, verbose=True).run() if self.screenwasoff: - RunCommand(keyInputCmd + ['26'], verbose=True).run() # Turn the screen back off \ No newline at end of file + RunCommand(keyInputCmd + ['26'], verbose=True).run() # Turn the screen back off From e2abd8dc7e569d9d46a63ddb857eaced82112a69 Mon Sep 17 00:00:00 2001 From: David Nguyen <87228593+davidnguyen-tech@users.noreply.github.com> Date: Thu, 30 Apr 2026 16:59:29 +0200 Subject: [PATCH 14/19] Address PR review findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Move setup_device() before first dotnet run so animations and screen timeout are normalized before the first measurement; defer activity resolution until after install - Preserve screenwasoff state across setup_device() so close_device can restore the original screen-off state on Helix devices - Add ensure_screen_on() before each incremental iteration - Add 10s timeout to adb getprop in emulator boot polling so a wedged adb daemon fails fast - Fix msbuild_args split in setup_helix.py to handle semicolon-delimited args (matching runner.py's split logic) - Always emit counters from AndroidInnerLoopParser even when Count==0 to keep per-iteration result arrays aligned across aggregation - Preserve top-level report fields (build, os, run, inLab) in the incremental aggregate report so PerfLab can classify the upload - Delete unused -dotnet-run.binlog after each dotnet run to save ~20 MB per scenario in Helix upload bandwidth - Fix dead CodegenType==JIT condition; apply R2R disable for all CoreCLR inner-loop runs - Add ANDROIDINNERLOOP to testtraits.py testtypes list - Validate --inner-loop-iterations >= 1 - Hoist pre_iteration closure outside the incremental loop - Rename publishTimes → buildDeployTimes in AndroidInnerLoopParser - Rename Time to Main → Time to Displayed to match the logcat metric - Update stale comments and warning messages Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../maui_scenarios_android_innerloop.proj | 10 +- .../mauiandroidinnerloop/setup_helix.py | 22 +++-- src/scenarios/shared/androidhelper.py | 18 ++-- src/scenarios/shared/runner.py | 75 +++++++++------ src/scenarios/shared/testtraits.py | 3 +- .../Util/Parsers/AndroidInnerLoopParser.cs | 91 +++++++------------ 6 files changed, 108 insertions(+), 111 deletions(-) diff --git a/eng/performance/maui_scenarios_android_innerloop.proj b/eng/performance/maui_scenarios_android_innerloop.proj index 40a74270d37..0a0de42afc8 100644 --- a/eng/performance/maui_scenarios_android_innerloop.proj +++ b/eng/performance/maui_scenarios_android_innerloop.proj @@ -16,11 +16,11 @@ <_MSBuildArgs Condition="'$(RuntimeFlavor)' == 'mono'">/p:UseMonoRuntime=true <_MSBuildArgs Condition="'$(RuntimeFlavor)' == 'coreclr'">/p:UseMonoRuntime=false - - <_MSBuildArgs Condition="'$(RuntimeFlavor)' == 'coreclr' and '$(CodegenType)' == 'JIT'">$(_MSBuildArgs) /p:PublishReadyToRun=false /p:PublishReadyToRunComposite=false + + <_MSBuildArgs Condition="'$(RuntimeFlavor)' == 'coreclr'">$(_MSBuildArgs);/p:PublishReadyToRun=false;/p:PublishReadyToRunComposite=false + Windows targets physical devices; Linux targets emulators. --> android-arm64 android-x64 <_MSBuildArgs>$(_MSBuildArgs) /p:RuntimeIdentifier=$(AndroidRid) @@ -34,8 +34,8 @@ $(RuntimeFlavor)_$(CodegenType) 10 + If the screen turns off mid-run, ActivityTaskManager never emits the 'Displayed' line + and measure_startup_from_logcat will time out. --> 1800000 diff --git a/src/scenarios/mauiandroidinnerloop/setup_helix.py b/src/scenarios/mauiandroidinnerloop/setup_helix.py index f0864c2860a..76e6a3aa94c 100644 --- a/src/scenarios/mauiandroidinnerloop/setup_helix.py +++ b/src/scenarios/mauiandroidinnerloop/setup_helix.py @@ -3,6 +3,7 @@ import os import platform +import re import stat import subprocess import sys @@ -77,7 +78,7 @@ def _setup_adb_windows(android_home): device_count = _count_adb_devices() log(f"Device count: {device_count}") if device_count == 0: - log("WARNING: No devices detected — build -t:Install will likely fail") + log("WARNING: No devices detected — dotnet run will likely fail") def _setup_adb_linux(): @@ -91,18 +92,23 @@ def _setup_adb_linux(): device_count = _count_adb_devices() log(f"Device count: {device_count}") if device_count == 0: - log("WARNING: No devices detected — build -t:Install will likely fail") + log("WARNING: No devices detected — dotnet run will likely fail") return # Wait for emulator boot log("Waiting for emulator to fully boot (up to 60s)...") boot_wait = 0 boot_completed = "" while boot_wait < 60: - result = subprocess.run( - ["adb", "shell", "getprop", "sys.boot_completed"], - capture_output=True, text=True, - ) - boot_completed = result.stdout.strip() + try: + result = subprocess.run( + ["adb", "shell", "getprop", "sys.boot_completed"], + capture_output=True, text=True, + timeout=10, + ) + boot_completed = result.stdout.strip() + except subprocess.TimeoutExpired: + log(" adb getprop timed out after 10s; retrying") + boot_completed = "" if boot_completed == "1": log(f"Emulator fully booted after {boot_wait}s") break @@ -268,7 +274,7 @@ def restore_packages(ctx): f"/p:TargetFrameworks={ctx['framework']}", ] if ctx["msbuild_args"]: - restore_args.extend(ctx["msbuild_args"].split()) + restore_args.extend(arg for arg in re.split(r'[;\s]+', ctx["msbuild_args"]) if arg) result = run_cmd(restore_args, check=False) if result.returncode != 0: log(f"STEP 2 FAILED with exit code {result.returncode}", tee=True) diff --git a/src/scenarios/shared/androidhelper.py b/src/scenarios/shared/androidhelper.py index 1c2ffda54ce..76ef37650d5 100644 --- a/src/scenarios/shared/androidhelper.py +++ b/src/scenarios/shared/androidhelper.py @@ -27,7 +27,6 @@ def setup_device(self, packagename: str, packagepath: str, animationsdisabled: b raise Exception("packagepath is required when skip_install is False") run_split_regex = r":\s(.+)" - self.screenwasoff = False self.packagename = packagename if not skip_xharness_warmup: @@ -196,6 +195,12 @@ def setup_device(self, packagename: str, packagepath: str, animationsdisabled: b # Test run to check if permissions are needed getLogger().info("Test run to check if permissions are needed") + keyInputCmd = xharness_adb() + [ + 'shell', + 'input', + 'keyevent' + ] + # -W in the start command waits for the app to finish initial draw. self.startappcommand = xharness_adb() + [ 'shell', @@ -277,9 +282,7 @@ def ensure_screen_on(self) -> None: if "mInteractive=false" in checkScreenOn.stdout: # Turn on the screen to make interactive and see if it worked getLogger().info("Screen was off, turning on.") - # Guard so a second call doesn't overwrite a True we already set. - if not self.screenwasoff: - self.screenwasoff = True + self.screenwasoff = True RunCommand(keyInputCmd + ['26'], verbose=True).run() # Press the power key RunCommand(keyInputCmd + ['82'], verbose=True).run() # Unlock with menu key (no-op on password lock) @@ -381,13 +384,6 @@ def close_device(self, skip_uninstall: bool = False): ] RunCommand(uninstallAppCmd, verbose=True).run() - - keyInputCmd = xharness_adb() + [ - 'shell', - 'input', - 'keyevent' - ] - # Restore Android package verifier settings try: getLogger().info("Restoring package verifier settings to pretest values") diff --git a/src/scenarios/shared/runner.py b/src/scenarios/shared/runner.py index 88fb767d6c0..1ab392b4329 100644 --- a/src/scenarios/shared/runner.py +++ b/src/scenarios/shared/runner.py @@ -184,8 +184,8 @@ def parseargs(self): androidinnerloopparser.add_argument('--configuration', '-c', help='Build configuration', dest='configuration', required=True) androidinnerloopparser.add_argument('--msbuild-args', help='Additional MSBuild arguments', dest='msbuildargs', default='') androidinnerloopparser.add_argument('--package-name', help='Android package name for startup measurement (e.g. com.companyname.mauiandroidinnerloop)', dest='packagename', required=True) - androidinnerloopparser.add_argument('--inner-loop-iterations', help='Number of incremental build+deploy+startup iterations (1+)', type=int, default=10, dest='innerloopiterations') - androidinnerloopparser.add_argument('--screen-timeout-ms', help='Screen timeout in milliseconds. Must be large enough so the display stays on for the entire scenario; a screen-off mid-run breaks cold startup measurement (am start reports LaunchState: UNKNOWN).', type=int, default=30 * 60 * 1000, dest='screentimeoutms') + androidinnerloopparser.add_argument('--inner-loop-iterations', help='Number of incremental build+deploy+startup iterations (1+)', type=int, default=10, dest='innerloopiterations', choices=range(1, 1001), metavar='N') + androidinnerloopparser.add_argument('--screen-timeout-ms', help='Screen timeout in milliseconds. Must be large enough so the display stays on for the entire scenario; if the screen turns off mid-run, ActivityTaskManager never emits the Displayed line and startup measurement times out.', type=int, default=30 * 60 * 1000, dest='screentimeoutms') self.add_common_arguments(androidinnerloopparser) args = parser.parse_args() @@ -1010,12 +1010,21 @@ def run(self): from shared.util import helixuploaddir import upload + def remove_dotnet_run_binlog(binlog_path): + """Delete the extra `*-dotnet-run.binlog` that are produced by `dotnet run`. + It only contains the _Run target firing am start; we measure startup via logcat + and don't parse it. + """ + aux = binlog_path[:-len('.binlog')] + '-dotnet-run.binlog' + if os.path.exists(aux): + os.remove(aux) + def merge_build_and_startup(build_report_path, startup_results, final_report_path): """Load the build metrics report, append a startup time counter, write to final path.""" with open(build_report_path, 'r') as f: report = json.load(f) startup_counter = { - "name": "Time to Main", + "name": "Time to Displayed", "topCounter": True, "defaultCounter": False, "higherIsBetter": False, @@ -1064,6 +1073,7 @@ def run_incremental_iteration(iteration, num_iterations, base_cmd, edit_pairs, if pre_launch_fn is not None: pre_launch_fn() subprocess.run(incremental_cmd, check=True) + remove_dotnet_run_binlog(iter_binlog) # Measure startup ms = measure_startup_fn(packagename, activityname) @@ -1084,9 +1094,13 @@ def run_incremental_iteration(iteration, num_iterations, base_cmd, edit_pairs, iter_data = json.load(f) test_obj = iter_data["tests"][0] counters = test_obj["counters"] - # Return test metadata (without counters) for building the final report - test_metadata = test_obj.copy() - test_metadata["counters"] = [] + # Capture top-level metadata (build, os, run, inLab) so the + # final aggregated report has the same shape as the first + # report — PerfLab needs these to classify the upload. + test_metadata = { + "test": {**test_obj, "counters": []}, + "top_level": {k: v for k, v in iter_data.items() if k != "tests"}, + } # Clean up temp report (leave binlog for later cleanup) if os.path.exists(iter_report): @@ -1100,31 +1114,28 @@ def run_incremental_iteration(iteration, num_iterations, base_cmd, edit_pairs, os.makedirs(const.TRACEDIR, exist_ok=True) first_binlog = os.path.join(const.TRACEDIR, 'first-build-and-deploy.binlog') - # Build the base MSBuild command. `dotnet run -p:WaitForExit=false` - # chains Build → Install → _Run in a single MSBuild invocation, - # mirroring the VS Code F5 path (see dotnet/android - # build-properties.md:1827-1843). WaitForExit=false returns once the - # activity is launched instead of blocking on logcat. + # Build the base MSBuild command. base_cmd = ['dotnet', 'run', '--project', self.csprojpath, '-p:WaitForExit=false', '--no-restore', '-c', self.configuration, '-f', self.framework] if self.msbuildargs: + # Split on semicolons and whitespace. NOTE: MSBuild args with + # embedded spaces (e.g. /p:Foo="a b") will be shredded; all + # current callers pass only /p:Key=Value forms without spaces. for arg in re.split(r'[;\s]+', self.msbuildargs): if arg.strip(): base_cmd.append(arg.strip()) # --- First build + deploy --- - # AndroidHelper is instantiated here (before first_cmd) so we can - # wake the screen and clear logcat BEFORE dotnet run fires am start. - # Physical Helix CI devices have screens OFF; ActivityTaskManager - # only emits 'Displayed' when the screen is on. + # Normalize device state BEFORE the first build so the first sample is measured under the same conditions as incrementals. + # Activity resolution is deferred until after install (requires the APK to be on the device). androidHelper = AndroidHelper() try: first_cmd = base_cmd + [f'-bl:{first_binlog}'] getLogger().info("First build+deploy: %s" % ' '.join(first_cmd)) - # Wake screen before first launch so ActivityTaskManager logs the 'Displayed' line. - # Physical Helix CI devices have screens OFF; Displayed is never logged with screen off. + androidHelper.setup_device(self.packagename, packagepath=None, animationsdisabled=True, skip_install=True, skip_xharness_warmup=True, skip_package_verifier=True, skip_test_launch=True, screen_timeout_ms=self.screentimeoutms) androidHelper.ensure_screen_on() androidHelper.clear_logcat() subprocess.run(first_cmd, check=True) + remove_dotnet_run_binlog(first_binlog) # Capture SDK versions from the just-built output. Other Android # scenarios do this in pre.py on the build machine, but our first @@ -1164,11 +1175,16 @@ def run_incremental_iteration(iteration, num_iterations, base_cmd, edit_pairs, # Never let version capture regress the measurement pipeline. getLogger().warning("Version capture failed, continuing without versions.json: %s" % ex) - # --- Device setup for physical devices (animations, screen timeout, activity resolution) --- - # Note: ensure_screen_on() was already called above before the first deploy. - androidHelper.setup_device(self.packagename, packagepath=None, animationsdisabled=True, skip_install=True, skip_xharness_warmup=True, skip_package_verifier=True, skip_test_launch=True, screen_timeout_ms=self.screentimeoutms) - - activityname = androidHelper.activityname + # --- Resolve activity (requires app installed by first_cmd) --- + getLogger().info("Resolving launchable activity after first install.") + resolve_cmd = xharness_adb() + [ + 'shell', + f'cmd package resolve-activity --brief {self.packagename} | tail -n 1' + ] + resolve_result = RunCommand(resolve_cmd, verbose=True) + resolve_result.run() + activityname = resolve_result.stdout.strip() + androidHelper.activityname = activityname getLogger().info("Using resolved activity: %s" % activityname) # --- First startup measurement (logcat-based) --- @@ -1216,13 +1232,17 @@ def run_incremental_iteration(iteration, num_iterations, base_cmd, edit_pairs, report_template = None # test metadata from first parsed report intermediate_files = [] # files to clean up + def pre_iteration(): + androidHelper.ensure_screen_on() + androidHelper.clear_logcat() + for iteration in range(1, num_iterations + 1): ms, counters, iter_binlog, test_metadata = run_incremental_iteration( iteration, num_iterations, base_cmd, edit_pairs, self.packagename, activityname, scenarioprefix, startup, self.traits, androidHelper.measure_startup_from_logcat, - pre_launch_fn=androidHelper.clear_logcat) + pre_launch_fn=pre_iteration) incremental_startup_results.append(ms) intermediate_files.append(iter_binlog) @@ -1248,7 +1268,7 @@ def run_incremental_iteration(iteration, num_iterations, base_cmd, edit_pairs, incremental_e2e_report = os.path.join(const.TRACEDIR, 'incremental-debug-e2e-perf-lab-report.json') final_counters = list(aggregated_counters.values()) final_counters.append({ - "name": "Time to Main", + "name": "Time to Displayed", "topCounter": True, "defaultCounter": False, "higherIsBetter": False, @@ -1256,8 +1276,11 @@ def run_incremental_iteration(iteration, num_iterations, base_cmd, edit_pairs, "results": incremental_startup_results }) if report_template is not None: - report_template["counters"] = final_counters - final_report_data = {"tests": [report_template]} + report_template["test"]["counters"] = final_counters + final_report_data = { + **report_template["top_level"], + "tests": [report_template["test"]], + } else: # Fallback: should not happen if at least 1 iteration ran final_report_data = {"tests": [{"counters": final_counters}]} diff --git a/src/scenarios/shared/testtraits.py b/src/scenarios/shared/testtraits.py index 3f2acef0fb9..93c4f57dc7f 100644 --- a/src/scenarios/shared/testtraits.py +++ b/src/scenarios/shared/testtraits.py @@ -10,7 +10,8 @@ const.SOD, const.INNERLOOP, const.DEVICESTARTUP, - const.BUILDTIME] + const.BUILDTIME, + const.ANDROIDINNERLOOP] class TestTraits: diff --git a/src/tools/ScenarioMeasurement/Util/Parsers/AndroidInnerLoopParser.cs b/src/tools/ScenarioMeasurement/Util/Parsers/AndroidInnerLoopParser.cs index d112a2bd451..8ee22d99f95 100644 --- a/src/tools/ScenarioMeasurement/Util/Parsers/AndroidInnerLoopParser.cs +++ b/src/tools/ScenarioMeasurement/Util/Parsers/AndroidInnerLoopParser.cs @@ -18,7 +18,7 @@ public class AndroidInnerLoopParser : IParser public IEnumerable Parse(string binlogFile, string processName, IList pids, string commandLine) { - var publishTimes = new List(); + var buildDeployTimes = new List(); // Build tasks (compilation) var cscTimes = new List(); @@ -131,75 +131,46 @@ public IEnumerable Parse(string binlogFile, string processName, IList 0) - yield return new Counter { Name = "Build+Deploy Time", MetricName = "s", DefaultCounter = true, TopCounter = true, Results = publishTimes.ToArray() }; + yield return new Counter { Name = "Build+Deploy Time", MetricName = "s", DefaultCounter = true, TopCounter = true, Results = buildDeployTimes.ToArray() }; // Build task counters - if (cscTimes.Count > 0) - yield return new Counter { Name = "Csc Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = cscTimes.ToArray() }; - if (xamlCTimes.Count > 0) - yield return new Counter { Name = "XamlC Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = xamlCTimes.ToArray() }; - if (generateJavaStubsTimes.Count > 0) - yield return new Counter { Name = "GenerateJavaStubs Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = generateJavaStubsTimes.ToArray() }; - if (linkAssembliesNoShrinkTimes.Count > 0) - yield return new Counter { Name = "LinkAssembliesNoShrink Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = linkAssembliesNoShrinkTimes.ToArray() }; - if (d8Times.Count > 0) - yield return new Counter { Name = "D8 Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = d8Times.ToArray() }; - if (javacTimes.Count > 0) - yield return new Counter { Name = "Javac Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = javacTimes.ToArray() }; - if (generateTypeMappingsTimes.Count > 0) - yield return new Counter { Name = "GenerateTypeMappings Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = generateTypeMappingsTimes.ToArray() }; - if (processAssembliesTimes.Count > 0) - yield return new Counter { Name = "ProcessAssemblies Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = processAssembliesTimes.ToArray() }; - if (generateJavaCallableWrappersTimes.Count > 0) - yield return new Counter { Name = "GenerateJavaCallableWrappers Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = generateJavaCallableWrappersTimes.ToArray() }; - if (filterAssembliesTimes.Count > 0) - yield return new Counter { Name = "FilterAssemblies Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = filterAssembliesTimes.ToArray() }; - if (waitForAppDetectionTimes.Count > 0) - yield return new Counter { Name = "WaitForAppDetection Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = waitForAppDetectionTimes.ToArray() }; - if (generateMainAndroidManifestTimes.Count > 0) - yield return new Counter { Name = "GenerateMainAndroidManifest Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = generateMainAndroidManifestTimes.ToArray() }; - if (resolveSdksTimes.Count > 0) - yield return new Counter { Name = "ResolveSdks Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = resolveSdksTimes.ToArray() }; - if (generateNativeApplicationConfigSourcesTimes.Count > 0) - yield return new Counter { Name = "GenerateNativeApplicationConfigSources Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = generateNativeApplicationConfigSourcesTimes.ToArray() }; + yield return new Counter { Name = "Csc Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = cscTimes.ToArray() }; + yield return new Counter { Name = "XamlC Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = xamlCTimes.ToArray() }; + yield return new Counter { Name = "GenerateJavaStubs Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = generateJavaStubsTimes.ToArray() }; + yield return new Counter { Name = "LinkAssembliesNoShrink Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = linkAssembliesNoShrinkTimes.ToArray() }; + yield return new Counter { Name = "D8 Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = d8Times.ToArray() }; + yield return new Counter { Name = "Javac Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = javacTimes.ToArray() }; + yield return new Counter { Name = "GenerateTypeMappings Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = generateTypeMappingsTimes.ToArray() }; + yield return new Counter { Name = "ProcessAssemblies Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = processAssembliesTimes.ToArray() }; + yield return new Counter { Name = "GenerateJavaCallableWrappers Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = generateJavaCallableWrappersTimes.ToArray() }; + yield return new Counter { Name = "FilterAssemblies Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = filterAssembliesTimes.ToArray() }; + yield return new Counter { Name = "WaitForAppDetection Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = waitForAppDetectionTimes.ToArray() }; + yield return new Counter { Name = "GenerateMainAndroidManifest Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = generateMainAndroidManifestTimes.ToArray() }; + yield return new Counter { Name = "ResolveSdks Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = resolveSdksTimes.ToArray() }; + yield return new Counter { Name = "GenerateNativeApplicationConfigSources Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = generateNativeApplicationConfigSourcesTimes.ToArray() }; // Build target counters - if (coreCompileTargetTimes.Count > 0) - yield return new Counter { Name = "CoreCompile Target Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = coreCompileTargetTimes.ToArray() }; - if (xamlCTargetTimes.Count > 0) - yield return new Counter { Name = "XamlC Target Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = xamlCTargetTimes.ToArray() }; - if (generateJavaStubsTargetTimes.Count > 0) - yield return new Counter { Name = "_GenerateJavaStubs Target Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = generateJavaStubsTargetTimes.ToArray() }; - if (linkAssembliesNoShrinkTargetTimes.Count > 0) - yield return new Counter { Name = "_LinkAssembliesNoShrink Target Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = linkAssembliesNoShrinkTargetTimes.ToArray() }; - if (compileToDalvikTargetTimes.Count > 0) - yield return new Counter { Name = "_CompileToDalvik Target Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = compileToDalvikTargetTimes.ToArray() }; - if (compileJavaTargetTimes.Count > 0) - yield return new Counter { Name = "_CompileJava Target Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = compileJavaTargetTimes.ToArray() }; + yield return new Counter { Name = "CoreCompile Target Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = coreCompileTargetTimes.ToArray() }; + yield return new Counter { Name = "XamlC Target Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = xamlCTargetTimes.ToArray() }; + yield return new Counter { Name = "_GenerateJavaStubs Target Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = generateJavaStubsTargetTimes.ToArray() }; + yield return new Counter { Name = "_LinkAssembliesNoShrink Target Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = linkAssembliesNoShrinkTargetTimes.ToArray() }; + yield return new Counter { Name = "_CompileToDalvik Target Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = compileToDalvikTargetTimes.ToArray() }; + yield return new Counter { Name = "_CompileJava Target Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = compileJavaTargetTimes.ToArray() }; // Deploy target counters - if (signTargetTimes.Count > 0) - yield return new Counter { Name = "_Sign Target Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = signTargetTimes.ToArray() }; - if (uploadTargetTimes.Count > 0) - yield return new Counter { Name = "_Upload Target Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = uploadTargetTimes.ToArray() }; - if (deployApkTargetTimes.Count > 0) - yield return new Counter { Name = "_DeployApk Target Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = deployApkTargetTimes.ToArray() }; - if (buildApkFastDevTargetTimes.Count > 0) - yield return new Counter { Name = "_BuildApkFastDev Target Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = buildApkFastDevTargetTimes.ToArray() }; + yield return new Counter { Name = "_Sign Target Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = signTargetTimes.ToArray() }; + yield return new Counter { Name = "_Upload Target Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = uploadTargetTimes.ToArray() }; + yield return new Counter { Name = "_DeployApk Target Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = deployApkTargetTimes.ToArray() }; + yield return new Counter { Name = "_BuildApkFastDev Target Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = buildApkFastDevTargetTimes.ToArray() }; // Task-level counters (granular) - if (fastDeployTimes.Count > 0) - yield return new Counter { Name = "FastDeploy Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = fastDeployTimes.ToArray() }; - if (androidSignPackageTimes.Count > 0) - yield return new Counter { Name = "AndroidSignPackage Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = androidSignPackageTimes.ToArray() }; - if (androidApkSignerTimes.Count > 0) - yield return new Counter { Name = "AndroidApkSigner Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = androidApkSignerTimes.ToArray() }; - if (aapt2LinkTimes.Count > 0) - yield return new Counter { Name = "Aapt2Link Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = aapt2LinkTimes.ToArray() }; + yield return new Counter { Name = "FastDeploy Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = fastDeployTimes.ToArray() }; + yield return new Counter { Name = "AndroidSignPackage Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = androidSignPackageTimes.ToArray() }; + yield return new Counter { Name = "AndroidApkSigner Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = androidApkSignerTimes.ToArray() }; + yield return new Counter { Name = "Aapt2Link Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = aapt2LinkTimes.ToArray() }; } } From 27ef49ad6ce5c529182ca45404316a3193af1706 Mon Sep 17 00:00:00 2001 From: David Nguyen <87228593+davidnguyen-tech@users.noreply.github.com> Date: Tue, 5 May 2026 13:58:18 +0200 Subject: [PATCH 15/19] Reject non-Debug BuildConfig for MAUI Android Inner Loop Inner loop measurement is intentionally Debug-only. Add a ValidateInnerLoopBuildConfig target that fails the build with a clear error if BuildConfig != Debug, so a future flip in sdk-perf-jobs.yml can't silently produce numbers that look like they came from a Debug run. Switch the work-item Command to '-c $(BuildConfig)' to match the pattern in maui_scenarios_android.proj and maui_scenarios_ios.proj. The Error target is the gate; the property is now the single source of truth for the configuration the work item runs against. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/performance/maui_scenarios_android_innerloop.proj | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/eng/performance/maui_scenarios_android_innerloop.proj b/eng/performance/maui_scenarios_android_innerloop.proj index 0a0de42afc8..9f96ccdf074 100644 --- a/eng/performance/maui_scenarios_android_innerloop.proj +++ b/eng/performance/maui_scenarios_android_innerloop.proj @@ -54,6 +54,11 @@ + + + + mauiandroidinnerloop @@ -80,7 +85,7 @@ $(_WindowsEnvVars);$(Python) setup_helix.py $(PERFLAB_Framework)-android "$(_MSBuildArgs)" - $(Python) test.py androidinnerloop --csproj-path app/MauiAndroidInnerLoop.csproj --edit-src "src/MainPage.xaml.cs;src/MainPage.xaml" --edit-dest "app/Pages/MainPage.xaml.cs;app/Pages/MainPage.xaml" --package-name com.companyname.mauiandroidinnerloop -f $(PERFLAB_Framework)-android -c Debug --msbuild-args "$(_MSBuildArgs)" --scenario-name "%(Identity)" --inner-loop-iterations $(InnerLoopIterations) --screen-timeout-ms $(ScreenTimeoutMs) $(ScenarioArgs) + $(Python) test.py androidinnerloop --csproj-path app/MauiAndroidInnerLoop.csproj --edit-src "src/MainPage.xaml.cs;src/MainPage.xaml" --edit-dest "app/Pages/MainPage.xaml.cs;app/Pages/MainPage.xaml" --package-name com.companyname.mauiandroidinnerloop -f $(PERFLAB_Framework)-android -c $(BuildConfig) --msbuild-args "$(_MSBuildArgs)" --scenario-name "%(Identity)" --inner-loop-iterations $(InnerLoopIterations) --screen-timeout-ms $(ScreenTimeoutMs) $(ScenarioArgs) $(Python) post.py output.log @@ -89,7 +94,7 @@ $(_LinuxEnvVars);$(Python) setup_helix.py $(PERFLAB_Framework)-android "$(_MSBuildArgs)" - $(Python) test.py androidinnerloop --csproj-path app/MauiAndroidInnerLoop.csproj --edit-src "src/MainPage.xaml.cs;src/MainPage.xaml" --edit-dest "app/Pages/MainPage.xaml.cs;app/Pages/MainPage.xaml" --package-name com.companyname.mauiandroidinnerloop -f $(PERFLAB_Framework)-android -c Debug --msbuild-args "$(_MSBuildArgs)" --scenario-name "%(Identity)" --inner-loop-iterations $(InnerLoopIterations) --screen-timeout-ms $(ScreenTimeoutMs) $(ScenarioArgs) + $(Python) test.py androidinnerloop --csproj-path app/MauiAndroidInnerLoop.csproj --edit-src "src/MainPage.xaml.cs;src/MainPage.xaml" --edit-dest "app/Pages/MainPage.xaml.cs;app/Pages/MainPage.xaml" --package-name com.companyname.mauiandroidinnerloop -f $(PERFLAB_Framework)-android -c $(BuildConfig) --msbuild-args "$(_MSBuildArgs)" --scenario-name "%(Identity)" --inner-loop-iterations $(InnerLoopIterations) --screen-timeout-ms $(ScreenTimeoutMs) $(ScenarioArgs) $(Python) post.py output.log From 57d68ff8b2e77afb588444197d6e2abc87dda70b Mon Sep 17 00:00:00 2001 From: David Nguyen <87228593+davidnguyen-tech@users.noreply.github.com> Date: Tue, 5 May 2026 16:22:14 +0200 Subject: [PATCH 16/19] Address PR review: align with Android verifier-disable + clarify semicolon-list args Drop skip_package_verifier=True from setup_device. Every other Android benchmark calls setup_device with the default (False), which disables the device-global package verifier and was added specifically to make startup testing less noisy. Our True flag was inherited from the pre-AndroidHelper local setup_measurement_device() and had no scenario-specific reason. Removing it brings inner-loop in line with the rest of the Android startup suite and avoids verifier overhead on each per-iteration reinstall. Update --edit-src / --edit-dest help strings to document the semicolon- separated convention and the positional pairing (impl uses args.editsrc.split(';') and pairs by index). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/scenarios/shared/runner.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/scenarios/shared/runner.py b/src/scenarios/shared/runner.py index 1ab392b4329..dcce26cfbf2 100644 --- a/src/scenarios/shared/runner.py +++ b/src/scenarios/shared/runner.py @@ -178,8 +178,8 @@ def parseargs(self): androidinnerloopparser = subparsers.add_parser(const.ANDROIDINNERLOOP, description='measure first and incremental deploy time via binlogs') androidinnerloopparser.add_argument('--csproj-path', help='Path to .csproj file to build', dest='csprojpath', required=True) - androidinnerloopparser.add_argument('--edit-src', help='Path to modified source file (copied before incremental deploy)', dest='editsrc', required=True) - androidinnerloopparser.add_argument('--edit-dest', help='Destination path for the modified file', dest='editdest', required=True) + androidinnerloopparser.add_argument('--edit-src', help='Semicolon-separated path(s) to modified source file(s) copied before each incremental deploy. Each entry pairs positionally with the same-index --edit-dest entry.', dest='editsrc', required=True) + androidinnerloopparser.add_argument('--edit-dest', help='Semicolon-separated destination path(s) for the modified file(s). Must contain the same number of entries as --edit-src.', dest='editdest', required=True) androidinnerloopparser.add_argument('--framework', '-f', help='Target framework (e.g., net10.0-android)', dest='framework', required=True) androidinnerloopparser.add_argument('--configuration', '-c', help='Build configuration', dest='configuration', required=True) androidinnerloopparser.add_argument('--msbuild-args', help='Additional MSBuild arguments', dest='msbuildargs', default='') @@ -1131,7 +1131,7 @@ def run_incremental_iteration(iteration, num_iterations, base_cmd, edit_pairs, try: first_cmd = base_cmd + [f'-bl:{first_binlog}'] getLogger().info("First build+deploy: %s" % ' '.join(first_cmd)) - androidHelper.setup_device(self.packagename, packagepath=None, animationsdisabled=True, skip_install=True, skip_xharness_warmup=True, skip_package_verifier=True, skip_test_launch=True, screen_timeout_ms=self.screentimeoutms) + androidHelper.setup_device(self.packagename, packagepath=None, animationsdisabled=True, skip_install=True, skip_xharness_warmup=True, skip_test_launch=True, screen_timeout_ms=self.screentimeoutms) androidHelper.ensure_screen_on() androidHelper.clear_logcat() subprocess.run(first_cmd, check=True) From 0f1a76c82465cf0e1ad603b4da8e14c1a1451e35 Mon Sep 17 00:00:00 2001 From: David Nguyen <87228593+davidnguyen-tech@users.noreply.github.com> Date: Tue, 5 May 2026 16:40:59 +0200 Subject: [PATCH 17/19] Revert verifier-disable change; keep semicolon-list help text Reverting the skip_package_verifier kwarg change from a807f347 after re-checking the history. The True flag was added deliberately in 26c3f746 (the AndroidHelper migration) to preserve the existing device-mutation footprint of this scenario: the prior local setup_measurement_device() function only touched screen-wake, animations, and screen_off_timeout, and never mutated verifier_verify_adb_installs or package_verifier_enable. The green baseline run was produced under those conditions and silently flipping verifier state on the Helix device for this scenario would change measurement conditions without a deliberate decision. The inner-loop scenario reinstalls the APK on every iteration, so a verifier-disable change here would compound across samples in a way that the cold-start scenarios (which install once and measure startup of the already-installed app) don't have to deal with. Worth a separate, intentional change with its own baseline if we want to make that switch later. The --edit-src / --edit-dest help text update from a807f347 is kept. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/scenarios/shared/runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scenarios/shared/runner.py b/src/scenarios/shared/runner.py index dcce26cfbf2..89ae1c3d2f4 100644 --- a/src/scenarios/shared/runner.py +++ b/src/scenarios/shared/runner.py @@ -1131,7 +1131,7 @@ def run_incremental_iteration(iteration, num_iterations, base_cmd, edit_pairs, try: first_cmd = base_cmd + [f'-bl:{first_binlog}'] getLogger().info("First build+deploy: %s" % ' '.join(first_cmd)) - androidHelper.setup_device(self.packagename, packagepath=None, animationsdisabled=True, skip_install=True, skip_xharness_warmup=True, skip_test_launch=True, screen_timeout_ms=self.screentimeoutms) + androidHelper.setup_device(self.packagename, packagepath=None, animationsdisabled=True, skip_install=True, skip_xharness_warmup=True, skip_package_verifier=True, skip_test_launch=True, screen_timeout_ms=self.screentimeoutms) androidHelper.ensure_screen_on() androidHelper.clear_logcat() subprocess.run(first_cmd, check=True) From d019e816eb131881736b35afb94c98844d85f423 Mon Sep 17 00:00:00 2001 From: David Nguyen <87228593+davidnguyen-tech@users.noreply.github.com> Date: Wed, 6 May 2026 09:44:33 +0200 Subject: [PATCH 18/19] Revert ValidateInnerLoopBuildConfig guard and -c $(BuildConfig) switch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reverts the proj-side changes from 8ffbbf4a after build 2967606 failed. Per @LoopedBard3's investigation: in the Helix flow, send_to_helix.py overwrites the BuildConfig env var with a long composite identifier used for log-directory naming (e.g. x64_main_maui_scenarios_android_innerloop), so by the time the Test target runs $(BuildConfig) is no longer 'Debug' and the validation target fires. The same property is then used for the work-item Command's -c argument, which would also pass garbage to the inner runner. Restoring the literal '-c Debug' that was working in build 2964110 and removing the ValidateInnerLoopBuildConfig target. A future guard would need a property that doesn't get clobbered by send_to_helix.py (e.g., a new pipeline variable mirrored from ${{ parameters.buildConfig }} in run-performance-job.yml) — out of scope for this PR. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/performance/maui_scenarios_android_innerloop.proj | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/eng/performance/maui_scenarios_android_innerloop.proj b/eng/performance/maui_scenarios_android_innerloop.proj index 9f96ccdf074..0a0de42afc8 100644 --- a/eng/performance/maui_scenarios_android_innerloop.proj +++ b/eng/performance/maui_scenarios_android_innerloop.proj @@ -54,11 +54,6 @@ - - - - mauiandroidinnerloop @@ -85,7 +80,7 @@ $(_WindowsEnvVars);$(Python) setup_helix.py $(PERFLAB_Framework)-android "$(_MSBuildArgs)" - $(Python) test.py androidinnerloop --csproj-path app/MauiAndroidInnerLoop.csproj --edit-src "src/MainPage.xaml.cs;src/MainPage.xaml" --edit-dest "app/Pages/MainPage.xaml.cs;app/Pages/MainPage.xaml" --package-name com.companyname.mauiandroidinnerloop -f $(PERFLAB_Framework)-android -c $(BuildConfig) --msbuild-args "$(_MSBuildArgs)" --scenario-name "%(Identity)" --inner-loop-iterations $(InnerLoopIterations) --screen-timeout-ms $(ScreenTimeoutMs) $(ScenarioArgs) + $(Python) test.py androidinnerloop --csproj-path app/MauiAndroidInnerLoop.csproj --edit-src "src/MainPage.xaml.cs;src/MainPage.xaml" --edit-dest "app/Pages/MainPage.xaml.cs;app/Pages/MainPage.xaml" --package-name com.companyname.mauiandroidinnerloop -f $(PERFLAB_Framework)-android -c Debug --msbuild-args "$(_MSBuildArgs)" --scenario-name "%(Identity)" --inner-loop-iterations $(InnerLoopIterations) --screen-timeout-ms $(ScreenTimeoutMs) $(ScenarioArgs) $(Python) post.py output.log @@ -94,7 +89,7 @@ $(_LinuxEnvVars);$(Python) setup_helix.py $(PERFLAB_Framework)-android "$(_MSBuildArgs)" - $(Python) test.py androidinnerloop --csproj-path app/MauiAndroidInnerLoop.csproj --edit-src "src/MainPage.xaml.cs;src/MainPage.xaml" --edit-dest "app/Pages/MainPage.xaml.cs;app/Pages/MainPage.xaml" --package-name com.companyname.mauiandroidinnerloop -f $(PERFLAB_Framework)-android -c $(BuildConfig) --msbuild-args "$(_MSBuildArgs)" --scenario-name "%(Identity)" --inner-loop-iterations $(InnerLoopIterations) --screen-timeout-ms $(ScreenTimeoutMs) $(ScenarioArgs) + $(Python) test.py androidinnerloop --csproj-path app/MauiAndroidInnerLoop.csproj --edit-src "src/MainPage.xaml.cs;src/MainPage.xaml" --edit-dest "app/Pages/MainPage.xaml.cs;app/Pages/MainPage.xaml" --package-name com.companyname.mauiandroidinnerloop -f $(PERFLAB_Framework)-android -c Debug --msbuild-args "$(_MSBuildArgs)" --scenario-name "%(Identity)" --inner-loop-iterations $(InnerLoopIterations) --screen-timeout-ms $(ScreenTimeoutMs) $(ScenarioArgs) $(Python) post.py output.log From 802b12fd84a9b308c76fb074043f6ac5a7645b51 Mon Sep 17 00:00:00 2001 From: David Nguyen <87228593+davidnguyen-tech@users.noreply.github.com> Date: Wed, 6 May 2026 11:57:53 +0200 Subject: [PATCH 19/19] Document why -c Debug is hardcoded in inner loop scenario Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/performance/maui_scenarios_android_innerloop.proj | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eng/performance/maui_scenarios_android_innerloop.proj b/eng/performance/maui_scenarios_android_innerloop.proj index 0a0de42afc8..a02e7353ddc 100644 --- a/eng/performance/maui_scenarios_android_innerloop.proj +++ b/eng/performance/maui_scenarios_android_innerloop.proj @@ -78,6 +78,7 @@ + $(_WindowsEnvVars);$(Python) setup_helix.py $(PERFLAB_Framework)-android "$(_MSBuildArgs)" $(Python) test.py androidinnerloop --csproj-path app/MauiAndroidInnerLoop.csproj --edit-src "src/MainPage.xaml.cs;src/MainPage.xaml" --edit-dest "app/Pages/MainPage.xaml.cs;app/Pages/MainPage.xaml" --package-name com.companyname.mauiandroidinnerloop -f $(PERFLAB_Framework)-android -c Debug --msbuild-args "$(_MSBuildArgs)" --scenario-name "%(Identity)" --inner-loop-iterations $(InnerLoopIterations) --screen-timeout-ms $(ScreenTimeoutMs) $(ScenarioArgs) @@ -87,6 +88,7 @@ + $(_LinuxEnvVars);$(Python) setup_helix.py $(PERFLAB_Framework)-android "$(_MSBuildArgs)" $(Python) test.py androidinnerloop --csproj-path app/MauiAndroidInnerLoop.csproj --edit-src "src/MainPage.xaml.cs;src/MainPage.xaml" --edit-dest "app/Pages/MainPage.xaml.cs;app/Pages/MainPage.xaml" --package-name com.companyname.mauiandroidinnerloop -f $(PERFLAB_Framework)-android -c Debug --msbuild-args "$(_MSBuildArgs)" --scenario-name "%(Identity)" --inner-loop-iterations $(InnerLoopIterations) --screen-timeout-ms $(ScreenTimeoutMs) $(ScenarioArgs)