diff --git a/eng/performance/maui_scenarios_android_innerloop.proj b/eng/performance/maui_scenarios_android_innerloop.proj new file mode 100644 index 00000000000..a02e7353ddc --- /dev/null +++ b/eng/performance/maui_scenarios_android_innerloop.proj @@ -0,0 +1,102 @@ + + + + + + + true + + + + <_MSBuildArgs Condition="'$(RuntimeFlavor)' == 'mono'">/p:UseMonoRuntime=true + <_MSBuildArgs Condition="'$(RuntimeFlavor)' == 'coreclr'">/p:UseMonoRuntime=false + + + <_MSBuildArgs Condition="'$(RuntimeFlavor)' == 'coreclr'">$(_MSBuildArgs);/p:PublishReadyToRun=false;/p:PublishReadyToRunComposite=false + + + android-arm64 + android-x64 + <_MSBuildArgs>$(_MSBuildArgs) /p:RuntimeIdentifier=$(AndroidRid) + + + <_MSBuildArgs>$(_MSBuildArgs) /p:SupportedOSPlatformVersion=23 + + <_MSBuildArgs Condition="!$(_MSBuildArgs.Contains('/p:TargetFrameworks='))">$(_MSBuildArgs) /p:TargetFrameworks=$(PERFLAB_Framework)-android + + $(RuntimeFlavor)_$(CodegenType) + 10 + + 1800000 + + + + + + + + + + + + 01:00 + + + + + + mauiandroidinnerloop + $(ScenariosDir)%(ScenarioDirectoryName) + + + + + + + $(Python) pre.py default -f $(PERFLAB_Framework) + %(PreparePayloadWorkItem.PayloadDirectory) + + + + + + <_WindowsEnvVars>set DOTNET_ROOT=%HELIX_CORRELATION_PAYLOAD%\dotnet;set DOTNET_CLI_TELEMETRY_OPTOUT=1;set DOTNET_MULTILEVEL_LOOKUP=0;set NUGET_PACKAGES=%HELIX_WORKITEM_ROOT%\.packages;set ANDROID_HOME=%HELIX_WORKITEM_ROOT%\android-sdk;set ANDROID_SDK_ROOT=%HELIX_WORKITEM_ROOT%\android-sdk;set JAVA_HOME=%HELIX_WORKITEM_ROOT%\jdk;set PATH=%HELIX_CORRELATION_PAYLOAD%\dotnet;;%HELIX_WORKITEM_ROOT%\android-sdk\platform-tools;;%HELIX_WORKITEM_ROOT%\jdk\bin;;%PATH% + <_LinuxEnvVars>export DOTNET_ROOT=$HELIX_CORRELATION_PAYLOAD/dotnet;export DOTNET_CLI_TELEMETRY_OPTOUT=1;export DOTNET_MULTILEVEL_LOOKUP=0;export NUGET_PACKAGES=$HELIX_WORKITEM_ROOT/.packages;export ANDROID_HOME=$HELIX_WORKITEM_ROOT/android-sdk;export ANDROID_SDK_ROOT=$HELIX_WORKITEM_ROOT/android-sdk;export JAVA_HOME=$HELIX_WORKITEM_ROOT/jdk;export ANDROID_SERIAL=emulator-5554;export PATH=$HELIX_CORRELATION_PAYLOAD/dotnet:$HELIX_WORKITEM_ROOT/android-sdk/platform-tools:$HELIX_WORKITEM_ROOT/jdk/bin:$PATH + + + + + + $(_WindowsEnvVars);$(Python) setup_helix.py $(PERFLAB_Framework)-android "$(_MSBuildArgs)" + $(Python) test.py androidinnerloop --csproj-path app/MauiAndroidInnerLoop.csproj --edit-src "src/MainPage.xaml.cs;src/MainPage.xaml" --edit-dest "app/Pages/MainPage.xaml.cs;app/Pages/MainPage.xaml" --package-name com.companyname.mauiandroidinnerloop -f $(PERFLAB_Framework)-android -c Debug --msbuild-args "$(_MSBuildArgs)" --scenario-name "%(Identity)" --inner-loop-iterations $(InnerLoopIterations) --screen-timeout-ms $(ScreenTimeoutMs) $(ScenarioArgs) + $(Python) post.py + output.log + + + + + + + $(_LinuxEnvVars);$(Python) setup_helix.py $(PERFLAB_Framework)-android "$(_MSBuildArgs)" + $(Python) test.py androidinnerloop --csproj-path app/MauiAndroidInnerLoop.csproj --edit-src "src/MainPage.xaml.cs;src/MainPage.xaml" --edit-dest "app/Pages/MainPage.xaml.cs;app/Pages/MainPage.xaml" --package-name com.companyname.mauiandroidinnerloop -f $(PERFLAB_Framework)-android -c Debug --msbuild-args "$(_MSBuildArgs)" --scenario-name "%(Identity)" --inner-loop-iterations $(InnerLoopIterations) --screen-timeout-ms $(ScreenTimeoutMs) $(ScenarioArgs) + $(Python) post.py + output.log + + + + + + diff --git a/eng/pipelines/sdk-perf-jobs.yml b/eng/pipelines/sdk-perf-jobs.yml index 4c3a62d1a72..dda52b3e49b 100644 --- a/eng/pipelines/sdk-perf-jobs.yml +++ b/eng/pipelines/sdk-perf-jobs.yml @@ -527,6 +527,84 @@ jobs: ${{ each parameter in parameters.jobParameters }}: ${{ parameter.key }}: ${{ parameter.value }} + # Maui Android inner loop benchmarks on devices (Mono Default) - Debug + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64-android-arm64-pixel + - win-x64-android-arm64-galaxy + isPublic: false + jobParameters: + runKind: maui_scenarios_android_innerloop + projectFileName: maui_scenarios_android_innerloop.proj + channels: + - main + runtimeFlavor: mono + codeGenType: Default + buildConfig: Debug + additionalJobIdentifier: Mono_Debug_InnerLoop + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # Maui Android inner loop benchmarks on devices (CoreCLR Default) - Debug + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64-android-arm64-pixel + - win-x64-android-arm64-galaxy + isPublic: false + jobParameters: + runKind: maui_scenarios_android_innerloop + projectFileName: maui_scenarios_android_innerloop.proj + channels: + - main + runtimeFlavor: coreclr + codeGenType: Default + buildConfig: Debug + additionalJobIdentifier: CoreCLR_Debug_InnerLoop + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # Maui Android inner loop benchmarks on emulator (Mono Default) - Debug + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - ubuntu-x64-android-emulator + isPublic: false + jobParameters: + runKind: maui_scenarios_android_innerloop + projectFileName: maui_scenarios_android_innerloop.proj + channels: + - main + runtimeFlavor: mono + codeGenType: Default + buildConfig: Debug + additionalJobIdentifier: Mono_Debug_InnerLoop_Emulator + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # Maui Android inner loop benchmarks on emulator (CoreCLR Default) - Debug + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - ubuntu-x64-android-emulator + isPublic: false + jobParameters: + runKind: maui_scenarios_android_innerloop + projectFileName: maui_scenarios_android_innerloop.proj + channels: + - main + runtimeFlavor: coreclr + codeGenType: Default + buildConfig: Debug + additionalJobIdentifier: CoreCLR_Debug_InnerLoop_Emulator + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + # Maui iOS scenario benchmarks (Mono - Default) - Debug - template: /eng/pipelines/templates/build-machine-matrix.yml parameters: diff --git a/eng/pipelines/templates/build-machine-matrix.yml b/eng/pipelines/templates/build-machine-matrix.yml index 71ffc558754..f558c755adb 100644 --- a/eng/pipelines/templates/build-machine-matrix.yml +++ b/eng/pipelines/templates/build-machine-matrix.yml @@ -146,6 +146,19 @@ jobs: machinePool: GalaxyA16 ${{ insert }}: ${{ parameters.jobParameters }} +- ${{ if and(containsValue(parameters.buildMachines, 'ubuntu-x64-android-emulator'), not(eq(parameters.isPublic, true))) }}: # Ubuntu x64 Android emulator only used in private builds currently + - template: ${{ parameters.jobTemplate }} + parameters: + osGroup: ubuntu + archType: x64 + osVersion: 2204 + pool: + vmImage: ubuntu-latest + container: ubuntu_x64_build_container + queue: Ubuntu.2204.Amd64.Android.36 + machinePool: AndroidEmulator + ${{ insert }}: ${{ parameters.jobParameters }} + - ${{ if and(containsValue(parameters.buildMachines, 'osx-x64-ios-arm64'), not(eq(parameters.isPublic, true))) }}: # iPhone ARM64 17 only used in private builds currently - template: ${{ parameters.jobTemplate }} parameters: diff --git a/eng/pipelines/templates/run-performance-job.yml b/eng/pipelines/templates/run-performance-job.yml index 4fe69d8c312..0045cdb5a03 100644 --- a/eng/pipelines/templates/run-performance-job.yml +++ b/eng/pipelines/templates/run-performance-job.yml @@ -190,7 +190,7 @@ jobs: - '--is-scenario' - ${{ if ne(length(parameters.runEnvVars), 0) }}: - "--run-env-vars ${{ join(' ', parameters.runEnvVars)}}" - - ${{ if and(in(parameters.runKind, 'maui_scenarios_ios', 'maui_scenarios_android'), ne(parameters.runtimeFlavor, '')) }}: + - ${{ if and(in(parameters.runKind, 'maui_scenarios_ios', 'maui_scenarios_android', 'maui_scenarios_android_innerloop'), ne(parameters.runtimeFlavor, '')) }}: - '--runtime-flavor ${{ parameters.runtimeFlavor }}' - ${{ if ne(parameters.osVersion, '') }}: - '--os-version ${{ parameters.osVersion }}' diff --git a/scripts/run_performance_job.py b/scripts/run_performance_job.py index 7e7e48c8c44..ad1c51908a8 100644 --- a/scripts/run_performance_job.py +++ b/scripts/run_performance_job.py @@ -570,9 +570,9 @@ def get_run_configurations( configurations["iOSLlvmBuild"] = str(ios_llvm_build) # .NET Android and .NET MAUI Android sample app scenarios - if run_kind == "maui_scenarios_android": + if run_kind in ["maui_scenarios_android", "maui_scenarios_android_innerloop"]: if not runtime_flavor in ("mono", "coreclr"): - raise Exception("Runtime flavor must be specified for maui_scenarios_android") + raise Exception(f"Runtime flavor must be specified for {run_kind}") configurations["CodegenType"] = str(codegen_type) configurations["RuntimeType"] = str(runtime_flavor) if build_config is not None and build_config != DEFAULT_BUILD_CONFIG: @@ -1190,7 +1190,7 @@ def publish_dotnet_app_to_payload(payload_dir_name: str, csproj_path: str, self_ verbose=True).run() # Search for additional binlogs generated by the maui scenarios prepare payload work items to copy to the artifacts log dir - if args.run_kind in ["maui_scenarios_android", "maui_scenarios_ios"]: + if args.run_kind in ["maui_scenarios_android", "maui_scenarios_ios", "maui_scenarios_android_innerloop"]: for binlog_path in glob(os.path.join(payload_dir, "scenarios_out", "**", "*.binlog"), recursive=True): shutil.copy(binlog_path, ci_artifacts_log_dir) diff --git a/src/scenarios/mauiandroidinnerloop/post.py b/src/scenarios/mauiandroidinnerloop/post.py new file mode 100644 index 00000000000..de98a0c507b --- /dev/null +++ b/src/scenarios/mauiandroidinnerloop/post.py @@ -0,0 +1,28 @@ +''' +post cleanup script +''' + +import subprocess +import sys +import traceback +from performance.logger import setup_loggers, getLogger +from shared.postcommands import clean_directories +from shared.util import xharness_adb +from test import EXENAME + +setup_loggers(True) +logger = getLogger(__name__) + +try: + # Uninstall the app from the connected device so re-runs start from a clean state + package_name = f'com.companyname.{EXENAME.lower()}' + logger.info(f"Uninstalling {package_name} from device") + subprocess.run(xharness_adb() + ['uninstall', package_name], check=False) + + logger.info("Shutting down dotnet build servers") + subprocess.run(['dotnet', 'build-server', 'shutdown'], check=False) + + clean_directories() +except Exception as e: + logger.error(f"Post cleanup failed: {e}\n{traceback.format_exc()}") + sys.exit(1) diff --git a/src/scenarios/mauiandroidinnerloop/pre.py b/src/scenarios/mauiandroidinnerloop/pre.py new file mode 100644 index 00000000000..e37d3a44b70 --- /dev/null +++ b/src/scenarios/mauiandroidinnerloop/pre.py @@ -0,0 +1,131 @@ +''' +pre-command: Set up a MAUI Android app for deploy measurement. +Creates the template (without restore) and prepares the modified file for incremental deploy. +NuGet packages are restored on the Helix machine, not shipped in the payload. +''' +import os +import shutil +import sys +from performance.logger import setup_loggers, getLogger +from shared import const +from shared.mauisharedpython import install_latest_maui, MauiNuGetConfigContext +from shared.precommands import PreCommands +from test import EXENAME + +setup_loggers(True) +logger = getLogger(__name__) +logger.info("Starting pre-command for MAUI Android deploy measurement") + +precommands = PreCommands() + +with MauiNuGetConfigContext(precommands.framework): + install_latest_maui( + precommands, + workloads=["microsoft.net.sdk.android"], + workload_name='maui-android', + ) + + # Log the generated rollback file for diagnostics. install_latest_maui + # writes this; if it's missing the install failed and we want to crash. + with open("rollback_maui.json", "r") as f: + logger.info(f"Generated rollback_maui.json contents:\n{f.read()}") + + precommands.print_dotnet_info() + + # Create template without restoring packages — packages will be restored + # on the Helix machine to avoid shipping ~1-2GB in the workitem payload. + precommands.new(template='maui', + output_dir=const.APPDIR, + bin_dir=const.BINDIR, + exename=EXENAME, + working_directory=sys.path[0], + no_restore=True, + extra_args=['-sc']) + + # Copy the merged NuGet.config into the app directory. This file contains + # MAUI NuGet feed URLs added by MauiNuGetConfigContext. The Helix machine + # needs these feeds during restore, and we must copy before the context + # manager restores the original NuGet.config. + repo_root = os.path.normpath(os.path.join(sys.path[0], '..', '..', '..')) + repo_nuget_config = os.path.join(repo_root, 'NuGet.config') + app_nuget_config = os.path.join(const.APPDIR, 'NuGet.config') + shutil.copy2(repo_nuget_config, app_nuget_config) + logger.info(f"Copied merged NuGet.config from {repo_nuget_config} to {app_nuget_config}") + + # Inject properties into the csproj so they apply to every command that + # targets this project (restore, build, install). + csproj_path = os.path.join(const.APPDIR, f'{EXENAME}.csproj') + with open(csproj_path, 'r') as f: + csproj_content = f.read() + + logger.info(f"Original .csproj content:\n{csproj_content}") + + injected_props = { + # Preview SDKs may lack prune-package-data files, causing NETSDK1226. + 'AllowMissingPrunePackageData': 'true', + # The perf repo globally disables the Roslyn compiler server to avoid + # BenchmarkDotNet file-locking issues. Re-enable it here to match real + # MAUI developer inner loop experience. + 'UseSharedCompilation': 'true', + } + csproj_modified = csproj_content + if '' not in csproj_modified: + raise Exception( + f"Cannot inject properties into {csproj_path}: " + f"no found in the generated template." + ) + for prop_name, prop_value in injected_props.items(): + if prop_name not in csproj_modified: + csproj_modified = csproj_modified.replace( + '', + f' <{prop_name}>{prop_value}\n ', + 1 # only the first PropertyGroup + ) + + with open(csproj_path, 'w') as f: + f.write(csproj_modified) + + logger.info(f"Updated {csproj_path} with injected properties") + logger.info(f"Modified .csproj content:\n{csproj_modified}") + + # Create modified source files in src/ for the incremental deploy simulation. + # The runner toggles between original and modified versions each iteration, + # exercising both the C# compiler (Csc) and XAML compiler (XamlC) paths. + src_dir = os.path.join(sys.path[0], const.SRCDIR) + os.makedirs(src_dir, exist_ok=True) + + # --- Modified MainPage.xaml.cs: add a debug line in the constructor --- + cs_original = os.path.join(const.APPDIR, 'Pages', 'MainPage.xaml.cs') + cs_modified = os.path.join(src_dir, 'MainPage.xaml.cs') + + with open(cs_original, 'r') as f: + cs_content = f.read() + + cs_modified_content = cs_content.replace( + 'InitializeComponent();', + 'InitializeComponent();\n\t\tSystem.Diagnostics.Debug.WriteLine("incremental-touch");' + ) + if cs_modified_content == cs_content: + raise Exception("Could not find 'InitializeComponent();' in %s — template may have changed" % cs_original) + + with open(cs_modified, 'w') as f: + f.write(cs_modified_content) + logger.info(f"Modified MainPage.xaml.cs written to {cs_modified}") + + # --- Modified MainPage.xaml: change a label's text --- + xaml_original = os.path.join(const.APPDIR, 'Pages', 'MainPage.xaml') + xaml_modified = os.path.join(src_dir, 'MainPage.xaml') + + with open(xaml_original, 'r') as f: + xaml_content = f.read() + + xaml_modified_content = xaml_content.replace( + 'Text="Task Categories"', + 'Text="Task Categories (updated)"' + ) + if xaml_modified_content == xaml_content: + raise Exception("Could not find 'Text=\"Task Categories\"' in %s — template may have changed" % xaml_original) + + with open(xaml_modified, 'w') as f: + f.write(xaml_modified_content) + logger.info(f"Modified MainPage.xaml written to {xaml_modified}") diff --git a/src/scenarios/mauiandroidinnerloop/setup_helix.py b/src/scenarios/mauiandroidinnerloop/setup_helix.py new file mode 100644 index 00000000000..76e6a3aa94c --- /dev/null +++ b/src/scenarios/mauiandroidinnerloop/setup_helix.py @@ -0,0 +1,324 @@ +#!/usr/bin/env python3 +"""setup_helix.py — Helix machine setup for MAUI Android inner loop (Windows + Linux).""" + +import os +import platform +import re +import stat +import subprocess +import sys +import time +from datetime import datetime + +# --- Constants --- +IS_WINDOWS = platform.system() == "Windows" +EXE = ".exe" if IS_WINDOWS else "" + +# --- Logging --- +_logfile = None + + +def log(msg, tee=False): + """Write *msg* with a timestamp to the log file.""" + line = f"[{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] {msg}" + if _logfile: + _logfile.write(line + "\n") + _logfile.flush() + if tee: + print(line, flush=True) + + +def log_raw(msg, tee=False): + """Write *msg* verbatim (no timestamp) to the log file.""" + if _logfile: + _logfile.write(msg + "\n") + _logfile.flush() + if tee: + print(msg, flush=True) + + +def run_cmd(args, check=True, **kwargs): + """Run a command, logging stdout/stderr. Returns CompletedProcess.""" + log(f"Running: {args}") + result = subprocess.run( + args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, + **kwargs, + ) + if result.stdout: + for line in result.stdout.splitlines(): + log_raw(line) + if check and result.returncode != 0: + raise subprocess.CalledProcessError(result.returncode, args, result.stdout) + return result + + +def _chmod_exec(path): + if not IS_WINDOWS and os.path.isfile(path): + os.chmod(path, os.stat(path).st_mode | stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH) + + +# --- ADB device setup --- +def _count_adb_devices(): + result = subprocess.run(["adb", "devices"], capture_output=True, text=True) + count = 0 + for line in result.stdout.splitlines()[1:]: + parts = line.split() + if len(parts) >= 2 and parts[1] == "device": + count += 1 + return count + + +def _setup_adb_windows(android_home): + """Windows: wait for a physical device, then verify.""" + log("Waiting for device (timeout 30s)...") + try: + run_cmd(["adb", "wait-for-device"], check=False, timeout=30) + except subprocess.TimeoutExpired: + log("WARNING: adb wait-for-device timed out") + device_count = _count_adb_devices() + log(f"Device count: {device_count}") + if device_count == 0: + log("WARNING: No devices detected — dotnet run will likely fail") + + +def _setup_adb_linux(): + """Linux: target emulator-5554, wait for device and boot.""" + log("ANDROID_SERIAL=emulator-5554 (set by PreCommands)") + log("Waiting for device (timeout 30s)...") + try: + run_cmd(["adb", "wait-for-device"], check=False, timeout=30) + except subprocess.TimeoutExpired: + log("WARNING: adb wait-for-device timed out") + device_count = _count_adb_devices() + log(f"Device count: {device_count}") + if device_count == 0: + log("WARNING: No devices detected — dotnet run will likely fail") + return + # Wait for emulator boot + log("Waiting for emulator to fully boot (up to 60s)...") + boot_wait = 0 + boot_completed = "" + while boot_wait < 60: + try: + result = subprocess.run( + ["adb", "shell", "getprop", "sys.boot_completed"], + capture_output=True, text=True, + timeout=10, + ) + boot_completed = result.stdout.strip() + except subprocess.TimeoutExpired: + log(" adb getprop timed out after 10s; retrying") + boot_completed = "" + if boot_completed == "1": + log(f"Emulator fully booted after {boot_wait}s") + break + time.sleep(5) + boot_wait += 5 + log(f" sys.boot_completed={boot_completed} (waited {boot_wait}s)") + if boot_completed != "1": + log(f"WARNING: Emulator did not report boot_completed=1 after 60s") + + +def _dump_log(): + if not _logfile: + return + _logfile.flush() + try: + with open(_logfile.name, "r") as f: + print(f.read()) + except Exception: + pass + + +# --- Orchestration --- +def print_diagnostics(): + log_raw("=== DIAGNOSTICS ===", tee=True) + log_raw(f"DOTNET_ROOT={os.environ.get('DOTNET_ROOT', '')}") + + +def setup_dotnet(correlation_payload): + """Return the path to the dotnet executable in the correlation payload.""" + dotnet_root = os.path.join(correlation_payload, "dotnet") + dotnet_exe = os.path.join(dotnet_root, f"dotnet{EXE}") + log(f"DOTNET_ROOT={dotnet_root}") + run_cmd([dotnet_exe, "--version"], check=False) + return dotnet_exe + + +def install_workload(ctx): + """Step 1: Install workloads via restore (dependencies) then install (pinned). + + First, ``workload restore`` installs whatever workloads the project + needs (including iOS/MacCatalyst if MAUI requires them) at whatever + version is available in the feeds. This satisfies dependency packages + that may not exist at the pinned version. + + Then, ``workload install maui-android --from-rollback-file`` pins the + android workload to the exact version from the rollback file we want + to test against. + """ + log_raw("=== STEP 1: Workload Install ===", tee=True) + + # 1a. Workload restore — satisfy all project dependencies first. + log("Step 1a: workload restore (satisfy project dependencies)", tee=True) + result = run_cmd( + [ctx["dotnet_exe"], "workload", "restore", ctx["csproj"], + "--configfile", ctx["nuget_config"], + f"-p:TargetFrameworks={ctx['framework']}"], + check=False, + ) + if result.returncode != 0: + log(f"WARNING: workload restore exited with code {result.returncode} " + "— continuing with pinned install", tee=True) + else: + log("Workload restore succeeded") + + # 1b. Workload install — pin maui-android to the rollback version. + rollback_file = os.path.join(ctx["workitem_root"], "rollback_maui.json") + log(f"Step 1b: workload install maui-android " + f"(pinned via {rollback_file})", tee=True) + with open(rollback_file, "r", encoding="utf-8") as f: + log_raw(f"rollback_maui.json contents:\n{f.read()}", tee=True) + result = run_cmd( + [ctx["dotnet_exe"], "workload", "install", "maui-android", + "--from-rollback-file", rollback_file, + "--configfile", ctx["nuget_config"]], + check=False, + ) + if result.returncode != 0: + log(f"STEP 1 FAILED: workload install exited with code " + f"{result.returncode}", tee=True) + _dump_log() + sys.exit(1) + log("Workload install (pinned maui-android) succeeded") + + +def install_android_dependencies(ctx): + """Restore the real MAUI csproj and run InstallAndroidDependencies. + + Uses the real project at ``ctx["csproj"]`` directly. + ``AllowMissingPrunePackageData=true`` bypasses NETSDK1226 on preview + SDKs that lack prune package data. + """ + log_raw("=== Installing Android SDK & Java Dependencies ===", tee=True) + + android_home = os.path.join(ctx["workitem_root"], "android-sdk") + java_home = os.path.join(ctx["workitem_root"], "jdk") + os.makedirs(android_home, exist_ok=True) + os.makedirs(java_home, exist_ok=True) + + csproj = ctx["csproj"] + log(f"Using project: {csproj}") + + # Restore the project. Override TargetFrameworks to android-only. + result = run_cmd( + [ctx["dotnet_exe"], "restore", csproj, + "--configfile", ctx["nuget_config"], + f"-p:TargetFrameworks={ctx['framework']}"], + check=False, + ) + if result.returncode != 0: + log("WARNING: restore failed — " + "InstallAndroidDependencies may not work") + + # Use dotnet msbuild (not dotnet build) to run only this target + # without the full build pipeline. + result = run_cmd( + [ctx["dotnet_exe"], "msbuild", csproj, + "-t:InstallAndroidDependencies", + f"/p:AndroidSdkDirectory={android_home}", + f"/p:JavaSdkDirectory={java_home}", + "/p:AcceptAndroidSdkLicenses=True", + f"/p:TargetFramework={ctx['framework']}"], + check=False, + ) + if result.returncode != 0: + log("InstallAndroidDependencies FAILED", tee=True) + _dump_log() + sys.exit(1) + log("Android SDK and Java dependencies installed successfully") + + # Set up paths for adb and log the values (env vars set by PreCommands) + platform_tools = os.path.join(android_home, "platform-tools") + log(f"ANDROID_HOME={android_home}") + log(f"JAVA_HOME={java_home}") + + _chmod_exec(os.path.join(platform_tools, "adb")) + ctx["android_home"] = android_home + + +def setup_android_sdk(ctx): + """Install Android SDK and Java via the built-in MSBuild target.""" + log_raw("=== Setting up Android SDK ===") + install_android_dependencies(ctx) + + +def setup_adb_device(ctx): + """Restart ADB server and run platform-specific device setup.""" + log_raw("=== ADB DEVICE SETUP ===") + run_cmd(["adb", "kill-server"], check=False) + run_cmd(["adb", "start-server"], check=False) + run_cmd(["adb", "devices", "-l"], check=False) + if IS_WINDOWS: + _setup_adb_windows(ctx["android_home"]) + else: + _setup_adb_linux() + + +def restore_packages(ctx): + """Step 2: Restore NuGet packages.""" + log_raw("=== STEP 2: Restore ===", tee=True) + restore_args = [ + ctx["dotnet_exe"], "restore", ctx["csproj"], + "--configfile", ctx["nuget_config"], + f"/p:TargetFrameworks={ctx['framework']}", + ] + if ctx["msbuild_args"]: + restore_args.extend(arg for arg in re.split(r'[;\s]+', ctx["msbuild_args"]) if arg) + result = run_cmd(restore_args, check=False) + if result.returncode != 0: + log(f"STEP 2 FAILED with exit code {result.returncode}", tee=True) + _dump_log() + sys.exit(2) + log("Restore succeeded") + + +# --- Main --- +def main(): + if len(sys.argv) < 3: + print(f"Usage: {sys.argv[0]} FRAMEWORK MSBUILD_ARGS", + file=sys.stderr) + sys.exit(1) + + global _logfile + upload_root = os.environ.get("HELIX_WORKITEM_UPLOAD_ROOT") + if upload_root: + _logfile = open(os.path.join(upload_root, "output.log"), "a") + + workitem_root = os.environ.get("HELIX_WORKITEM_ROOT", ".") + correlation_payload = os.environ.get("HELIX_CORRELATION_PAYLOAD", ".") + ctx = { + "framework": sys.argv[1], + "msbuild_args": sys.argv[2], + "workitem_root": workitem_root, + "correlation_payload": correlation_payload, + "nuget_config": os.path.join(workitem_root, "app", "NuGet.config"), + "csproj": os.path.join(workitem_root, "app", "MauiAndroidInnerLoop.csproj"), + } + + ctx["dotnet_exe"] = setup_dotnet(correlation_payload) + print_diagnostics() + + install_workload(ctx) + setup_android_sdk(ctx) + run_cmd([ctx["dotnet_exe"], "--info"], check=False) + setup_adb_device(ctx) + restore_packages(ctx) + + log_raw("=== SETUP SUCCEEDED ===", tee=True) + _dump_log() + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/scenarios/mauiandroidinnerloop/test.py b/src/scenarios/mauiandroidinnerloop/test.py new file mode 100644 index 00000000000..ddaf0bda63f --- /dev/null +++ b/src/scenarios/mauiandroidinnerloop/test.py @@ -0,0 +1,14 @@ +''' +MAUI Android Inner Loop (Debug End-2-End) Time Measurement +Orchestrates first build-deploy-startup → file edit → incremental build-deploy-startup → parse binlogs and startup times. +''' +import os +from shared.runner import TestTraits, Runner + +EXENAME = 'MauiAndroidInnerLoop' + +if __name__ == "__main__": + traits = TestTraits(exename=EXENAME, + guiapp='false', + ) + Runner(traits).run() diff --git a/src/scenarios/shared/androidhelper.py b/src/scenarios/shared/androidhelper.py index df3e0ef4d52..76ef37650d5 100644 --- a/src/scenarios/shared/androidhelper.py +++ b/src/scenarios/shared/androidhelper.py @@ -1,4 +1,5 @@ import re +import subprocess import time from performance.common import RunCommand from logging import getLogger @@ -21,56 +22,62 @@ def __init__(self): self.startverifierverifyadbinstalls = None self.startpackageverifierenable = None - def setup_device(self, packagename: str, packagepath: str, animationsdisabled: bool, forcewaitstart: bool = True): + def setup_device(self, packagename: str, packagepath: str, animationsdisabled: bool, forcewaitstart: bool = True, skip_install: bool = False, skip_xharness_warmup: bool = False, skip_package_verifier: bool = False, skip_test_launch: bool = False, screen_timeout_ms: int = 2 * 60 * 1000): + if not skip_install and packagepath is None: + raise Exception("packagepath is required when skip_install is False") + run_split_regex = r":\s(.+)" - self.screenwasoff = False self.packagename = packagename - # Try calling xharness with stdout=None and stderr=None to hopefully bypass the hang - getLogger().info("Clearing xharness stdout and stderr to avoid hang") - cmdline = xharness_adb() + [ - 'shell', - 'echo', 'Hello World' - ] - RunCommand(cmdline, verbose=False).run() - getLogger().info("Ran echo command to clear stdout and stderr") - - cmdline = xharness_adb() + [ - 'shell', - 'wm', - 'size' - ] - RunCommand(cmdline, verbose=True).run() - - # Capture and disable Android package verifier settings to avoid prompts/overhead during installs - try: - getLogger().info("Capturing current package verifier settings") - cmdline = xharness_adb() + [ - 'shell', 'settings', 'get', 'global', 'verifier_verify_adb_installs' - ] - get_verifier_adb_cmd = RunCommand(cmdline, verbose=True) - get_verifier_adb_cmd.run() - self.startverifierverifyadbinstalls = get_verifier_adb_cmd.stdout.strip() - + if not skip_xharness_warmup: + # Try calling xharness with stdout=None and stderr=None to hopefully bypass the hang + getLogger().info("Clearing xharness stdout and stderr to avoid hang") cmdline = xharness_adb() + [ - 'shell', 'settings', 'get', 'global', 'package_verifier_enable' + 'shell', + 'echo', 'Hello World' ] - get_pkg_verifier_cmd = RunCommand(cmdline, verbose=True) - get_pkg_verifier_cmd.run() - self.startpackageverifierenable = get_pkg_verifier_cmd.stdout.strip() + RunCommand(cmdline, verbose=False).run() + getLogger().info("Ran echo command to clear stdout and stderr") - getLogger().info("Disabling package verifier settings for the run") cmdline = xharness_adb() + [ - 'shell', 'settings', 'put', 'global', 'verifier_verify_adb_installs', '0' + 'shell', + 'wm', + 'size' ] RunCommand(cmdline, verbose=True).run() - cmdline = xharness_adb() + [ - 'shell', 'settings', 'put', 'global', 'package_verifier_enable', '0' - ] - RunCommand(cmdline, verbose=True).run() - except CalledProcessError: - # Best-effort: don't fail setup if device doesn't support these keys; proceed with the run - getLogger().warning("Failed to update package verifier settings; continuing without changes", exc_info=True) + + # Capture and disable Android package verifier settings to avoid prompts/overhead during installs + if skip_package_verifier: + getLogger().info("Skipping package verifier changes") + else: + try: + getLogger().info("Capturing current package verifier settings") + cmdline = xharness_adb() + [ + 'shell', 'settings', 'get', 'global', 'verifier_verify_adb_installs' + ] + get_verifier_adb_cmd = RunCommand(cmdline, verbose=True) + get_verifier_adb_cmd.run() + self.startverifierverifyadbinstalls = get_verifier_adb_cmd.stdout.strip() + + cmdline = xharness_adb() + [ + 'shell', 'settings', 'get', 'global', 'package_verifier_enable' + ] + get_pkg_verifier_cmd = RunCommand(cmdline, verbose=True) + get_pkg_verifier_cmd.run() + self.startpackageverifierenable = get_pkg_verifier_cmd.stdout.strip() + + getLogger().info("Disabling package verifier settings for the run") + cmdline = xharness_adb() + [ + 'shell', 'settings', 'put', 'global', 'verifier_verify_adb_installs', '0' + ] + RunCommand(cmdline, verbose=True).run() + cmdline = xharness_adb() + [ + 'shell', 'settings', 'put', 'global', 'package_verifier_enable', '0' + ] + RunCommand(cmdline, verbose=True).run() + except CalledProcessError: + # Best-effort: don't fail setup if device doesn't support these keys; proceed with the run + getLogger().warning("Failed to update package verifier settings; continuing without changes", exc_info=True) # Get animation values getLogger().info("Getting Values we will need set specifically") @@ -106,7 +113,7 @@ def setup_device(self, packagename: str, packagepath: str, animationsdisabled: b animationValue = 0 else: animationValue = 1 - minimumTimeoutValue = 2 * 60 * 1000 # milliseconds + minimumTimeoutValue = screen_timeout_ms cmdline = xharness_adb() + [ 'shell', 'settings', 'put', 'global', 'window_animation_scale', str(animationValue) ] @@ -157,19 +164,21 @@ def setup_device(self, packagename: str, packagepath: str, animationsdisabled: b self.packagename ] - installCmd = xharnesscommand() + [ - 'android', - 'install', - '--app', packagepath, - '--package-name', - self.packagename, - '-o', - const.TRACEDIR, - '-v' - ] - RunCommand(installCmd, verbose=True).run() + if not skip_install: + installCmd = xharnesscommand() + [ + 'android', + 'install', + '--app', packagepath, + '--package-name', + self.packagename, + '-o', + const.TRACEDIR, + '-v' + ] + RunCommand(installCmd, verbose=True).run() + getLogger().info("Completed install.") - getLogger().info("Completed install, running shell.") + getLogger().info("Resolving launchable activity.") cmdline = xharness_adb() + [ 'shell', f'cmd package resolve-activity --brief {self.packagename} | tail -n 1' @@ -178,7 +187,85 @@ def setup_device(self, packagename: str, packagepath: str, animationsdisabled: b getActivity.run() getLogger().info(f"Target Activity {getActivity.stdout}") - # More setup stuff + self.ensure_screen_on() + + self.activityname = getActivity.stdout.strip() + + if not skip_test_launch: + # Test run to check if permissions are needed + getLogger().info("Test run to check if permissions are needed") + + keyInputCmd = xharness_adb() + [ + 'shell', + 'input', + 'keyevent' + ] + + # -W in the start command waits for the app to finish initial draw. + self.startappcommand = xharness_adb() + [ + 'shell', + 'am', + 'start-activity', + '-W', + '-n', + self.activityname + ] + + testRun = RunCommand(self.startappcommand, verbose=True) + testRun.run() + testRunStats = re.findall(run_split_regex, testRun.stdout) # Split results saving value (List: Starting, Status, LaunchState, Activity, TotalTime, WaitTime) + getLogger().info(f"Test run activity: {testRunStats[3]}") + time.sleep(10) # Add delay to ensure app is fully installed and give it some time to settle + + RunCommand(self.stopappcommand, verbose=True).run() + if "com.google.android.permissioncontroller" in testRunStats[3]: + # On perm screen, use the buttons to close it. it will stay away until the app is reinstalled + RunCommand(keyInputCmd + ['22'], verbose=True).run() # Select next button + time.sleep(1) + RunCommand(keyInputCmd + ['22'], verbose=True).run() # Select next button + time.sleep(1) + RunCommand(keyInputCmd + ['66'], verbose=True).run() # Press enter to close main perm screen + time.sleep(1) + RunCommand(keyInputCmd + ['22'], verbose=True).run() # Select next button + time.sleep(1) + RunCommand(keyInputCmd + ['66'], verbose=True).run() # Press enter to close out of second screen + time.sleep(1) + + # Check to make sure it worked + testRun = RunCommand(self.startappcommand, verbose=True) + testRun.run() + testRunStats = re.findall(run_split_regex, testRun.stdout) + getLogger().info(f"Test run activity: {testRunStats[3]}") + RunCommand(self.stopappcommand, verbose=True).run() + + if "com.google.android.permissioncontroller" in testRunStats[3]: + getLogger().exception("Failed to get past permission screen, run locally to see if enough next button presses were used.") + raise Exception("Failed to get past permission screen, run locally to see if enough next button presses were used.") + + self.startappcommand = xharness_adb() + [ + 'shell', + 'am', + 'start-activity' + ] + if forcewaitstart: + self.startappcommand.append('-W') + + self.startappcommand += [ + '-n', + self.activityname + ] + + def ensure_screen_on(self) -> None: + """Wake the device screen and unlock it if it is off. + + Must be called before any launch we want ActivityTaskManager to log a + 'Displayed' line for — Android never emits that line when the screen is + off. Physical Helix CI devices start with screens off. + + Sets self.screenwasoff = True the FIRST time the screen is found off so + close_device() can restore it. A second call when the screen is already + on leaves the flag unchanged. + """ checkScreenOnCmd = xharness_adb() + [ 'shell', 'dumpsys input_method | grep mInteractive' @@ -192,12 +279,12 @@ def setup_device(self, packagename: str, packagepath: str, animationsdisabled: b 'keyevent' ] - if "mInteractive=false" in checkScreenOn.stdout: + if "mInteractive=false" in checkScreenOn.stdout: # Turn on the screen to make interactive and see if it worked getLogger().info("Screen was off, turning on.") self.screenwasoff = True - RunCommand(keyInputCmd + ['26'], verbose=True).run() # Press the power key - RunCommand(keyInputCmd + ['82'], verbose=True).run() # Unlock the screen with menu key (only works if it is not a password lock) + RunCommand(keyInputCmd + ['26'], verbose=True).run() # Press the power key + RunCommand(keyInputCmd + ['82'], verbose=True).run() # Unlock with menu key (no-op on password lock) checkScreenOn = RunCommand(checkScreenOnCmd, verbose=True) checkScreenOn.run() @@ -205,89 +292,97 @@ def setup_device(self, packagename: str, packagepath: str, animationsdisabled: b getLogger().exception("Failed to make screen interactive.") raise Exception("Failed to make screen interactive.") - # Actual testing some run stuff - getLogger().info("Test run to check if permissions are needed") - self.activityname = getActivity.stdout.strip() + def clear_logcat(self) -> None: + """Clear the logcat ring buffer before a measured launch. - # -W in the start command waits for the app to finish initial draw. - self.startappcommand = xharness_adb() + [ + Called before every dotnet-run invocation so stale 'Displayed' lines + from previous iterations don't contaminate the next measurement. + """ + RunCommand(xharness_adb() + ['logcat', '-c'], verbose=True).run() + + def measure_startup_from_logcat(self, packagename: str, activityname: str, timeout_s: int = 30) -> int: + """Measure app startup time from logcat's 'Displayed' line. + + Polls 'adb shell logcat -d' for the 'Displayed /: +NNNms' + line emitted by ActivityTaskManager (API 29+) / ActivityManager (API + 23-28) after the first frame is drawn. + + Prerequisites (caller's responsibility): + - ensure_screen_on() called before the launch — Android never logs + 'Displayed' when the screen is off. + - clear_logcat() called immediately before the launch — otherwise a + stale line from a previous iteration is returned instantly. + + activityname is accepted to preserve runner call compatibility. It is + not used in the grep; packagename alone uniquely identifies the app and + is robust to activity-name format variations. + + Returns startup time in milliseconds. + Raises Exception with a logcat tail on timeout. + """ + poll_cmd = xharness_adb() + [ 'shell', - 'am', - 'start-activity', - '-W', - '-n', - self.activityname + # Use packagename (not activityname) to match regardless of whether + # the activity is reported as '.MainActivity' or the fully-qualified + # 'com.company.app.MainActivity'. + f"logcat -d | grep -E 'ActivityTaskManager|ActivityManager' | grep 'Displayed {packagename}/'" ] - testRun = RunCommand(self.startappcommand, verbose=True) - testRun.run() - testRunStats = re.findall(run_split_regex, testRun.stdout) # Split results saving value (List: Starting, Status, LaunchState, Activity, TotalTime, WaitTime) - getLogger().info(f"Test run activity: {testRunStats[3]}") - time.sleep(10) # Add delay to ensure app is fully installed and give it some time to settle - - RunCommand(self.stopappcommand, verbose=True).run() - if "com.google.android.permissioncontroller" in testRunStats[3]: - # On perm screen, use the buttons to close it. it will stay away until the app is reinstalled - RunCommand(keyInputCmd + ['22'], verbose=True).run() # Select next button - time.sleep(1) - RunCommand(keyInputCmd + ['22'], verbose=True).run() # Select next button - time.sleep(1) - RunCommand(keyInputCmd + ['66'], verbose=True).run() # Press enter to close main perm screen - time.sleep(1) - RunCommand(keyInputCmd + ['22'], verbose=True).run() # Select next button - time.sleep(1) - RunCommand(keyInputCmd + ['66'], verbose=True).run() # Press enter to close out of second screen + deadline = time.time() + timeout_s + result = None + while time.time() < deadline: + # Use subprocess.run directly instead of RunCommand because grep + # returns exit code 1 when no lines match, which RunCommand treats + # as an error and raises CalledProcessError. + result = subprocess.run(poll_cmd, capture_output=True, text=True) + if result.stdout.strip(): + break time.sleep(1) + else: + # Dump recent logcat to help diagnose screen-off or timing issues + debug_cmd = xharness_adb() + ['shell', 'logcat -d -t 40'] + debug_result = subprocess.run(debug_cmd, capture_output=True, text=True) + raise Exception( + "Timed out waiting for 'Displayed %s/' in logcat after %d seconds.\n" + "Last 40 logcat lines:\n%s" % (packagename, timeout_s, debug_result.stdout) + ) - # Check to make sure it worked - testRun = RunCommand(self.startappcommand, verbose=True) - testRun.run() - testRunStats = re.findall(run_split_regex, testRun.stdout) - getLogger().info(f"Test run activity: {testRunStats[3]}") - RunCommand(self.stopappcommand, verbose=True).run() - - if "com.google.android.permissioncontroller" in testRunStats[3]: - getLogger().exception("Failed to get past permission screen, run locally to see if enough next button presses were used.") - raise Exception("Failed to get past permission screen, run locally to see if enough next button presses were used.") - - self.startappcommand = xharness_adb() + [ - 'shell', - 'am', - 'start-activity' - ] - if forcewaitstart: - self.startappcommand.append('-W') + # Parse '+NNNms' or '+Ns NNNms' from ActivityTaskManager/ActivityManager. + dirty_capture = re.search(r"\+(\d*s?\d+)ms", result.stdout) + if dirty_capture: + capture_list = dirty_capture.group(1).split('s') + if len(capture_list) == 1: + startup_ms = int(capture_list[0]) + elif len(capture_list) == 2: + startup_ms = int(capture_list[0]) * 1000 + int(capture_list[1].zfill(3)) + else: + raise Exception("Android time capture failed! Unexpected format: %s" % dirty_capture.group(0)) + getLogger().info("Startup time (logcat Displayed): %d ms" % startup_ms) + return startup_ms - self.startappcommand += [ - '-n', - self.activityname - ] + raise Exception( + "Logcat returned output but no '+NNNms' pattern found.\nOutput: %s" % result.stdout + ) - def close_device(self): + def close_device(self, skip_uninstall: bool = False): keyInputCmd = xharness_adb() + [ 'shell', 'input', 'keyevent' ] - getLogger().info("Stopping App for uninstall") - RunCommand(self.stopappcommand, verbose=True).run() - - getLogger().info("Uninstalling app") - uninstallAppCmd = xharnesscommand() + [ - 'android', - 'uninstall', - '--package-name', - self.packagename - ] - RunCommand(uninstallAppCmd, verbose=True).run() - - - keyInputCmd = xharness_adb() + [ - 'shell', - 'input', - 'keyevent' - ] + if not skip_uninstall: + getLogger().info("Stopping App for uninstall") + RunCommand(self.stopappcommand, verbose=True).run() + + getLogger().info("Uninstalling app") + uninstallAppCmd = xharnesscommand() + [ + 'android', + 'uninstall', + '--package-name', + self.packagename + ] + RunCommand(uninstallAppCmd, verbose=True).run() # Restore Android package verifier settings try: @@ -343,4 +438,4 @@ def close_device(self): RunCommand(cmdline, verbose=True).run() if self.screenwasoff: - RunCommand(keyInputCmd + ['26'], verbose=True).run() # Turn the screen back off \ No newline at end of file + RunCommand(keyInputCmd + ['26'], verbose=True).run() # Turn the screen back off diff --git a/src/scenarios/shared/const.py b/src/scenarios/shared/const.py index 074fa7fdf89..3492887ff13 100644 --- a/src/scenarios/shared/const.py +++ b/src/scenarios/shared/const.py @@ -19,6 +19,7 @@ ANDROIDINSTRUMENTATION = "androidinstrumentation" DEVICEPOWERCONSUMPTION = "devicepowerconsumption" BUILDTIME = "buildtime" +ANDROIDINNERLOOP = "androidinnerloop" SCENARIO_NAMES = {STARTUP: 'Startup', SDK: 'SDK', @@ -27,7 +28,8 @@ INNERLOOP: 'Innerloop', INNERLOOPMSBUILD: 'InnerLoopMsBuild', DOTNETWATCH: 'DotnetWatch', - BUILDTIME: 'BuildTime'} + BUILDTIME: 'BuildTime', + ANDROIDINNERLOOP: 'AndroidInnerLoop'} BINDIR = 'bin' PUBDIR = 'pub' diff --git a/src/scenarios/shared/mauisharedpython.py b/src/scenarios/shared/mauisharedpython.py index 3b80dcb91cb..e91c113c08d 100644 --- a/src/scenarios/shared/mauisharedpython.py +++ b/src/scenarios/shared/mauisharedpython.py @@ -388,29 +388,49 @@ def __exit__(self, exc_type, exc_val, exc_tb): def install_latest_maui( precommands: PreCommands, - feed=extract_latest_dotnet_feed_from_nuget_config(path=os.path.join(get_repo_root_path(), "NuGet.config")) + feed: str | None = None, + workloads: list[str] | None = None, + workload_name: str = 'maui' ): ''' - Install the latest maui workload using the provided feed. - This function will create a rollback file and install the maui workload using that file. + Install the latest MAUI workload using the provided feed. + Creates a rollback file and installs the workload using that file. + + Args: + precommands: PreCommands instance for running dotnet commands. + feed: NuGet feed URL to resolve workload packages from. + If None, resolves from NuGet.config at call time. + workloads: NuGet SDK package names to resolve (e.g., + ["microsoft.net.sdk.android"]). + If None, defaults to the full set of 6 MAUI workloads. + workload_name: The dotnet CLI workload name to install + (e.g., 'maui', 'maui-android'). ''' - getLogger().info("########## Installing latest MAUI workload ##########") + getLogger().info(f"########## Installing latest {workload_name} workload ##########") if precommands.has_workload: - getLogger().info("Skipping maui installation due to --has-workload=true") + getLogger().info(f"Skipping {workload_name} installation due to --has-workload=true") return - maui_rollback_dict: dict[str, str] = { - "microsoft.net.sdk.android" : "", - "microsoft.net.sdk.ios" : "", - "microsoft.net.sdk.maccatalyst" : "", - "microsoft.net.sdk.macos" : "", - "microsoft.net.sdk.maui" : "", - "microsoft.net.sdk.tvos" : "" - } + if feed is None: + feed = extract_latest_dotnet_feed_from_nuget_config( + path=os.path.join(get_repo_root_path(), "NuGet.config") + ) + + if workloads is None: + workloads = [ + "microsoft.net.sdk.android", + "microsoft.net.sdk.ios", + "microsoft.net.sdk.maccatalyst", + "microsoft.net.sdk.macos", + "microsoft.net.sdk.maui", + "microsoft.net.sdk.tvos", + ] + + maui_rollback_dict = {name: "" for name in workloads} - getLogger().info(f"Installing the latest maui workload from feed {feed}") + getLogger().info(f"Installing the latest {workload_name} workload from feed {feed}") # Get the latest published version of the maui workloads for workload in maui_rollback_dict.keys(): @@ -453,10 +473,10 @@ def install_latest_maui( getLogger().debug(f"Extracted .NET version '{dotnet_version}' from SDK version '{sdk_version}'") else: getLogger().error(f"Unable to find .NET version in SDK version '{sdk_version}' for package {package['id']}") - raise Exception("Unable to find .NET version in SDK version") + raise Exception(f"Unable to find .NET version in SDK version '{sdk_version}' for package {package['id']}") else: getLogger().error(f"Unable to find .NET SDK version in package ID: {package['id']}") - raise Exception("Unable to find .NET SDK version in package ID") + raise Exception(f"Unable to find .NET SDK version in package ID: {package['id']}") # Filter out packages that have lower 'dotnet_version' than the rest of the packages # Sometimes feed can contain packages from previous release versions, so we need to filter them out @@ -514,6 +534,6 @@ def install_latest_maui( getLogger().info("Created rollback_maui.json file") # Install the workload using the rollback file - getLogger().info("Installing maui workload with rollback file") - precommands.install_workload('maui', ['--from-rollback-file', 'rollback_maui.json']) - getLogger().info("########## Finished installing latest MAUI workload ##########") + getLogger().info(f"Installing {workload_name} workload with rollback file") + precommands.install_workload(workload_name, ['--from-rollback-file', 'rollback_maui.json']) + getLogger().info(f"########## Finished installing latest {workload_name} workload ##########") diff --git a/src/scenarios/shared/runner.py b/src/scenarios/shared/runner.py index 165f0e882ba..89ae1c3d2f4 100644 --- a/src/scenarios/shared/runner.py +++ b/src/scenarios/shared/runner.py @@ -28,6 +28,7 @@ from performance.common import RunCommand, iswin, extension, helixworkitemroot from performance.logger import setup_loggers from shared.testtraits import TestTraits, testtypes +from shared.versionmanager import versions_write_json, versions_read_json_file_save_env, get_sdk_versions from subprocess import CalledProcessError @@ -174,6 +175,19 @@ def parseargs(self): buildtimeparser.add_argument('--binlog-path', help='Location of binlog', dest='binlogpath') self.add_common_arguments(buildtimeparser) + androidinnerloopparser = subparsers.add_parser(const.ANDROIDINNERLOOP, + description='measure first and incremental deploy time via binlogs') + androidinnerloopparser.add_argument('--csproj-path', help='Path to .csproj file to build', dest='csprojpath', required=True) + androidinnerloopparser.add_argument('--edit-src', help='Semicolon-separated path(s) to modified source file(s) copied before each incremental deploy. Each entry pairs positionally with the same-index --edit-dest entry.', dest='editsrc', required=True) + androidinnerloopparser.add_argument('--edit-dest', help='Semicolon-separated destination path(s) for the modified file(s). Must contain the same number of entries as --edit-src.', dest='editdest', required=True) + androidinnerloopparser.add_argument('--framework', '-f', help='Target framework (e.g., net10.0-android)', dest='framework', required=True) + androidinnerloopparser.add_argument('--configuration', '-c', help='Build configuration', dest='configuration', required=True) + androidinnerloopparser.add_argument('--msbuild-args', help='Additional MSBuild arguments', dest='msbuildargs', default='') + androidinnerloopparser.add_argument('--package-name', help='Android package name for startup measurement (e.g. com.companyname.mauiandroidinnerloop)', dest='packagename', required=True) + androidinnerloopparser.add_argument('--inner-loop-iterations', help='Number of incremental build+deploy+startup iterations (1+)', type=int, default=10, dest='innerloopiterations', choices=range(1, 1001), metavar='N') + androidinnerloopparser.add_argument('--screen-timeout-ms', help='Screen timeout in milliseconds. Must be large enough so the display stays on for the entire scenario; if the screen turns off mid-run, ActivityTaskManager never emits the Displayed line and startup measurement times out.', type=int, default=30 * 60 * 1000, dest='screentimeoutms') + self.add_common_arguments(androidinnerloopparser) + args = parser.parse_args() if not args.testtype: @@ -196,7 +210,18 @@ def parseargs(self): if self.testtype == const.BUILDTIME: self.binlogpath = args.binlogpath - + + if self.testtype == const.ANDROIDINNERLOOP: + self.csprojpath = args.csprojpath + self.editsrcs = args.editsrc.split(';') if args.editsrc else [] + self.editdests = args.editdest.split(';') if args.editdest else [] + self.framework = args.framework + self.configuration = args.configuration + self.msbuildargs = args.msbuildargs or os.environ.get('PERFLAB_MSBUILD_ARGS', '') + self.packagename = args.packagename + self.innerloopiterations = args.innerloopiterations + self.screentimeoutms = args.screentimeoutms + if self.testtype == const.DEVICESTARTUP: self.packagepath = args.packagepath self.packagename = args.packagename @@ -974,4 +999,322 @@ def run(self): if not (self.binlogpath and os.path.exists(os.path.join(const.TRACEDIR, self.binlogpath))): raise Exception("For build time measurements a valid binlog path must be provided.") self.traits.add_traits(overwrite=True, apptorun="app", startupmetric=const.BUILDTIME, tracename=self.binlogpath, scenarioname=self.scenarioname) - startup.parsetraces(self.traits) \ No newline at end of file + startup.parsetraces(self.traits) + + elif self.testtype == const.ANDROIDINNERLOOP: + import hashlib + import subprocess + from shutil import copytree + from performance.common import runninginlab + from performance.constants import UPLOAD_CONTAINER, UPLOAD_STORAGE_URI, UPLOAD_QUEUE + from shared.util import helixuploaddir + import upload + + def remove_dotnet_run_binlog(binlog_path): + """Delete the extra `*-dotnet-run.binlog` that are produced by `dotnet run`. + It only contains the _Run target firing am start; we measure startup via logcat + and don't parse it. + """ + aux = binlog_path[:-len('.binlog')] + '-dotnet-run.binlog' + if os.path.exists(aux): + os.remove(aux) + + def merge_build_and_startup(build_report_path, startup_results, final_report_path): + """Load the build metrics report, append a startup time counter, write to final path.""" + with open(build_report_path, 'r') as f: + report = json.load(f) + startup_counter = { + "name": "Time to Displayed", + "topCounter": True, + "defaultCounter": False, + "higherIsBetter": False, + "metricName": "ms", + "results": startup_results + } + # Report structure: { "tests": [ { "counters": [...] } ] } + report["tests"][0]["counters"].append(startup_counter) + with open(final_report_path, 'w') as f: + json.dump(report, f, indent=2) + getLogger().info("Merged report written to: %s" % final_report_path) + + def run_incremental_iteration(iteration, num_iterations, base_cmd, edit_pairs, + packagename, activityname, + scenarioprefix, startup, traits, measure_startup_fn, + pre_launch_fn=None): + """Run one incremental build+deploy+startup iteration. + + edit_pairs is a list of (dest_path, original_content, modified_content) tuples. + pre_launch_fn, if provided, is called before each dotnet run (e.g. to clear + the logcat buffer so stale 'Displayed' lines don't pollute the measurement). + Returns (startup_ms, counters_list, binlog_path, test_metadata). + """ + import subprocess + + getLogger().info("=== Incremental iteration %d/%d ===" % (iteration, num_iterations)) + + # Toggle source files + for dest, original, modified in edit_pairs: + if iteration % 2 == 1: + with open(dest, 'w') as f: + f.write(modified) + content_hash = hashlib.md5(modified.encode()).hexdigest()[:8] + getLogger().info("Applied modified source: %s (hash=%s, len=%d)" % (dest, content_hash, len(modified))) + else: + with open(dest, 'w') as f: + f.write(original) + content_hash = hashlib.md5(original.encode()).hexdigest()[:8] + getLogger().info("Restored original source: %s (hash=%s, len=%d)" % (dest, content_hash, len(original))) + + # Incremental build+deploy with per-iteration binlog + iter_binlog_name = 'incremental-build-and-deploy-%d.binlog' % iteration + iter_binlog = os.path.join(const.TRACEDIR, iter_binlog_name) + incremental_cmd = base_cmd + [f'-bl:{iter_binlog}'] + getLogger().info("Incremental build+deploy: %s" % ' '.join(incremental_cmd)) + if pre_launch_fn is not None: + pre_launch_fn() + subprocess.run(incremental_cmd, check=True) + remove_dotnet_run_binlog(iter_binlog) + + # Measure startup + ms = measure_startup_fn(packagename, activityname) + getLogger().info("Incremental iteration %d/%d: build+deploy done, startup: %d ms" % (iteration, num_iterations, ms)) + + # Parse this iteration's binlog → temp build report + iter_report_name = 'incremental-build-report-%d.json' % iteration + iter_report = os.path.join(const.TRACEDIR, iter_report_name) + startup.reportjson = iter_report + traits.add_traits(overwrite=True, apptorun="app", startupmetric=const.ANDROIDINNERLOOP, + tracename=iter_binlog_name, + scenarioname=scenarioprefix + " - Incremental Build and Deploy", + upload_to_perflab_container=False) + startup.parsetraces(traits, copy_traces=False) + + # Extract build counters and test metadata from temp report + with open(iter_report, 'r') as f: + iter_data = json.load(f) + test_obj = iter_data["tests"][0] + counters = test_obj["counters"] + # Capture top-level metadata (build, os, run, inLab) so the + # final aggregated report has the same shape as the first + # report — PerfLab needs these to classify the upload. + test_metadata = { + "test": {**test_obj, "counters": []}, + "top_level": {k: v for k, v in iter_data.items() if k != "tests"}, + } + + # Clean up temp report (leave binlog for later cleanup) + if os.path.exists(iter_report): + os.remove(iter_report) + getLogger().info("Removed temp report: %s" % iter_report) + + return ms, counters, iter_binlog, test_metadata + + scenarioprefix = self.scenarioname or "MAUI Android Build and Deploy" + + os.makedirs(const.TRACEDIR, exist_ok=True) + first_binlog = os.path.join(const.TRACEDIR, 'first-build-and-deploy.binlog') + + # Build the base MSBuild command. + base_cmd = ['dotnet', 'run', '--project', self.csprojpath, '-p:WaitForExit=false', '--no-restore', '-c', self.configuration, '-f', self.framework] + if self.msbuildargs: + # Split on semicolons and whitespace. NOTE: MSBuild args with + # embedded spaces (e.g. /p:Foo="a b") will be shredded; all + # current callers pass only /p:Key=Value forms without spaces. + for arg in re.split(r'[;\s]+', self.msbuildargs): + if arg.strip(): + base_cmd.append(arg.strip()) + + # --- First build + deploy --- + # Normalize device state BEFORE the first build so the first sample is measured under the same conditions as incrementals. + # Activity resolution is deferred until after install (requires the APK to be on the device). + androidHelper = AndroidHelper() + try: + first_cmd = base_cmd + [f'-bl:{first_binlog}'] + getLogger().info("First build+deploy: %s" % ' '.join(first_cmd)) + androidHelper.setup_device(self.packagename, packagepath=None, animationsdisabled=True, skip_install=True, skip_xharness_warmup=True, skip_package_verifier=True, skip_test_launch=True, screen_timeout_ms=self.screentimeoutms) + androidHelper.ensure_screen_on() + androidHelper.clear_logcat() + subprocess.run(first_cmd, check=True) + remove_dotnet_run_binlog(first_binlog) + + # Capture SDK versions from the just-built output. Other Android + # scenarios do this in pre.py on the build machine, but our first + # build runs on Helix and IS the measured build, so we have to + # do it here. Discover the RID folder the build actually produced + # (android-arm64 on physical devices, android-x64 on emulator). + try: + framework_obj_root = os.path.join(os.path.dirname(self.csprojpath), 'obj', self.configuration, self.framework) + abi_by_rid = {'android-arm': 'armeabi-v7a', 'android-arm64': 'arm64-v8a', 'android-x86': 'x86', 'android-x64': 'x86_64'} + + dll_folder = None + for rid_dir in sorted(glob.glob(os.path.join(framework_obj_root, 'android-*'))): + rid = os.path.basename(rid_dir) + linked = os.path.join(rid_dir, 'linked') + if os.path.isdir(linked): + dll_folder = linked + break + abi = abi_by_rid.get(rid) + if abi: + staging = os.path.join(rid_dir, 'android', 'assets', abi) + if os.path.isdir(staging): + dll_folder = staging + break + if dll_folder is None: + raise FileNotFoundError("No android-* RID folder with DLLs found under %s" % framework_obj_root) + + getLogger().info("Capturing SDK versions from: %s" % dll_folder) + version_dict = get_sdk_versions(dll_folder) + + os.makedirs(const.TRACEDIR, exist_ok=True) + versions_path = os.path.join(const.TRACEDIR, 'versions.json') + versions_write_json(version_dict, versions_path) + # Surface PERFLAB_DATA_* env vars so the Startup tool's Reporter + # picks them up into build.AdditionalData. + versions_read_json_file_save_env(versions_path) + except Exception as ex: + # Never let version capture regress the measurement pipeline. + getLogger().warning("Version capture failed, continuing without versions.json: %s" % ex) + + # --- Resolve activity (requires app installed by first_cmd) --- + getLogger().info("Resolving launchable activity after first install.") + resolve_cmd = xharness_adb() + [ + 'shell', + f'cmd package resolve-activity --brief {self.packagename} | tail -n 1' + ] + resolve_result = RunCommand(resolve_cmd, verbose=True) + resolve_result.run() + activityname = resolve_result.stdout.strip() + androidHelper.activityname = activityname + getLogger().info("Using resolved activity: %s" % activityname) + + # --- First startup measurement (logcat-based) --- + # `dotnet run -p:WaitForExit=false` issued `am start -S` (no -W). + # Poll logcat for ActivityTaskManager's 'Displayed /: +NNNms' line. + # Screen was guaranteed on by ensure_screen_on() above. + first_startup_ms = androidHelper.measure_startup_from_logcat(self.packagename, activityname) + getLogger().info("First deploy startup: %d ms" % first_startup_ms) + + # --- Parse first build report --- + startup = StartupWrapper() + first_build_report = os.path.join(const.TRACEDIR, 'first-build-and-deploy-perf-lab-report.json') + startup.reportjson = first_build_report + saved_upload = self.traits.upload_to_perflab_container + self.traits.add_traits(overwrite=True, apptorun="app", startupmetric=const.ANDROIDINNERLOOP, + tracename='first-build-and-deploy.binlog', + scenarioname=scenarioprefix + " - First Build and Deploy", + upload_to_perflab_container=False) + startup.parsetraces(self.traits, copy_traces=False) + + # Merge first build metrics + startup → first e2e report + first_e2e_report = os.path.join(const.TRACEDIR, 'first-debug-e2e-perf-lab-report.json') + merge_build_and_startup(first_build_report, [first_startup_ms], first_e2e_report) + + # --- Incremental loop --- + num_iterations = self.innerloopiterations + getLogger().info("Starting incremental loop: %d iterations" % num_iterations) + + # Build list of (dest, original_content, modified_content) tuples for toggling + if len(self.editsrcs) != len(self.editdests): + raise Exception("--edit-src and --edit-dest must have the same number of semicolon-separated paths") + edit_pairs = [] + for src, dest in zip(self.editsrcs, self.editdests): + original = None + modified = None + with open(dest, 'r') as f: + original = f.read() + with open(src, 'r') as f: + modified = f.read() + edit_pairs.append((dest, original, modified)) + getLogger().info("Edit pair: %s <-> %s" % (src, dest)) + + incremental_startup_results = [] + aggregated_counters = {} # counter_name -> aggregated counter dict + report_template = None # test metadata from first parsed report + intermediate_files = [] # files to clean up + + def pre_iteration(): + androidHelper.ensure_screen_on() + androidHelper.clear_logcat() + + for iteration in range(1, num_iterations + 1): + ms, counters, iter_binlog, test_metadata = run_incremental_iteration( + iteration, num_iterations, base_cmd, + edit_pairs, + self.packagename, activityname, scenarioprefix, startup, self.traits, + androidHelper.measure_startup_from_logcat, + pre_launch_fn=pre_iteration) + + incremental_startup_results.append(ms) + intermediate_files.append(iter_binlog) + + # Save test metadata from the first iteration + if report_template is None: + report_template = test_metadata + + for counter in counters: + name = counter["name"] + if name not in aggregated_counters: + aggregated_counters[name] = { + "name": name, + "topCounter": counter.get("topCounter", False), + "defaultCounter": counter.get("defaultCounter", False), + "higherIsBetter": counter.get("higherIsBetter", False), + "metricName": counter.get("metricName", "ms"), + "results": [] + } + aggregated_counters[name]["results"].extend(counter.get("results", [])) + + # --- Aggregate incremental results --- + incremental_e2e_report = os.path.join(const.TRACEDIR, 'incremental-debug-e2e-perf-lab-report.json') + final_counters = list(aggregated_counters.values()) + final_counters.append({ + "name": "Time to Displayed", + "topCounter": True, + "defaultCounter": False, + "higherIsBetter": False, + "metricName": "ms", + "results": incremental_startup_results + }) + if report_template is not None: + report_template["test"]["counters"] = final_counters + final_report_data = { + **report_template["top_level"], + "tests": [report_template["test"]], + } + else: + # Fallback: should not happen if at least 1 iteration ran + final_report_data = {"tests": [{"counters": final_counters}]} + with open(incremental_e2e_report, 'w') as f: + json.dump(final_report_data, f, indent=2) + getLogger().info("Final incremental E2E report written to: %s" % incremental_e2e_report) + + # --- Cleanup and upload --- + # Clean up intermediates from TRACEDIR + for f_path in intermediate_files + [first_build_report]: + if f_path.endswith('.binlog'): + getLogger().info("Keeping binlog for upload: %s" % f_path) + continue + if os.path.exists(f_path): + os.remove(f_path) + getLogger().info("Removed intermediate: %s" % f_path) + + # Wipe helix upload traces dir so copytree repopulates it cleanly + if runninginlab(): + traces_upload = os.path.join(helixuploaddir() or '', 'traces') + if os.path.exists(traces_upload): + rmtree(traces_upload) + + # Final upload + self.traits.add_traits(overwrite=True, upload_to_perflab_container=saved_upload) + helix_upload_dir = helixuploaddir() + if runninginlab() and helix_upload_dir is not None: + copytree(const.TRACEDIR, os.path.join(helix_upload_dir, 'traces'), dirs_exist_ok=True) + if self.traits.upload_to_perflab_container: + for report_path in [first_e2e_report, incremental_e2e_report]: + upload_code = upload.upload(report_path, UPLOAD_CONTAINER, UPLOAD_QUEUE, UPLOAD_STORAGE_URI) + getLogger().info("Upload code for %s: %s" % (os.path.basename(report_path), upload_code)) + if upload_code != 0: + sys.exit(upload_code) + + finally: + androidHelper.close_device(skip_uninstall=True) diff --git a/src/scenarios/shared/startup.py b/src/scenarios/shared/startup.py index da0fa548680..331b90aadde 100644 --- a/src/scenarios/shared/startup.py +++ b/src/scenarios/shared/startup.py @@ -56,7 +56,7 @@ def __init__(self): def _setstartuppath(self, path: str): self.startuppath = os.path.join(path, "Startup%s" % extension()) - def parsetraces(self, traits: TestTraits): + def parsetraces(self, traits: TestTraits, copy_traces: bool = True): directory = TRACEDIR if traits.tracefolder: directory = TRACEDIR + '/' + traits.tracefolder @@ -83,9 +83,16 @@ def parsetraces(self, traits: TestTraits): # rethrow the original exception raise + # copy_traces=False skips the per-call TRACEDIR -> Helix upload-dir copy and + # the perflab container upload. Callers that parse repeatedly (e.g. the + # inner loop scenario) should pass False and do one final copy at the end + # to avoid O(N^2) I/O over accumulating binlogs. + if not copy_traces: + return + helix_upload_dir = helixuploaddir() if runninginlab() and helix_upload_dir is not None: - copytree(TRACEDIR, os.path.join(helix_upload_dir, 'traces')) + copytree(TRACEDIR, os.path.join(helix_upload_dir, 'traces'), dirs_exist_ok=True) if traits.upload_to_perflab_container: import upload upload_code = upload.upload(self.reportjson, upload_container, UPLOAD_QUEUE, UPLOAD_STORAGE_URI) @@ -163,7 +170,7 @@ def runtests(self, traits: TestTraits): helix_upload_dir = helixuploaddir() if runninginlab() and helix_upload_dir is not None: - copytree(TRACEDIR, os.path.join(helix_upload_dir, 'traces')) + copytree(TRACEDIR, os.path.join(helix_upload_dir, 'traces'), dirs_exist_ok=True) if traits.upload_to_perflab_container: import upload upload_code = upload.upload(self.reportjson, upload_container, UPLOAD_QUEUE, UPLOAD_STORAGE_URI) diff --git a/src/scenarios/shared/testtraits.py b/src/scenarios/shared/testtraits.py index 3f2acef0fb9..93c4f57dc7f 100644 --- a/src/scenarios/shared/testtraits.py +++ b/src/scenarios/shared/testtraits.py @@ -10,7 +10,8 @@ const.SOD, const.INNERLOOP, const.DEVICESTARTUP, - const.BUILDTIME] + const.BUILDTIME, + const.ANDROIDINNERLOOP] class TestTraits: diff --git a/src/tools/ScenarioMeasurement/Startup/Startup.cs b/src/tools/ScenarioMeasurement/Startup/Startup.cs index 3c8a180561b..1e6f6a73bd7 100644 --- a/src/tools/ScenarioMeasurement/Startup/Startup.cs +++ b/src/tools/ScenarioMeasurement/Startup/Startup.cs @@ -27,6 +27,7 @@ enum MetricType WinUIBlazor, TimeToMain2, BuildTime, + AndroidInnerLoop, } public class InnerLoopMarkerEventSource : EventSource @@ -291,6 +292,7 @@ static void checkArg(string arg, string name) MetricType.WinUIBlazor => new WinUIBlazorParser(), MetricType.TimeToMain2 => new TimeToMain2Parser(AddTestProcessEnvironmentVariable), MetricType.BuildTime => new BuildTimeParser(), + MetricType.AndroidInnerLoop => new AndroidInnerLoopParser(), _ => throw new ArgumentOutOfRangeException(), }; diff --git a/src/tools/ScenarioMeasurement/Util/Parsers/AndroidInnerLoopParser.cs b/src/tools/ScenarioMeasurement/Util/Parsers/AndroidInnerLoopParser.cs new file mode 100644 index 00000000000..8ee22d99f95 --- /dev/null +++ b/src/tools/ScenarioMeasurement/Util/Parsers/AndroidInnerLoopParser.cs @@ -0,0 +1,176 @@ +using System; +using System.Collections.Generic; +using System.IO; +using Microsoft.Build.Logging.StructuredLogger; +using StructuredLogViewer; +using Microsoft.Diagnostics.Tracing; +using Reporting; + +namespace ScenarioMeasurement; + +/// +/// Parses Android inner loop (build+deploy) target and task durations from a binary log file. +/// +public class AndroidInnerLoopParser : IParser +{ + public void EnableKernelProvider(ITraceSession kernel) { throw new NotImplementedException(); } + public void EnableUserProviders(ITraceSession user) { throw new NotImplementedException(); } + + public IEnumerable Parse(string binlogFile, string processName, IList pids, string commandLine) + { + var buildDeployTimes = new List(); + + // Build tasks (compilation) + var cscTimes = new List(); + var xamlCTimes = new List(); + var generateJavaStubsTimes = new List(); + var linkAssembliesNoShrinkTimes = new List(); + var d8Times = new List(); + var javacTimes = new List(); + var generateTypeMappingsTimes = new List(); + var processAssembliesTimes = new List(); + var generateJavaCallableWrappersTimes = new List(); + var filterAssembliesTimes = new List(); + var waitForAppDetectionTimes = new List(); + var generateMainAndroidManifestTimes = new List(); + var resolveSdksTimes = new List(); + var generateNativeApplicationConfigSourcesTimes = new List(); + + // Deploy tasks + var fastDeployTimes = new List(); + var androidSignPackageTimes = new List(); + var androidApkSignerTimes = new List(); + var aapt2LinkTimes = new List(); + + // Build targets + var coreCompileTargetTimes = new List(); + var xamlCTargetTimes = new List(); + var generateJavaStubsTargetTimes = new List(); + var linkAssembliesNoShrinkTargetTimes = new List(); + var compileToDalvikTargetTimes = new List(); + var compileJavaTargetTimes = new List(); + + // Deploy targets + var signTargetTimes = new List(); + var uploadTargetTimes = new List(); + var deployApkTargetTimes = new List(); + var buildApkFastDevTargetTimes = new List(); + + if (File.Exists(binlogFile)) + { + var build = BinaryLog.ReadBuild(binlogFile); + BuildAnalyzer.AnalyzeBuild(build); + + foreach (var task in build.FindChildrenRecursive()) + { + var name = task.Name; + var s = task.Duration.TotalMilliseconds / 1000.0; + + if (name.Equals("Csc", StringComparison.OrdinalIgnoreCase)) + cscTimes.Add(s); + else if (name.Equals("XamlCTask", StringComparison.OrdinalIgnoreCase)) + xamlCTimes.Add(s); + else if (name.Equals("GenerateJavaStubs", StringComparison.OrdinalIgnoreCase)) + generateJavaStubsTimes.Add(s); + else if (name.Equals("LinkAssembliesNoShrink", StringComparison.OrdinalIgnoreCase)) + linkAssembliesNoShrinkTimes.Add(s); + else if (name.Equals("D8", StringComparison.OrdinalIgnoreCase)) + d8Times.Add(s); + else if (name.Equals("Javac", StringComparison.OrdinalIgnoreCase)) + javacTimes.Add(s); + else if (name.Equals("GenerateTypeMappings", StringComparison.OrdinalIgnoreCase)) + generateTypeMappingsTimes.Add(s); + else if (name.Equals("ProcessAssemblies", StringComparison.OrdinalIgnoreCase)) + processAssembliesTimes.Add(s); + else if (name.Equals("GenerateJavaCallableWrappers", StringComparison.OrdinalIgnoreCase)) + generateJavaCallableWrappersTimes.Add(s); + else if (name.Equals("FilterAssemblies", StringComparison.OrdinalIgnoreCase)) + filterAssembliesTimes.Add(s); + else if (name.Equals("WaitForAppDetection", StringComparison.OrdinalIgnoreCase)) + waitForAppDetectionTimes.Add(s); + else if (name.Equals("GenerateMainAndroidManifest", StringComparison.OrdinalIgnoreCase)) + generateMainAndroidManifestTimes.Add(s); + else if (name.Equals("ResolveSdks", StringComparison.OrdinalIgnoreCase)) + resolveSdksTimes.Add(s); + else if (name.Equals("GenerateNativeApplicationConfigSources", StringComparison.OrdinalIgnoreCase)) + generateNativeApplicationConfigSourcesTimes.Add(s); + else if (name.Equals("FastDeploy", StringComparison.OrdinalIgnoreCase)) + fastDeployTimes.Add(s); + else if (name.Equals("AndroidSignPackage", StringComparison.OrdinalIgnoreCase)) + androidSignPackageTimes.Add(s); + else if (name.Equals("AndroidApkSigner", StringComparison.OrdinalIgnoreCase)) + androidApkSignerTimes.Add(s); + else if (name.Equals("Aapt2Link", StringComparison.OrdinalIgnoreCase)) + aapt2LinkTimes.Add(s); + } + + foreach (var target in build.FindChildrenRecursive()) + { + var name = target.Name; + var s = target.Duration.TotalMilliseconds / 1000.0; + + if (name.Equals("CoreCompile", StringComparison.Ordinal)) + coreCompileTargetTimes.Add(s); + else if (name.Equals("XamlC", StringComparison.Ordinal)) + xamlCTargetTimes.Add(s); + else if (name.Equals("_GenerateJavaStubs", StringComparison.Ordinal)) + generateJavaStubsTargetTimes.Add(s); + else if (name.Equals("_LinkAssembliesNoShrink", StringComparison.Ordinal)) + linkAssembliesNoShrinkTargetTimes.Add(s); + else if (name.Equals("_CompileToDalvik", StringComparison.Ordinal)) + compileToDalvikTargetTimes.Add(s); + else if (name.Equals("_CompileJava", StringComparison.Ordinal)) + compileJavaTargetTimes.Add(s); + else if (name.Equals("_Sign", StringComparison.Ordinal)) + signTargetTimes.Add(s); + else if (name.Equals("_Upload", StringComparison.Ordinal)) + uploadTargetTimes.Add(s); + else if (name.Equals("_DeployApk", StringComparison.Ordinal)) + deployApkTargetTimes.Add(s); + else if (name.Equals("_BuildApkFastDev", StringComparison.Ordinal)) + buildApkFastDevTargetTimes.Add(s); + } + + buildDeployTimes.Add(build.Duration.TotalMilliseconds / 1000.0); + } + + // Overall duration + yield return new Counter { Name = "Build+Deploy Time", MetricName = "s", DefaultCounter = true, TopCounter = true, Results = buildDeployTimes.ToArray() }; + + // Build task counters + yield return new Counter { Name = "Csc Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = cscTimes.ToArray() }; + yield return new Counter { Name = "XamlC Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = xamlCTimes.ToArray() }; + yield return new Counter { Name = "GenerateJavaStubs Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = generateJavaStubsTimes.ToArray() }; + yield return new Counter { Name = "LinkAssembliesNoShrink Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = linkAssembliesNoShrinkTimes.ToArray() }; + yield return new Counter { Name = "D8 Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = d8Times.ToArray() }; + yield return new Counter { Name = "Javac Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = javacTimes.ToArray() }; + yield return new Counter { Name = "GenerateTypeMappings Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = generateTypeMappingsTimes.ToArray() }; + yield return new Counter { Name = "ProcessAssemblies Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = processAssembliesTimes.ToArray() }; + yield return new Counter { Name = "GenerateJavaCallableWrappers Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = generateJavaCallableWrappersTimes.ToArray() }; + yield return new Counter { Name = "FilterAssemblies Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = filterAssembliesTimes.ToArray() }; + yield return new Counter { Name = "WaitForAppDetection Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = waitForAppDetectionTimes.ToArray() }; + yield return new Counter { Name = "GenerateMainAndroidManifest Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = generateMainAndroidManifestTimes.ToArray() }; + yield return new Counter { Name = "ResolveSdks Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = resolveSdksTimes.ToArray() }; + yield return new Counter { Name = "GenerateNativeApplicationConfigSources Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = generateNativeApplicationConfigSourcesTimes.ToArray() }; + + // Build target counters + yield return new Counter { Name = "CoreCompile Target Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = coreCompileTargetTimes.ToArray() }; + yield return new Counter { Name = "XamlC Target Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = xamlCTargetTimes.ToArray() }; + yield return new Counter { Name = "_GenerateJavaStubs Target Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = generateJavaStubsTargetTimes.ToArray() }; + yield return new Counter { Name = "_LinkAssembliesNoShrink Target Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = linkAssembliesNoShrinkTargetTimes.ToArray() }; + yield return new Counter { Name = "_CompileToDalvik Target Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = compileToDalvikTargetTimes.ToArray() }; + yield return new Counter { Name = "_CompileJava Target Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = compileJavaTargetTimes.ToArray() }; + + // Deploy target counters + yield return new Counter { Name = "_Sign Target Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = signTargetTimes.ToArray() }; + yield return new Counter { Name = "_Upload Target Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = uploadTargetTimes.ToArray() }; + yield return new Counter { Name = "_DeployApk Target Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = deployApkTargetTimes.ToArray() }; + yield return new Counter { Name = "_BuildApkFastDev Target Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = buildApkFastDevTargetTimes.ToArray() }; + + // Task-level counters (granular) + yield return new Counter { Name = "FastDeploy Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = fastDeployTimes.ToArray() }; + yield return new Counter { Name = "AndroidSignPackage Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = androidSignPackageTimes.ToArray() }; + yield return new Counter { Name = "AndroidApkSigner Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = androidApkSignerTimes.ToArray() }; + yield return new Counter { Name = "Aapt2Link Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = aapt2LinkTimes.ToArray() }; + } +}