add SPLIT_CI option for using split unit tests

chingor13 · chingor13 · commit 61f4130395aa · 2026-03-26T16:45:51.000Z
diff --git a/monorepo-migration/migrate.sh b/monorepo-migration/migrate.sh
@@ -59,6 +59,7 @@ UPDATE_GENERATION_CONFIG_SCRIPT="$TRANSFORM_SCRIPT_DIR/update_generation_config.
 UPDATE_OWLBOT_HERMETIC_SCRIPT="$TRANSFORM_SCRIPT_DIR/update_owlbot_hermetic.py"
 TRANSFORM_OWLBOT_SCRIPT="$TRANSFORM_SCRIPT_DIR/update_owlbot.py"
 UPDATE_DEP_MGMT_SCRIPT="$TRANSFORM_SCRIPT_DIR/update_dependency_management.py"
+UPDATE_CI_FILTERS_SCRIPT="$TRANSFORM_SCRIPT_DIR/update_ci_filters.py"
 
 # Track number of commits made by this script
 COMMIT_COUNT=0
@@ -378,8 +379,8 @@ if [ -d "$SOURCE_REPO_NAME/.github/workflows" ]; then
         if [ -f "$workflow" ]; then
             filename=$(basename "$workflow")
 
-            if [[ "${filename}" == "ci.yaml" && "${SKIP_CI_WORKFLOW}" == "true" ]]; then
-                echo "Skipping ci.yaml workflow as requested by user"
+            if [[ "${filename}" == "ci.yaml" && ( "${SKIP_CI_WORKFLOW}" == "true" || "${SPLIT_CI}" == "true" ) ]]; then
+                echo "Skipping ci.yaml workflow as requested by user (SKIP_CI_WORKFLOW=${SKIP_CI_WORKFLOW}, SPLIT_CI=${SPLIT_CI})"
                 continue
             fi
 
@@ -412,6 +413,19 @@ if [ -d "$SOURCE_REPO_NAME/.github/workflows" ]; then
     COMMIT_COUNT=$((COMMIT_COUNT + 1))
 fi
 
+# 7.5b Update central ci.yaml filters if SPLIT_CI is true
+if [ "${SPLIT_CI}" == "true" ]; then
+    CI_YAML=".github/workflows/ci.yaml"
+    if [ -f "$CI_YAML" ]; then
+        echo "Adding $SOURCE_REPO_NAME to $CI_YAML filters..."
+        python3 "$UPDATE_CI_FILTERS_SCRIPT" "$CI_YAML" "$SOURCE_REPO_NAME"
+        echo "Committing ci.yaml filter update..."
+        git add "$CI_YAML"
+        git commit -n --no-gpg-sign -m "chore($SOURCE_REPO_NAME): add to filters in $CI_YAML"
+        COMMIT_COUNT=$((COMMIT_COUNT + 1))
+    fi
+fi
+
 # 7.6 Update generation_config.yaml
 echo "Updating generation_config.yaml..."
 SOURCE_CONFIG="$SOURCE_REPO_NAME/generation_config.yaml"
@@ -524,7 +538,11 @@ modernize_and_extract() {
 # 7.11 Modernize root pom.xml
 echo "Modernizing root pom.xml..."
 PARENT_VERSION=$(grep -m 1 "<version>.*{x-version-update:google-cloud-java:current}" google-cloud-jar-parent/pom.xml | sed -E 's/.*<version>(.*)<\/version>.*/\1/')
-modernize_and_extract "$SOURCE_REPO_NAME/pom.xml" "$PARENT_VERSION" --source-repo "$SOURCE_REPO_NAME"
+EXTRA_MODERNIZE_ARGS=""
+if [ "${SPLIT_CI}" == "true" ]; then
+    EXTRA_MODERNIZE_ARGS="--add-bulk-tests-profile"
+fi
+modernize_and_extract "$SOURCE_REPO_NAME/pom.xml" "$PARENT_VERSION" --source-repo "$SOURCE_REPO_NAME" $EXTRA_MODERNIZE_ARGS
 
 echo "Committing root pom.xml modernization..."
 git add "$SOURCE_REPO_NAME/pom.xml"
diff --git a/monorepo-migration/modernize_pom.py b/monorepo-migration/modernize_pom.py
@@ -171,7 +171,7 @@ def get_monorepo_versions(monorepo_root='.'):
                 versions[artifactId] = version
     return versions
 
-def modernize_pom(file_path, parent_version, source_repo_name=None, parent_artifactId='google-cloud-jar-parent', relative_path='../google-cloud-jar-parent/pom.xml', monorepo_versions=None, parent_managed_deps=None, keep_parent=False):
+def modernize_pom(file_path, parent_version, source_repo_name=None, parent_artifactId='google-cloud-jar-parent', relative_path='../google-cloud-jar-parent/pom.xml', monorepo_versions=None, parent_managed_deps=None, keep_parent=False, add_bulk_tests_profile=False):
     with open(file_path, 'r') as f:
         lines = f.readlines()
 
@@ -196,6 +196,15 @@ def modernize_pom(file_path, parent_version, source_repo_name=None, parent_artif
     should_preserve = False
     current_group_id = None
     has_version = False
+    in_profiles = False
+    has_bulk_tests_profile = False
+    profiles_lines = []
+    
+    # Check if bulkTests profile already exists if we might add it
+    if add_bulk_tests_profile:
+        bulk_tests_pattern = r'<id>\s*bulkTests\s*</id>'
+        if re.search(bulk_tests_pattern, "".join(lines)):
+            has_bulk_tests_profile = True
 
     for line in lines:
         # URL Modernization
@@ -231,6 +240,17 @@ def modernize_pom(file_path, parent_version, source_repo_name=None, parent_artif
         if '<id>native-test</id>' in line:
             line = line.replace('<id>native-test</id>', '<id>native</id>')
 
+        # Track if we are in profiles section to potentially add bulkTests there
+        if '<profiles>' in line:
+            in_profiles = True
+        if '</profiles>' in line:
+            in_profiles = False
+            if add_bulk_tests_profile and not has_bulk_tests_profile:
+                # Add bulkTests profile before closing </profiles>
+                indent = "    "
+                line = f"{indent}<profile>\n{indent}  <id>bulkTests</id>\n{indent}  <properties>\n{indent}    <skipTests>true</skipTests>\n{indent}  </properties>\n{indent}</profile>\n{line}"
+                has_bulk_tests_profile = True
+
         # Parent section modernization
         if not keep_parent:
             if '<parent>' in line and not in_parent:
@@ -469,6 +489,25 @@ def modernize_pom(file_path, parent_version, source_repo_name=None, parent_artif
 
         new_lines.append(line)
 
+    # If bulkTests profile was requested but no <profiles> section existed
+    if add_bulk_tests_profile and not has_bulk_tests_profile:
+        # Find the last line that isn't </project> and empty space
+        for i in range(len(new_lines) - 1, -1, -1):
+            if '</project>' in new_lines[i]:
+                indent = "  "
+                profile_block = (
+                    f"\n{indent}<profiles>\n"
+                    f"{indent}  <profile>\n"
+                    f"{indent}    <id>bulkTests</id>\n"
+                    f"{indent}    <properties>\n"
+                    f"{indent}      <skipTests>true</skipTests>\n"
+                    f"{indent}    </properties>\n"
+                    f"{indent}  </profile>\n"
+                    f"{indent}</profiles>\n"
+                )
+                new_lines.insert(i, profile_block)
+                break
+
     with open(file_path, 'w') as f:
         # Clean up double empty lines potentially introduced by pruning
         content = "".join(new_lines)
@@ -483,6 +522,7 @@ def modernize_pom(file_path, parent_version, source_repo_name=None, parent_artif
     parser.add_argument("--parent-artifactId", default="google-cloud-jar-parent", help="Artifact ID of the parent POM.")
     parser.add_argument("--relative-path", default="../google-cloud-jar-parent/pom.xml", help="Relative path to the parent POM.")
     parser.add_argument("--keep-parent", action="store_true", help="Keep the existing parent section.")
+    parser.add_argument("--add-bulk-tests-profile", action="store_true", help="Add bulkTests profile to skip tests.")
 
     args = parser.parse_args()
 
@@ -506,6 +546,7 @@ def modernize_pom(file_path, parent_version, source_repo_name=None, parent_artif
         args.relative_path,
         monorepo_versions,
         parent_managed_deps,
-        args.keep_parent
+        args.keep_parent,
+        args.add_bulk_tests_profile
     )
 
diff --git a/monorepo-migration/update_ci_filters.py b/monorepo-migration/update_ci_filters.py
@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import os
+
+def update_ci_filters(ci_yaml, repo_name):
+    if not os.path.exists(ci_yaml):
+        print(f"Error: {ci_yaml} does not exist.")
+        return
+
+    with open(ci_yaml, "r") as f:
+        lines = f.readlines()
+
+    final_lines = []
+    in_filters = False
+    filter_lines = []
+    new_filter = f"          {repo_name}: {repo_name}/**\n"
+
+    for line in lines:
+        if "filters: |" in line:
+            final_lines.append(line)
+            in_filters = True
+            continue
+        
+        if in_filters:
+            # End of filter block is an empty line or line with less indentation
+            if line.strip() == "" or not line.startswith("          "):
+                if not any(f"          {repo_name}:" in l for l in filter_lines):
+                    filter_lines.append(new_filter)
+                filter_lines.sort()
+                final_lines.extend(filter_lines)
+                final_lines.append(line)
+                in_filters = False
+            else:
+                filter_lines.append(line)
+        else:
+            final_lines.append(line)
+
+    if in_filters:
+        if not any(f"          {repo_name}:" in l for l in filter_lines):
+            filter_lines.append(new_filter)
+        filter_lines.sort()
+        final_lines.extend(filter_lines)
+
+    with open(ci_yaml, "w") as f:
+        f.writelines(final_lines)
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Update central ci.yaml filters.")
+    parser.add_argument("ci_yaml", help="Path to the ci.yaml file.")
+    parser.add_argument("repo_name", help="Name of the repository to add to filters.")
+    args = parser.parse_args()
+    
+    update_ci_filters(args.ci_yaml, args.repo_name)
