feat: integrate environment substitution, add port polling utility, update Cloud Build symlinks,

Author: omkargaikwad23

cloudbuild.yaml

@@ -26,12 +26,18 @@ steps:
       - '-c'
       - |
         set -e
+        # Workaround for evalbench bug: settings are only applied if path basename matches extension ID
+        ln -s /workspace /workspace/cloud-sql-postgresql
         cd /evalbench
 
         export EVAL_GCP_PROJECT_ID=$PROJECT_ID
         export EVAL_GCP_PROJECT_REGION=us-central1
         # Maps the decrypted DB_PASSWORD to the exact variable expected by gemini_cli and extension skills
         export CLOUD_SQL_POSTGRES_PASSWORD=$$DB_PASSWORD
+
+        # Substitute environment variables in model_config.yaml
+        python3 /workspace/evals/substitute_env.py
+
         export EVALBENCH_INSECURE=True
         export EVALBENCH_HOST=0.0.0.0
         cd evalbench
@@ -42,28 +48,14 @@ steps:
 
         echo "Starting Evaluation Server in background..."
         python3 -u ./eval_server.py --localhost </dev/null 2>&1 | tee server.log &
-        
+
         echo "Waiting for port 50051 to open..."
-        python3 -c "
-        import socket
-        import time
-        for i in range(20):
-            try:
-                s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-                s.connect(('127.0.0.1', 50051))
-                print('Port is open!')
-                exit(0)
-            except Exception as e:
-                print(f'Port not open yet: {e}')
-                time.sleep(1)
-        print('Port failed to open')
-        exit(1)
-        " || { echo "Server failed to bind port. Check logs above."; exit 1; }
-        
+        python3 /workspace/evals/wait_for_port.py || { echo "Server failed to bind port."; exit 1; }
+
         echo "Server is running. Launching Evaluation Client..."
         cd /evalbench
         export PYTHONPATH=./evalbench:./evalbench/evalproto
-        
+
         python3 evalbench/client/eval_client.py --experiment=/workspace/evals/run_config.yaml --endpoint=local || { echo "Client failed! Server logs:"; cat /evalbench/evalbench/server.log; exit 1; }
 
 availableSecrets:

evals/model_config.yaml

@@ -20,12 +20,13 @@ env:
   GOOGLE_GENAI_USE_VERTEXAI: "true"
 setup:
   extensions:
-    "/workspace":
+    # Points to the symlink created in cloudbuild.yaml to match the extension ID
+    "/workspace/cloud-sql-postgresql":
       settings:
         CLOUD_SQL_POSTGRES_PROJECT: "ext-test-cloud-sql-postgres"
         CLOUD_SQL_POSTGRES_INSTANCE: "daily-ci-evals-db"
         CLOUD_SQL_POSTGRES_REGION: "us-central1"
         CLOUD_SQL_POSTGRES_DATABASE: "postgres"
         CLOUD_SQL_POSTGRES_USER: "postgres"
-        CLOUD_SQL_POSTGRES_PASSWORD: ${CLOUD_SQL_POSTGRES_PASSWORD}
+        CLOUD_SQL_POSTGRES_PASSWORD: '${CLOUD_SQL_POSTGRES_PASSWORD}'
         CLOUD_SQL_POSTGRES_IP_TYPE: "PUBLIC"

evals/substitute_env.py

@@ -0,0 +1,17 @@
+import os
+import re
+
+def main():
+    yaml_path = '/workspace/evals/model_config.yaml'
+    if os.path.exists(yaml_path):
+        with open(yaml_path, 'r') as f:
+            content = f.read()
+        content = re.sub(r'\${(\w+)}', lambda m: os.environ.get(m.group(1), m.group(0)), content)
+        with open(yaml_path, 'w') as f:
+            f.write(content)
+        print(f"Successfully substituted environment variables in {yaml_path}")
+    else:
+        print(f"File not found: {yaml_path}")
+
+if __name__ == '__main__':
+    main()

evals/wait_for_port.py

@@ -0,0 +1,20 @@
+import socket
+import time
+import sys
+
+def main():
+    for i in range(20):
+        try:
+            s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+            s.settimeout(1)
+            s.connect(('127.0.0.1', 50051))
+            print('Port 50051 is open!')
+            sys.exit(0)
+        except Exception:
+            print('Port not open yet, retrying...')
+            time.sleep(1)
+    print('Port failed to open')
+    sys.exit(1)
+
+if __name__ == '__main__':
+    main()