diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 402e43c..d871aaa 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -21,17 +21,52 @@ steps: - name: 'us-central1-docker.pkg.dev/cloud-db-nl2sql/evalbench/eval_server:latest' entrypoint: 'bash' # Decrypts the secret from Secret Manager into the DB_PASSWORD environment variable - secretEnv: ['DB_PASSWORD'] + secretEnv: ['DB_PASSWORD', 'GITHUB_TOKEN'] args: - '-c' - | set -e + + # Only run on release branches + if [[ "$_HEAD_BRANCH" != release-please-* ]]; then + echo "Not a release-please branch. Exiting." + exit 0 + fi + echo "Release branch detected. Fetching PR data from GitHub API..." + + # Fetch PR data and status code + HTTP_STATUS=$(curl -s -o pr_data.json -w "%{http_code}" -H "Authorization: token $$GITHUB_TOKEN" \ + "https://api.github.com/repos/$REPO_FULL_NAME/pulls/$_PR_NUMBER") + + if [ "$$HTTP_STATUS" -ne 200 ]; then + echo "Error fetching PR data: HTTP $$HTTP_STATUS" + cat pr_data.json + exit 1 + fi + + PR_DATA=$(cat pr_data.json) + + # Extract labels and title from PR data (Use $$ to escape bash variables) + PR_LABELS=$(echo "$$PR_DATA" | jq -r '[.labels[].name] | join(",")') + PR_TITLE=$(echo "$$PR_DATA" | jq -r '.title') + + # Determine Release Version (Use double quotes and $$ for bash variables) + if [[ "$$PR_LABELS" == *"autorelease: triggered"* ]]; then + if [[ "$$PR_TITLE" =~ release\ ([0-9]+\.[0-9]+\.[0-9]+) ]]; then + export RELEASE_VERSION="$${BASH_REMATCH[1]}" + else + export RELEASE_VERSION="unknown" + fi + else + export RELEASE_VERSION="unknown" + fi + # Workaround for evalbench bug: settings are only applied if path basename matches extension ID ln -s /workspace /workspace/cloud-sql-postgresql cd /evalbench export EVAL_GCP_PROJECT_ID=$PROJECT_ID - export EVAL_GCP_PROJECT_REGION=us-central1 + export EVAL_GCP_PROJECT_REGION=$_CLOUD_SQL_REGION export GOOGLE_CLOUD_PROJECT=$PROJECT_ID export CLOUD_SQL_POSTGRES_PROJECT=$PROJECT_ID export CLOUD_SQL_POSTGRES_INSTANCE=$_CLOUD_SQL_INSTANCE @@ -43,6 +78,9 @@ steps: # Maps the decrypted DB_PASSWORD to the exact variable expected by gemini_cli and extension skills export CLOUD_SQL_POSTGRES_PASSWORD=$$DB_PASSWORD + # Combine CI metadata with run config + cat /workspace/evals/ci_metadata.yaml >> /workspace/evals/run_config.yaml + # Substitute environment variables in model_config.yaml python3 /workspace/evals/substitute_env.py @@ -58,3 +96,5 @@ availableSecrets: secretManager: - versionName: projects/$PROJECT_ID/secrets/daily-ci-evals-db-password/versions/latest env: 'DB_PASSWORD' + - versionName: projects/$PROJECT_ID/secrets/GITHUB_TOKEN/versions/latest + env: 'GITHUB_TOKEN' diff --git a/evals/ci_metadata.yaml b/evals/ci_metadata.yaml new file mode 100644 index 0000000..0fc1249 --- /dev/null +++ b/evals/ci_metadata.yaml @@ -0,0 +1,22 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +############################################################ +### CI Metadata (Repository Specific) +### Note: These fields are used for version tracking in BQ +### and are not part of the core Evalbench schema. +############################################################ + +extension_id: cloud-sql-postgresql +release_version: ${RELEASE_VERSION} \ No newline at end of file diff --git a/evals/run_config.yaml b/evals/run_config.yaml index b83b7e6..0f45e6e 100644 --- a/evals/run_config.yaml +++ b/evals/run_config.yaml @@ -12,14 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -extension_id: cloud-sql-postgresql - dataset_config: /workspace/evals/dataset.json dataset_format: gemini-cli-format orchestrator: geminicli model_config: /workspace/evals/model_config.yaml -# You can reference default simulated user models provided by the evalbench repo: simulated_user_model_config: /workspace/evals/gemini_2.5_pro_model.yaml scorers: diff --git a/evals/substitute_env.py b/evals/substitute_env.py index 3ef2295..f10c8e3 100644 --- a/evals/substitute_env.py +++ b/evals/substitute_env.py @@ -2,16 +2,17 @@ import re def main(): - yaml_path = '/workspace/evals/model_config.yaml' - if os.path.exists(yaml_path): - with open(yaml_path, 'r') as f: - content = f.read() - content = re.sub(r'\${(\w+)}', lambda m: os.environ.get(m.group(1), m.group(0)), content) - with open(yaml_path, 'w') as f: - f.write(content) - print(f"Successfully substituted environment variables in {yaml_path}") - else: - print(f"File not found: {yaml_path}") + yaml_paths = ['/workspace/evals/model_config.yaml', '/workspace/evals/run_config.yaml'] + for yaml_path in yaml_paths: + if os.path.exists(yaml_path): + with open(yaml_path, 'r') as f: + content = f.read() + content = re.sub(r'\${(\w+)}', lambda m: os.environ.get(m.group(1), m.group(0)), content) + with open(yaml_path, 'w') as f: + f.write(content) + print(f"Successfully substituted environment variables in {yaml_path}") + else: + print(f"File not found: {yaml_path}") if __name__ == '__main__': main() \ No newline at end of file