Add CI checks for stale generated docs and proto files (#9199)

* Add CI checks for stale generated docs and proto files

* Self review

* Review

* Fix

* Fix

* Fix

* Fix

* nits

Author: begelundmuller

.github/workflows/docs-check.yml

@@ -4,13 +4,33 @@ on:
   pull_request:
     paths:
       - ".github/workflows/docs-check.yml"
-      - "docs/**"
       - "cli/**"
+      - "docs/**"
+      - "runtime/parser/schema/**"
 env:
   NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }}
   NETLIFY_SITE_ID: 23baf08e-2d3e-44db-8bd4-938e54467a29
 
 jobs:
+  check:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: 1.25
+
+      - name: Generate docs
+        run: make docs.generate
+
+      - name: Check for uncommitted changes
+        run: |
+          git diff --exit-code || (echo "Generated docs are out of date. Please run 'make docs.generate' locally and commit the changes." && exit 1)
+          test -z "$(git status --porcelain)" || (echo "Generated docs are out of date. Please run 'make docs.generate' locally and commit the changes." && git status --porcelain && exit 1)
+
   build:
     runs-on: ubuntu-latest
     steps:
@@ -23,20 +43,6 @@ jobs:
           node-version-file: '.nvmrc'
           cache: 'npm'
 
-      # - name: Filter modified codepaths
-      #   uses: dorny/paths-filter@v3
-      #   id: filter
-      #   with:
-      #     filters: |
-      #       cli:
-      #         - "cli/**"
-
-      # - name: Check if new files were created
-      #   if: ${{ steps.filter.outputs.cli == 'true' }}
-      #   run: |
-      #     make docs.generate
-      #     git diff --exit-code || (echo "New files were created, please run 'make docs.generate' locally and commit the changes" && exit 0)
-
       - name: Build docs
         run: |-
           npm ci -w docs

.github/workflows/proto-check.yml

@@ -2,7 +2,12 @@ name: Check .proto files
 on:
   pull_request:
     paths:
+      - ".github/workflows/proto-check.yml"
       - "proto/**"
+      - "scripts/convert-openapi-v2-to-v3/**"
+      - "web-admin/src/client/**"
+      - "web-common/src/proto/**"
+      - "web-common/src/runtime-client/**"
 jobs:
   check:
     runs-on: ubuntu-latest
@@ -12,3 +17,33 @@ jobs:
       - uses: bufbuild/buf-lint-action@v1
         with:
           input: "proto"
+
+  generate:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: 1.25
+
+      - name: Set up buf
+        uses: bufbuild/buf-setup-action@v1
+
+      - name: Set up NodeJS
+        uses: actions/setup-node@v4
+        with:
+          node-version-file: '.nvmrc'
+          cache: 'npm'
+
+      - name: Generate proto
+        run: make proto.generate
+        env:
+          HUSKY: 0
+
+      - name: Check for uncommitted changes
+        run: |
+          git diff --exit-code || (echo "Generated proto files are out of date. Please run 'make proto.generate' locally and commit the changes." && exit 1)
+          test -z "$(git status --porcelain)" || (echo "Generated proto files are out of date. Please run 'make proto.generate' locally and commit the changes." && git status --porcelain && exit 1)

Makefile

@@ -10,20 +10,8 @@ cli-only:
 cli: cli.prepare
 	go build -o rill cli/main.go 
 
-KEEP_DIRS := rill-openrtb-prog-ads rill-github-analytics rill-cost-monitoring
-
 .PHONY: cli.prepare
-cli.prepare:
-	@set -e; \
-	rm -rf runtime/pkg/examples/embed/dist || true; \
-	mkdir -p runtime/pkg/examples/embed/dist; \
-	# Create a temp dir (GNU mktemp first, then BSD/macOS fallback)
-	TMP_CLONE_DIR=$$(mktemp -d 2>/dev/null || mktemp -d -t rill-examples); \
-	trap 'rm -rf "$$TMP_CLONE_DIR"' EXIT; \
-	git clone --quiet --depth=1 https://github.com/rilldata/rill-examples.git "$$TMP_CLONE_DIR"; \
-	for d in $(KEEP_DIRS); do \
-		cp -R "$$TMP_CLONE_DIR/$$d" runtime/pkg/examples/embed/dist/; \
-	done
+cli.prepare: runtime.examples.embed
 	npm install
 	npm run build
 	rm -rf cli/pkg/web/embed/dist || true
@@ -42,13 +30,15 @@ coverage.go:
 	go tool cover -func coverage/go.out
 
 .PHONY: docs.generate
-docs.generate:
+docs.generate: runtime.examples.embed
 	# Temporarily replaces ~/.rill/config.yaml to avoid including user-defined defaults in generated docs.
-	# Sets version to the latest tag to simulate a production build, where certain commands are hidden.
+	#
+	# Sets main.Version to a fixed tag to simulate a production build, where certain commands are hidden.
+	# Not using scripts/versiontag.sh since the actual version should not be emitted to the generated files as it would go stale on the next release.
 	rm -rf docs/docs/reference/cli/*.md docs/docs/reference/project-files/*.md
 	if [ -f ~/.rill/config.yaml ]; then mv ~/.rill/config.yaml ~/.rill/config.yaml.tmp; fi;
-	go run -ldflags="-X main.Version=$(shell scripts/versiontag.sh)" ./cli docs generate-cli docs/docs/reference/cli/
-	go run -ldflags="-X main.Version=$(shell scripts/versiontag.sh)" ./cli docs generate-project docs/docs/reference/project-files/
+	RILL_DOCS_GENERATE=true go run -ldflags="-X main.Version=1.0.0" ./cli docs generate-cli docs/docs/reference/cli/
+	RILL_DOCS_GENERATE=true go run -ldflags="-X main.Version=1.0.0" ./cli docs generate-project docs/docs/reference/project-files/
 	if [ -f ~/.rill/config.yaml.tmp ]; then mv ~/.rill/config.yaml.tmp ~/.rill/config.yaml; fi;
 
 .PHONY: proto.generate
@@ -59,12 +49,25 @@ proto.generate:
 	cd proto && buf generate --template buf.gen.runtime.yaml --path rill/runtime
 	cd proto && buf generate --template buf.gen.local.yaml --path rill/local
 	cd proto && buf generate --template buf.gen.ui.yaml
-	go run -ldflags="-X main.Version=$(shell scripts/versiontag.sh)" \
-		scripts/convert-openapi-v2-to-v3/convert.go --force \
+	go run scripts/convert-openapi-v2-to-v3/convert.go --force \
 		proto/gen/rill/admin/v1/admin.swagger.yaml proto/gen/rill/admin/v1/openapi.yaml
-	go run -ldflags="-X main.Version=$(shell scripts/versiontag.sh)" \
-		scripts/convert-openapi-v2-to-v3/convert.go --force --public-only \
+	go run scripts/convert-openapi-v2-to-v3/convert.go --force --public-only \
 		proto/gen/rill/admin/v1/admin.swagger.yaml proto/gen/rill/admin/v1/public.openapi.yaml
+	npm install
 	npm run generate:runtime-client -w web-common
 	npm run generate:client -w web-admin
-	
+
+KEEP_EXAMPLES := rill-openrtb-prog-ads rill-github-analytics rill-cost-monitoring
+
+.PHONY: runtime.examples.embed
+runtime.examples.embed:
+	@set -e; \
+	rm -rf runtime/pkg/examples/embed/dist || true; \
+	mkdir -p runtime/pkg/examples/embed/dist; \
+	# Create a temp dir (GNU mktemp first, then BSD/macOS fallback)
+	TMP_CLONE_DIR=$$(mktemp -d 2>/dev/null || mktemp -d -t rill-examples); \
+	trap 'rm -rf "$$TMP_CLONE_DIR"' EXIT; \
+	git clone --quiet --depth=1 https://github.com/rilldata/rill-examples.git "$$TMP_CLONE_DIR"; \
+	for d in $(KEEP_EXAMPLES); do \
+		cp -R "$$TMP_CLONE_DIR/$$d" runtime/pkg/examples/embed/dist/; \
+	done

cli/pkg/cmdutil/helper.go

@@ -694,6 +694,12 @@ func hashStr(ss ...string) string {
 }
 
 // isTerminal reports whether both stdin and stdout are connected to an interactive terminal.
+// It can be overridden by setting RILL_DOCS_GENERATE=true (used in CI for docs generation).
 func isTerminal() bool {
+	if os.Getenv("RILL_DOCS_GENERATE") == "true" {
+		// Used for generating docs with `make docs.generate`.
+		// This ensures --interactive defaults to "true", which makes the generated docs appear the way the CLI help menus appear to a real user (e.g. strips agent instructions).
+		return true
+	}
 	return term.IsTerminal(int(os.Stdin.Fd())) && term.IsTerminal(int(os.Stdout.Fd()))
 }

proto/gen/rill/admin/v1/openapi.yaml

@@ -2321,7 +2321,7 @@ externalDocs:
 info:
     description: Rill Admin API enables programmatic management of Rill Cloud resources, including organizations, projects, and user access. It provides endpoints for creating, updating, and deleting these resources, as well as managing authentication and permissions.
     title: Rill Admin API
-    version: v0.84.4
+    version: 1.0.0
 openapi: 3.0.3
 paths:
     /v1/ai/complete:

proto/gen/rill/admin/v1/public.openapi.yaml

@@ -2321,7 +2321,7 @@ externalDocs:
 info:
     description: Rill Admin API enables programmatic management of Rill Cloud resources, including organizations, projects, and user access. It provides endpoints for creating, updating, and deleting these resources, as well as managing authentication and permissions.
     title: Rill Admin API
-    version: v0.84.4
+    version: 1.0.0
 openapi: 3.0.3
 paths:
     /v1/ai/complete: {}

scripts/convert-openapi-v2-to-v3/convert.go

@@ -14,8 +14,6 @@ import (
 	"gopkg.in/yaml.v3"
 )
 
-var Version string
-
 func main() {
 	var force bool
 	var publicOnly bool
@@ -100,10 +98,8 @@ func convertOpenAPIDocs(input, output string, force, publicOnly bool) error {
 	// Prune to only keep public visibility
 	prunePublicOnly(convertedDocs, publicOnly)
 
-	// Inject version if set
-	if Version != "" {
-		convertedDocs.Info.Version = Version
-	}
+	// Use a fixed version for the OpenAPI spec to avoid patch releases causing changes to the generated document
+	convertedDocs.Info.Version = "1.0.0"
 
 	if err = os.MkdirAll(filepath.Dir(output), 0o755); err != nil {
 		return fmt.Errorf("unable to create output directory: %w", err)