Add benchmarks for end-to-end tests and examples (#611)

* endtoend: Add benchmarks for all end-to-end tests

* workflows: Run the benchmarks on master / tags

Author: kyleconroy

.github/workflows/equinox.yml

@@ -9,6 +9,17 @@ on:
 
 jobs:
 
+  benchmark:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Go 1.14
+      uses: actions/setup-go@v2
+      with:
+        go-version: '1.14'
+    - uses: equinox-io/setup@master
+    - run: go test -bench . -json -benchmem -run=XXX ./...
+
   macos:
     name: release --platforms darwin
     runs-on: macos-latest
@@ -28,7 +39,7 @@ jobs:
   linux:
     name: release --platforms linux
     runs-on: ubuntu-latest
-    needs: [macos]
+    needs: [macos, benchmark]
     steps:
     - uses: actions/checkout@v2
     - name: Set up Go 1.14

internal/endtoend/endtoend_test.go

@@ -44,6 +44,30 @@ func TestExamples(t *testing.T) {
 	}
 }
 
+func BenchmarkExamples(b *testing.B) {
+	examples, err := filepath.Abs(filepath.Join("..", "..", "examples"))
+	if err != nil {
+		b.Fatal(err)
+	}
+	files, err := ioutil.ReadDir(examples)
+	if err != nil {
+		b.Fatal(err)
+	}
+	for _, replay := range files {
+		if !replay.IsDir() {
+			continue
+		}
+		tc := replay.Name()
+		b.Run(tc, func(b *testing.B) {
+			path := filepath.Join(examples, tc)
+			var stderr bytes.Buffer
+			for i := 0; i < b.N; i++ {
+				cmd.Generate(cmd.Env{}, path, &stderr)
+			}
+		})
+	}
+}
+
 func TestReplay(t *testing.T) {
 	t.Parallel()
 	var dirs []string
@@ -139,3 +163,30 @@ func expectedStderr(t *testing.T, dir string) string {
 	}
 	return ""
 }
+
+func BenchmarkReplay(b *testing.B) {
+	var dirs []string
+	err := filepath.Walk("testdata", func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if info.Name() == "sqlc.json" || info.Name() == "sqlc.yaml" {
+			dirs = append(dirs, filepath.Dir(path))
+			return filepath.SkipDir
+		}
+		return nil
+	})
+	if err != nil {
+		b.Fatal(err)
+	}
+	for _, replay := range dirs {
+		tc := replay
+		b.Run(tc, func(b *testing.B) {
+			path, _ := filepath.Abs(tc)
+			var stderr bytes.Buffer
+			for i := 0; i < b.N; i++ {
+				cmd.Generate(cmd.Env{}, path, &stderr)
+			}
+		})
+	}
+}