Merge pull request #124 from mxlint/bugfix/memory-usage

limited concurrency and better memory footprint

Author: xiwenc

README.md

@@ -82,6 +82,8 @@ lint:
   jsonFile: ""
   ignoreNoqa: false
   noCache: false
+  concurrency: 4
+  regoTrace: false
   skip:
     example/doc:
       - rule: "001_002"
@@ -102,6 +104,8 @@ Notes:
 - `rules.rulesets` are synchronized into `rules.path` before linting.
 - `lint.skip` supports skipping by document path (relative to `modelsource`) and rule number.
 - `lint.noCache` disables lint result cache when set to `true`.
+- `lint.concurrency` limits how many rules are evaluated in parallel. Lower values reduce peak memory usage for large models.
+- `lint.regoTrace` enables OPA tracing for Rego rules. Keep it `false` for normal runs to reduce memory overhead.
 
 ---
 

default.yaml

@@ -6,6 +6,9 @@ lint:
   xunitReport: ""
   jsonFile: ""
   ignoreNoqa: false
+  noCache: false
+  concurrency: 4
+  regoTrace: false
   skip: {}
 cache:
   directory: .mendix-cache/mxlint

lint/cache.go

@@ -4,6 +4,7 @@ import (
 	"crypto/sha256"
 	"encoding/json"
 	"fmt"
+	"io"
 	"os"
 	"path/filepath"
 	"strings"
@@ -71,12 +72,18 @@ func getCachePath(cacheKey CacheKey) (string, error) {
 
 // computeFileHash computes SHA256 hash of a file's contents
 func computeFileHash(filePath string) (string, error) {
-	content, err := os.ReadFile(filePath)
+	file, err := os.Open(filePath)
 	if err != nil {
 		return "", err
 	}
-	hash := sha256.Sum256(content)
-	return fmt.Sprintf("%x", hash), nil
+	defer file.Close()
+
+	hasher := sha256.New()
+	if _, err := io.Copy(hasher, file); err != nil {
+		return "", err
+	}
+
+	return fmt.Sprintf("%x", hasher.Sum(nil)), nil
 }
 
 // createCacheKey creates a cache key from rule and input file paths
@@ -226,4 +233,3 @@ func GetCacheStats() (int, int64, error) {
 
 	return fileCount, totalSize, err
 }
-

lint/config.go

@@ -38,6 +38,9 @@ type ConfigLintSpec struct {
 	XunitReport string                      `yaml:"xunitReport"`
 	JSONFile    string                      `yaml:"jsonFile"`
 	IgnoreNoqa  *bool                       `yaml:"ignoreNoqa"`
+	NoCache     *bool                       `yaml:"noCache"`
+	Concurrency *int                        `yaml:"concurrency"`
+	RegoTrace   *bool                       `yaml:"regoTrace"`
 	Skip        map[string][]ConfigSkipRule `yaml:"skip"`
 }
 
@@ -293,11 +296,14 @@ func mergeConfig(base *Config, overlay *Config) {
 	if overlay.Lint.IgnoreNoqa != nil {
 		base.Lint.IgnoreNoqa = overlay.Lint.IgnoreNoqa
 	}
-	if strings.TrimSpace(overlay.Cache.Directory) != "" {
-		base.Cache.Directory = strings.TrimSpace(overlay.Cache.Directory)
+	if overlay.Lint.NoCache != nil {
+		base.Lint.NoCache = overlay.Lint.NoCache
 	}
-	if overlay.Cache.Enable != nil {
-		base.Cache.Enable = overlay.Cache.Enable
+	if overlay.Lint.Concurrency != nil {
+		base.Lint.Concurrency = overlay.Lint.Concurrency
+	}
+	if overlay.Lint.RegoTrace != nil {
+		base.Lint.RegoTrace = overlay.Lint.RegoTrace
 	}
 
 	if overlay.Serve.Port != nil {

lint/config_test.go

@@ -326,3 +326,27 @@ func TestLoadMergedConfig_NormalizesSkipMapKeys(t *testing.T) {
 		t.Fatalf("unexpected unnormalized skip key present: %#v", cfg.Lint.Skip)
 	}
 }
+
+func TestLoadMergedConfig_LintConcurrencyAndTrace(t *testing.T) {
+	projectDir := t.TempDir()
+	setDefaultConfigForTest(t, "")
+	projectConfig := `lint:
+  concurrency: 2
+  regoTrace: true
+`
+	if err := os.WriteFile(filepath.Join(projectDir, "mxlint.yaml"), []byte(projectConfig), 0644); err != nil {
+		t.Fatalf("failed to write project config: %v", err)
+	}
+
+	cfg, err := LoadMergedConfig(projectDir)
+	if err != nil {
+		t.Fatalf("LoadMergedConfig returned error: %v", err)
+	}
+
+	if cfg.Lint.Concurrency == nil || *cfg.Lint.Concurrency != 2 {
+		t.Fatalf("expected lint.concurrency=2, got %#v", cfg.Lint.Concurrency)
+	}
+	if cfg.Lint.RegoTrace == nil || *cfg.Lint.RegoTrace != true {
+		t.Fatalf("expected lint.regoTrace=true, got %#v", cfg.Lint.RegoTrace)
+	}
+}

lint/lint.go

@@ -48,11 +48,16 @@ func EvalAllWithResults(rulesPath string, modelSourcePath string, xunitReport st
 	// Create a mutex to safely print testsuites
 	var printMutex sync.Mutex
 
+	maxConcurrency := effectiveLintConcurrency(len(rules))
+	sem := make(chan struct{}, maxConcurrency)
+
 	// Launch goroutines to evaluate rules in parallel
 	for i, rule := range rules {
+		sem <- struct{}{}
 		wg.Add(1)
 		go func(index int, r Rule) {
 			defer wg.Done()
+			defer func() { <-sem }()
 
 			testsuite, err := evalTestsuite(r, modelSourcePath, ignoreNoqa, useCache)
 			if err != nil {
@@ -156,11 +161,16 @@ func EvalAll(rulesPath string, modelSourcePath string, xunitReport string, jsonF
 	// Create a mutex to safely print testsuites
 	var printMutex sync.Mutex
 
+	maxConcurrency := effectiveLintConcurrency(len(rules))
+	sem := make(chan struct{}, maxConcurrency)
+
 	// Launch goroutines to evaluate rules in parallel
 	for i, rule := range rules {
+		sem <- struct{}{}
 		wg.Add(1)
 		go func(index int, r Rule) {
 			defer wg.Done()
+			defer func() { <-sem }()
 
 			testsuite, err := evalTestsuite(r, modelSourcePath, ignoreNoqa, useCache)
 			if err != nil {

lint/lint_rego.go

@@ -51,12 +51,15 @@ func evalTestcase_Rego(rulePath string, queryString string, inputFilePath string
 	ctx := context.Background()
 
 	startTime := time.Now()
-	r := rego.New(
+	regoOptions := []func(*rego.Rego){
 		rego.Query(queryString),
 		rego.Module(rulePath, regoContent),
 		rego.Input(data),
-		rego.Trace(true),
-	)
+	}
+	if regoTraceEnabled() {
+		regoOptions = append(regoOptions, rego.Trace(true))
+	}
+	r := rego.New(regoOptions...)
 
 	rs, err := r.Eval(ctx)
 	if err != nil {

lint/options.go

@@ -0,0 +1,36 @@
+package lint
+
+import "runtime"
+
+const defaultMaxLintConcurrency = 4
+
+func effectiveLintConcurrency(ruleCount int) int {
+	if ruleCount <= 0 {
+		return 1
+	}
+
+	cfg := getConfig()
+	if cfg != nil && cfg.Lint.Concurrency != nil && *cfg.Lint.Concurrency > 0 {
+		if *cfg.Lint.Concurrency > ruleCount {
+			return ruleCount
+		}
+		return *cfg.Lint.Concurrency
+	}
+
+	auto := runtime.GOMAXPROCS(0)
+	if auto < 1 {
+		auto = 1
+	}
+	if auto > defaultMaxLintConcurrency {
+		auto = defaultMaxLintConcurrency
+	}
+	if auto > ruleCount {
+		auto = ruleCount
+	}
+	return auto
+}
+
+func regoTraceEnabled() bool {
+	cfg := getConfig()
+	return cfg != nil && cfg.Lint.RegoTrace != nil && *cfg.Lint.RegoTrace
+}

lint/options_test.go

@@ -0,0 +1,70 @@
+package lint
+
+import "testing"
+
+func intPtr(v int) *int {
+	return &v
+}
+
+func boolPtr(v bool) *bool {
+	return &v
+}
+
+func TestEffectiveLintConcurrency_DefaultIsBounded(t *testing.T) {
+	SetConfig(&Config{})
+	t.Cleanup(func() {
+		SetConfig(&Config{})
+	})
+
+	value := effectiveLintConcurrency(100)
+	if value < 1 || value > defaultMaxLintConcurrency {
+		t.Fatalf("expected default concurrency within [1,%d], got %d", defaultMaxLintConcurrency, value)
+	}
+}
+
+func TestEffectiveLintConcurrency_UsesConfigWhenProvided(t *testing.T) {
+	SetConfig(&Config{
+		Lint: ConfigLintSpec{
+			Concurrency: intPtr(2),
+		},
+	})
+	t.Cleanup(func() {
+		SetConfig(&Config{})
+	})
+
+	value := effectiveLintConcurrency(10)
+	if value != 2 {
+		t.Fatalf("expected configured concurrency 2, got %d", value)
+	}
+}
+
+func TestEffectiveLintConcurrency_CapsToRuleCount(t *testing.T) {
+	SetConfig(&Config{
+		Lint: ConfigLintSpec{
+			Concurrency: intPtr(8),
+		},
+	})
+	t.Cleanup(func() {
+		SetConfig(&Config{})
+	})
+
+	value := effectiveLintConcurrency(3)
+	if value != 3 {
+		t.Fatalf("expected concurrency capped to rule count 3, got %d", value)
+	}
+}
+
+func TestRegoTraceEnabled(t *testing.T) {
+	SetConfig(&Config{
+		Lint: ConfigLintSpec{
+			RegoTrace: boolPtr(true),
+		},
+	})
+	t.Cleanup(func() {
+		SetConfig(&Config{})
+	})
+
+	if !regoTraceEnabled() {
+		t.Fatal("expected regoTraceEnabled to return true")
+	}
+}