From c3e12577c0b2cb9dfa3fed2f9f7d3ab8c98edd61 Mon Sep 17 00:00:00 2001 From: 2pk03 Date: Fri, 13 Mar 2026 09:27:34 +0100 Subject: [PATCH 1/3] Add local commit checks --- Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/Makefile b/Makefile index 2b4adb0..93581a2 100644 --- a/Makefile +++ b/Makefile @@ -171,7 +171,6 @@ test-nested-modules: ## Run go test across nested Go modules under addons/proces ( cd $$dir && go test ./... ); \ done; \ echo "nested module tests passed." - vet: ## Run go vet @echo "==> go vet" @go vet ./... From a15c13acefe62f0058a8427798b0980399559775 Mon Sep 17 00:00:00 2001 From: 2pk03 Date: Thu, 16 Apr 2026 10:26:22 +0200 Subject: [PATCH 2/3] fix(security): remediate dependabot alerts --- addons/processors/iceberg-processor/go.mod | 36 ++++----- addons/processors/iceberg-processor/go.sum | 74 +++++++++--------- addons/processors/sql-processor/go.mod | 34 ++++----- addons/processors/sql-processor/go.sum | 70 +++++++++-------- examples/E10_java-kafka-client-demo/pom.xml | 2 +- examples/E30_flink-kafscale-demo/pom.xml | 2 +- examples/E40_spark-kafscale-demo/pom.xml | 2 +- go.mod | 33 ++++---- go.sum | 84 +++++++++------------ lfs-client-sdk/java/pom.xml | 2 +- 10 files changed, 159 insertions(+), 180 deletions(-) diff --git a/addons/processors/iceberg-processor/go.mod b/addons/processors/iceberg-processor/go.mod index bd42a6c..056c5ed 100644 --- a/addons/processors/iceberg-processor/go.mod +++ b/addons/processors/iceberg-processor/go.mod @@ -6,9 +6,9 @@ require ( github.com/KafScale/platform v1.5.0 github.com/apache/arrow-go/v18 v18.4.1 github.com/apache/iceberg-go v0.4.0 - github.com/aws/aws-sdk-go-v2 v1.41.2 + github.com/aws/aws-sdk-go-v2 v1.41.5 github.com/aws/aws-sdk-go-v2/config v1.32.9 - github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0 + github.com/aws/aws-sdk-go-v2/service/s3 v1.97.3 github.com/prometheus/client_golang v1.23.2 github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 go.etcd.io/etcd/api/v3 v3.6.8 @@ -42,23 +42,23 @@ require ( github.com/antlr4-go/antlr/v4 v4.13.1 // indirect github.com/apache/thrift v0.22.0 // indirect github.com/aws/aws-sdk-go v1.55.7 // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.19.10 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.18 // indirect github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.84 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.18 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.18 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.5 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.18 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 // indirect github.com/aws/aws-sdk-go-v2/service/signin v1.0.6 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.30.11 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.15 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.41.7 // indirect - github.com/aws/smithy-go v1.24.1 // indirect + github.com/aws/smithy-go v1.24.2 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5 // indirect @@ -71,7 +71,7 @@ require ( github.com/envoyproxy/go-control-plane/envoy v1.36.0 // indirect github.com/envoyproxy/protoc-gen-validate v1.3.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect - github.com/go-jose/go-jose/v4 v4.1.3 // indirect + github.com/go-jose/go-jose/v4 v4.1.4 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-viper/mapstructure/v2 v2.4.0 // indirect @@ -127,11 +127,11 @@ require ( go.opentelemetry.io/contrib/detectors/gcp v1.39.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.62.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.62.0 // indirect - go.opentelemetry.io/otel v1.40.0 // indirect - go.opentelemetry.io/otel/metric v1.40.0 // indirect - go.opentelemetry.io/otel/sdk v1.40.0 // indirect - go.opentelemetry.io/otel/sdk/metric v1.40.0 // indirect - go.opentelemetry.io/otel/trace v1.40.0 // indirect + go.opentelemetry.io/otel v1.43.0 // indirect + go.opentelemetry.io/otel/metric v1.43.0 // indirect + go.opentelemetry.io/otel/sdk v1.43.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.43.0 // indirect + go.opentelemetry.io/otel/trace v1.43.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.1 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect @@ -142,7 +142,7 @@ require ( golang.org/x/net v0.49.0 // indirect golang.org/x/oauth2 v0.34.0 // indirect golang.org/x/sync v0.19.0 // indirect - golang.org/x/sys v0.41.0 // indirect + golang.org/x/sys v0.42.0 // indirect golang.org/x/telemetry v0.0.0-20260109210033-bd525da824e2 // indirect golang.org/x/term v0.40.0 // indirect golang.org/x/text v0.34.0 // indirect diff --git a/addons/processors/iceberg-processor/go.sum b/addons/processors/iceberg-processor/go.sum index 7c0553c..868de46 100644 --- a/addons/processors/iceberg-processor/go.sum +++ b/addons/processors/iceberg-processor/go.sum @@ -96,10 +96,10 @@ github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmms github.com/atomicgo/cursor v0.0.1/go.mod h1:cBON2QmmrysudxNBFthvMtN32r3jxVRIvzkUiF/RuIk= github.com/aws/aws-sdk-go v1.55.7 h1:UJrkFq7es5CShfBwlWAC8DA077vp8PyVbQd3lqLiztE= github.com/aws/aws-sdk-go v1.55.7/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= -github.com/aws/aws-sdk-go-v2 v1.41.2 h1:LuT2rzqNQsauaGkPK/7813XxcZ3o3yePY0Iy891T2ls= -github.com/aws/aws-sdk-go-v2 v1.41.2/go.mod h1:IvvlAZQXvTXznUPfRVfryiG1fbzE2NGK6m9u39YQ+S4= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4= +github.com/aws/aws-sdk-go-v2 v1.41.5 h1:dj5kopbwUsVUVFgO4Fi5BIT3t4WyqIDjGKCangnV/yY= +github.com/aws/aws-sdk-go-v2 v1.41.5/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 h1:eBMB84YGghSocM7PsjmmPffTa+1FBUeNvGvFou6V/4o= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI= github.com/aws/aws-sdk-go-v2/config v1.32.9 h1:ktda/mtAydeObvJXlHzyGpK1xcsLaP16zfUPDGoW90A= github.com/aws/aws-sdk-go-v2/config v1.32.9/go.mod h1:U+fCQ+9QKsLW786BCfEjYRj34VVTbPdsLP3CHSYXMOI= github.com/aws/aws-sdk-go-v2/credentials v1.19.10 h1:EEhmEUFCE1Yhl7vDhNOI5OCL/iKMdkkYFTRpZXNw7m8= @@ -108,26 +108,26 @@ github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.18 h1:Ii4s+Sq3yDfaMLpjrJsqD6 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.18/go.mod h1:6x81qnY++ovptLE6nWQeWrpXxbnlIex+4H4eYYGcqfc= github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.84 h1:cTXRdLkpBanlDwISl+5chq5ui1d1YWg4PWMR9c3kXyw= github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.84/go.mod h1:kwSy5X7tfIHN39uucmjQVs2LvDdXEjQucgQQEqCggEo= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.18 h1:F43zk1vemYIqPAwhjTjYIz0irU2EY7sOb/F5eJ3HuyM= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.18/go.mod h1:w1jdlZXrGKaJcNoL+Nnrj+k5wlpGXqnNrKoP22HvAug= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.18 h1:xCeWVjj0ki0l3nruoyP2slHsGArMxeiiaoPN5QZH6YQ= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.18/go.mod h1:r/eLGuGCBw6l36ZRWiw6PaZwPXb6YOj+i/7MizNl5/k= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 h1:Rgg6wvjjtX8bNHcvi9OnXWwcE0a2vGpbwmtICOsvcf4= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21/go.mod h1:A/kJFst/nm//cyqonihbdpQZwiUhhzpqTsdbhDdRF9c= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 h1:PEgGVtPoB6NTpPrBgqSE5hE/o47Ij9qk/SEZFbUOe9A= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21/go.mod h1:p+hz+PRAYlY3zcpJhPwXlLC4C+kqn70WIHwnzAfs6ps= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 h1:JqcdRG//czea7Ppjb+g/n4o8i/R50aTBHkA7vu0lK+k= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17/go.mod h1:CO+WeGmIdj/MlPel2KwID9Gt7CNq4M65HUfBW97liM0= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 h1:rWyie/PxDRIdhNf4DzRk0lvjVOqFJuNnO8WwaIRVxzQ= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22/go.mod h1:zd/JsJ4P7oGfUhXn1VyLqaRZwPmZwg44Jf2dS84Dm3Y= github.com/aws/aws-sdk-go-v2/service/glue v1.129.1 h1:43/6Yay8BWMwCq5Ow9pSTcumKROQdqe5DxnS/44LODQ= github.com/aws/aws-sdk-go-v2/service/glue v1.129.1/go.mod h1:iH5M4d6X8IdmFUwOVdnoCEt7eqhjYZuw4gEI0ebsQjs= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.5 h1:CeY9LUdur+Dxoeldqoun6y4WtJ3RQtzk0JMP2gfUay0= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.5/go.mod h1:AZLZf2fMaahW5s/wMRciu1sYbdsikT/UHwbUjOdEVTc= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 h1:Z5EiPIzXKewUQK0QTMkutjiaPVeVYXX7KIqhXu/0fXs= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8/go.mod h1:FsTpJtvC4U1fyDXk7c71XoDv3HlRm8V3NiYLeYLh5YE= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.18 h1:LTRCYFlnnKFlKsyIQxKhJuDuA3ZkrDQMRYm6rXiHlLY= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.18/go.mod h1:XhwkgGG6bHSd00nO/mexWTcTjgd6PjuvWQMqSn2UaEk= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 h1:bGeHBsGZx0Dvu/eJC0Lh9adJa3M1xREcndxLNZlve2U= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17/go.mod h1:dcW24lbU0CzHusTE8LLHhRLI42ejmINN8Lcr22bwh/g= -github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0 h1:oeu8VPlOre74lBA/PMhxa5vewaMIMmILM+RraSyB8KA= -github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0/go.mod h1:5jggDlZ2CLQhwJBiZJb4vfk4f0GxWdEDruWKEJ1xOdo= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 h1:JRaIgADQS/U6uXDqlPiefP32yXTda7Kqfx+LgspooZM= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13/go.mod h1:CEuVn5WqOMilYl+tbccq8+N2ieCy0gVn3OtRb0vBNNM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 h1:c31//R3xgIJMSC8S6hEVq+38DcvUlgFY0FM6mSI5oto= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21/go.mod h1:r6+pf23ouCB718FUxaqzZdbpYFyDtehyZcmP5KL9FkA= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 h1:ZlvrNcHSFFWURB8avufQq9gFsheUgjVD9536obIknfM= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21/go.mod h1:cv3TNhVrssKR0O/xxLJVRfd2oazSnZnkUeTf6ctUwfQ= +github.com/aws/aws-sdk-go-v2/service/s3 v1.97.3 h1:HwxWTbTrIHm5qY+CAEur0s/figc3qwvLWsNkF4RPToo= +github.com/aws/aws-sdk-go-v2/service/s3 v1.97.3/go.mod h1:uoA43SdFwacedBfSgfFSjjCvYe8aYBS7EnU5GZ/YKMM= github.com/aws/aws-sdk-go-v2/service/signin v1.0.6 h1:MzORe+J94I+hYu2a6XmV5yC9huoTv8NRcCrUNedDypQ= github.com/aws/aws-sdk-go-v2/service/signin v1.0.6/go.mod h1:hXzcHLARD7GeWnifd8j9RWqtfIgxj4/cAtIVIK7hg8g= github.com/aws/aws-sdk-go-v2/service/sso v1.30.11 h1:7oGD8KPfBOJGXiCoRKrrrQkbvCp8N++u36hrLMPey6o= @@ -136,8 +136,8 @@ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.15 h1:edCcNp9eGIUDUCrzoCu1jWA github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.15/go.mod h1:lyRQKED9xWfgkYC/wmmYfv7iVIM68Z5OQ88ZdcV1QbU= github.com/aws/aws-sdk-go-v2/service/sts v1.41.7 h1:NITQpgo9A5NrDZ57uOWj+abvXSb83BbyggcUBVksN7c= github.com/aws/aws-sdk-go-v2/service/sts v1.41.7/go.mod h1:sks5UWBhEuWYDPdwlnRFn1w7xWdH29Jcpe+/PJQefEs= -github.com/aws/smithy-go v1.24.1 h1:VbyeNfmYkWoxMVpGUAbQumkODcYmfMRfZ8yQiH30SK0= -github.com/aws/smithy-go v1.24.1/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0= +github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng= +github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/buger/goterm v1.0.4 h1:Z9YvGmOih81P0FbVtEYTFF6YsSgxSUKEhf/f9bTMXbY= @@ -237,8 +237,8 @@ github.com/fvbommel/sortorder v1.1.0 h1:fUmoe+HLsBTctBDoaBwpQo5N+nrCp8g/BjKb/6ZQ github.com/fvbommel/sortorder v1.1.0/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0= github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= -github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs= -github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= +github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= +github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -628,8 +628,8 @@ go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0. go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.56.0/go.mod h1:3qi2EEwMgB4xnKgPLqsDP3j9qxnHDZeHsnAxfjQqTko= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.62.0 h1:Hf9xI/XLML9ElpiHVDNwvqI0hIFlzV8dgIr35kV1kRU= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.62.0/go.mod h1:NfchwuyNoMcZ5MLHwPrODwUF1HWCXWrL31s8gSAdIKY= -go.opentelemetry.io/otel v1.40.0 h1:oA5YeOcpRTXq6NN7frwmwFR0Cn3RhTVZvXsP4duvCms= -go.opentelemetry.io/otel v1.40.0/go.mod h1:IMb+uXZUKkMXdPddhwAHm6UfOwJyh4ct1ybIlV14J0g= +go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= +go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.37.0 h1:zG8GlgXCJQd5BU98C0hZnBbElszTmUgCNCfYneaDL0A= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.37.0/go.mod h1:hOfBCz8kv/wuq73Mx2H2QnWokh/kHZxkh6SNF2bdKtw= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.31.0 h1:ZsXq73BERAiNuuFXYqP4MR5hBrjXfMGSO+Cx7qoOZiM= @@ -642,14 +642,14 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.31.0 h1:lUsI2 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.31.0/go.mod h1:2HpZxxQurfGxJlJDblybejHB6RX6pmExPNe517hREw4= go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.37.0 h1:6VjV6Et+1Hd2iLZEPtdV7vie80Yyqf7oikJLjQ/myi0= go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.37.0/go.mod h1:u8hcp8ji5gaM/RfcOo8z9NMnf1pVLfVY7lBY2VOGuUU= -go.opentelemetry.io/otel/metric v1.40.0 h1:rcZe317KPftE2rstWIBitCdVp89A2HqjkxR3c11+p9g= -go.opentelemetry.io/otel/metric v1.40.0/go.mod h1:ib/crwQH7N3r5kfiBZQbwrTge743UDc7DTFVZrrXnqc= -go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8= -go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE= -go.opentelemetry.io/otel/sdk/metric v1.40.0 h1:mtmdVqgQkeRxHgRv4qhyJduP3fYJRMX4AtAlbuWdCYw= -go.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2rhUqdWNoKcYrtcBPg= -go.opentelemetry.io/otel/trace v1.40.0 h1:WA4etStDttCSYuhwvEa8OP8I5EWu24lkOzp+ZYblVjw= -go.opentelemetry.io/otel/trace v1.40.0/go.mod h1:zeAhriXecNGP/s2SEG3+Y8X9ujcJOTqQ5RgdEJcawiA= +go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= +go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= +go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= +go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= +go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= +go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= +go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= +go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= go.opentelemetry.io/proto/otlp v1.7.1 h1:gTOMpGDb0WTBOP8JaO72iL3auEZhVmAQg4ipjOVAtj4= go.opentelemetry.io/proto/otlp v1.7.1/go.mod h1:b2rVh6rfI/s2pHWNlB7ILJcRALpcNDzKhACevjI+ZnE= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -723,8 +723,8 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= -golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= +golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/telemetry v0.0.0-20260109210033-bd525da824e2 h1:O1cMQHRfwNpDfDJerqRoE2oD+AFlyid87D40L/OkkJo= golang.org/x/telemetry v0.0.0-20260109210033-bd525da824e2/go.mod h1:b7fPSJ0pKZ3ccUh8gnTONJxhn3c/PS6tyzQvyqw4iA8= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -774,8 +774,6 @@ google.golang.org/genproto/googleapis/api v0.0.0-20251213004720-97cd9d5aeac2 h1: google.golang.org/genproto/googleapis/api v0.0.0-20251213004720-97cd9d5aeac2/go.mod h1:+rXWjjaukWZun3mLfjmVnQi18E1AsFbDN9QdJ5YXLto= google.golang.org/genproto/googleapis/rpc v0.0.0-20251213004720-97cd9d5aeac2 h1:2I6GHUeJ/4shcDpoUlLs/2WPnhg7yJwvXtqcMJt9liA= google.golang.org/genproto/googleapis/rpc v0.0.0-20251213004720-97cd9d5aeac2/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= -google.golang.org/grpc v1.79.1 h1:zGhSi45ODB9/p3VAawt9a+O/MULLl9dpizzNNpq7flY= -google.golang.org/grpc v1.79.1/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE= google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= diff --git a/addons/processors/sql-processor/go.mod b/addons/processors/sql-processor/go.mod index 03f2b5b..dbc93fd 100644 --- a/addons/processors/sql-processor/go.mod +++ b/addons/processors/sql-processor/go.mod @@ -1,11 +1,11 @@ module github.com/kafscale/platform/addons/processors/sql-processor -go 1.24.0 +go 1.25.0 require ( - github.com/aws/aws-sdk-go-v2 v1.32.6 + github.com/aws/aws-sdk-go-v2 v1.41.5 github.com/aws/aws-sdk-go-v2/config v1.28.6 - github.com/aws/aws-sdk-go-v2/service/s3 v1.58.2 + github.com/aws/aws-sdk-go-v2/service/s3 v1.97.3 github.com/jackc/pgproto3/v2 v2.3.4-0.20250125160525-bc041643406d github.com/prometheus/client_golang v1.19.0 go.etcd.io/etcd/client/v3 v3.5.13 @@ -14,21 +14,21 @@ require ( ) require ( - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.3 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.17.47 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.21 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.25 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.15 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.17 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.6 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.15 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.24.7 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.6 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.33.2 // indirect - github.com/aws/smithy-go v1.22.1 // indirect + github.com/aws/smithy-go v1.24.2 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect @@ -69,19 +69,19 @@ require ( go.etcd.io/etcd/raft/v3 v3.5.13 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.0 // indirect - go.opentelemetry.io/otel v1.40.0 // indirect + go.opentelemetry.io/otel v1.43.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.20.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.20.0 // indirect - go.opentelemetry.io/otel/metric v1.40.0 // indirect - go.opentelemetry.io/otel/sdk v1.40.0 // indirect - go.opentelemetry.io/otel/trace v1.40.0 // indirect + go.opentelemetry.io/otel/metric v1.43.0 // indirect + go.opentelemetry.io/otel/sdk v1.43.0 // indirect + go.opentelemetry.io/otel/trace v1.43.0 // indirect go.opentelemetry.io/proto/otlp v1.0.0 // indirect go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect go.uber.org/zap v1.17.0 // indirect golang.org/x/crypto v0.46.0 // indirect golang.org/x/net v0.48.0 // indirect - golang.org/x/sys v0.40.0 // indirect + golang.org/x/sys v0.42.0 // indirect golang.org/x/text v0.32.0 // indirect golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d // indirect diff --git a/addons/processors/sql-processor/go.sum b/addons/processors/sql-processor/go.sum index 90df3f1..93e6c6a 100644 --- a/addons/processors/sql-processor/go.sum +++ b/addons/processors/sql-processor/go.sum @@ -7,42 +7,42 @@ cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCB github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/aws/aws-sdk-go-v2 v1.32.6 h1:7BokKRgRPuGmKkFMhEg/jSul+tB9VvXhcViILtfG8b4= -github.com/aws/aws-sdk-go-v2 v1.32.6/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.3 h1:tW1/Rkad38LA15X4UQtjXZXNKsCgkshC3EbmcUmghTg= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.3/go.mod h1:UbnqO+zjqk3uIt9yCACHJ9IVNhyhOCnYk8yA19SAWrM= +github.com/aws/aws-sdk-go-v2 v1.41.5 h1:dj5kopbwUsVUVFgO4Fi5BIT3t4WyqIDjGKCangnV/yY= +github.com/aws/aws-sdk-go-v2 v1.41.5/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 h1:eBMB84YGghSocM7PsjmmPffTa+1FBUeNvGvFou6V/4o= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI= github.com/aws/aws-sdk-go-v2/config v1.28.6 h1:D89IKtGrs/I3QXOLNTH93NJYtDhm8SYa9Q5CsPShmyo= github.com/aws/aws-sdk-go-v2/config v1.28.6/go.mod h1:GDzxJ5wyyFSCoLkS+UhGB0dArhb9mI+Co4dHtoTxbko= github.com/aws/aws-sdk-go-v2/credentials v1.17.47 h1:48bA+3/fCdi2yAwVt+3COvmatZ6jUDNkDTIsqDiMUdw= github.com/aws/aws-sdk-go-v2/credentials v1.17.47/go.mod h1:+KdckOejLW3Ks3b0E3b5rHsr2f9yuORBum0WPnE5o5w= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.21 h1:AmoU1pziydclFT/xRV+xXE/Vb8fttJCLRPv8oAkprc0= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.21/go.mod h1:AjUdLYe4Tgs6kpH4Bv7uMZo7pottoyHMn4eTcIcneaY= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25 h1:s/fF4+yDQDoElYhfIVvSNyeCydfbuTKzhxSXDXCPasU= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25/go.mod h1:IgPfDv5jqFIzQSNbUEMoitNooSMXjRSDkhXv8jiROvU= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.25 h1:ZntTCl5EsYnhN/IygQEUugpdwbhdkom9uHcbCftiGgA= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.25/go.mod h1:DBdPrgeocww+CSl1C8cEV8PN1mHMBhuCDLpXezyvWkE= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 h1:Rgg6wvjjtX8bNHcvi9OnXWwcE0a2vGpbwmtICOsvcf4= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21/go.mod h1:A/kJFst/nm//cyqonihbdpQZwiUhhzpqTsdbhDdRF9c= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 h1:PEgGVtPoB6NTpPrBgqSE5hE/o47Ij9qk/SEZFbUOe9A= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21/go.mod h1:p+hz+PRAYlY3zcpJhPwXlLC4C+kqn70WIHwnzAfs6ps= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.15 h1:Z5r7SycxmSllHYmaAZPpmN8GviDrSGhMS6bldqtXZPw= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.15/go.mod h1:CetW7bDE00QoGEmPUoZuRog07SGVAUVW6LFpNP0YfIg= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 h1:iXtILhvDxB6kPvEXgsDhGaZCSC6LQET5ZHSdJozeI0Y= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1/go.mod h1:9nu0fVANtYiAePIBh2/pFUSwtJ402hLnp854CNoDOeE= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.17 h1:YPYe6ZmvUfDDDELqEKtAd6bo8zxhkm+XEFEzQisqUIE= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.17/go.mod h1:oBtcnYua/CgzCWYN7NZ5j7PotFDaFSUjCYVTtfyn7vw= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.6 h1:50+XsN70RS7dwJ2CkVNXzj7U2L1HKP8nqTd3XWEXBN4= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.6/go.mod h1:WqgLmwY7so32kG01zD8CPTJWVWM+TzJoOVHwTg4aPug= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.15 h1:246A4lSTXWJw/rmlQI+TT2OcqeDMKBdyjEQrafMaQdA= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.15/go.mod h1:haVfg3761/WF7YPuJOER2MP0k4UAXyHaLclKXB6usDg= -github.com/aws/aws-sdk-go-v2/service/s3 v1.58.2 h1:sZXIzO38GZOU+O0C+INqbH7C2yALwfMWpd64tONS/NE= -github.com/aws/aws-sdk-go-v2/service/s3 v1.58.2/go.mod h1:Lcxzg5rojyVPU/0eFwLtcyTaek/6Mtic5B1gJo7e/zE= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 h1:rWyie/PxDRIdhNf4DzRk0lvjVOqFJuNnO8WwaIRVxzQ= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22/go.mod h1:zd/JsJ4P7oGfUhXn1VyLqaRZwPmZwg44Jf2dS84Dm3Y= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 h1:JRaIgADQS/U6uXDqlPiefP32yXTda7Kqfx+LgspooZM= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13/go.mod h1:CEuVn5WqOMilYl+tbccq8+N2ieCy0gVn3OtRb0vBNNM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 h1:c31//R3xgIJMSC8S6hEVq+38DcvUlgFY0FM6mSI5oto= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21/go.mod h1:r6+pf23ouCB718FUxaqzZdbpYFyDtehyZcmP5KL9FkA= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 h1:ZlvrNcHSFFWURB8avufQq9gFsheUgjVD9536obIknfM= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21/go.mod h1:cv3TNhVrssKR0O/xxLJVRfd2oazSnZnkUeTf6ctUwfQ= +github.com/aws/aws-sdk-go-v2/service/s3 v1.97.3 h1:HwxWTbTrIHm5qY+CAEur0s/figc3qwvLWsNkF4RPToo= +github.com/aws/aws-sdk-go-v2/service/s3 v1.97.3/go.mod h1:uoA43SdFwacedBfSgfFSjjCvYe8aYBS7EnU5GZ/YKMM= github.com/aws/aws-sdk-go-v2/service/sso v1.24.7 h1:rLnYAfXQ3YAccocshIH5mzNNwZBkBo+bP6EhIxak6Hw= github.com/aws/aws-sdk-go-v2/service/sso v1.24.7/go.mod h1:ZHtuQJ6t9A/+YDuxOLnbryAmITtr8UysSny3qcyvJTc= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.6 h1:JnhTZR3PiYDNKlXy50/pNeix9aGMo6lLpXwJ1mw8MD4= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.6/go.mod h1:URronUEGfXZN1VpdktPSD1EkAL9mfrV+2F4sjH38qOY= github.com/aws/aws-sdk-go-v2/service/sts v1.33.2 h1:s4074ZO1Hk8qv65GqNXqDjmkf4HSQqJukaLuuW0TpDA= github.com/aws/aws-sdk-go-v2/service/sts v1.33.2/go.mod h1:mVggCnIWoM09jP71Wh+ea7+5gAp53q+49wDFs1SW5z8= -github.com/aws/smithy-go v1.22.1 h1:/HPHZQ0g7f4eUeK6HKglFz8uwVfZKgoI25rb/J+dnro= -github.com/aws/smithy-go v1.22.1/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= +github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng= +github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= @@ -118,8 +118,6 @@ github.com/jackc/chunkreader/v2 v2.0.0 h1:DUwgMQuuPnS0rhMXenUtZpqZqrR/30NWY+qQvT github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= -github.com/jackc/pgproto3/v2 v2.3.3 h1:1HLSx5H+tXR9pW3in3zaztoEwQYRC9SQaYUHjTSUOag= -github.com/jackc/pgproto3/v2 v2.3.3/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgproto3/v2 v2.3.4-0.20250125160525-bc041643406d h1:tfqquD0m2XihRYJ4SQ5dS3J0l4vIMXUT2dhvBib4d0Q= github.com/jackc/pgproto3/v2 v2.3.4-0.20250125160525-bc041643406d/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ= @@ -197,20 +195,20 @@ go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.0 h1:PzIubN4/sjByhDRHLviCjJuweBXWFZWhghjg7cS28+M= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.0/go.mod h1:Ct6zzQEuGK3WpJs2n4dn+wfJYzd/+hNnxMRTWjGn30M= -go.opentelemetry.io/otel v1.40.0 h1:oA5YeOcpRTXq6NN7frwmwFR0Cn3RhTVZvXsP4duvCms= -go.opentelemetry.io/otel v1.40.0/go.mod h1:IMb+uXZUKkMXdPddhwAHm6UfOwJyh4ct1ybIlV14J0g= +go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= +go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.20.0 h1:DeFD0VgTZ+Cj6hxravYYZE2W4GlneVH81iAOPjZkzk8= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.20.0/go.mod h1:GijYcYmNpX1KazD5JmWGsi4P7dDTTTnfv1UbGn84MnU= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.20.0 h1:gvmNvqrPYovvyRmCSygkUDyL8lC5Tl845MLEwqpxhEU= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.20.0/go.mod h1:vNUq47TGFioo+ffTSnKNdob241vePmtNZnAODKapKd0= -go.opentelemetry.io/otel/metric v1.40.0 h1:rcZe317KPftE2rstWIBitCdVp89A2HqjkxR3c11+p9g= -go.opentelemetry.io/otel/metric v1.40.0/go.mod h1:ib/crwQH7N3r5kfiBZQbwrTge743UDc7DTFVZrrXnqc= -go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8= -go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE= -go.opentelemetry.io/otel/sdk/metric v1.40.0 h1:mtmdVqgQkeRxHgRv4qhyJduP3fYJRMX4AtAlbuWdCYw= -go.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2rhUqdWNoKcYrtcBPg= -go.opentelemetry.io/otel/trace v1.40.0 h1:WA4etStDttCSYuhwvEa8OP8I5EWu24lkOzp+ZYblVjw= -go.opentelemetry.io/otel/trace v1.40.0/go.mod h1:zeAhriXecNGP/s2SEG3+Y8X9ujcJOTqQ5RgdEJcawiA= +go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= +go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= +go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= +go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= +go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= +go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= +go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= +go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= @@ -267,8 +265,8 @@ golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= -golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= +golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= diff --git a/examples/E10_java-kafka-client-demo/pom.xml b/examples/E10_java-kafka-client-demo/pom.xml index 65182a3..43ac625 100644 --- a/examples/E10_java-kafka-client-demo/pom.xml +++ b/examples/E10_java-kafka-client-demo/pom.xml @@ -27,7 +27,7 @@ 17 17 - 3.9.1 + 3.9.2 2.0.9 diff --git a/examples/E30_flink-kafscale-demo/pom.xml b/examples/E30_flink-kafscale-demo/pom.xml index 0f303bc..1f86ad1 100644 --- a/examples/E30_flink-kafscale-demo/pom.xml +++ b/examples/E30_flink-kafscale-demo/pom.xml @@ -12,7 +12,7 @@ 11 1.18.1 3.1.0-1.18 - 3.9.1 + 3.9.2 UTF-8 com.example.kafscale.flink.WordCountJob diff --git a/examples/E40_spark-kafscale-demo/pom.xml b/examples/E40_spark-kafscale-demo/pom.xml index a31810e..5983649 100644 --- a/examples/E40_spark-kafscale-demo/pom.xml +++ b/examples/E40_spark-kafscale-demo/pom.xml @@ -12,7 +12,7 @@ 11 3.5.1 2.12 - 3.9.1 + 3.9.2 3.2.0 UTF-8 com.example.kafscale.spark.WordCountSparkJob diff --git a/go.mod b/go.mod index 9f96fed..de9e138 100644 --- a/go.mod +++ b/go.mod @@ -3,11 +3,11 @@ module github.com/KafScale/platform go 1.25.2 require ( - github.com/aws/aws-sdk-go-v2 v1.41.2 + github.com/aws/aws-sdk-go-v2 v1.41.5 github.com/aws/aws-sdk-go-v2/config v1.32.9 github.com/aws/aws-sdk-go-v2/credentials v1.19.10 - github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0 - github.com/aws/smithy-go v1.24.1 + github.com/aws/aws-sdk-go-v2/service/s3 v1.97.3 + github.com/aws/smithy-go v1.24.2 github.com/google/uuid v1.6.0 github.com/modelcontextprotocol/go-sdk v1.4.1 github.com/prometheus/client_golang v1.23.2 @@ -28,16 +28,16 @@ require ( ) require ( - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.18 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.18 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.18 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.5 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.18 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 // indirect github.com/aws/aws-sdk-go-v2/service/signin v1.0.6 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.30.11 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.15 // indirect @@ -100,13 +100,12 @@ require ( go.etcd.io/raft/v3 v3.6.0 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 // indirect - go.opentelemetry.io/otel v1.40.0 // indirect + go.opentelemetry.io/otel v1.43.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0 // indirect - go.opentelemetry.io/otel/metric v1.40.0 // indirect - go.opentelemetry.io/otel/sdk v1.40.0 // indirect - go.opentelemetry.io/otel/sdk/metric v1.40.0 // indirect - go.opentelemetry.io/otel/trace v1.40.0 // indirect + go.opentelemetry.io/otel/metric v1.43.0 // indirect + go.opentelemetry.io/otel/sdk v1.43.0 // indirect + go.opentelemetry.io/otel/trace v1.43.0 // indirect go.opentelemetry.io/proto/otlp v1.5.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect @@ -114,7 +113,7 @@ require ( golang.org/x/crypto v0.48.0 // indirect golang.org/x/net v0.49.0 // indirect golang.org/x/oauth2 v0.34.0 // indirect - golang.org/x/sys v0.41.0 // indirect + golang.org/x/sys v0.42.0 // indirect golang.org/x/term v0.40.0 // indirect golang.org/x/text v0.34.0 // indirect golang.org/x/time v0.14.0 // indirect diff --git a/go.sum b/go.sum index f5fe5f1..1fb25e8 100644 --- a/go.sum +++ b/go.sum @@ -1,33 +1,33 @@ github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= -github.com/aws/aws-sdk-go-v2 v1.41.2 h1:LuT2rzqNQsauaGkPK/7813XxcZ3o3yePY0Iy891T2ls= -github.com/aws/aws-sdk-go-v2 v1.41.2/go.mod h1:IvvlAZQXvTXznUPfRVfryiG1fbzE2NGK6m9u39YQ+S4= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4= +github.com/aws/aws-sdk-go-v2 v1.41.5 h1:dj5kopbwUsVUVFgO4Fi5BIT3t4WyqIDjGKCangnV/yY= +github.com/aws/aws-sdk-go-v2 v1.41.5/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 h1:eBMB84YGghSocM7PsjmmPffTa+1FBUeNvGvFou6V/4o= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI= github.com/aws/aws-sdk-go-v2/config v1.32.9 h1:ktda/mtAydeObvJXlHzyGpK1xcsLaP16zfUPDGoW90A= github.com/aws/aws-sdk-go-v2/config v1.32.9/go.mod h1:U+fCQ+9QKsLW786BCfEjYRj34VVTbPdsLP3CHSYXMOI= github.com/aws/aws-sdk-go-v2/credentials v1.19.10 h1:EEhmEUFCE1Yhl7vDhNOI5OCL/iKMdkkYFTRpZXNw7m8= github.com/aws/aws-sdk-go-v2/credentials v1.19.10/go.mod h1:RnnlFCAlxQCkN2Q379B67USkBMu1PipEEiibzYN5UTE= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.18 h1:Ii4s+Sq3yDfaMLpjrJsqD6SmG/Wq/P5L/hw2qa78UAY= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.18/go.mod h1:6x81qnY++ovptLE6nWQeWrpXxbnlIex+4H4eYYGcqfc= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.18 h1:F43zk1vemYIqPAwhjTjYIz0irU2EY7sOb/F5eJ3HuyM= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.18/go.mod h1:w1jdlZXrGKaJcNoL+Nnrj+k5wlpGXqnNrKoP22HvAug= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.18 h1:xCeWVjj0ki0l3nruoyP2slHsGArMxeiiaoPN5QZH6YQ= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.18/go.mod h1:r/eLGuGCBw6l36ZRWiw6PaZwPXb6YOj+i/7MizNl5/k= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 h1:Rgg6wvjjtX8bNHcvi9OnXWwcE0a2vGpbwmtICOsvcf4= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21/go.mod h1:A/kJFst/nm//cyqonihbdpQZwiUhhzpqTsdbhDdRF9c= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 h1:PEgGVtPoB6NTpPrBgqSE5hE/o47Ij9qk/SEZFbUOe9A= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21/go.mod h1:p+hz+PRAYlY3zcpJhPwXlLC4C+kqn70WIHwnzAfs6ps= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 h1:JqcdRG//czea7Ppjb+g/n4o8i/R50aTBHkA7vu0lK+k= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17/go.mod h1:CO+WeGmIdj/MlPel2KwID9Gt7CNq4M65HUfBW97liM0= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.5 h1:CeY9LUdur+Dxoeldqoun6y4WtJ3RQtzk0JMP2gfUay0= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.5/go.mod h1:AZLZf2fMaahW5s/wMRciu1sYbdsikT/UHwbUjOdEVTc= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 h1:Z5EiPIzXKewUQK0QTMkutjiaPVeVYXX7KIqhXu/0fXs= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8/go.mod h1:FsTpJtvC4U1fyDXk7c71XoDv3HlRm8V3NiYLeYLh5YE= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.18 h1:LTRCYFlnnKFlKsyIQxKhJuDuA3ZkrDQMRYm6rXiHlLY= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.18/go.mod h1:XhwkgGG6bHSd00nO/mexWTcTjgd6PjuvWQMqSn2UaEk= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 h1:bGeHBsGZx0Dvu/eJC0Lh9adJa3M1xREcndxLNZlve2U= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17/go.mod h1:dcW24lbU0CzHusTE8LLHhRLI42ejmINN8Lcr22bwh/g= -github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0 h1:oeu8VPlOre74lBA/PMhxa5vewaMIMmILM+RraSyB8KA= -github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0/go.mod h1:5jggDlZ2CLQhwJBiZJb4vfk4f0GxWdEDruWKEJ1xOdo= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 h1:rWyie/PxDRIdhNf4DzRk0lvjVOqFJuNnO8WwaIRVxzQ= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22/go.mod h1:zd/JsJ4P7oGfUhXn1VyLqaRZwPmZwg44Jf2dS84Dm3Y= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 h1:JRaIgADQS/U6uXDqlPiefP32yXTda7Kqfx+LgspooZM= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13/go.mod h1:CEuVn5WqOMilYl+tbccq8+N2ieCy0gVn3OtRb0vBNNM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 h1:c31//R3xgIJMSC8S6hEVq+38DcvUlgFY0FM6mSI5oto= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21/go.mod h1:r6+pf23ouCB718FUxaqzZdbpYFyDtehyZcmP5KL9FkA= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 h1:ZlvrNcHSFFWURB8avufQq9gFsheUgjVD9536obIknfM= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21/go.mod h1:cv3TNhVrssKR0O/xxLJVRfd2oazSnZnkUeTf6ctUwfQ= +github.com/aws/aws-sdk-go-v2/service/s3 v1.97.3 h1:HwxWTbTrIHm5qY+CAEur0s/figc3qwvLWsNkF4RPToo= +github.com/aws/aws-sdk-go-v2/service/s3 v1.97.3/go.mod h1:uoA43SdFwacedBfSgfFSjjCvYe8aYBS7EnU5GZ/YKMM= github.com/aws/aws-sdk-go-v2/service/signin v1.0.6 h1:MzORe+J94I+hYu2a6XmV5yC9huoTv8NRcCrUNedDypQ= github.com/aws/aws-sdk-go-v2/service/signin v1.0.6/go.mod h1:hXzcHLARD7GeWnifd8j9RWqtfIgxj4/cAtIVIK7hg8g= github.com/aws/aws-sdk-go-v2/service/sso v1.30.11 h1:7oGD8KPfBOJGXiCoRKrrrQkbvCp8N++u36hrLMPey6o= @@ -36,8 +36,8 @@ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.15 h1:edCcNp9eGIUDUCrzoCu1jWA github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.15/go.mod h1:lyRQKED9xWfgkYC/wmmYfv7iVIM68Z5OQ88ZdcV1QbU= github.com/aws/aws-sdk-go-v2/service/sts v1.41.7 h1:NITQpgo9A5NrDZ57uOWj+abvXSb83BbyggcUBVksN7c= github.com/aws/aws-sdk-go-v2/service/sts v1.41.7/go.mod h1:sks5UWBhEuWYDPdwlnRFn1w7xWdH29Jcpe+/PJQefEs= -github.com/aws/smithy-go v1.24.1 h1:VbyeNfmYkWoxMVpGUAbQumkODcYmfMRfZ8yQiH30SK0= -github.com/aws/smithy-go v1.24.1/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0= +github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng= +github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= @@ -134,8 +134,6 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/modelcontextprotocol/go-sdk v1.3.1 h1:TfqtNKOIWN4Z1oqmPAiWDC2Jq7K9OdJaooe0teoXASI= -github.com/modelcontextprotocol/go-sdk v1.3.1/go.mod h1:DgVX498dMD8UJlseK1S5i1T4tFz2fkBk4xogC3D15nw= github.com/modelcontextprotocol/go-sdk v1.4.1 h1:M4x9GyIPj+HoIlHNGpK2hq5o3BFhC+78PkEaldQRphc= github.com/modelcontextprotocol/go-sdk v1.4.1/go.mod h1:Bo/mS87hPQqHSRkMv4dQq1XCu6zv4INdXnFZabkNU6s= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -168,8 +166,6 @@ github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0t github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/segmentio/asm v1.1.3 h1:WM03sfUOENvvKexOLp+pCqgb/WDjsi7EK8gIsICtzhc= github.com/segmentio/asm v1.1.3/go.mod h1:Ld3L4ZXGNcSLRg4JBsZ3//1+f/TjYl0Mzen/DQy1EJg= -github.com/segmentio/encoding v0.5.3 h1:OjMgICtcSFuNvQCdwqMCv9Tg7lEOXGwm1J5RPQccx6w= -github.com/segmentio/encoding v0.5.3/go.mod h1:HS1ZKa3kSN32ZHVZ7ZLPLXWvOVIiZtyJnO1gPH1sKt0= github.com/segmentio/encoding v0.5.4 h1:OW1VRern8Nw6ITAtwSZ7Idrl3MXCFwXHPgqESYfvNt0= github.com/segmentio/encoding v0.5.4/go.mod h1:HS1ZKa3kSN32ZHVZ7ZLPLXWvOVIiZtyJnO1gPH1sKt0= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= @@ -226,30 +222,20 @@ go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 h1:x7wzEgXfnzJcHDwStJT+mxOz4etr2EcexjqhBvmoakw= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0/go.mod h1:rg+RlpR5dKwaS95IyyZqj5Wd4E13lk/msnTS0Xl9lJM= -go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48= -go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8= -go.opentelemetry.io/otel v1.40.0 h1:oA5YeOcpRTXq6NN7frwmwFR0Cn3RhTVZvXsP4duvCms= -go.opentelemetry.io/otel v1.40.0/go.mod h1:IMb+uXZUKkMXdPddhwAHm6UfOwJyh4ct1ybIlV14J0g= +go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= +go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0 h1:OeNbIYk/2C15ckl7glBlOBp5+WlYsOElzTNmiPW/x60= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0/go.mod h1:7Bept48yIeqxP2OZ9/AqIpYS94h2or0aB4FypJTc8ZM= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0 h1:tgJ0uaNS4c98WRNUEx5U3aDlrDOI5Rs+1Vifcw4DJ8U= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0/go.mod h1:U7HYyW0zt/a9x5J1Kjs+r1f/d4ZHnYFclhYY2+YbeoE= -go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF0Ls5Q0= -go.opentelemetry.io/otel/metric v1.39.0/go.mod h1:jrZSWL33sD7bBxg1xjrqyDjnuzTUB0x1nBERXd7Ftcs= -go.opentelemetry.io/otel/metric v1.40.0 h1:rcZe317KPftE2rstWIBitCdVp89A2HqjkxR3c11+p9g= -go.opentelemetry.io/otel/metric v1.40.0/go.mod h1:ib/crwQH7N3r5kfiBZQbwrTge743UDc7DTFVZrrXnqc= -go.opentelemetry.io/otel/sdk v1.39.0 h1:nMLYcjVsvdui1B/4FRkwjzoRVsMK8uL/cj0OyhKzt18= -go.opentelemetry.io/otel/sdk v1.39.0/go.mod h1:vDojkC4/jsTJsE+kh+LXYQlbL8CgrEcwmt1ENZszdJE= -go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8= -go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE= -go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2WKg+sEJTtB8= -go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew= -go.opentelemetry.io/otel/sdk/metric v1.40.0 h1:mtmdVqgQkeRxHgRv4qhyJduP3fYJRMX4AtAlbuWdCYw= -go.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2rhUqdWNoKcYrtcBPg= -go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI= -go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA= -go.opentelemetry.io/otel/trace v1.40.0 h1:WA4etStDttCSYuhwvEa8OP8I5EWu24lkOzp+ZYblVjw= -go.opentelemetry.io/otel/trace v1.40.0/go.mod h1:zeAhriXecNGP/s2SEG3+Y8X9ujcJOTqQ5RgdEJcawiA= +go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= +go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= +go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= +go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= +go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= +go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= +go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= +go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= go.opentelemetry.io/proto/otlp v1.5.0 h1:xJvq7gMzB31/d406fB8U5CBdyQGw4P399D1aQWU/3i4= go.opentelemetry.io/proto/otlp v1.5.0/go.mod h1:keN8WnHxOy8PG0rQZjJJ5A2ebUoafqWp0eVQ4yIXvJ4= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -294,8 +280,8 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= -golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= +golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg= golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM= @@ -324,8 +310,6 @@ google.golang.org/genproto/googleapis/api v0.0.0-20251213004720-97cd9d5aeac2 h1: google.golang.org/genproto/googleapis/api v0.0.0-20251213004720-97cd9d5aeac2/go.mod h1:+rXWjjaukWZun3mLfjmVnQi18E1AsFbDN9QdJ5YXLto= google.golang.org/genproto/googleapis/rpc v0.0.0-20251213004720-97cd9d5aeac2 h1:2I6GHUeJ/4shcDpoUlLs/2WPnhg7yJwvXtqcMJt9liA= google.golang.org/genproto/googleapis/rpc v0.0.0-20251213004720-97cd9d5aeac2/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= -google.golang.org/grpc v1.79.1 h1:zGhSi45ODB9/p3VAawt9a+O/MULLl9dpizzNNpq7flY= -google.golang.org/grpc v1.79.1/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE= google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= diff --git a/lfs-client-sdk/java/pom.xml b/lfs-client-sdk/java/pom.xml index 9169291..60bbed2 100644 --- a/lfs-client-sdk/java/pom.xml +++ b/lfs-client-sdk/java/pom.xml @@ -29,7 +29,7 @@ limitations under the License. 17 17 UTF-8 - 4.1.1 + 4.1.2 2.31.5 2.17.2 5.11.0 From f0a00ec39c2a68328035036502cb1974d16d0a4a Mon Sep 17 00:00:00 2001 From: 2pk03 Date: Wed, 13 May 2026 15:19:38 +0200 Subject: [PATCH 3/3] add topic recovery --- cmd/kafscale-cli/main.go | 379 ++++++++++++++++++++++++++++++++++ cmd/kafscale-cli/main_test.go | 367 ++++++++++++++++++++++++++++++++ docs/operations.md | 21 ++ pkg/storage/recovery.go | 306 +++++++++++++++++++++++++++ pkg/storage/recovery_test.go | 110 ++++++++++ 5 files changed, 1183 insertions(+) create mode 100644 cmd/kafscale-cli/main.go create mode 100644 cmd/kafscale-cli/main_test.go create mode 100644 pkg/storage/recovery.go create mode 100644 pkg/storage/recovery_test.go diff --git a/cmd/kafscale-cli/main.go b/cmd/kafscale-cli/main.go new file mode 100644 index 0000000..d8c9921 --- /dev/null +++ b/cmd/kafscale-cli/main.go @@ -0,0 +1,379 @@ +// Copyright 2026 Alexander Alten (novatechflow), NovaTechflow (novatechflow.com). +// This project is supported and financed by Scalytics, Inc. (www.scalytics.io). +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "flag" + "fmt" + "io" + "os" + "path" + "slices" + "strconv" + "strings" + "time" + + metadatapb "github.com/KafScale/platform/pkg/gen/metadata" + "github.com/KafScale/platform/pkg/metadata" + "github.com/KafScale/platform/pkg/protocol" + "github.com/KafScale/platform/pkg/storage" +) + +var ( + newS3Client = storage.NewS3Client + newEtcdStore = metadata.NewEtcdStore + newMemoryS3 = func() storage.S3Client { return storage.NewMemoryS3Client() } +) + +type restoreConfig struct { + SourceTopic string + SourceNamespace string + TargetTopic string + TargetNamespace string + RestoreTo time.Time + Partitions []int32 +} + +func main() { + ctx := context.Background() + if err := run(ctx, os.Args[1:], os.Stdout, os.Stderr); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func run(ctx context.Context, args []string, stdout io.Writer, stderr io.Writer) error { + if len(args) == 0 { + writeUsage(stderr) + return fmt.Errorf("command required") + } + switch args[0] { + case "restore": + return runRestoreCommand(ctx, args[1:], stdout) + case "-h", "--help", "help": + writeUsage(stdout) + return nil + default: + writeUsage(stderr) + return fmt.Errorf("unknown command %q", args[0]) + } +} + +func writeUsage(w io.Writer) { + _, _ = fmt.Fprintln(w, "usage: kafscale-cli restore --topic --target-topic --to ") +} + +func runRestoreCommand(ctx context.Context, args []string, stdout io.Writer) error { + fs := flag.NewFlagSet("restore", flag.ContinueOnError) + fs.SetOutput(io.Discard) + + var ( + sourceTopic = fs.String("topic", "", "Source topic to recover from") + sourceNamespace = fs.String("namespace", envOrDefault("KAFSCALE_NAMESPACE", "default"), "Source namespace") + targetTopic = fs.String("target-topic", "", "Target topic to create and populate") + targetNamespace = fs.String("target-namespace", "", "Target namespace (defaults to source namespace)") + restoreToRaw = fs.String("to", "", "Restore cutoff in RFC3339 format") + partitionsRaw = fs.String("partitions", "", "Optional comma-separated partition list") + ) + if err := fs.Parse(args); err != nil { + return err + } + if *sourceTopic == "" { + return fmt.Errorf("--topic is required") + } + if *targetTopic == "" { + return fmt.Errorf("--target-topic is required") + } + if *restoreToRaw == "" { + return fmt.Errorf("--to is required") + } + restoreTo, err := time.Parse(time.RFC3339, *restoreToRaw) + if err != nil { + return fmt.Errorf("parse --to: %w", err) + } + partitions, err := parsePartitions(*partitionsRaw) + if err != nil { + return err + } + if *targetNamespace == "" { + *targetNamespace = *sourceNamespace + } + + s3Client, err := s3ClientFromEnv(ctx) + if err != nil { + return err + } + + store, err := metadataStoreFromEnv(ctx) + if err != nil { + return err + } + defer func() { _ = store.EtcdClient().Close() }() + + return executeRestore(ctx, stdout, restoreConfig{ + SourceTopic: *sourceTopic, + SourceNamespace: *sourceNamespace, + TargetTopic: *targetTopic, + TargetNamespace: *targetNamespace, + RestoreTo: restoreTo, + Partitions: partitions, + }, s3Client, store) +} + +func s3ClientFromEnv(ctx context.Context) (storage.S3Client, error) { + if envBoolDefault("KAFSCALE_USE_MEMORY_S3", false) { + return newMemoryS3(), nil + } + return newS3Client(ctx, storage.S3Config{ + Bucket: strings.TrimSpace(os.Getenv("KAFSCALE_S3_BUCKET")), + Region: strings.TrimSpace(os.Getenv("KAFSCALE_S3_REGION")), + Endpoint: strings.TrimSpace(os.Getenv("KAFSCALE_S3_ENDPOINT")), + ForcePathStyle: envBoolDefault("KAFSCALE_S3_PATH_STYLE", strings.TrimSpace(os.Getenv("KAFSCALE_S3_ENDPOINT")) != ""), + AccessKeyID: strings.TrimSpace(os.Getenv("KAFSCALE_S3_ACCESS_KEY")), + SecretAccessKey: strings.TrimSpace(os.Getenv("KAFSCALE_S3_SECRET_KEY")), + SessionToken: strings.TrimSpace(os.Getenv("KAFSCALE_S3_SESSION_TOKEN")), + KMSKeyARN: strings.TrimSpace(os.Getenv("KAFSCALE_S3_KMS_ARN")), + MaxConnections: envIntDefault("KAFSCALE_S3_CONCURRENCY", 16), + }) +} + +func metadataStoreFromEnv(ctx context.Context) (*metadata.EtcdStore, error) { + return newEtcdStore(ctx, metadata.ClusterMetadata{}, metadata.EtcdStoreConfig{ + Endpoints: splitCSV(strings.TrimSpace(os.Getenv("KAFSCALE_ETCD_ENDPOINTS"))), + Username: strings.TrimSpace(os.Getenv("KAFSCALE_ETCD_USERNAME")), + Password: strings.TrimSpace(os.Getenv("KAFSCALE_ETCD_PASSWORD")), + DialTimeout: 5 * time.Second, + }) +} + +func executeRestore(ctx context.Context, stdout io.Writer, cfg restoreConfig, s3Client storage.S3Client, store *metadata.EtcdStore) error { + if store == nil { + return fmt.Errorf("metadata store required") + } + if s3Client == nil { + return fmt.Errorf("s3 client required") + } + + sourceMeta, err := store.Metadata(ctx, []string{cfg.SourceTopic}) + if err != nil { + return err + } + if len(sourceMeta.Topics) == 0 || sourceMeta.Topics[0].ErrorCode != 0 { + return metadata.ErrUnknownTopic + } + + sourcePartitions := make(map[int32]struct{}, len(sourceMeta.Topics[0].Partitions)) + for _, partition := range sourceMeta.Topics[0].Partitions { + sourcePartitions[partition.Partition] = struct{}{} + } + for _, partition := range cfg.Partitions { + if _, ok := sourcePartitions[partition]; !ok { + return fmt.Errorf("partition %d not present in source topic %s", partition, cfg.SourceTopic) + } + } + + sourceCfg, err := store.FetchTopicConfig(ctx, cfg.SourceTopic) + if err != nil { + return err + } + + if _, err := store.CreateTopic(ctx, metadata.TopicSpec{ + Name: cfg.TargetTopic, + NumPartitions: sourceCfg.Partitions, + ReplicationFactor: int16(sourceCfg.ReplicationFactor), + }); err != nil { + return err + } + + targetCfg := cloneTopicConfig(sourceCfg) + targetCfg.Name = cfg.TargetTopic + targetCfg.CreatedAt = time.Now().UTC().Format(time.RFC3339) + if err := store.UpdateTopicConfig(ctx, targetCfg); err != nil { + return err + } + + result, err := storage.RecoverTopicToTimestamp(ctx, s3Client, storage.TopicRecoveryConfig{ + SourceNamespace: cfg.SourceNamespace, + SourceTopic: cfg.SourceTopic, + TargetNamespace: cfg.TargetNamespace, + TargetTopic: cfg.TargetTopic, + RestoreTo: cfg.RestoreTo, + Partitions: cfg.Partitions, + }) + if err != nil { + return err + } + + targetMeta, err := store.Metadata(ctx, []string{cfg.TargetTopic}) + if err != nil { + return err + } + if len(targetMeta.Topics) == 0 || targetMeta.Topics[0].ErrorCode != 0 { + return metadata.ErrUnknownTopic + } + + recoveredByPartition := make(map[int32]storage.RecoveredPartition, len(result.Partitions)) + for _, partition := range result.Partitions { + recoveredByPartition[partition.Partition] = partition + if partition.LastOffset >= 0 { + if err := store.UpdateOffsets(ctx, cfg.TargetTopic, partition.Partition, partition.LastOffset); err != nil { + return err + } + } + } + + if err := writePartitionStates(ctx, store, cfg.TargetTopic, targetMeta.Topics[0].Partitions, recoveredByPartition); err != nil { + return err + } + + _, _ = fmt.Fprintf(stdout, "restored %s to %s up to %s\n", cfg.SourceTopic, cfg.TargetTopic, cfg.RestoreTo.UTC().Format(time.RFC3339)) + for _, partition := range result.Partitions { + _, _ = fmt.Fprintf(stdout, "partition=%d segments=%d last_offset=%d\n", partition.Partition, partition.SegmentsCopied, partition.LastOffset) + } + return nil +} + +func writePartitionStates(ctx context.Context, store *metadata.EtcdStore, topic string, partitions []protocol.MetadataPartition, recovered map[int32]storage.RecoveredPartition) error { + for _, partition := range partitions { + summary, ok := recovered[partition.Partition] + state := &metadatapb.PartitionState{ + Topic: topic, + Partition: partition.Partition, + LeaderBroker: fmt.Sprintf("%d", partition.Leader), + LeaderEpoch: partition.LeaderEpoch, + } + if ok && summary.LastOffset >= 0 { + state.LogEndOffset = summary.LastOffset + 1 + state.HighWatermark = summary.LastOffset + 1 + if len(summary.Segments) > 0 { + last := summary.Segments[len(summary.Segments)-1] + state.ActiveSegment = path.Base(last.TargetKey) + state.Segments = make([]*metadatapb.SegmentInfo, 0, len(summary.Segments)) + for _, segment := range summary.Segments { + state.Segments = append(state.Segments, &metadatapb.SegmentInfo{ + BaseOffset: segment.BaseOffset, + SizeBytes: segment.SizeBytes, + CreatedAt: segment.CreatedAt.UTC().Format(time.RFC3339), + }) + } + } + } + payload, err := metadata.EncodePartitionState(state) + if err != nil { + return err + } + putCtx, cancel := context.WithTimeout(ctx, 3*time.Second) + _, err = store.EtcdClient().Put(putCtx, metadata.PartitionStateKey(topic, partition.Partition), string(payload)) + cancel() + if err != nil { + return err + } + } + return nil +} + +func cloneTopicConfig(cfg *metadatapb.TopicConfig) *metadatapb.TopicConfig { + if cfg == nil { + return nil + } + cloned := &metadatapb.TopicConfig{ + Name: cfg.Name, + Partitions: cfg.Partitions, + ReplicationFactor: cfg.ReplicationFactor, + RetentionMs: cfg.RetentionMs, + RetentionBytes: cfg.RetentionBytes, + SegmentBytes: cfg.SegmentBytes, + CreatedAt: cfg.CreatedAt, + Config: make(map[string]string, len(cfg.Config)), + } + for key, value := range cfg.Config { + cloned.Config[key] = value + } + return cloned +} + +func parsePartitions(raw string) ([]int32, error) { + if strings.TrimSpace(raw) == "" { + return nil, nil + } + parts := splitCSV(raw) + out := make([]int32, 0, len(parts)) + seen := make(map[int32]struct{}, len(parts)) + for _, part := range parts { + parsed, err := strconv.ParseInt(part, 10, 32) + if err != nil { + return nil, fmt.Errorf("parse partition %q: %w", part, err) + } + val := int32(parsed) + if _, ok := seen[val]; ok { + continue + } + seen[val] = struct{}{} + out = append(out, val) + } + slices.Sort(out) + return out, nil +} + +func splitCSV(raw string) []string { + if strings.TrimSpace(raw) == "" { + return nil + } + parts := strings.Split(raw, ",") + out := make([]string, 0, len(parts)) + for _, part := range parts { + val := strings.TrimSpace(part) + if val != "" { + out = append(out, val) + } + } + return out +} + +func envOrDefault(key string, fallback string) string { + if val := strings.TrimSpace(os.Getenv(key)); val != "" { + return val + } + return fallback +} + +func envBoolDefault(key string, fallback bool) bool { + raw := strings.TrimSpace(os.Getenv(key)) + if raw == "" { + return fallback + } + switch strings.ToLower(raw) { + case "1", "true", "yes", "on": + return true + case "0", "false", "no", "off": + return false + default: + return fallback + } +} + +func envIntDefault(key string, fallback int) int { + raw := strings.TrimSpace(os.Getenv(key)) + if raw == "" { + return fallback + } + val, err := strconv.Atoi(raw) + if err != nil || val <= 0 { + return fallback + } + return val +} diff --git a/cmd/kafscale-cli/main_test.go b/cmd/kafscale-cli/main_test.go new file mode 100644 index 0000000..0a547a9 --- /dev/null +++ b/cmd/kafscale-cli/main_test.go @@ -0,0 +1,367 @@ +// Copyright 2026 Alexander Alten (novatechflow), NovaTechflow (novatechflow.com). +// This project is supported and financed by Scalytics, Inc. (www.scalytics.io). +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "context" + "errors" + "strings" + "testing" + "time" + + clientv3 "go.etcd.io/etcd/client/v3" + + "github.com/KafScale/platform/internal/testutil" + metadatapb "github.com/KafScale/platform/pkg/gen/metadata" + "github.com/KafScale/platform/pkg/metadata" + "github.com/KafScale/platform/pkg/protocol" + "github.com/KafScale/platform/pkg/storage" + "github.com/twmb/franz-go/pkg/kmsg" +) + +func TestRunHelpAndUnknownCommand(t *testing.T) { + var stdout bytes.Buffer + if err := run(context.Background(), []string{"help"}, &stdout, &bytes.Buffer{}); err != nil { + t.Fatalf("run help: %v", err) + } + if !strings.Contains(stdout.String(), "usage: kafscale-cli restore") { + t.Fatalf("unexpected help output: %s", stdout.String()) + } + + err := run(context.Background(), []string{"wat"}, &bytes.Buffer{}, &bytes.Buffer{}) + if err == nil || !strings.Contains(err.Error(), "unknown command") { + t.Fatalf("expected unknown command error, got %v", err) + } +} + +func TestRunRestoreCommandUsesInjectedClients(t *testing.T) { + endpoints := testutil.StartEmbeddedEtcd(t) + ctx := context.Background() + + store, err := metadata.NewEtcdStore(ctx, metadata.ClusterMetadata{ + Brokers: []protocol.MetadataBroker{ + {NodeID: 1, Host: "broker-0", Port: 9092}, + }, + ControllerID: 1, + }, metadata.EtcdStoreConfig{Endpoints: endpoints}) + if err != nil { + t.Fatalf("NewEtcdStore: %v", err) + } + defer func() { _ = store.EtcdClient().Close() }() + + if _, err := store.CreateTopic(ctx, metadata.TopicSpec{ + Name: "orders", + NumPartitions: 1, + ReplicationFactor: 1, + }); err != nil { + t.Fatalf("CreateTopic: %v", err) + } + + s3 := storage.NewMemoryS3Client() + artifact, err := storage.BuildSegment(storage.SegmentWriterConfig{IndexIntervalMessages: 1}, []storage.RecordBatch{ + { + BaseOffset: 0, + LastOffsetDelta: 0, + MessageCount: 1, + Bytes: make([]byte, 70), + }, + }, time.Date(2026, 5, 13, 12, 0, 0, 0, time.UTC)) + if err != nil { + t.Fatalf("BuildSegment: %v", err) + } + if err := s3.UploadSegment(ctx, "default/orders/0/segment-00000000000000000000.kfs", artifact.SegmentBytes); err != nil { + t.Fatalf("UploadSegment: %v", err) + } + if err := s3.UploadIndex(ctx, "default/orders/0/segment-00000000000000000000.index", artifact.IndexBytes); err != nil { + t.Fatalf("UploadIndex: %v", err) + } + + prevS3 := newS3Client + prevStore := newEtcdStore + prevMemory := newMemoryS3 + t.Cleanup(func() { + newS3Client = prevS3 + newEtcdStore = prevStore + newMemoryS3 = prevMemory + }) + newS3Client = func(context.Context, storage.S3Config) (storage.S3Client, error) { + return s3, nil + } + newEtcdStore = func(context.Context, metadata.ClusterMetadata, metadata.EtcdStoreConfig) (*metadata.EtcdStore, error) { + return store, nil + } + newMemoryS3 = func() storage.S3Client { return s3 } + + t.Setenv("KAFSCALE_S3_BUCKET", "bucket") + t.Setenv("KAFSCALE_S3_REGION", "us-east-1") + t.Setenv("KAFSCALE_ETCD_ENDPOINTS", strings.Join(endpoints, ",")) + + var stdout bytes.Buffer + if err := run(context.Background(), []string{ + "restore", + "--topic", "orders", + "--target-topic", "orders-wrapper", + "--to", "2026-05-13T12:05:00Z", + }, &stdout, &bytes.Buffer{}); err != nil { + t.Fatalf("run restore: %v", err) + } + if !strings.Contains(stdout.String(), "orders-wrapper") { + t.Fatalf("unexpected restore output: %s", stdout.String()) + } +} + +func TestRunRestoreCommandRequiresTopic(t *testing.T) { + err := runRestoreCommand(context.Background(), []string{ + "--target-topic", "orders-restore", + "--to", "2026-05-13T12:05:00Z", + }, &bytes.Buffer{}) + if err == nil || !strings.Contains(err.Error(), "--topic is required") { + t.Fatalf("expected missing topic error, got %v", err) + } +} + +func TestS3ClientFromEnvUsesMemoryToggle(t *testing.T) { + prevMemory := newMemoryS3 + t.Cleanup(func() { newMemoryS3 = prevMemory }) + + mem := storage.NewMemoryS3Client() + newMemoryS3 = func() storage.S3Client { return mem } + t.Setenv("KAFSCALE_USE_MEMORY_S3", "true") + + client, err := s3ClientFromEnv(context.Background()) + if err != nil { + t.Fatalf("s3ClientFromEnv: %v", err) + } + if client != mem { + t.Fatal("expected injected memory s3 client") + } +} + +func TestMetadataStoreFromEnvPassesConfig(t *testing.T) { + prevStore := newEtcdStore + t.Cleanup(func() { newEtcdStore = prevStore }) + + t.Setenv("KAFSCALE_ETCD_ENDPOINTS", "http://a:2379,http://b:2379") + t.Setenv("KAFSCALE_ETCD_USERNAME", "user") + t.Setenv("KAFSCALE_ETCD_PASSWORD", "pass") + + var got metadata.EtcdStoreConfig + sentinel := errors.New("stop") + newEtcdStore = func(_ context.Context, _ metadata.ClusterMetadata, cfg metadata.EtcdStoreConfig) (*metadata.EtcdStore, error) { + got = cfg + return nil, sentinel + } + + err := func() error { + _, err := metadataStoreFromEnv(context.Background()) + return err + }() + if !errors.Is(err, sentinel) { + t.Fatalf("expected sentinel error, got %v", err) + } + if len(got.Endpoints) != 2 || got.Endpoints[0] != "http://a:2379" || got.Endpoints[1] != "http://b:2379" { + t.Fatalf("unexpected endpoints: %+v", got.Endpoints) + } + if got.Username != "user" || got.Password != "pass" { + t.Fatalf("unexpected auth config: %+v", got) + } +} + +func TestExecuteRestoreCreatesRecoveredTopic(t *testing.T) { + endpoints := testutil.StartEmbeddedEtcd(t) + ctx := context.Background() + + store, err := metadata.NewEtcdStore(ctx, metadata.ClusterMetadata{ + Brokers: []protocol.MetadataBroker{ + {NodeID: 1, Host: "broker-0", Port: 9092}, + }, + ControllerID: 1, + }, metadata.EtcdStoreConfig{Endpoints: endpoints}) + if err != nil { + t.Fatalf("NewEtcdStore: %v", err) + } + defer func() { _ = store.EtcdClient().Close() }() + + if _, err := store.CreateTopic(ctx, metadata.TopicSpec{ + Name: "orders", + NumPartitions: 1, + ReplicationFactor: 1, + }); err != nil { + t.Fatalf("CreateTopic: %v", err) + } + cfg, err := store.FetchTopicConfig(ctx, "orders") + if err != nil { + t.Fatalf("FetchTopicConfig: %v", err) + } + cfg.RetentionMs = 60000 + cfg.Config = map[string]string{"cleanup.policy": "delete"} + if err := store.UpdateTopicConfig(ctx, cfg); err != nil { + t.Fatalf("UpdateTopicConfig: %v", err) + } + + s3 := storage.NewMemoryS3Client() + artifact, err := storage.BuildSegment(storage.SegmentWriterConfig{IndexIntervalMessages: 1}, []storage.RecordBatch{ + { + BaseOffset: 0, + LastOffsetDelta: 0, + MessageCount: 1, + Bytes: make([]byte, 70), + }, + }, time.Date(2026, 5, 13, 12, 0, 0, 0, time.UTC)) + if err != nil { + t.Fatalf("BuildSegment: %v", err) + } + if err := s3.UploadSegment(ctx, "default/orders/0/segment-00000000000000000000.kfs", artifact.SegmentBytes); err != nil { + t.Fatalf("UploadSegment: %v", err) + } + if err := s3.UploadIndex(ctx, "default/orders/0/segment-00000000000000000000.index", artifact.IndexBytes); err != nil { + t.Fatalf("UploadIndex: %v", err) + } + + var stdout bytes.Buffer + if err := executeRestore(ctx, &stdout, restoreConfig{ + SourceTopic: "orders", + SourceNamespace: "default", + TargetTopic: "orders-restored", + TargetNamespace: "default", + RestoreTo: time.Date(2026, 5, 13, 12, 5, 0, 0, time.UTC), + }, s3, store); err != nil { + t.Fatalf("executeRestore: %v", err) + } + if !strings.Contains(stdout.String(), "restored orders to orders-restored") { + t.Fatalf("unexpected stdout: %s", stdout.String()) + } + + meta, err := store.Metadata(ctx, []string{"orders-restored"}) + if err != nil { + t.Fatalf("Metadata: %v", err) + } + if len(meta.Topics) != 1 || meta.Topics[0].ErrorCode != 0 || meta.Topics[0].Topic == nil || *meta.Topics[0].Topic != "orders-restored" { + t.Fatalf("unexpected target topic metadata: %+v", meta.Topics) + } + + targetCfg, err := store.FetchTopicConfig(ctx, "orders-restored") + if err != nil { + t.Fatalf("FetchTopicConfig target: %v", err) + } + if targetCfg.RetentionMs != 60000 { + t.Fatalf("expected retention to be copied, got %d", targetCfg.RetentionMs) + } + if targetCfg.Config["cleanup.policy"] != "delete" { + t.Fatalf("expected config to be copied, got %+v", targetCfg.Config) + } + + nextOffset, err := store.NextOffset(ctx, "orders-restored", 0) + if err != nil { + t.Fatalf("NextOffset: %v", err) + } + if nextOffset != 1 { + t.Fatalf("expected next offset 1, got %d", nextOffset) + } + + if _, err := s3.DownloadSegment(ctx, "default/orders-restored/0/segment-00000000000000000000.kfs", nil); err != nil { + t.Fatalf("DownloadSegment restored: %v", err) + } + + cli, err := clientv3.New(clientv3.Config{Endpoints: endpoints, DialTimeout: 3 * time.Second}) + if err != nil { + t.Fatalf("new etcd client: %v", err) + } + defer func() { _ = cli.Close() }() + + resp, err := cli.Get(ctx, metadata.PartitionStateKey("orders-restored", 0)) + if err != nil { + t.Fatalf("get partition state: %v", err) + } + if len(resp.Kvs) != 1 { + t.Fatalf("expected partition state, got %d", len(resp.Kvs)) + } + state, err := metadata.DecodePartitionState(resp.Kvs[0].Value) + if err != nil { + t.Fatalf("DecodePartitionState: %v", err) + } + if state.LogEndOffset != 1 || state.HighWatermark != 1 { + t.Fatalf("unexpected partition offsets: %+v", state) + } + if state.ActiveSegment != "segment-00000000000000000000.kfs" { + t.Fatalf("unexpected active segment: %+v", state) + } +} + +func TestCloneTopicConfig(t *testing.T) { + cfg := &metadatapb.TopicConfig{ + Name: "orders", + Partitions: 1, + ReplicationFactor: 1, + RetentionMs: 42, + Config: map[string]string{"a": "b"}, + } + cloned := cloneTopicConfig(cfg) + if cloned == cfg { + t.Fatal("expected clone to allocate a new object") + } + cloned.Config["a"] = "c" + if cfg.Config["a"] != "b" { + t.Fatal("expected source config map to stay untouched") + } +} + +func TestParsePartitions(t *testing.T) { + partitions, err := parsePartitions("2,0,2,1") + if err != nil { + t.Fatalf("parsePartitions: %v", err) + } + if want := []int32{0, 1, 2}; len(partitions) != len(want) || partitions[0] != want[0] || partitions[1] != want[1] || partitions[2] != want[2] { + t.Fatalf("unexpected partitions: %+v", partitions) + } +} + +func TestExecuteRestoreRejectsUnknownPartition(t *testing.T) { + endpoints := testutil.StartEmbeddedEtcd(t) + ctx := context.Background() + + store, err := metadata.NewEtcdStore(ctx, metadata.ClusterMetadata{ + Brokers: []protocol.MetadataBroker{ + {NodeID: 1, Host: "broker-0", Port: 9092}, + }, + ControllerID: 1, + Topics: []protocol.MetadataTopic{ + { + Topic: kmsg.StringPtr("orders"), + Partitions: []protocol.MetadataPartition{ + {Partition: 0, Leader: 1, Replicas: []int32{1}, ISR: []int32{1}}, + }, + }, + }, + }, metadata.EtcdStoreConfig{Endpoints: endpoints}) + if err != nil { + t.Fatalf("NewEtcdStore: %v", err) + } + defer func() { _ = store.EtcdClient().Close() }() + + err = executeRestore(ctx, &bytes.Buffer{}, restoreConfig{ + SourceTopic: "orders", + SourceNamespace: "default", + TargetTopic: "orders-restored", + TargetNamespace: "default", + RestoreTo: time.Now().UTC(), + Partitions: []int32{99}, + }, storage.NewMemoryS3Client(), store) + if err == nil || !strings.Contains(err.Error(), "partition 99") { + t.Fatalf("expected unknown partition error, got %v", err) + } +} diff --git a/docs/operations.md b/docs/operations.md index 83abcb3..11f1926 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -233,6 +233,27 @@ When the KafScale operator manages etcd, each cluster pod runs ```restore init c The restore image must include `/bin/sh` and `etcdctl`. Override with `KAFSCALE_OPERATOR_ETCD_SNAPSHOT_ETCDCTL_IMAGE` if you use a custom image. +### Topic Recovery On The DR Spine + +For broker topic data, KafScale now supports **segment-granular recovery into a new topic** on the DR side. This is intentionally not an in-place rollback on the primary cluster. + +Use `kafscale-cli restore` to create a fresh target topic, copy `.kfs` segment/index pairs up to a cutoff timestamp, and set the recovered topic's next offsets: + +```bash +kafscale-cli restore \ + --topic orders \ + --target-topic orders-restore-20260513 \ + --to 2026-05-13T14:23:00Z +``` + +Operational semantics: + +- Recovery runs against the existing KafScale S3 + etcd control plane, including `KAFSCALE_S3_BUCKET`, `KAFSCALE_S3_REGION`, `KAFSCALE_S3_ENDPOINT`, `KAFSCALE_S3_PATH_STYLE`, and `KAFSCALE_ETCD_ENDPOINTS`. +- The target topic must be new. KafScale refuses to restore over an existing persisted topic. +- Recovery is **segment-granular**. The cutoff uses the immutable segment creation time, then copies contiguous segment/index pairs up to the first segment created after that timestamp. +- Offsets are preserved inside the recovered topic so replay, validation, and downstream cutover can happen without rewriting the source topic. +- The safer pattern is restore, validate, then cut consumers or downstream jobs over deliberately. + ### Consumer Offsets After Restore Etcd restores recover committed consumer offsets. If a consumer has **no committed offsets**, it may start at the end and see zero records even though data exists in S3. In production: diff --git a/pkg/storage/recovery.go b/pkg/storage/recovery.go new file mode 100644 index 0000000..743bc3b --- /dev/null +++ b/pkg/storage/recovery.go @@ -0,0 +1,306 @@ +// Copyright 2026 Alexander Alten (novatechflow), NovaTechflow (novatechflow.com). +// This project is supported and financed by Scalytics, Inc. (www.scalytics.io). +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "bytes" + "context" + "encoding/binary" + "fmt" + "path" + "sort" + "strconv" + "strings" + "time" +) + +const segmentHeaderLen = 32 + +// TopicRecoveryConfig defines a segment-granular topic restore into a new topic. +type TopicRecoveryConfig struct { + SourceNamespace string + SourceTopic string + TargetNamespace string + TargetTopic string + RestoreTo time.Time + Partitions []int32 +} + +// RecoveredSegment describes one copied segment/index pair. +type RecoveredSegment struct { + Partition int32 + BaseOffset int64 + LastOffset int64 + SizeBytes int64 + CreatedAt time.Time + SourceKey string + TargetKey string + SourceIndex string + TargetIndex string +} + +// RecoveredPartition summarizes copied data for one partition. +type RecoveredPartition struct { + Partition int32 + SegmentsCopied int + LastOffset int64 + Segments []RecoveredSegment +} + +// TopicRecoveryResult is the outcome of a restore run. +type TopicRecoveryResult struct { + SourceNamespace string + SourceTopic string + TargetNamespace string + TargetTopic string + RestoreTo time.Time + SegmentsCopied int + Partitions []RecoveredPartition +} + +type sourceSegment struct { + RecoveredSegment +} + +// RecoverTopicToTimestamp copies immutable segment/index pairs from one topic to +// another up to the first segment created after the requested cutoff. +func RecoverTopicToTimestamp(ctx context.Context, s3 S3Client, cfg TopicRecoveryConfig) (*TopicRecoveryResult, error) { + if s3 == nil { + return nil, fmt.Errorf("s3 client required") + } + if cfg.SourceTopic == "" { + return nil, fmt.Errorf("source topic required") + } + if cfg.TargetTopic == "" { + return nil, fmt.Errorf("target topic required") + } + if cfg.RestoreTo.IsZero() { + return nil, fmt.Errorf("restore timestamp required") + } + if cfg.SourceNamespace == "" { + cfg.SourceNamespace = "default" + } + if cfg.TargetNamespace == "" { + cfg.TargetNamespace = cfg.SourceNamespace + } + if cfg.SourceNamespace == cfg.TargetNamespace && cfg.SourceTopic == cfg.TargetTopic { + return nil, fmt.Errorf("target topic must differ from source topic") + } + + targetPrefix := path.Join(cfg.TargetNamespace, cfg.TargetTopic) + existing, err := s3.ListSegments(ctx, targetPrefix) + if err != nil { + return nil, err + } + for _, obj := range existing { + if strings.HasSuffix(obj.Key, ".kfs") { + return nil, fmt.Errorf("target topic already has persisted segments under %s", targetPrefix) + } + } + + sourcePrefix := path.Join(cfg.SourceNamespace, cfg.SourceTopic) + objects, err := s3.ListSegments(ctx, sourcePrefix) + if err != nil { + return nil, err + } + + allowedPartitions := make(map[int32]struct{}, len(cfg.Partitions)) + for _, partition := range cfg.Partitions { + allowedPartitions[partition] = struct{}{} + } + + segmentsByPartition := make(map[int32][]sourceSegment) + for _, obj := range objects { + if !strings.HasSuffix(obj.Key, ".kfs") { + continue + } + segment, err := inspectSourceSegment(ctx, s3, obj, cfg.SourceNamespace, cfg.SourceTopic) + if err != nil { + return nil, err + } + if len(allowedPartitions) > 0 { + if _, ok := allowedPartitions[segment.Partition]; !ok { + continue + } + } + segmentsByPartition[segment.Partition] = append(segmentsByPartition[segment.Partition], segment) + } + + partitions := make([]int32, 0, len(segmentsByPartition)) + for partition := range segmentsByPartition { + partitions = append(partitions, partition) + } + sort.Slice(partitions, func(i, j int) bool { return partitions[i] < partitions[j] }) + + result := &TopicRecoveryResult{ + SourceNamespace: cfg.SourceNamespace, + SourceTopic: cfg.SourceTopic, + TargetNamespace: cfg.TargetNamespace, + TargetTopic: cfg.TargetTopic, + RestoreTo: cfg.RestoreTo.UTC(), + Partitions: make([]RecoveredPartition, 0, len(partitions)), + } + + for _, partition := range partitions { + segments := segmentsByPartition[partition] + sort.Slice(segments, func(i, j int) bool { return segments[i].BaseOffset < segments[j].BaseOffset }) + + summary := RecoveredPartition{ + Partition: partition, + LastOffset: -1, + Segments: make([]RecoveredSegment, 0, len(segments)), + } + for _, segment := range segments { + if segment.CreatedAt.After(cfg.RestoreTo) { + break + } + + segmentBytes, err := s3.DownloadSegment(ctx, segment.SourceKey, nil) + if err != nil { + return nil, err + } + indexBytes, err := s3.DownloadIndex(ctx, segment.SourceIndex) + if err != nil { + return nil, err + } + + targetSegmentKey := segmentObjectKey(cfg.TargetNamespace, cfg.TargetTopic, partition, segment.BaseOffset) + targetIndexKey := segmentIndexKey(cfg.TargetNamespace, cfg.TargetTopic, partition, segment.BaseOffset) + if err := s3.UploadSegment(ctx, targetSegmentKey, segmentBytes); err != nil { + return nil, err + } + if err := s3.UploadIndex(ctx, targetIndexKey, indexBytes); err != nil { + return nil, err + } + + copied := segment.RecoveredSegment + copied.TargetKey = targetSegmentKey + copied.TargetIndex = targetIndexKey + summary.Segments = append(summary.Segments, copied) + summary.SegmentsCopied++ + summary.LastOffset = copied.LastOffset + result.SegmentsCopied++ + } + result.Partitions = append(result.Partitions, summary) + } + + return result, nil +} + +func inspectSourceSegment(ctx context.Context, s3 S3Client, obj S3Object, namespace string, topic string) (sourceSegment, error) { + partition, baseOffset, err := parseSegmentLocation(obj.Key, namespace, topic) + if err != nil { + return sourceSegment{}, err + } + if obj.Size < segmentFooterLen { + return sourceSegment{}, fmt.Errorf("segment %s too small", obj.Key) + } + + headerBytes, err := s3.DownloadSegment(ctx, obj.Key, &ByteRange{Start: 0, End: segmentHeaderLen - 1}) + if err != nil { + return sourceSegment{}, err + } + createdAt, err := parseSegmentHeaderCreatedAt(headerBytes) + if err != nil { + return sourceSegment{}, err + } + + start := obj.Size - segmentFooterLen + footerBytes, err := s3.DownloadSegment(ctx, obj.Key, &ByteRange{Start: start, End: obj.Size - 1}) + if err != nil { + return sourceSegment{}, err + } + lastOffset, err := parseSegmentFooter(footerBytes) + if err != nil { + return sourceSegment{}, err + } + + return sourceSegment{ + RecoveredSegment: RecoveredSegment{ + Partition: partition, + BaseOffset: baseOffset, + LastOffset: lastOffset, + SizeBytes: obj.Size, + CreatedAt: createdAt, + SourceKey: obj.Key, + SourceIndex: segmentIndexKey(namespace, topic, partition, baseOffset), + }, + }, nil +} + +func parseSegmentLocation(key string, namespace string, topic string) (int32, int64, error) { + prefix := path.Join(namespace, topic) + "/" + if !strings.HasPrefix(key, prefix) { + return 0, 0, fmt.Errorf("segment %s not under %s", key, prefix) + } + trimmed := strings.TrimPrefix(key, prefix) + parts := strings.Split(trimmed, "/") + if len(parts) != 2 { + return 0, 0, fmt.Errorf("segment %s has unexpected layout", key) + } + partition, err := strconv.ParseInt(parts[0], 10, 32) + if err != nil { + return 0, 0, fmt.Errorf("parse partition from %s: %w", key, err) + } + baseOffset, ok := parseSegmentBaseOffset(parts[1]) + if !ok { + return 0, 0, fmt.Errorf("parse base offset from %s", key) + } + return int32(partition), baseOffset, nil +} + +func parseSegmentHeaderCreatedAt(data []byte) (time.Time, error) { + if len(data) < segmentHeaderLen { + return time.Time{}, fmt.Errorf("header too small") + } + reader := bytes.NewReader(data) + magic := make([]byte, len(segmentMagic)) + if _, err := reader.Read(magic); err != nil { + return time.Time{}, err + } + if string(magic) != segmentMagic { + return time.Time{}, fmt.Errorf("invalid segment magic") + } + var version uint16 + if err := binary.Read(reader, binary.BigEndian, &version); err != nil { + return time.Time{}, err + } + var flags uint16 + if err := binary.Read(reader, binary.BigEndian, &flags); err != nil { + return time.Time{}, err + } + var baseOffset int64 + if err := binary.Read(reader, binary.BigEndian, &baseOffset); err != nil { + return time.Time{}, err + } + var messageCount int32 + if err := binary.Read(reader, binary.BigEndian, &messageCount); err != nil { + return time.Time{}, err + } + var createdAtMillis int64 + if err := binary.Read(reader, binary.BigEndian, &createdAtMillis); err != nil { + return time.Time{}, err + } + return time.UnixMilli(createdAtMillis).UTC(), nil +} + +func segmentObjectKey(namespace string, topic string, partition int32, baseOffset int64) string { + return path.Join(namespace, topic, fmt.Sprintf("%d", partition), fmt.Sprintf("segment-%020d.kfs", baseOffset)) +} + +func segmentIndexKey(namespace string, topic string, partition int32, baseOffset int64) string { + return path.Join(namespace, topic, fmt.Sprintf("%d", partition), fmt.Sprintf("segment-%020d.index", baseOffset)) +} diff --git a/pkg/storage/recovery_test.go b/pkg/storage/recovery_test.go new file mode 100644 index 0000000..47e2de6 --- /dev/null +++ b/pkg/storage/recovery_test.go @@ -0,0 +1,110 @@ +// Copyright 2026 Alexander Alten (novatechflow), NovaTechflow (novatechflow.com). +// This project is supported and financed by Scalytics, Inc. (www.scalytics.io). +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "context" + "testing" + "time" +) + +func TestRecoverTopicToTimestampCopiesEligibleSegments(t *testing.T) { + s3 := NewMemoryS3Client() + older := time.Date(2026, 5, 13, 12, 0, 0, 0, time.UTC) + newer := older.Add(30 * time.Minute) + + uploadRecoverySegment(t, s3, "default", "orders", 0, 0, older) + uploadRecoverySegment(t, s3, "default", "orders", 0, 1, newer) + + result, err := RecoverTopicToTimestamp(context.Background(), s3, TopicRecoveryConfig{ + SourceNamespace: "default", + SourceTopic: "orders", + TargetNamespace: "default", + TargetTopic: "orders-restore", + RestoreTo: older.Add(10 * time.Minute), + }) + if err != nil { + t.Fatalf("RecoverTopicToTimestamp: %v", err) + } + if result.SegmentsCopied != 1 { + t.Fatalf("expected 1 copied segment, got %d", result.SegmentsCopied) + } + if len(result.Partitions) != 1 { + t.Fatalf("expected 1 partition summary, got %d", len(result.Partitions)) + } + partition := result.Partitions[0] + if partition.Partition != 0 { + t.Fatalf("expected partition 0, got %d", partition.Partition) + } + if partition.LastOffset != 0 { + t.Fatalf("expected last offset 0, got %d", partition.LastOffset) + } + if partition.SegmentsCopied != 1 { + t.Fatalf("expected 1 copied segment for partition, got %d", partition.SegmentsCopied) + } + + segKey := "default/orders-restore/0/segment-00000000000000000000.kfs" + if _, err := s3.DownloadSegment(context.Background(), segKey, nil); err != nil { + t.Fatalf("download restored segment: %v", err) + } + if _, err := s3.DownloadIndex(context.Background(), "default/orders-restore/0/segment-00000000000000000000.index"); err != nil { + t.Fatalf("download restored index: %v", err) + } + if _, err := s3.DownloadSegment(context.Background(), "default/orders-restore/0/segment-00000000000000000001.kfs", nil); err == nil { + t.Fatal("expected newer segment to be excluded") + } +} + +func TestRecoverTopicToTimestampRejectsExistingTarget(t *testing.T) { + s3 := NewMemoryS3Client() + created := time.Date(2026, 5, 13, 12, 0, 0, 0, time.UTC) + uploadRecoverySegment(t, s3, "default", "orders", 0, 0, created) + uploadRecoverySegment(t, s3, "default", "orders-restore", 0, 0, created) + + _, err := RecoverTopicToTimestamp(context.Background(), s3, TopicRecoveryConfig{ + SourceNamespace: "default", + SourceTopic: "orders", + TargetNamespace: "default", + TargetTopic: "orders-restore", + RestoreTo: created.Add(time.Minute), + }) + if err == nil { + t.Fatal("expected existing target restore to fail") + } +} + +func uploadRecoverySegment(t *testing.T, s3 *MemoryS3Client, namespace string, topic string, partition int32, baseOffset int64, created time.Time) { + t.Helper() + + artifact, err := BuildSegment(SegmentWriterConfig{IndexIntervalMessages: 1}, []RecordBatch{ + { + BaseOffset: baseOffset, + LastOffsetDelta: 0, + MessageCount: 1, + Bytes: make([]byte, 70), + }, + }, created) + if err != nil { + t.Fatalf("BuildSegment: %v", err) + } + + if err := s3.UploadSegment(context.Background(), segmentObjectKey(namespace, topic, partition, baseOffset), artifact.SegmentBytes); err != nil { + t.Fatalf("UploadSegment: %v", err) + } + if err := s3.UploadIndex(context.Background(), segmentIndexKey(namespace, topic, partition, baseOffset), artifact.IndexBytes); err != nil { + t.Fatalf("UploadIndex: %v", err) + } +}