diff --git a/cmd/neofs-node/config.go b/cmd/neofs-node/config.go index fc815bace0..250f41c1e3 100644 --- a/cmd/neofs-node/config.go +++ b/cmd/neofs-node/config.go @@ -41,6 +41,7 @@ import ( "github.com/nspcc-dev/neofs-node/pkg/services/replicator" trustcontroller "github.com/nspcc-dev/neofs-node/pkg/services/reputation/local/controller" truststorage "github.com/nspcc-dev/neofs-node/pkg/services/reputation/local/storage" + "github.com/nspcc-dev/neofs-node/pkg/services/sidechain" "github.com/nspcc-dev/neofs-node/pkg/timers" "github.com/nspcc-dev/neofs-node/pkg/util" "github.com/nspcc-dev/neofs-node/pkg/util/state" @@ -175,6 +176,7 @@ type shared struct { control *controlSvc.Server metaService *meta.Meta + sidechain *sidechain.SideChain containerPayments *paymentChecker } @@ -200,7 +202,6 @@ type cfg struct { // configuration of the internal // services cfgGRPC cfgGRPC - cfgMeta cfgMeta cfgMorph cfgMorph cfgBalance cfgBalance cfgContainer cfgContainer @@ -253,10 +254,6 @@ func (g *cfgGRPC) registerService(f func(*grpc.Server)) { } } -type cfgMeta struct { - network meta.NeoFSNetwork -} - type cfgMorph struct { client *client.Client @@ -731,16 +728,6 @@ func (c *cfg) configWatcher(ctx context.Context) { continue } - // Meta service - - var p meta.Parameters - p.NeoEnpoints = c.appCfg.FSChain.Endpoints - err = c.metaService.Reload(p) - if err != nil { - c.log.Error("failed to reload meta service configuration", zap.Error(err)) - continue - } - // gRPC if err = reloadGRPC(c, oldGRPC); err != nil { diff --git a/cmd/neofs-node/config/meta/config_test.go b/cmd/neofs-node/config/meta/config_test.go index 3ab4217cc3..59ac8a75b6 100644 --- a/cmd/neofs-node/config/meta/config_test.go +++ b/cmd/neofs-node/config/meta/config_test.go @@ -12,12 +12,18 @@ func TestLoggerSection_Level(t *testing.T) { t.Run("defaults", func(t *testing.T) { emptyConfig := configtest.EmptyConfig() require.Equal(t, "", emptyConfig.Meta.Path) + require.Zero(t, emptyConfig.Meta.SeedPort) + require.Nil(t, emptyConfig.Meta.P2PAddresses) + require.Nil(t, emptyConfig.Meta.RPCAddresses) }) const path = "../../../../config/example/node" var fileConfigTest = func(c *config.Config) { require.Equal(t, "path/to/meta", c.Meta.Path) + require.Equal(t, uint64(20334), c.Meta.SeedPort) + require.Equal(t, []string{"node1:20334", "node2:20334"}, c.Meta.P2PAddresses) + require.Equal(t, []string{"localhost:30334"}, c.Meta.RPCAddresses) } configtest.ForEachFileType(path, fileConfigTest) diff --git a/cmd/neofs-node/config/meta/meta.go b/cmd/neofs-node/config/meta/meta.go index a3a0a3049c..e8e690caf8 100644 --- a/cmd/neofs-node/config/meta/meta.go +++ b/cmd/neofs-node/config/meta/meta.go @@ -3,4 +3,17 @@ package metaconfig // Meta contains configuration for Meta service. type Meta struct { Path string `mapstructure:"path"` + + // SeedPort of metadata chain network. Addresses will be used the same as + // the configured ones in FS chain. + SeedPort uint64 `mapstructure:"seed_port"` + + // Network addresses to listen Neo metadata P2P on list in the form of + // "[host]:[port][:announcedPort]". + P2PAddresses []string `mapstructure:"p2p_addresses"` + + // Neo RPC service configuration. + // + // Optional. + RPCAddresses []string `mapstructure:"rpc_addresses"` } diff --git a/cmd/neofs-node/main.go b/cmd/neofs-node/main.go index 8d051519cb..5825816b3a 100644 --- a/cmd/neofs-node/main.go +++ b/cmd/neofs-node/main.go @@ -143,7 +143,7 @@ func initApp(c *cfg) { initAndLog(c, "accounting", initAccountingService) initAndLog(c, "session", initSessionService) initAndLog(c, "reputation", initReputationService) - initAndLog(c, "meta", initMeta) + initAndLog(c, "meta sidechain", initMeta) initAndLog(c, "object", initObjectService) initAndLog(c, "morph notifications", listenMorphNotifications) diff --git a/cmd/neofs-node/meta.go b/cmd/neofs-node/meta.go index 43d7c963de..859f250bc4 100644 --- a/cmd/neofs-node/meta.go +++ b/cmd/neofs-node/meta.go @@ -4,14 +4,24 @@ import ( "bytes" "context" "fmt" + "math" + "net" + "path" "slices" + "strconv" "sync" + "time" + "github.com/nspcc-dev/neo-go/pkg/config" + "github.com/nspcc-dev/neo-go/pkg/core/native/noderoles" + "github.com/nspcc-dev/neo-go/pkg/core/storage/dbconfig" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neofs-node/pkg/core/container" "github.com/nspcc-dev/neofs-node/pkg/core/netmap" cntClient "github.com/nspcc-dev/neofs-node/pkg/morph/client/container" - "github.com/nspcc-dev/neofs-node/pkg/services/meta" + meta "github.com/nspcc-dev/neofs-node/pkg/services/meta" getsvc "github.com/nspcc-dev/neofs-node/pkg/services/object/get" + "github.com/nspcc-dev/neofs-node/pkg/services/sidechain" containerSDK "github.com/nspcc-dev/neofs-sdk-go/container" cid "github.com/nspcc-dev/neofs-sdk-go/container/id" netmapsdk "github.com/nspcc-dev/neofs-sdk-go/netmap" @@ -21,43 +31,6 @@ import ( "golang.org/x/sync/errgroup" ) -func initMeta(c *cfg) { - if c.cfgMorph.client == nil { - initMorphComponents(c) - } - - c.cfgMeta.network = &neofsNetwork{ - key: c.binPublicKey, - cnrClient: c.cCli, - containers: c.cnrSrc, - network: c.netMapSource, - header: c.cfgObject.getSvc, - } - - var err error - p := meta.Parameters{ - Logger: c.log.With(zap.String("service", "meta data")), - Network: c.cfgMeta.network, - Timeout: c.appCfg.FSChain.DialTimeout, - NeoEnpoints: c.appCfg.FSChain.Endpoints, - ContainerHash: c.containerSH, - NetmapHash: c.netmapSH, - RootPath: c.appCfg.Meta.Path, - } - if p.RootPath == "" { - p.RootPath = "metadata" - } - c.metaService, err = meta.New(p) - fatalOnErr(err) - - c.workers = append(c.workers, newWorkerFromFunc(func(ctx context.Context) { - err = c.metaService.Run(ctx) - if err != nil { - c.internalErr <- fmt.Errorf("meta data service error: %w", err) - } - })) -} - type neofsNetwork struct { key []byte @@ -90,7 +63,7 @@ func (c *neofsNetwork) Head(ctx context.Context, cID cid.ID, oID oid.ID) (object return *hw.h, nil } -func (c *neofsNetwork) IsMineWithMeta(id cid.ID, cData []byte) (bool, error) { +func (c *neofsNetwork) IsMineWithMeta(id cid.ID, _ []byte) (bool, error) { curEpoch, err := c.network.Epoch() if err != nil { return false, fmt.Errorf("read current NeoFS epoch: %w", err) @@ -99,10 +72,9 @@ func (c *neofsNetwork) IsMineWithMeta(id cid.ID, cData []byte) (bool, error) { if err != nil { return false, fmt.Errorf("read network map at the current epoch #%d: %w", curEpoch, err) } - var cnr containerSDK.Container - err = cnr.Unmarshal(cData) + cnr, err := c.containers.Get(id) if err != nil { - return false, fmt.Errorf("unmarshal container: %w", err) + return false, fmt.Errorf("get container: %w", err) } return c.isMineWithMeta(id, cnr, networkMap), nil } @@ -190,3 +162,185 @@ func (c *neofsNetwork) List(e uint64) (map[cid.ID]struct{}, error) { return res, nil } + +func initMeta(c *cfg) { + l := c.log.With(zap.String("component", "metadata chain (SN)")) + + v, err := c.cfgMorph.client.GetVersion() + fatalOnErr(err) + + fsChainProtocol := v.Protocol + standByCommittee := make([]string, 0, len(v.Protocol.StandbyCommittee)) + for _, c := range v.Protocol.StandbyCommittee { + standByCommittee = append(standByCommittee, c.StringCompressed()) + } + + if c.appCfg.Meta.SeedPort >= math.MaxUint16 { + fatalOnErr(fmt.Errorf("seed port exceeds max number %d: %d", math.MaxUint16, c.appCfg.Meta.SeedPort)) + } + + seedList, err := changePort(fsChainProtocol.SeedList, uint16(c.appCfg.Meta.SeedPort)) + fatalOnErr(err) + + var chainCfg = config.Config{ + ProtocolConfiguration: config.ProtocolConfiguration{ + CommitteeHistory: nil, + Genesis: config.Genesis{ + MaxTraceableBlocks: fsChainProtocol.MaxTraceableBlocks, + MaxValidUntilBlockIncrement: fsChainProtocol.MaxValidUntilBlockIncrement, + Roles: map[noderoles.Role]keys.PublicKeys{ + noderoles.P2PNotary: v.Protocol.StandbyCommittee, + noderoles.NeoFSAlphabet: v.Protocol.StandbyCommittee, + }, + TimePerBlock: 50 * time.Millisecond, + }, + Magic: fsChainProtocol.Network + 1, + MemPoolSize: 0, + Hardforks: nil, + InitialGASSupply: 0, + P2PNotaryRequestPayloadPoolSize: 1000, + MaxBlockSize: 0, + MaxBlockSystemFee: 0, + MaxTraceableBlocks: fsChainProtocol.MaxTraceableBlocks, + MaxTransactionsPerBlock: 0, + MaxValidUntilBlockIncrement: fsChainProtocol.MaxValidUntilBlockIncrement, + P2PSigExtensions: true, + P2PStateExchangeExtensions: false, + NeoFSStateSyncExtensions: false, + ReservedAttributes: false, + SeedList: seedList, + StandbyCommittee: standByCommittee, + StateRootInHeader: false, + StateSyncInterval: 0, + TimePerBlock: 50 * time.Millisecond, + MaxTimePerBlock: 20 * time.Second, + ValidatorsCount: uint32(len(standByCommittee)), + ValidatorsHistory: nil, + VerifyTransactions: true, + }, + ApplicationConfiguration: config.ApplicationConfiguration{ + P2P: config.P2P{ + Addresses: c.appCfg.Meta.P2PAddresses, + MinPeers: max(1, len(seedList)), + BroadcastTxsBatchDelay: 5 * time.Millisecond, + AttemptConnPeers: 100, + }, + Relay: true, + Oracle: config.OracleConfiguration{}, + P2PNotary: config.P2PNotary{}, + StateRoot: config.StateRoot{}, + NeoFSBlockFetcher: config.NeoFSBlockFetcher{}, + NeoFSStateFetcher: config.NeoFSStateFetcher{}, + }, + } + + var cfgDB = dbconfig.DBConfiguration{ + Type: dbconfig.BoltDB, + BoltDBOptions: dbconfig.BoltDBOptions{ + FilePath: path.Join(c.appCfg.Meta.Path, "chain_bolt.db"), + }, + } + chainCfg.ApplicationConfiguration.DBConfiguration = cfgDB + + if len(c.appCfg.Meta.RPCAddresses) > 0 { + var rpcConfig config.RPC + rpcConfig.BasicService = config.BasicService{ + Enabled: true, + Addresses: c.appCfg.Meta.RPCAddresses, + } + + chainCfg.ApplicationConfiguration.RPC = rpcConfig + } + + applyMetachainDefaults(&chainCfg) + + err = chainCfg.ProtocolConfiguration.Validate() + fatalOnErr(err) + + ch, err := sidechain.New(chainCfg, l, c.internalErr) + fatalOnErr(err) + + c.sidechain = ch + + c.workers = append(c.workers, &workerFromFunc{ + fn: func(ctx context.Context) { + err := ch.Run(ctx) + if err != nil { + c.internalErr <- err + } + }, + }) + c.closers = append(c.closers, func() { + c.sidechain.Stop() + }) + + c.metaService, err = meta.New(meta.Parameters{ + Logger: c.log.With(zap.String("component", "metadata service")), + Chain: c.sidechain, + Path: c.appCfg.Meta.Path, + Network: &neofsNetwork{ + key: c.binPublicKey, + cnrClient: c.cCli, + containers: c.cnrSrc, + network: c.netMapSource, + header: c.cfgObject.getSvc, + }, + }) + fatalOnErr(err) + + c.workers = append(c.workers, newWorkerFromFunc(func(ctx context.Context) { + err = c.metaService.Run(ctx) + if err != nil { + c.internalErr <- fmt.Errorf("meta data service error: %w", err) + } + })) +} + +func changePort(addrs []string, port uint16) ([]string, error) { + res := slices.Clone(addrs) + for i := range res { + host, _, err := net.SplitHostPort(res[i]) + if err != nil { + return nil, fmt.Errorf("[%d] address ('%s') cannot be parsed: %w", i, res[i], err) + } + res[i] = net.JoinHostPort(host, strconv.FormatUint(uint64(port), 10)) + } + + return res, nil +} + +func applyMetachainDefaults(cfg *config.Config) { + if cfg.ApplicationConfiguration.P2P.MaxPeers == 0 { + cfg.ApplicationConfiguration.P2P.MaxPeers = 100 + } + if cfg.ApplicationConfiguration.P2P.AttemptConnPeers == 0 { + cfg.ApplicationConfiguration.P2P.AttemptConnPeers = cfg.ApplicationConfiguration.P2P.MinPeers + 10 + } + if cfg.ApplicationConfiguration.P2P.DialTimeout == 0 { + cfg.ApplicationConfiguration.P2P.DialTimeout = time.Minute + } + if cfg.ApplicationConfiguration.P2P.ProtoTickInterval == 0 { + cfg.ApplicationConfiguration.P2P.ProtoTickInterval = 2 * time.Second + } + if cfg.ProtocolConfiguration.MaxTraceableBlocks == 0 { + cfg.ProtocolConfiguration.MaxTraceableBlocks = 17280 + } + if cfg.ProtocolConfiguration.MaxValidUntilBlockIncrement == 0 { + cfg.ProtocolConfiguration.MaxValidUntilBlockIncrement = 8640 + } + if cfg.ApplicationConfiguration.P2P.PingInterval == 0 { + cfg.ApplicationConfiguration.P2P.PingInterval = 30 * time.Second + } + if cfg.ApplicationConfiguration.P2P.PingTimeout == 0 { + cfg.ApplicationConfiguration.P2P.PingTimeout = time.Minute + } + if cfg.ApplicationConfiguration.RPC.MaxWebSocketClients == 0 { + cfg.ApplicationConfiguration.RPC.MaxWebSocketClients = 64 + } + if cfg.ApplicationConfiguration.RPC.SessionPoolSize == 0 { + cfg.ApplicationConfiguration.RPC.SessionPoolSize = 20 + } + if cfg.ApplicationConfiguration.RPC.MaxGasInvoke == 0 { + cfg.ApplicationConfiguration.RPC.MaxGasInvoke = 100 + } +} diff --git a/cmd/neofs-node/object.go b/cmd/neofs-node/object.go index 8e70c94131..abc881f5c7 100644 --- a/cmd/neofs-node/object.go +++ b/cmd/neofs-node/object.go @@ -25,7 +25,7 @@ import ( containerClient "github.com/nspcc-dev/neofs-node/pkg/morph/client/container" netmapClient "github.com/nspcc-dev/neofs-node/pkg/morph/client/netmap" "github.com/nspcc-dev/neofs-node/pkg/morph/event" - "github.com/nspcc-dev/neofs-node/pkg/services/meta" + meta "github.com/nspcc-dev/neofs-node/pkg/services/meta" objectService "github.com/nspcc-dev/neofs-node/pkg/services/object" "github.com/nspcc-dev/neofs-node/pkg/services/object/acl" v2 "github.com/nspcc-dev/neofs-node/pkg/services/object/acl/v2" @@ -263,14 +263,10 @@ func initObjectService(c *cfg) { fatalOnErr(err) c.cfgObject.containerNodes = cnrNodes - mNumber, err := c.cli.MagicNumber() - fatalOnErr(err) - os := &objectSource{signer: neofsecdsa.SignerRFC6979(c.key.PrivateKey), get: sGet} sPut := putsvc.NewService(&transport{clients: putConstructor}, c, c.metaService, initQuotas(c.cCli, c.cfgObject.quotasTTL), c.containerPayments, - putsvc.WithNetworkMagic(mNumber), putsvc.WithKeyStorage(keyStorage), putsvc.WithClientConstructor(putConstructor), putsvc.WithContainerClient(c.cCli), @@ -345,7 +341,7 @@ func initObjectService(c *cfg) { putSvc: sPut, keys: keyStorage, } - server := objectService.New(objSvc, mNumber, c.cfgObject.pool.search, fsChain, storage, c.metaService, c.key.PrivateKey, c.metricsCollector, aclChecker, aclSvc, coreConstructor) + server := objectService.New(objSvc, c.cfgObject.pool.search, fsChain, storage, c.metaService, c.key.PrivateKey, c.metricsCollector, aclChecker, aclSvc, coreConstructor) os.server = server svcDesc := protoobject.ObjectService_ServiceDesc diff --git a/config/example/node.env b/config/example/node.env index 40f334e997..647c395251 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -26,6 +26,9 @@ NEOFS_NODE_PERSISTENT_STATE_PATH=/state # Meta data section NEOFS_METADATA_PATH=path/to/meta +NEOFS_METADATA_SEED_PORT=20334 +NEOFS_METADATA_P2P_ADDRESSES="node1:20334 node2:20334" +NEOFS_METADATA_RPC_ADDRESSES="localhost:30334" # gRPC section ## 0 server diff --git a/config/example/node.json b/config/example/node.json index c9449378fe..7a9c5bbdb7 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -42,7 +42,15 @@ } }, "metadata": { - "path": "path/to/meta" + "path": "path/to/meta", + "seed_port": "20334", + "p2p_addresses": [ + "node1:20334", + "node2:20334" + ], + "rpc_addresses": [ + "localhost:30334" + ] }, "grpc": [ { diff --git a/config/example/node.yaml b/config/example/node.yaml index 09c7fc14b2..2756647130 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -100,6 +100,11 @@ object: metadata: path: path/to/meta # path to meta data storages, required + seed_port: 20334 + p2p_addresses: + - "node1:20334" + - "node2:20334" + rpc_addresses: "localhost:30334" storage: # note: shard configuration can be omitted for relay node (see `node.relay`) diff --git a/pkg/core/object/replicate.go b/pkg/core/object/replicate.go index 7c49360bf8..96155a3ed5 100644 --- a/pkg/core/object/replicate.go +++ b/pkg/core/object/replicate.go @@ -3,6 +3,16 @@ package objectcore import ( "fmt" + "github.com/nspcc-dev/neo-go/pkg/core/native" + "github.com/nspcc-dev/neo-go/pkg/core/state" + "github.com/nspcc-dev/neo-go/pkg/core/transaction" + "github.com/nspcc-dev/neo-go/pkg/crypto/hash" + "github.com/nspcc-dev/neo-go/pkg/io" + "github.com/nspcc-dev/neo-go/pkg/smartcontract" + "github.com/nspcc-dev/neo-go/pkg/smartcontract/callflag" + "github.com/nspcc-dev/neo-go/pkg/util" + "github.com/nspcc-dev/neo-go/pkg/vm/emit" + "github.com/nspcc-dev/neo-go/pkg/vm/opcode" "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" cid "github.com/nspcc-dev/neofs-sdk-go/container/id" "github.com/nspcc-dev/neofs-sdk-go/object" @@ -25,7 +35,43 @@ const ( typeKey = "type" ) -// EncodeReplicationMetaInfo uses NEO's map (strict order) serialized format as a raw +var metaHash = state.CreateNativeContractHash("MetaData") + +// MetaVerifScript build verification script for metadata chain transactions. +func MetaVerifScript(cID []byte, placementVectorsNumber int) []byte { + var ( + verifScriptBuf = io.NewBufBinWriter() + writer = verifScriptBuf.BinWriter + ) + emit.Int(writer, int64(placementVectorsNumber)) // sigs array length + emit.Opcodes(writer, opcode.PACK) + emit.Bytes(writer, cID) + emit.Int(writer, 2) // number or args + emit.Opcodes(writer, opcode.PACK) + emit.AppCallNoArgs(writer, metaHash, "verifyPlacementSignatures", callflag.ReadOnly) + + return verifScriptBuf.Bytes() +} + +// EncodeChainMetaInfo creates transaction for metadata and byte slice +// that should be signed for this transaction. +func EncodeChainMetaInfo(numberOfPlacementVectors int, cID cid.ID, oID, firstPart, previousPart oid.ID, pSize uint64, typ object.Type, + deleted, locked oid.ID, vub uint64, magicNumber uint32) (*transaction.Transaction, []byte) { + metadata := EncodeObjectMetadata(cID, oID, firstPart, previousPart, pSize, typ, deleted, locked, vub, magicNumber) + + verifScript := MetaVerifScript(cID[:], numberOfPlacementVectors) + tx, err := objectTransaction(hash.Hash160(verifScript), metadata, uint32(vub)) + if err != nil { + panic(fmt.Errorf("making transaction: %w", err)) + } + tx.Scripts = append(tx.Scripts[:0], transaction.Witness{ + VerificationScript: verifScript, + }) + + return tx, hash.GetSignedData(magicNumber, tx) +} + +// EncodeObjectMetadata uses NEO's map (strict order) serialized format as a raw // representation of object's meta information. // // This (ordered) format is used (keys are strings): @@ -40,8 +86,8 @@ const ( // "deleted": [OPTIONAL] array of _raw_ object IDs // "locked": [OPTIONAL] array of _raw_ object IDs // "type": [OPTIONAL] object type enumeration -func EncodeReplicationMetaInfo(cID cid.ID, oID, firstPart, previousPart oid.ID, pSize uint64, typ object.Type, - deleted, locked []oid.ID, vub uint64, magicNumber uint32) []byte { +func EncodeObjectMetadata(cID cid.ID, oID, firstPart, previousPart oid.ID, pSize uint64, typ object.Type, + deleted, locked oid.ID, vub uint64, magicNumber uint32) []byte { kvs := []stackitem.MapElement{ kv(cidKey, cID[:]), kv(oidKey, oID[:]), @@ -56,11 +102,11 @@ func EncodeReplicationMetaInfo(cID cid.ID, oID, firstPart, previousPart oid.ID, if !previousPart.IsZero() { kvs = append(kvs, kv(previousPartKey, previousPart[:])) } - if len(deleted) > 0 { - kvs = append(kvs, oidsKV(deletedKey, deleted)) + if !deleted.IsZero() { + kvs = append(kvs, kv(deletedKey, deleted[:])) } - if len(locked) > 0 { - kvs = append(kvs, oidsKV(lockedKey, locked)) + if !locked.IsZero() { + kvs = append(kvs, kv(lockedKey, locked[:])) } if typ != object.TypeRegular { kvs = append(kvs, kv(typeKey, uint32(typ))) @@ -77,18 +123,28 @@ func EncodeReplicationMetaInfo(cID cid.ID, oID, firstPart, previousPart oid.ID, return result } +func objectTransaction(acc util.Uint160, metaData []byte, vub uint32) (*transaction.Transaction, error) { + script, err := smartcontract.CreateCallScript(metaHash, "submitObjectPut", metaData) + if err != nil { + return nil, fmt.Errorf("making transaction script: %w", err) + } + + tx := transaction.New(script, 0) + tx.Nonce = vub + tx.ValidUntilBlock = vub + tx.Signers = append(tx.Signers, transaction.Signer{ + Account: acc, + Scopes: transaction.Global, + }) + tx.SystemFee = 30 * native.GASFactor + tx.NetworkFee = 30 * native.GASFactor + + return tx, nil +} + func kv(k string, value any) stackitem.MapElement { return stackitem.MapElement{ Key: stackitem.Make(k), Value: stackitem.Make(value), } } - -func oidsKV(fieldKey string, oIDs []oid.ID) stackitem.MapElement { - res := make([]stackitem.Item, 0, len(oIDs)) - for _, oID := range oIDs { - res = append(res, stackitem.NewByteArray(oID[:])) - } - - return kv(fieldKey, res) -} diff --git a/pkg/core/object/replicate_test.go b/pkg/core/object/replicate_test.go index 67eaffaa90..e49194f05a 100644 --- a/pkg/core/object/replicate_test.go +++ b/pkg/core/object/replicate_test.go @@ -23,8 +23,8 @@ type m struct { first oid.ID prev oid.ID - deleted []oid.ID - locked []oid.ID + deleted oid.ID + locked oid.ID typ object.Type } @@ -37,8 +37,8 @@ func TestMetaInfo(t *testing.T) { magic: rand.Uint32(), first: oidtest.ID(), prev: oidtest.ID(), - deleted: oidtest.IDs(10), - locked: oidtest.IDs(10), + deleted: oidtest.ID(), + locked: oidtest.ID(), typ: object.TypeTombstone, } @@ -49,9 +49,9 @@ func TestMetaInfo(t *testing.T) { t.Run("no optional", func(t *testing.T) { meta.first = oid.ID{} meta.prev = oid.ID{} - meta.deleted = nil - meta.deleted = nil - meta.locked = nil + meta.deleted = oid.ID{} + meta.deleted = oid.ID{} + meta.locked = oid.ID{} meta.typ = object.TypeRegular testMeta(t, meta, false) @@ -59,7 +59,7 @@ func TestMetaInfo(t *testing.T) { } func testMeta(t *testing.T, m m, full bool) { - raw := EncodeReplicationMetaInfo(m.cID, m.oID, m.first, m.prev, m.size, m.typ, m.deleted, m.locked, m.vub, m.magic) + raw := EncodeObjectMetadata(m.cID, m.oID, m.first, m.prev, m.size, m.typ, m.deleted, m.locked, m.vub, m.magic) item, err := stackitem.Deserialize(raw) require.NoError(t, err) @@ -94,26 +94,11 @@ func testMeta(t *testing.T, m m, full bool) { require.Equal(t, m.prev[:], mm[6].Value.Value().([]byte)) require.Equal(t, deletedKey, string(mm[7].Key.Value().([]byte))) - require.Equal(t, m.deleted, stackItemToOIDs(t, mm[7].Value)) + require.Equal(t, m.deleted[:], mm[7].Value.Value().([]byte)) require.Equal(t, lockedKey, string(mm[8].Key.Value().([]byte))) - require.Equal(t, m.locked, stackItemToOIDs(t, mm[8].Value)) + require.Equal(t, m.locked[:], mm[8].Value.Value().([]byte)) require.Equal(t, typeKey, string(mm[9].Key.Value().([]byte))) require.Equal(t, int(m.typ), int(mm[9].Value.Value().(*big.Int).Uint64())) } - -func stackItemToOIDs(t *testing.T, value stackitem.Item) []oid.ID { - value, ok := value.(*stackitem.Array) - require.True(t, ok) - - vv := value.Value().([]stackitem.Item) - res := make([]oid.ID, 0, len(vv)) - - for _, v := range vv { - raw := v.Value().([]byte) - res = append(res, oid.ID(raw)) - } - - return res -} diff --git a/pkg/services/meta/blocks.go b/pkg/services/meta/blocks.go index 404ae0371d..352bbd987a 100644 --- a/pkg/services/meta/blocks.go +++ b/pkg/services/meta/blocks.go @@ -2,167 +2,70 @@ package meta import ( "context" - "fmt" + "time" "github.com/nspcc-dev/neo-go/pkg/core/block" - "github.com/nspcc-dev/neo-go/pkg/neorpc" - cid "github.com/nspcc-dev/neofs-sdk-go/container/id" + "github.com/nspcc-dev/neo-go/pkg/core/state" oid "github.com/nspcc-dev/neofs-sdk-go/object/id" "go.uber.org/zap" ) -func (m *Meta) handleBlock(b *block.Header) error { - h := b.Hash() - ind := b.Index - l := m.l.With(zap.Stringer("block hash", h), zap.Uint32("index", ind)) - l.Debug("handling block") - - m.cliM.RLock() - res, err := m.ws.GetBlockNotifications(h, &neorpc.NotificationFilter{ - Contract: &m.cnrH, - }) - if err != nil { - m.cliM.RUnlock() - return fmt.Errorf("fetching %s block: %w", h, err) - } - m.cliM.RUnlock() - - if len(res.Application) == 0 { - l.Debug("no notifications for block") - return nil - } - - evsByCID := make(map[cid.ID]blockObjEvents) - for _, n := range res.Application { - l := m.l.With(zap.Stringer("tx", n.Container)) - - switch n.Name { - case objPutEvName: - ev, err := parseObjNotification(n) - if err != nil { - l.Error("invalid object notification received", zap.Error(err)) - continue - } - - if magic := uint32(ev.network.Uint64()); magic != m.magicNumber { - l.Warn("skipping object notification with wrong magic number", zap.Uint32("expected", m.magicNumber), zap.Uint32("got", magic)) - continue - } - - m.notifier.notifyReceived(oid.NewAddress(ev.cID, ev.oID)) - - m.stM.RLock() - _, ok := m.storages[ev.cID] - m.stM.RUnlock() - if !ok { - l.Debug("skipping object notification", zap.Stringer("container", ev.cID)) - continue - } - - m.l.Debug("received object notification", zap.Stringer("address", oid.NewAddress(ev.cID, ev.oID))) - - blockEvents, ok := evsByCID[ev.cID] - if !ok { - blockEvents = blockObjEvents{cID: ev.cID, bInd: ind} - } - blockEvents.evs = append(blockEvents.evs, ev) - evsByCID[ev.cID] = blockEvents - case cnrDeleteName, cnrRmName: - ev, err := parseCnrNotification(n) - if err != nil { - l.Error("invalid container removal notification received", zap.Error(err)) - continue - } - - m.stM.RLock() - _, ok := m.storages[ev.cID] - m.stM.RUnlock() - if !ok { - l.Debug("skipping container notification", zap.Stringer("container", ev.cID)) - continue - } - - err = m.dropContainer(ev.cID) - if err != nil { - l.Error("deleting container failed", zap.Error(err)) - continue - } - - l.Debug("deleted container", zap.Stringer("cID", ev.cID)) - case cnrPutName, cnrCrtName: - ev, err := parseCnrNotification(n) - if err != nil { - l.Error("invalid container notification received", zap.Error(err)) - continue - } - - go m.addContainerIfMine(l, ev.cID) - default: - l.Debug("skip notification", zap.String("event name", n.Name)) - continue - } - } - - if len(evsByCID) == 0 { - return nil - } - - for _, ev := range evsByCID { - m.blockEventsBuff <- ev - } - - l.Debug("handled block successfully", zap.Int("num of notifications", len(res.Application))) - - return nil -} - -type blockObjEvents struct { - cID cid.ID - bInd uint32 - evs []objEvent -} - -func (m *Meta) blockStorer(ctx context.Context, buff <-chan blockObjEvents) { +func (m *Meta) blockHandler(ctx context.Context, buff <-chan *block.Header) { + prevBlockFetchTime := time.Now() for { - if len(buff) == blockBuffSize { - m.l.Warn("block notifications buffer has been completely filled") + if len(buff) >= blockBuffSize-1 { + m.l.Warn("block header buffer has been completely filled") } select { case <-ctx.Done(): return - case blockEvs := <-buff: - m.stM.RLock() - st, ok := m.storages[blockEvs.cID] - m.stM.RUnlock() - if !ok { - m.l.Debug("do not store inactual events", zap.Stringer("cID", blockEvs.cID), zap.Uint32("events from block", blockEvs.bInd)) - continue - } + case b := <-buff: + blockReceivedAfter := time.Since(prevBlockFetchTime) + prevBlockFetchTime = time.Now() + + h := b.Hash() + ind := b.Index + m.l.Debug("received block", zap.Stringer("block hash", h), zap.Uint32("index", ind)) - st.putObjects(ctx, m.l.With(zap.String("storage", st.path)), blockEvs.bInd, blockEvs.evs, m.net) + m.chainHeigh.Store(ind) - m.l.Debug("stored container's notification for block successfully", - zap.Int("num of notifications", len(blockEvs.evs)), - zap.Stringer("cID", blockEvs.cID), - zap.Uint32("events from block", blockEvs.bInd)) + m.metrics.newBlockFetchTime.Observe(blockReceivedAfter.Seconds()) } } } -func (m *Meta) blockHandler(ctx context.Context, buff <-chan *block.Header) { +func (m *Meta) notificationHandler(ctx context.Context, buff <-chan *state.ContainedNotificationEvent) { for { - if len(buff) == blockBuffSize { - m.l.Warn("block header buffer has been completely filled") + if len(buff) >= notificationBuffSize-1 { + m.l.Warn("notification buffer has been completely filled") } select { case <-ctx.Done(): return - case b := <-buff: - err := m.handleBlock(b) - if err != nil { - m.l.Error("block handling failed", zap.Error(err)) + case n := <-buff: + l := m.l.With(zap.Stringer("tx", n.Container)) + + switch n.Name { + case objPutEvName: + ev, err := parseObjNotification(*n) + if err != nil { + l.Error("invalid object notification received", zap.Error(err)) + continue + } + + if magic := uint32(ev.network.Uint64()); magic != m.magicNumber { + l.Warn("skipping object notification with wrong magic number", zap.Uint32("expected", m.magicNumber), zap.Uint32("got", magic)) + continue + } + + addr := oid.NewAddress(ev.cID, ev.oID) + m.notifier.notifyReceived(addr) + + l.Debug("object notification successfully handled", zap.Stringer("address", addr)) + default: + l.Debug("skip notification", zap.String("event name", n.Name)) continue } } diff --git a/pkg/services/meta/containers.go b/pkg/services/meta/containers.go deleted file mode 100644 index 79db5e75ef..0000000000 --- a/pkg/services/meta/containers.go +++ /dev/null @@ -1,717 +0,0 @@ -package meta - -import ( - "bytes" - "context" - "errors" - "fmt" - "maps" - "math/big" - "os" - "path" - "slices" - "strconv" - "sync" - - "github.com/nspcc-dev/neo-go/pkg/core/mpt" - "github.com/nspcc-dev/neo-go/pkg/core/storage" - "github.com/nspcc-dev/neo-go/pkg/core/storage/dbconfig" - "github.com/nspcc-dev/neo-go/pkg/util" - objectcore "github.com/nspcc-dev/neofs-node/pkg/core/object" - cid "github.com/nspcc-dev/neofs-sdk-go/container/id" - "github.com/nspcc-dev/neofs-sdk-go/object" - oid "github.com/nspcc-dev/neofs-sdk-go/object/id" - "github.com/nspcc-dev/neofs-sdk-go/version" - "go.uber.org/zap" -) - -type containerStorage struct { - m sync.RWMutex - mptOpsBatch map[string][]byte - - l *zap.Logger - path string - mpt *mpt.Trie - db storage.Store -} - -func (s *containerStorage) drop() error { - s.m.Lock() - defer s.m.Unlock() - - err := s.db.Close() - if err != nil { - return fmt.Errorf("close container storage: %w", err) - } - - err = os.RemoveAll(s.path) - if err != nil { - return fmt.Errorf("remove container storage: %w", err) - } - - return nil -} - -type eventWithMptKVs struct { - ev objEvent - additionalKVs map[string][]byte -} - -func (s *containerStorage) putObjects(ctx context.Context, l *zap.Logger, bInd uint32, ee []objEvent, net NeoFSNetwork) { - s.m.Lock() - defer s.m.Unlock() - - // raw indexes are responsible for object validation and only after - // object is taken as a valid one, it goes to the slower-on-read MPT - // storage via objCh - objCh := make(chan eventWithMptKVs, len(ee)) - - var wg sync.WaitGroup - - wg.Go(func() { - err := s.putRawIndexes(ctx, l, ee, net, objCh) - if err != nil { - l.Error("failed to put raw indexes", zap.Error(err)) - } - }) - wg.Go(func() { - err := s.putMPTIndexes(bInd, objCh) - if err != nil { - l.Error("failed to put mpt indexes", zap.Error(err)) - } - }) - wg.Wait() -} - -// lock should be taken. -func (s *containerStorage) putMPTIndexes(bInd uint32, ch <-chan eventWithMptKVs) error { - for evWithKeys := range ch { - maps.Copy(s.mptOpsBatch, evWithKeys.additionalKVs) - - e := evWithKeys.ev - commsuffix := e.oID[:] - - // batching that is implemented for MPT ignores key's first byte - - s.mptOpsBatch[string(append([]byte{0, oidIndex}, commsuffix...))] = []byte{} - if len(e.deletedObjects) > 0 { - s.mptOpsBatch[string(append([]byte{0, deletedIndex}, commsuffix...))] = e.deletedObjects - } - if len(e.lockedObjects) > 0 { - s.mptOpsBatch[string(append([]byte{0, lockedIndex}, commsuffix...))] = e.lockedObjects - } - s.mptOpsBatch[string(append([]byte{0, sizeIndex}, commsuffix...))] = e.size.Bytes() - if len(e.firstObject) > 0 { - s.mptOpsBatch[string(append([]byte{0, firstPartIndex}, commsuffix...))] = e.firstObject - } - if len(e.prevObject) > 0 { - s.mptOpsBatch[string(append([]byte{0, previousPartIndex}, commsuffix...))] = e.prevObject - } - if e.typ != object.TypeRegular { - s.mptOpsBatch[string(append([]byte{0, typeIndex}, commsuffix...))] = []byte{byte(e.typ)} - } - } - - root := s.mpt.StateRoot() - s.mpt.Store.Put([]byte{rootKey}, root[:]) - - _, err := s.mpt.PutBatch(mpt.MapToMPTBatch(s.mptOpsBatch)) - if err != nil { - return fmt.Errorf("put batch to MPT storage: %w", err) - } - clear(s.mptOpsBatch) - - s.mpt.Flush(bInd) - - return nil -} - -// lock should be taken. -func (s *containerStorage) putRawIndexes(ctx context.Context, l *zap.Logger, ee []objEvent, net NeoFSNetwork, res chan<- eventWithMptKVs) (finalErr error) { - batch := make(map[string][]byte) - defer func() { - close(res) - - if finalErr == nil && len(batch) > 0 { - err := s.db.PutChangeSet(batch, nil) - if err != nil { - finalErr = fmt.Errorf("put change set to DB: %w", err) - } - } - }() - - for _, e := range ee { - err := isOpAllowed(s.db, e) - if err != nil { - l.Warn("skip object", zap.Stringer("oid", e.oID), zap.String("reason", err.Error())) - continue - } - - evWithMpt := eventWithMptKVs{ev: e} - - h, err := net.Head(ctx, e.cID, e.oID) - if err != nil { - // TODO define behavior with status (non-network) errors; maybe it is near #3140 - return fmt.Errorf("HEAD %s object: %w", e.oID, err) - } - - commsuffix := e.oID[:] - - batch[string(append([]byte{oidIndex}, commsuffix...))] = []byte{} - if len(e.deletedObjects) > 0 { - batch[string(append([]byte{deletedIndex}, commsuffix...))] = e.deletedObjects - evWithMpt.additionalKVs, err = s.deleteObjectsOps(batch, e.deletedObjects, false) - if err != nil { - l.Error("cleaning deleted object", zap.Stringer("oid", e.oID), zap.Error(err)) - continue - } - } - if len(e.lockedObjects) > 0 { - batch[string(append([]byte{lockedIndex}, commsuffix...))] = e.lockedObjects - - for locked := range slices.Chunk(e.lockedObjects, oid.Size) { - batch[string(append([]byte{lockedByIndex}, locked...))] = commsuffix - } - } - - err = objectcore.VerifyHeaderForMetadata(h) - if err != nil { - l.Error("header verification", zap.Stringer("oid", e.oID), zap.Error(err)) - continue - } - - res <- evWithMpt - - fillObjectIndex(batch, h, false) - } - - return finalErr -} - -func isOpAllowed(db storage.Store, e objEvent) error { - if len(e.deletedObjects) == 0 && len(e.lockedObjects) == 0 { - return nil - } - - key := make([]byte, 1+oid.Size) - - for obj := range slices.Chunk(e.deletedObjects, oid.Size) { - copy(key[1:], obj) - - // delete object that does not exist - key[0] = oidIndex - _, err := db.Get(key) - if err != nil { - if errors.Is(err, storage.ErrKeyNotFound) { - return fmt.Errorf("%s object-to-delete is missing", oid.ID(obj)) - } - return fmt.Errorf("%s object-to-delete's presence check: %w", oid.ID(obj), err) - } - - // delete object that is locked - key[0] = lockedByIndex - v, err := db.Get(key) - if err != nil { - if errors.Is(err, storage.ErrKeyNotFound) { - continue - } - return fmt.Errorf("%s object-to-delete's lock status check: %w", oid.ID(obj), err) - } - return fmt.Errorf("%s object-to-delete is locked by %s", oid.ID(obj), oid.ID(v)) - } - - for obj := range slices.Chunk(e.lockedObjects, oid.Size) { - copy(key[1:], obj) - - // lock object that does not exist - key[0] = oidIndex - _, err := db.Get(key) - if err != nil { - return fmt.Errorf("%s object-to-lock's presence check: %w", oid.ID(obj), err) - } - } - - return nil -} - -const binPropertyMarker = "1" // ROOT, PHY, etc. - -func fillObjectIndex(batch map[string][]byte, h object.Object, isParent bool) { - id := h.GetID() - typ := h.Type() - owner := h.Owner() - creationEpoch := h.CreationEpoch() - pSize := h.PayloadSize() - fPart := h.GetFirstID() - parID := h.GetParentID() - par := h.Parent() - hasParent := par != nil - phy := !isParent - pldHash, _ := h.PayloadChecksum() - var ver version.Version - if v := h.Version(); v != nil { - ver = *v - } - - oidKey := [1 + oid.Size]byte{oidIndex} - copy(oidKey[1:], id[:]) - batch[string(oidKey[:])] = []byte{} - - putPlainAttribute(batch, id, object.FilterVersion, ver.String()) - putPlainAttribute(batch, id, object.FilterOwnerID, string(owner[:])) - putPlainAttribute(batch, id, object.FilterType, typ.String()) - putIntAttribute(batch, id, object.FilterCreationEpoch, strconv.FormatUint(creationEpoch, 10), new(big.Int).SetUint64(creationEpoch)) - putIntAttribute(batch, id, object.FilterPayloadSize, strconv.FormatUint(pSize, 10), new(big.Int).SetUint64(pSize)) - putPlainAttribute(batch, id, object.FilterPayloadChecksum, string(pldHash.Value())) - if !fPart.IsZero() { - putPlainAttribute(batch, id, object.FilterFirstSplitObject, string(fPart[:])) - } - if !parID.IsZero() { - putPlainAttribute(batch, id, object.FilterParentID, string(parID[:])) - } - if !hasParent && fPart.IsZero() && typ == object.TypeRegular { - putPlainAttribute(batch, id, object.FilterRoot, binPropertyMarker) - } - if phy { - putPlainAttribute(batch, id, object.FilterPhysical, binPropertyMarker) - } - for _, a := range h.Attributes() { - ak, av := a.Key(), a.Value() - if n, isInt := parseInt(av); isInt && intWithinLimits(n) { - putIntAttribute(batch, id, ak, av, n) - } else { - putPlainAttribute(batch, id, ak, av) - } - } - - if hasParent && !parID.IsZero() { - fillObjectIndex(batch, *par, true) - } -} - -func (s *containerStorage) deleteObjectsOps(dbKV map[string][]byte, objects []byte, canDeleteLockObjects bool) (map[string][]byte, error) { - rng := storage.SeekRange{} - mptKV := make(map[string][]byte) - - if len(objects) == 0 { - return mptKV, nil - } - objects = s.expandChildren(objects) - - // nil value means "delete" operation - - for len(objects) > 0 { - o := objects[:oid.Size] - objects = objects[oid.Size:] - rng.Start = append([]byte{oidIndex}, o...) - stopKey := lastObjectKey(o) - - var objFound bool - var err error - - s.db.Seek(rng, func(k, v []byte) bool { - if bytes.Compare(k, stopKey) > 0 { - return false - } - if !bytes.HasPrefix(k[1:], o) { - return true - } - - if !objFound { - objFound = true - - // size index is the only index that is common for both storages - // but is stored in completely different forms - mptKV[string(append([]byte{0, sizeIndex}, o...))] = nil - } - - dbKV[string(k)] = nil - - switch pref := k[0]; pref { - // DB-only keys - case firstPartIndex, previousPartIndex, typeIndex, lockedByIndex: - case lockedIndex: - if canDeleteLockObjects { - mptKV[string(append([]byte{0}, k...))] = nil - for lockedObj := range slices.Chunk(v, oid.Size) { - dbKV[string(append([]byte{lockedByIndex}, lockedObj...))] = nil - } - } - // common keys for DB and MPT storages - case oidIndex, deletedIndex: - mptKV[string(append([]byte{0}, k...))] = nil - // DB reversed indexes - case oidToAttrIndex: - withoutOID := k[1+oid.Size:] - i := bytes.Index(withoutOID, objectcore.MetaAttributeDelimiter) - if i < 0 { - err = fmt.Errorf("unexpected attribute index without delimiter: %s", string(k)) - return false - } - attrK := withoutOID[:i] - attrV := withoutOID[i+attributeDelimiterLen:] - - // drop reverse plain index - keyToDrop := make([]byte, 0, len(k)+len(objectcore.MetaAttributeDelimiter)) - keyToDrop = append(keyToDrop, attrPlainToOIDIndex) - keyToDrop = append(keyToDrop, withoutOID...) - keyToDrop = append(keyToDrop, objectcore.MetaAttributeDelimiter...) - keyToDrop = append(keyToDrop, o...) - - dbKV[string(keyToDrop)] = nil - - if vInt, isInt := parseInt(string(attrV)); isInt && intWithinLimits(vInt) { - keyToDrop = slices.Grow(keyToDrop, 1+len(attrK)+attributeDelimiterLen+intValLen+oid.Size) - // drop reverse int index - keyToDrop = keyToDrop[:0] - keyToDrop = append(keyToDrop, attrIntToOIDIndex) - keyToDrop = append(keyToDrop, attrK...) - keyToDrop = append(keyToDrop, objectcore.MetaAttributeDelimiter...) - keyToDrop = keyToDrop[:len(keyToDrop)+intValLen] - putBigInt(keyToDrop[len(keyToDrop)-intValLen:], vInt) - keyToDrop = append(keyToDrop, o...) - - dbKV[string(keyToDrop)] = nil - } - default: - err = fmt.Errorf("unexpected index prefix: %d", k[0]) - return false - } - - return true - }) - if err != nil { - return nil, err - } - } - - return mptKV, nil -} - -// lastObjectKey returns the least possible key in sorted DB list that -// proves there will not be information about the object anymore. -func lastObjectKey(rawOID []byte) []byte { - res := make([]byte, 0, len(rawOID)+1) - res = append(res, lastEnumIndex-1) - - return append(res, rawOID...) -} - -func storageForContainer(l *zap.Logger, rootPath string, cID cid.ID) (*containerStorage, error) { - p := path.Join(rootPath, cID.EncodeToString()) - - st, err := storage.NewBoltDBStore(dbconfig.BoltDBOptions{FilePath: p, ReadOnly: false}) - if err != nil { - return nil, fmt.Errorf("open bolt store at %q: %w", p, err) - } - - var prevRootNode mpt.Node - root, err := st.Get([]byte{rootKey}) - if !errors.Is(err, storage.ErrKeyNotFound) { - if err != nil { - return nil, fmt.Errorf("get state root from db: %w", err) - } - - if len(root) != util.Uint256Size { - return nil, fmt.Errorf("root hash from db is %d bytes long, expect %d", len(root), util.Uint256Size) - } - - prevRootNode = mpt.NewHashNode([util.Uint256Size]byte(root)) - } - - return &containerStorage{ - l: l.With(zap.Stringer("cid", cID)), - path: p, - mpt: mpt.NewTrie(prevRootNode, mpt.ModeLatest, storage.NewMemCachedStore(st)), - db: st, - mptOpsBatch: make(map[string][]byte), - }, nil -} - -const ( - intValLen = 33 // prefix byte for sign + fixed256 in attrIntToOIDIndex - attributeDelimiterLen = 1 - attrIDFixedLen = 1 + oid.Size + attributeDelimiterLen -) - -func parseInt(s string) (*big.Int, bool) { - return new(big.Int).SetString(s, 10) -} - -var ( - maxUint256 = new(big.Int).SetBytes([]byte{255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255}) - maxUint256Neg = new(big.Int).Neg(maxUint256) -) - -func intWithinLimits(n *big.Int) bool { - return n.Cmp(maxUint256Neg) >= 0 && n.Cmp(maxUint256) <= 0 -} - -func putPlainAttribute(batch map[string][]byte, id oid.ID, k, v string) { - resKey := make([]byte, 0, 1+2*attributeDelimiterLen+oid.Size+len(k)+len(v)) - - // PREFIX_OID_ATTR_DELIM_VAL - resKey = append(resKey, oidToAttrIndex) - resKey = append(resKey, id[:]...) - resKey = append(resKey, k...) - resKey = append(resKey, objectcore.MetaAttributeDelimiter...) - resKey = append(resKey, v...) - - batch[string(resKey)] = []byte{} - resKey = resKey[:0] - - // PREFIX_ATTR_DELIM_VAL_DELIM_OID - resKey = append(resKey, attrPlainToOIDIndex) - resKey = append(resKey, k...) - resKey = append(resKey, objectcore.MetaAttributeDelimiter...) - resKey = append(resKey, v...) - resKey = append(resKey, objectcore.MetaAttributeDelimiter...) - resKey = append(resKey, id[:]...) - - batch[string(resKey)] = []byte{} -} - -func putIntAttribute(batch map[string][]byte, id oid.ID, k, vRaw string, vParsed *big.Int) { - putPlainAttribute(batch, id, k, vRaw) - - resKey := make([]byte, 0, 1+len(k)+attributeDelimiterLen+intValLen+oid.Size) - - // PREFIX_ATTR_DELIM_VAL_OID - resKey = append(resKey, attrIntToOIDIndex) - resKey = append(resKey, k...) - resKey = append(resKey, objectcore.MetaAttributeDelimiter...) - - resKey = resKey[:len(resKey)+intValLen] - putBigInt(resKey[len(resKey)-intValLen:], vParsed) - - resKey = append(resKey, id[:]...) - batch[string(resKey)] = []byte{} -} - -func putBigInt(b []byte, bInt *big.Int) { - neg := bInt.Sign() < 0 - if neg { - b[0] = 0 - } else { - b[0] = 1 - } - bInt.FillBytes(b[1:]) - if neg { - for i := range b[1:] { - b[1+i] = ^b[1+i] - } - } -} - -func (s *containerStorage) handleNewEpoch(e uint64) error { - s.m.Lock() - defer s.m.Unlock() - - var err error - oidsToDelete := make(map[oid.ID]struct{}) - var eBig big.Int - eBig.SetUint64(e) - lastValidEpochUint256 := make([]byte, intValLen) - putBigInt(lastValidEpochUint256, &eBig) - - var rng storage.SeekRange - rng.Prefix = slices.Concat([]byte{attrIntToOIDIndex}, []byte(object.AttributeExpirationEpoch), objectcore.MetaAttributeDelimiter) - commPrefLen := len(rng.Prefix) - - s.db.Seek(rng, func(k, _ []byte) bool { - if len(k) != commPrefLen+intValLen+oid.Size { - err = fmt.Errorf("unknown expiration attr index with %d len", len(k)) - return false - } - if bytes.Compare(k[commPrefLen:commPrefLen+intValLen], lastValidEpochUint256) >= 0 { - return false - } - oidsToDelete[oid.ID(k[commPrefLen+intValLen:])] = struct{}{} - - return true - }) - if err != nil { - return err - } - - if len(oidsToDelete) == 0 { - return nil - } - - rawOIDs := make([]byte, 0, oid.Size*len(oidsToDelete)) - - // check locked status - for oID := range oidsToDelete { - rawOID := oID[:] - - lock, err := s.db.Get(append([]byte{lockedByIndex}, rawOID...)) - if err != nil { - if errors.Is(err, storage.ErrKeyNotFound) { - rawOIDs = append(rawOIDs, rawOID...) - continue - } - return fmt.Errorf("check %s object-to-delete lock status: %w", oID, err) - } - - _, ok := oidsToDelete[oid.ID(lock)] - if ok { - // if lock expires in the same epoch, object is free to delete - rawOIDs = append(rawOIDs, rawOID...) - } - } - - dbBatch := make(map[string][]byte) - mptBatch, err := s.deleteObjectsOps(dbBatch, rawOIDs, true) - if err != nil { - return fmt.Errorf("making delete operations batch: %w", err) - } - - var wg sync.WaitGroup - - var mptErr error - var dbErr error - - wg.Go(func() { - _, err := s.mpt.PutBatch(mpt.MapToMPTBatch(mptBatch)) - if err != nil { - mptErr = fmt.Errorf("mpt delete operations application: %w", err) - } - }) - wg.Go(func() { - err := s.db.PutChangeSet(dbBatch, nil) - if err != nil { - dbErr = fmt.Errorf("raw db delete operations application: %w", err) - } - }) - - wg.Wait() - - return errors.Join(mptErr, dbErr) -} - -func (s *containerStorage) expandChildren(rootOIDs []byte) []byte { - // sorting before every SEEK is used for exact single operation for every - // children searching subroutine; search is done in 3 steps: - // 1. root -> last/link object search - // 2. last/link -> first part ID search - // 3. first part ID -> all children search - - var childrenWithParentID [][]byte - var rng storage.SeekRange - rng.Prefix = slices.Concat([]byte{attrPlainToOIDIndex}, []byte(object.FilterParentID), objectcore.MetaAttributeDelimiter) - rootsSorted := slices.SortedFunc(slices.Chunk(rootOIDs, oid.Size), bytes.Compare) - rng.Start = rootsSorted[0] - keyLenExp := len(rng.Prefix) + len(rng.Start) + attributeDelimiterLen + oid.Size - - s.db.Seek(rng, func(k, _ []byte) bool { - if len(k) != keyLenExp { - s.l.Warn("unexpected parent ID index's len", - zap.Int("expected", keyLenExp), - zap.Int("actual", len(k)), - zap.String("key", fmt.Sprintf("%x", k)), - ) - - return true - } - currRoot := k[len(rng.Prefix) : len(rng.Prefix)+oid.Size] - for len(rootsSorted) > 0 { - switch bytes.Compare(currRoot, rootsSorted[0]) { - case -1: - return true - case +1: - rootsSorted = rootsSorted[1:] - continue - case 0: - childrenWithParentID = append(childrenWithParentID, slices.Clone(k[len(rng.Prefix)+oid.Size+attributeDelimiterLen:])) - rootsSorted = rootsSorted[1:] - return true - } - } - - return len(rootsSorted) != 0 - }) - - if len(childrenWithParentID) == 0 { - // all objects are roots, no additional children - return rootOIDs - } - - firstPartOIDs := make([][]byte, 0, len(childrenWithParentID)) - slices.SortFunc(childrenWithParentID, bytes.Compare) - rng.Prefix = []byte{oidToAttrIndex} - rng.Start = slices.Concat(childrenWithParentID[0], []byte(object.FilterFirstSplitObject), objectcore.MetaAttributeDelimiter) - keyLenExp = len(rng.Prefix) + len(rng.Start) + oid.Size - - s.db.Seek(rng, func(k, _ []byte) bool { - if len(k) != keyLenExp { - s.l.Warn("unexpected first object ID index's len", - zap.Int("expected", keyLenExp), - zap.Int("actual", len(k)), - zap.Int("index prefix", oidToAttrIndex), - zap.String("key", fmt.Sprintf("%x", k)), - ) - - return true - } - currChild := k[1 : 1+oid.Size] - for len(childrenWithParentID) > 0 { - switch bytes.Compare(currChild, childrenWithParentID[0]) { - case -1: - case +1: - // should never happen, it means we have info - // about child object with parent ID but child - // object does not have first part ID, nothing - // can be done, just skip - childrenWithParentID = childrenWithParentID[1:] - continue - case 0: - firstPartOIDs = append(firstPartOIDs, k[1+oid.Size+len(object.FilterFirstSplitObject)+attributeDelimiterLen:]) - childrenWithParentID = childrenWithParentID[1:] - } - } - - return len(childrenWithParentID) != 0 - }) - - if len(firstPartOIDs) == 0 { - // unexpected but nothing to do - return rootOIDs - } - - slices.SortFunc(firstPartOIDs, bytes.Compare) - children := slices.Concat(firstPartOIDs...) // first objects are children too - rng.Prefix = slices.Concat([]byte{attrPlainToOIDIndex}, []byte(object.FilterFirstSplitObject), objectcore.MetaAttributeDelimiter) - rng.Start = firstPartOIDs[0] - keyLenExp = len(rng.Prefix) + len(rng.Start) + attributeDelimiterLen + oid.Size - - s.db.Seek(rng, func(k, _ []byte) bool { - if len(k) != keyLenExp { - s.l.Warn("unexpected first object ID index's len", - zap.Int("expected", keyLenExp), - zap.Int("actual", len(k)), - zap.Int("index prefix", attrPlainToOIDIndex), - zap.String("key", fmt.Sprintf("%x", k)), - ) - - return true - } - currChild := k[len(rng.Prefix) : len(rng.Prefix)+oid.Size] - for len(firstPartOIDs) > 0 { - switch bytes.Compare(currChild, firstPartOIDs[0]) { - case -1: - return true - case +1: - firstPartOIDs = firstPartOIDs[1:] - continue - case 0: - children = append(children, slices.Clone(k[len(rng.Prefix)+oid.Size+attributeDelimiterLen:])...) - return true - } - } - - return false - }) - - return slices.Concat(rootOIDs, children) -} diff --git a/pkg/services/meta/containers_test.go b/pkg/services/meta/containers_test.go deleted file mode 100644 index c818bd0acd..0000000000 --- a/pkg/services/meta/containers_test.go +++ /dev/null @@ -1,297 +0,0 @@ -package meta - -import ( - "bytes" - "context" - "fmt" - "math/big" - "strconv" - "testing" - - "github.com/nspcc-dev/neo-go/pkg/core/storage" - cid "github.com/nspcc-dev/neofs-sdk-go/container/id" - cidtest "github.com/nspcc-dev/neofs-sdk-go/container/id/test" - "github.com/nspcc-dev/neofs-sdk-go/object" - oid "github.com/nspcc-dev/neofs-sdk-go/object/id" - objecttest "github.com/nspcc-dev/neofs-sdk-go/object/test" - "github.com/stretchr/testify/require" - "go.uber.org/zap/zaptest" -) - -func objsToAddrMap(oo []object.Object) map[oid.Address]object.Object { - res := make(map[oid.Address]object.Object) - for _, o := range oo { - res[oid.NewAddress(o.GetContainerID(), o.GetID())] = o - } - return res -} - -func setExpiration(o *object.Object, epoch uint64) { - var attr object.Attribute - - attr.SetKey(object.AttributeExpirationEpoch) - attr.SetValue(strconv.FormatUint(epoch, 10)) - - o.SetAttributes(append(o.Attributes(), attr)...) -} - -func TestObjectExpiration(t *testing.T) { - t.Run("expired objects", func(t *testing.T) { - const objNum = 10 - cID := cidtest.ID() - var oo []object.Object - for i := range objNum { - o := objecttest.Object() - o.ResetRelations() - o.SetContainerID(cID) - setExpiration(&o, uint64(i)) - - oo = append(oo, o) - } - - st, err := storageForContainer(zaptest.NewLogger(t), t.TempDir(), cID) - require.NoError(t, err) - t.Cleanup(func() { - _ = st.drop() - }) - - net := testNetwork{} - net.setContainers([]cid.ID{cID}) - net.setObjects(objsToAddrMap(oo)) - ee := make([]objEvent, 0, objNum) - for _, o := range oo { - ee = append(ee, objEvent{ - cID: cID, - oID: o.GetID(), - size: big.NewInt(testObjectSize), - network: big.NewInt(testNetworkMagic), - }) - } - - st.putObjects(context.Background(), zaptest.NewLogger(t), 0, ee, &net) - - for i, o := range oo { - oIDToDelete := o.GetID() - kExpect := append([]byte{oidIndex}, oIDToDelete[:]...) - - _, err = st.db.Get(kExpect) - require.NoError(t, err) - - err = st.handleNewEpoch(uint64(i + 1)) - require.NoError(t, err) - - _, err = st.db.Get(kExpect) - require.ErrorIs(t, err, storage.ErrKeyNotFound, fmt.Sprintf("%d object was not expired", i)) - } - - // all objects now are expired, empty db is expected - st.db.Seek(storage.SeekRange{}, func(k, v []byte) bool { - require.Fail(t, "no KV after expirations are expected") - return true - }) - }) - - t.Run("lock expiration", func(t *testing.T) { - cID := cidtest.ID() - - const ( - objExp = iota - lockExp - ) - - o := objecttest.Object() - oID := o.GetID() - o.ResetRelations() - o.SetContainerID(cID) - setExpiration(&o, objExp) - lock := objecttest.Object() - lock.ResetRelations() - lock.SetContainerID(cID) - setExpiration(&lock, lockExp) - - st, err := storageForContainer(zaptest.NewLogger(t), t.TempDir(), cID) - require.NoError(t, err) - t.Cleanup(func() { - _ = st.drop() - }) - - net := testNetwork{} - net.setContainers([]cid.ID{cID}) - net.setObjects(objsToAddrMap([]object.Object{o, lock})) - - eObj := objEvent{ - cID: cID, - oID: oID, - size: big.NewInt(testObjectSize), - network: big.NewInt(testNetworkMagic), - } - eLock := objEvent{ - cID: cID, - oID: lock.GetID(), - size: big.NewInt(testObjectSize), - network: big.NewInt(testNetworkMagic), - lockedObjects: oID[:], - } - - st.putObjects(context.Background(), zaptest.NewLogger(t), 0, []objEvent{eObj}, &net) - st.putObjects(context.Background(), zaptest.NewLogger(t), 0, []objEvent{eLock}, &net) - - kExpect := append([]byte{oidIndex}, oID[:]...) - - err = st.handleNewEpoch(uint64(objExp + 1)) - require.NoError(t, err) - - _, err = st.db.Get(kExpect) - require.NoError(t, err, "locked object expired") - - err = st.handleNewEpoch(uint64(lockExp + 1)) - require.NoError(t, err) - - _, err = st.db.Get(kExpect) - require.ErrorIs(t, err, storage.ErrKeyNotFound, "unlocked object has not expired") - - // all objects now are expired, empty db is expected - st.db.Seek(storage.SeekRange{}, func(k, v []byte) bool { - require.Fail(t, "no KV after expirations are expected") - return true - }) - }) -} - -func objectChain(cID cid.ID, length int) (object.Object, []object.Object) { - reset := func(o *object.Object) { - o.SetContainerID(cID) - o.ResetRelations() - o.SetType(object.TypeRegular) - } - - root := objecttest.Object() - reset(&root) - first := objecttest.Object() - reset(&first) - - children := make([]object.Object, length-1) - children[0] = first - for i := range children { - if i != 0 { - children[i] = objecttest.Object() - reset(&children[i]) - children[i].SetFirstID(first.GetID()) - children[i].SetPreviousID(children[i-1].GetID()) - } - if i == len(children)-1 { - children[i].SetParentID(root.GetID()) - children[i].SetParent(&root) - } - } - - link := objecttest.Object() - reset(&link) - link.SetFirstID(first.GetID()) - link.SetParentID(root.GetID()) - link.SetParent(&root) - link.SetType(object.TypeLink) - - return root, append(children, link) -} - -type bigObj struct { - root object.Object - children []object.Object -} - -func TestBigObjects(t *testing.T) { - cID := cidtest.ID() - l := zaptest.NewLogger(t) - ctx := context.Background() - - var bigObjs []bigObj - for range 10 { - root, children := objectChain(cID, 10) - bigObjs = append(bigObjs, bigObj{root, children}) - } - - var rawObjSlice []object.Object - for _, obj := range bigObjs { - rawObjSlice = append(rawObjSlice, obj.children...) - } - - net := testNetwork{} - net.setContainers([]cid.ID{cID}) - net.setObjects(objsToAddrMap(rawObjSlice)) - ee := make([]objEvent, 0, len(rawObjSlice)) - for _, o := range rawObjSlice { - ev := objEvent{ - cID: cID, - oID: o.GetID(), - size: big.NewInt(testObjectSize), - network: big.NewInt(testNetworkMagic), - } - ev.typ = o.Type() - if id := o.GetPreviousID(); !id.IsZero() { - ev.prevObject = id[:] - } - if id := o.GetFirstID(); !id.IsZero() { - ev.firstObject = id[:] - } - - ee = append(ee, ev) - } - - st, err := storageForContainer(zaptest.NewLogger(t), t.TempDir(), cID) - require.NoError(t, err) - t.Cleanup(func() { - _ = st.drop() - }) - - st.putObjects(ctx, l, 0, ee, &net) - - tsObj := objecttest.Object() - tsObj.ResetRelations() - tsObj.SetContainerID(cID) - tsObj.SetType(object.TypeTombstone) - tsEv := objEvent{ - cID: cID, - oID: tsObj.GetID(), - size: big.NewInt(testObjectSize), - network: big.NewInt(testNetworkMagic), - } - net.setObjects(objsToAddrMap([]object.Object{tsObj})) - - for _, bigO := range bigObjs { - rID := bigO.root.GetID() - tsEv.deletedObjects = rID[:] - - st.putObjects(ctx, l, 0, []objEvent{tsEv}, &net) - - k := make([]byte, 1+oid.Size) - k[0] = oidIndex - - for _, child := range bigO.children { - chID := child.GetID() - copy(k[1:], chID[:]) - - _, err = st.db.Get(k) - require.ErrorIs(t, err, storage.ErrKeyNotFound) - } - } - - // only last operation from TS should be kept - tsID := tsObj.GetID() - st.db.Seek(storage.SeekRange{}, func(k, _ []byte) bool { - switch k[0] { - case oidIndex, oidToAttrIndex, deletedIndex: - if bytes.Equal(k[1:1+oid.Size], tsID[:]) { - return true - } - case attrIntToOIDIndex, attrPlainToOIDIndex: - if bytes.Equal(k[len(k)-oid.Size:], tsID[:]) { - return true - } - default: - } - - t.Fatalf("empty db is expected after full clean, found key: %d", k[0]) - return true - }) -} diff --git a/pkg/services/meta/meta.go b/pkg/services/meta/meta.go index 79db3e133f..0f23a15ad0 100644 --- a/pkg/services/meta/meta.go +++ b/pkg/services/meta/meta.go @@ -4,85 +4,106 @@ import ( "context" "errors" "fmt" - "os" + "math" + "path" + "slices" "sync" + "sync/atomic" "time" "github.com/nspcc-dev/neo-go/pkg/core/block" "github.com/nspcc-dev/neo-go/pkg/core/state" - "github.com/nspcc-dev/neo-go/pkg/neorpc" - "github.com/nspcc-dev/neo-go/pkg/neorpc/result" - "github.com/nspcc-dev/neo-go/pkg/rpcclient/invoker" + "github.com/nspcc-dev/neo-go/pkg/core/transaction" + "github.com/nspcc-dev/neo-go/pkg/io" "github.com/nspcc-dev/neo-go/pkg/util" + "github.com/nspcc-dev/neo-go/pkg/vm/emit" + "github.com/nspcc-dev/neo-go/pkg/vm/opcode" + "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" + "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/common" + metabase "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase" cid "github.com/nspcc-dev/neofs-sdk-go/container/id" + neofscrypto "github.com/nspcc-dev/neofs-sdk-go/crypto" "github.com/nspcc-dev/neofs-sdk-go/object" oid "github.com/nspcc-dev/neofs-sdk-go/object/id" "go.uber.org/zap" - "golang.org/x/sync/errgroup" ) const ( - // raw storage prefixes. - - // rootKey is the key for the last known state root in KV data base - // associated with MPT. - rootKey = 0xff - // notificationBuffSize is a nesessary buffer for neo-go's client proper // notification work; it is required to always read notifications without // any blocking or making additional RPC. - notificationBuffSize = 100 + notificationBuffSize = 10000 ) -// NeoFSNetwork describes current NeoFS storage network state. -type NeoFSNetwork interface { - // Epoch returns current epoch in the NeoFS network. - Epoch() (uint64, error) - // List returns node's containers that support chain-based meta data and - // any error that does not allow listing. - List(uint64) (map[cid.ID]struct{}, error) - // IsMineWithMeta checks if the given container has meta enabled and current - // node belongs to it. - IsMineWithMeta(cid.ID, []byte) (bool, error) - // Head returns actual object header from the NeoFS network (non-local - // objects should also be returned). Missing, removed object statuses - // must be reported according to API statuses from SDK. - Head(context.Context, cid.ID, oid.ID) (object.Object, error) -} - -// wsClient is for test purposes only. -type wsClient interface { - invoker.RPCInvoke - - GetBlockNotifications(blockHash util.Uint256, filters *neorpc.NotificationFilter) (*result.BlockNotifications, error) - GetVersion() (*result.Version, error) +type ( + // NeoFSNetwork describes current NeoFS storage network state. + NeoFSNetwork interface { + // Head returns actual object header from the NeoFS network (non-local + // objects should also be returned). Missing, removed object statuses + // must be reported according to API statuses from SDK. + Head(context.Context, cid.ID, oid.ID) (object.Object, error) + + // IsMineWithMeta checks if the given container has meta enabled and current + // node belongs to it. + IsMineWithMeta(cid.ID, []byte) (bool, error) + } - ReceiveHeadersOfAddedBlocks(flt *neorpc.BlockFilter, rcvr chan<- *block.Header) (string, error) - ReceiveExecutionNotifications(flt *neorpc.NotificationFilter, rcvr chan<- *state.ContainedNotificationEvent) (string, error) - Unsubscribe(id string) error + // MetaChain describes metadata chain. + MetaChain interface { + // Magic must return metadata chain magic number. + Magic() uint32 + // Height must return actual chain height. + Height() uint32 + // AddTx must add transaction to the chain without blocking. + AddTx(tx *transaction.Transaction) error + // SubscribeForBlocks must subscribe for new block headers. + // Block shouldbe sent to provided channel. + SubscribeForBlocks(ch chan *block.Header) + // SubscribeForNotifications must subscribe for new chain notifications. + // Notifications should be sent to provided channel. + SubscribeForNotifications(ch chan *state.ContainedNotificationEvent) + } +) - Close() +func newNotifier(metaSvc *Meta) *objectNotifier { + return &objectNotifier{ + metaSvc: metaSvc, + subs: make(map[oid.Address]objSubInfo), + } } -func newNotifier() objectNotifier { - return objectNotifier{ - notifications: make(chan oid.Address, 1024), - subs: make(map[oid.Address]chan<- struct{}), - } +type objSubInfo struct { + txH util.Uint256 + ch chan<- struct{} + timeSubscriptionStarted time.Time + blockSubscriptionStarted uint32 + objHeader object.Object } type objectNotifier struct { - notifications chan oid.Address + metaSvc *Meta m sync.Mutex - subs map[oid.Address]chan<- struct{} + subs map[oid.Address]objSubInfo } -func (on *objectNotifier) subscribe(addr oid.Address, ch chan<- struct{}) { +func (on *objectNotifier) subscribe(o object.Object, ch chan<- struct{}, h util.Uint256) { + var ( + subTime = time.Now() + subBlock = on.metaSvc.chainHeigh.Load() + addr = o.Address() + ) + on.m.Lock() defer on.m.Unlock() - on.subs[addr] = ch + on.subs[addr] = objSubInfo{ + txH: h, + ch: ch, + timeSubscriptionStarted: subTime, + blockSubscriptionStarted: subBlock, + objHeader: o, + } } func (on *objectNotifier) unsubscribe(addr oid.Address) { @@ -93,295 +114,217 @@ func (on *objectNotifier) unsubscribe(addr oid.Address) { } func (on *objectNotifier) notifyReceived(addr oid.Address) { + var ( + timeTook time.Duration + blocksTook uint32 + currHeight = on.metaSvc.chainHeigh.Load() + ) + on.m.Lock() - defer on.m.Unlock() - ch, ok := on.subs[addr] + sub, ok := on.subs[addr] if ok { - close(ch) + close(sub.ch) delete(on.subs, addr) + + timeTook = time.Since(sub.timeSubscriptionStarted) + blocksTook = currHeight - sub.blockSubscriptionStarted } + + on.m.Unlock() + + if ok { + on.metaSvc.taskQueue <- storageTask{addr: addr, o: &sub.objHeader} + } else { + on.metaSvc.taskQueue <- storageTask{addr: addr} + } + + on.metaSvc.l.Debug("object notification handled", + zap.Stringer("addr", addr), + zap.Duration("timeTook", timeTook), + zap.Uint32("blocksTook", blocksTook), + zap.String("txHash", sub.txH.StringLE())) } -// Meta handles object meta information received from FS chain and object -// storages. Chain information is stored in Merkle-Patricia Tries. Full objects -// index is built and stored as a simple KV storage. +// Meta handles object meta information received from metadata chain and object +// storages. It must be created using [New]. type Meta struct { - l *zap.Logger - rootPath string - netmapH util.Uint160 - cnrH util.Uint160 - net NeoFSNetwork + l *zap.Logger + metrics metrics - stM sync.RWMutex - storages map[cid.ID]*containerStorage + net NeoFSNetwork - timeout time.Duration + chainHeigh atomic.Uint32 + chain MetaChain magicNumber uint32 - cliM sync.RWMutex - ws wsClient - blockSubID string - cnrSubID string - cnrCrtSubID string bCh chan *block.Header - cnrPutEv chan *state.ContainedNotificationEvent - epochEv chan *state.ContainedNotificationEvent - - notifier objectNotifier + evsCh chan *state.ContainedNotificationEvent - blockHeadersBuff chan *block.Header - blockEventsBuff chan blockObjEvents + taskQueue chan storageTask - // runtime reload fields - cfgM sync.RWMutex - endpoints []string + metabase *metabase.DB + notifier *objectNotifier } -const blockBuffSize = 1024 +const blockBuffSize = 10000 -// Parameters groups arguments for [New] call. +// Parameters groups arguments for [New] call. Logger, Chain and Network +// must not be nil, path must not be empty. type Parameters struct { - Logger *zap.Logger - Network NeoFSNetwork - Timeout time.Duration - ContainerHash util.Uint160 - NetmapHash util.Uint160 - RootPath string - - // fields that support runtime reload - NeoEnpoints []string + Logger *zap.Logger + Chain MetaChain + Path string + Network NeoFSNetwork } func validatePrm(p Parameters) error { - if p.RootPath == "" { - return errors.New("empty path") - } if p.Logger == nil { return errors.New("missing logger") } - if len(p.NeoEnpoints) == 0 { - return errors.New("no endpoints to NeoFS chain network") - } - if p.Network == nil { - return errors.New("missing NeoFS network state") + if p.Chain == nil { + return errors.New("missing metadata chain") } - if (p.ContainerHash == util.Uint160{}) { - return errors.New("missing container contract hash") + if p.Path == "" { + return errors.New("empty metadata path") } - if (p.NetmapHash == util.Uint160{}) { - return errors.New("missing netmap contract hash") + if p.Network == nil { + return errors.New("missing NeoFS network") } return nil } -// New makes [Meta]. +// this is only neeeded to treat every object as a non-expired one in metabase +// GC routines do not relate to meta service directly. +type epochStateStub struct{} + +func (e epochStateStub) CurrentEpoch() uint64 { + return math.MaxUint64 +} + +// New makes [Meta] using [Parameters]. [metabase.DB] is created, opened and +// inited when called. func New(p Parameters) (*Meta, error) { err := validatePrm(p) if err != nil { return nil, err } - - storagesFS, err := os.ReadDir(p.RootPath) - if err != nil && !os.IsNotExist(err) { - return nil, fmt.Errorf("read existing container storages: %w", err) - } - storagesRead := make(map[cid.ID]*containerStorage) - for _, s := range storagesFS { - sName := s.Name() - cID, err := cid.DecodeString(sName) - if err != nil { - p.Logger.Warn("skip unknown container storage entity", zap.String("name", sName), zap.Error(err)) - continue - } - - st, err := storageForContainer(p.Logger, p.RootPath, cID) - if err != nil { - p.Logger.Warn("skip container storage that cannot be read", zap.String("name", sName), zap.Error(err)) - continue - } - - storagesRead[cID] = st - } - - storages := storagesRead - defer func() { - if err != nil { - for _, st := range storages { - _ = st.db.Close() - } - } - }() - - e, err := p.Network.Epoch() + metaDB := metabase.New( + metabase.WithPath(path.Join(p.Path, "metadataDB.bolt")), + metabase.WithLogger( + p.Logger.With(zap.String("component", "metadata storage"))), + metabase.WithEpochState(epochStateStub{}), + ) + err = metaDB.Open(false) if err != nil { - return nil, fmt.Errorf("read current NeoFS epoch: %w", err) + _ = metaDB.Close() + return nil, fmt.Errorf("failed to open metabase: %w", err) } - cnrsNetwork, err := p.Network.List(e) + var dbIDRaw [common.IDSize]byte + copy(dbIDRaw[:], "metadataobjectDB") + dbID, err := common.NewIDFromBytes(dbIDRaw[:]) if err != nil { - return nil, fmt.Errorf("listing node's containers: %w", err) + _ = metaDB.Close() + panic(fmt.Errorf("failed to create metabase ID: %w", err)) } - for cID := range storagesRead { - if _, ok := cnrsNetwork[cID]; !ok { - err = storagesRead[cID].drop() - if err != nil { - p.Logger.Warn("could not drop container storage", zap.Stringer("cID", cID), zap.Error(err)) - } - - delete(storagesRead, cID) - } + err = metaDB.Init(dbID) + if err != nil { + _ = metaDB.Close() + return nil, fmt.Errorf("failed to init metabase: %w", err) } - for cID := range cnrsNetwork { - if _, ok := storages[cID]; !ok { - st, err := storageForContainer(p.Logger, p.RootPath, cID) - if err != nil { - return nil, fmt.Errorf("open container storage %s: %w", cID, err) - } - - storages[cID] = st - } + m := &Meta{ + l: p.Logger, + chain: p.Chain, + net: p.Network, + bCh: make(chan *block.Header, blockBuffSize), + evsCh: make(chan *state.ContainedNotificationEvent, notificationBuffSize), + taskQueue: make(chan storageTask, notificationBuffSize), + metabase: metaDB, } + notifier := newNotifier(m) + m.notifier = notifier - return &Meta{ - l: p.Logger, - rootPath: p.RootPath, - netmapH: p.NetmapHash, - cnrH: p.ContainerHash, - net: p.Network, - endpoints: p.NeoEnpoints, - timeout: p.Timeout, - bCh: make(chan *block.Header, notificationBuffSize), - cnrPutEv: make(chan *state.ContainedNotificationEvent, notificationBuffSize), - epochEv: make(chan *state.ContainedNotificationEvent, notificationBuffSize), - blockHeadersBuff: make(chan *block.Header, blockBuffSize), - blockEventsBuff: make(chan blockObjEvents, blockBuffSize), - storages: storages, - notifier: newNotifier(), - }, nil -} + m.addMetrics() -// Reload updates service in runtime. -// Currently supported fields: -// - endpoints -func (m *Meta) Reload(p Parameters) error { - m.cfgM.Lock() - defer m.cfgM.Unlock() - - m.endpoints = p.NeoEnpoints - - return nil + return m, nil } -// Run starts notification handling. Must be called only on instances created -// with [New]. Blocked until context is done. -func (m *Meta) Run(ctx context.Context) error { - defer func() { - m.stM.Lock() - for _, st := range m.storages { - st.m.Lock() - _ = st.db.Close() - st.m.Unlock() - } - clear(m.storages) - - m.stM.Unlock() - }() - - var err error - m.ws, err = m.connect(ctx) - if err != nil { - return fmt.Errorf("connect to NEO RPC: %w", err) - } - defer m.ws.Close() +// MagicNumber returns metadata chain's magic number. +func (m *Meta) MagicNumber() uint32 { + return m.magicNumber +} - v, err := m.ws.GetVersion() - if err != nil { - return fmt.Errorf("get version: %w", err) - } - m.magicNumber = uint32(v.Protocol.Network) +// Height returns current metadata chain block height. +func (m *Meta) Height() uint32 { + return m.chain.Height() +} - m.stM.RLock() - hasContainers := len(m.storages) > 0 - m.stM.RUnlock() +// IndexedSignature is undexed signature, pointing at place of signature's +// public keys in a sorted (by these public keys) placement vector. It will be +// used as an optimized signatures check. +type IndexedSignature struct { + Index uint8 + Signature neofscrypto.Signature +} - if hasContainers { - m.blockSubID, err = m.subscribeForBlocks(m.bCh) - if err != nil { - return fmt.Errorf("block subscription: %w", err) - } - } else { - err = m.subscribeForNewContainers() - if err != nil { - return fmt.Errorf("new container subscription: %w", err) +// SubmitObjectPut sends transaction to metadata chain. Transaction must be +// completed excepting [transaction.Witness.InvocationScript] that will be +// filled using provided signatures. signatures must be a two two-dimensional +// array that corresponds to placement vectors for a container that tx was +// made for. Signature optimized sorting is not this func's responsibility. +func (m *Meta) SubmitObjectPut(tx *transaction.Transaction, signatures [][]IndexedSignature) error { + var ( + invokBuff = io.NewBufBinWriter() + writer = invokBuff.BinWriter + ) + for _, signature := range slices.Backward(signatures) { + vectorLen := len(signature) + for j := vectorLen - 1; j >= 0; j-- { + emit.Array(writer, + stackitem.Make(signature[j].Index), // singature's index + stackitem.Make(signature[j].Signature.Value()), // signature + ) } + emit.Int(writer, int64(vectorLen)) + emit.Opcodes(writer, opcode.PACK) } - err = m.subscribeEvents() - if err != nil { - return fmt.Errorf("subscribe for meta notifications: %w", err) + if invokBuff.Err != nil { + panic(invokBuff.Err) } - var wg sync.WaitGroup - - wg.Go(func() { m.flusher(ctx) }) - wg.Go(func() { m.blockHandler(ctx, m.blockHeadersBuff) }) - wg.Go(func() { m.blockStorer(ctx, m.blockEventsBuff) }) + tx.Scripts[0].InvocationScript = invokBuff.Bytes() - err = m.listenNotifications(ctx) - wg.Wait() + m.l.Debug("sending transaction to chain...", zap.String("txHash", tx.Hash().StringLE())) + now := time.Now() + err := m.chain.AddTx(tx) + took := time.Since(now) + m.l.Debug("sent transaction to chain", zap.String("txHash", tx.Hash().StringLE()), zap.Duration("took", took), zap.Error(err)) return err } -func (m *Meta) flusher(ctx context.Context) { - const ( - flushInterval = time.Second - collapseDepth = 10 - ) - - t := time.NewTicker(flushInterval) - - for { - select { - case <-t.C: - m.stM.RLock() - - var wg errgroup.Group - wg.SetLimit(1024) - - for _, st := range m.storages { - if st == nil { - panic(fmt.Errorf("nil container storage: %s", st.path)) - } - - wg.Go(func() error { - st.m.Lock() - defer st.m.Unlock() - - st.mpt.Collapse(collapseDepth) +// Run starts notification handling. Must be called only on instances created +// with [New]. Blocked until context is done. Cancelling the context stops +// the service. +func (m *Meta) Run(ctx context.Context) error { + m.magicNumber = m.chain.Magic() - _, err := st.mpt.Store.PersistSync() - if err != nil { - return fmt.Errorf("persisting %q storage: %w", st.path, err) - } + var wg sync.WaitGroup - return nil - }) - } + wg.Go(func() { m.blockHandler(ctx, m.bCh) }) + wg.Go(func() { m.notificationHandler(ctx, m.evsCh) }) + wg.Go(func() { m.storager(ctx, m.taskQueue) }) - err := wg.Wait() + m.chain.SubscribeForBlocks(m.bCh) + m.chain.SubscribeForNotifications(m.evsCh) - m.stM.RUnlock() + wg.Wait() - if err != nil { - m.l.Error("storage flusher failed", zap.Error(err)) - continue - } + close(m.bCh) + close(m.evsCh) + close(m.taskQueue) - t.Reset(flushInterval) - case <-ctx.Done(): - return - } - } + return m.metabase.Close() } diff --git a/pkg/services/meta/metrics.go b/pkg/services/meta/metrics.go new file mode 100644 index 0000000000..3828134a8b --- /dev/null +++ b/pkg/services/meta/metrics.go @@ -0,0 +1,42 @@ +package meta + +import "github.com/prometheus/client_golang/prometheus" + +const ( + nameSpace = "neofs_node" + subsystem = "metadata_chain" +) + +type metrics struct { + newBlockFetchTime prometheus.Histogram + objAcceptTime prometheus.Histogram + objAcceptBlocks prometheus.Histogram +} + +func (m *Meta) addMetrics() { + m.metrics = metrics{ + newBlockFetchTime: prometheus.NewHistogram(prometheus.HistogramOpts{ + Namespace: nameSpace, + Subsystem: subsystem, + Name: "new_block_fetch_time", + Help: "Time to receive new block after the previous one, seconds", + Buckets: []float64{0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1, 0.15}, + }), + objAcceptTime: prometheus.NewHistogram(prometheus.HistogramOpts{ + Namespace: nameSpace, + Subsystem: subsystem, + Name: "obj_accept_time", + Help: "Time b/w object is sent to chain and transaction with it is received, seconds", + Buckets: []float64{0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1, 0.15}, + }), + objAcceptBlocks: prometheus.NewHistogram(prometheus.HistogramOpts{ + Namespace: nameSpace, + Subsystem: subsystem, + Name: "obj_accept_blocks", + Help: "Number of blocks b/w object is sent to chain and transaction with it is received, blocks", + Buckets: []float64{0.5, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, + }), + } + + prometheus.MustRegister(m.metrics.newBlockFetchTime, m.metrics.objAcceptTime, m.metrics.objAcceptBlocks) +} diff --git a/pkg/services/meta/notifications.go b/pkg/services/meta/notifications.go index 3437fff810..738477a183 100644 --- a/pkg/services/meta/notifications.go +++ b/pkg/services/meta/notifications.go @@ -1,304 +1,21 @@ package meta import ( - "context" - "errors" "fmt" "math/big" - "slices" - "sync" - "time" - "github.com/nspcc-dev/neo-go/pkg/core/block" "github.com/nspcc-dev/neo-go/pkg/core/state" - "github.com/nspcc-dev/neo-go/pkg/neorpc" - "github.com/nspcc-dev/neo-go/pkg/rpcclient" - "github.com/nspcc-dev/neo-go/pkg/rpcclient/invoker" + "github.com/nspcc-dev/neo-go/pkg/util" "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" - "github.com/nspcc-dev/neofs-contract/rpc/container" cid "github.com/nspcc-dev/neofs-sdk-go/container/id" "github.com/nspcc-dev/neofs-sdk-go/object" oid "github.com/nspcc-dev/neofs-sdk-go/object/id" - "go.uber.org/zap" ) const ( - objPutEvName = "ObjectPut" - cnrDeleteName = "DeleteSuccess" - cnrRmName = "Removed" - cnrPutName = "PutSuccess" - cnrCrtName = "Created" - newEpochName = "NewEpoch" -) - -// subscribeForBlocks reqauires [Meta.cliM] to be taken. -func (m *Meta) subscribeForBlocks(ch chan<- *block.Header) (string, error) { - m.l.Debug("subscribe for blocks") - return m.ws.ReceiveHeadersOfAddedBlocks(nil, ch) -} - -// unsubscribeFromBlocks requires [Meta.cliM] to be taken. -func (m *Meta) unsubscribeFromBlocks() { - var err error - m.l.Debug("unsubscribing from blocks") - - err = m.ws.Unsubscribe(m.blockSubID) - if err != nil { - m.l.Warn("could not unsubscribe from blocks", zap.String("ID", m.blockSubID), zap.Error(err)) - return - } - - m.blockSubID = "" - - m.l.Debug("successfully unsubscribed from blocks") -} - -// subscribeForNewContainers requires [Meta.cliM] to be taken. -func (m *Meta) subscribeForNewContainers() error { - m.l.Debug("subscribing for containers") - - cnrPutEv := cnrPutName - var err1 error - m.cnrSubID, err1 = m.ws.ReceiveExecutionNotifications(&neorpc.NotificationFilter{Contract: &m.cnrH, Name: &cnrPutEv}, m.cnrPutEv) - - cnrCrtEv := cnrCrtName - var err2 error - m.cnrCrtSubID, err2 = m.ws.ReceiveExecutionNotifications(&neorpc.NotificationFilter{Contract: &m.cnrH, Name: &cnrCrtEv}, m.cnrPutEv) - - return errors.Join(err1, err2) -} - -// unsubscribeFromNewContainers requires [Meta.cliM] to be taken. -func (m *Meta) unsubscribeFromNewContainers() { - m.l.Debug("unsubscribing from containers") - - if err := m.ws.Unsubscribe(m.cnrCrtSubID); err != nil { - m.l.Error("could not unsubscribe from containers, ignore", zap.String("event", cnrCrtName), zap.String("sub", m.cnrCrtSubID), zap.Error(err)) - } - - err := m.ws.Unsubscribe(m.cnrSubID) - if err != nil { - m.l.Error("could not unsubscribe from containers", zap.String("event", cnrPutName), zap.String("sub", m.cnrSubID), zap.Error(err)) - return - } - - m.cnrSubID = "" - - m.l.Debug("successfully unsubscribed from containers") -} - -func (m *Meta) subscribeEvents() error { - epochEv := newEpochName - _, err := m.ws.ReceiveExecutionNotifications(&neorpc.NotificationFilter{Contract: &m.netmapH, Name: &epochEv}, m.epochEv) - if err != nil { - return fmt.Errorf("subscribe for epoch notifications: %w", err) - } - - return nil -} - -func (m *Meta) listenNotifications(ctx context.Context) error { - for { - select { - case h, ok := <-m.bCh: - if !ok { - err := m.reconnect(ctx) - if err != nil { - return err - } - - continue - } - - m.blockHeadersBuff <- h - case aer, ok := <-m.cnrPutEv: - if !ok { - err := m.reconnect(ctx) - if err != nil { - return err - } - - continue - } - - m.cliM.RLock() - alreadyListenToContainers := m.blockSubID != "" - m.cliM.RUnlock() - if alreadyListenToContainers { - // container will be handled - continue - } - - l := m.l.With(zap.Stringer("notification container", aer.Container)) - - ev, err := parseCnrNotification(*aer) - if err != nil { - l.Error("invalid container notification received", zap.Error(err)) - continue - } - - go m.addContainerIfMine(l, ev.cID) - case aer, ok := <-m.epochEv: - if !ok { - err := m.reconnect(ctx) - if err != nil { - return err - } - - continue - } - - l := m.l.With(zap.Stringer("notification container", aer.Container)) - - epoch, err := parseEpochNotification(aer) - if err != nil { - l.Error("invalid new epoch notification received", zap.Error(err)) - continue - } - - go func() { - err = m.handleEpochNotification(epoch) - if err != nil { - l.Error("handling new epoch notification", zap.Uint64("epoch", epoch), zap.Error(err)) - return - } - }() - case <-ctx.Done(): - m.l.Info("stop listening meta notifications") - return nil - } - } -} - -func (m *Meta) addContainerIfMine(l *zap.Logger, cID cid.ID) { - m.cliM.RLock() - reader := container.NewReader(invoker.New(m.ws, nil), m.cnrH) - cData, err := reader.GetContainerData(cID[:]) - m.cliM.RUnlock() - if err != nil { - l.Error("can't get container data", zap.Stringer("cid", cID), zap.Error(err)) - return - } - - ok, err := m.net.IsMineWithMeta(cID, cData) - if err != nil { - l.Error("failed to check container relation to node", zap.Stringer("cid", cID), zap.Error(err)) - return - } - if !ok { - return - } - - err = m.addContainer(cID) - if err != nil { - l.Error("can't add new container storage", zap.Stringer("cid", cID), zap.Error(err)) - return - } - - l.Debug("added container storage", zap.Stringer("cid", cID)) -} - -func (m *Meta) reconnect(ctx context.Context) error { - m.l.Warn("reconnecting to web socket client due to connection lost") - - m.cliM.Lock() - defer m.cliM.Unlock() - - var err error - m.ws, err = m.connect(ctx) - if err != nil { - return fmt.Errorf("reconnecting to web socket: %w", err) - } - - m.stM.RLock() - if len(m.storages) > 0 { - m.bCh = make(chan *block.Header, notificationBuffSize) - m.blockSubID, err = m.subscribeForBlocks(m.bCh) - if err != nil { - m.stM.RUnlock() - return fmt.Errorf("subscription for blocks: %w", err) - } - } else { - m.cnrPutEv = make(chan *state.ContainedNotificationEvent, notificationBuffSize) - err = m.subscribeForNewContainers() - if err != nil { - m.stM.RUnlock() - return fmt.Errorf("subscription for containers: %w", err) - } - } - m.stM.RUnlock() - - m.epochEv = make(chan *state.ContainedNotificationEvent, notificationBuffSize) - - err = m.subscribeEvents() - if err != nil { - return fmt.Errorf("subscribe for meta notifications: %w", err) - } - - return nil -} - -func (m *Meta) connect(ctx context.Context) (*rpcclient.WSClient, error) { - m.cfgM.RLock() - endpoints := slices.Clone(m.endpoints) - m.cfgM.RUnlock() - - var cli *rpcclient.WSClient - var err error -outer: - for { - for _, e := range endpoints { - cli, err = rpcclient.NewWS(ctx, e, rpcclient.WSOptions{ - Options: rpcclient.Options{ - DialTimeout: m.timeout, - }, - }) - if err == nil { - break outer - } - - m.l.Warn("creating rpc client", zap.String("endpoint", e), zap.Error(err)) - } - - const reconnectionCooldown = time.Second * 5 - m.l.Error("FS chain reconnection failed", zap.Duration("cooldown time", reconnectionCooldown)) - - select { - case <-ctx.Done(): - return nil, ctx.Err() - case <-time.After(reconnectionCooldown): - } - } - - err = cli.Init() - if err != nil { - return nil, fmt.Errorf("web socket client initializing: %w", err) - } - - return cli, nil -} - -const ( - // MPT-only key prefixes. - oidIndex = iota - attrIntToOIDIndex - attrPlainToOIDIndex - oidToAttrIndex - sizeIndex - firstPartIndex - previousPartIndex - deletedIndex - lockedIndex - typeIndex - - // storage-only key prefixes. - lockedByIndex - - lastEnumIndex -) + objPutEvName = "ObjectPut" -const ( - // meta map keys from FS chain. + // meta map keys from metadata chain. sizeKey = "size" networkMagicKey = "network" firstPartKey = "firstPart" @@ -309,15 +26,15 @@ const ( ) type objEvent struct { - cID cid.ID - oID oid.ID - size *big.Int - network *big.Int - firstObject []byte - prevObject []byte - deletedObjects []byte - lockedObjects []byte - typ object.Type + cID cid.ID + oID oid.ID + size *big.Int + network *big.Int + firstObject []byte + prevObject []byte + deletedObject []byte + lockedObject []byte + typ object.Type } func parseObjNotification(ev state.ContainedNotificationEvent) (objEvent, error) { @@ -403,26 +120,18 @@ func parseObjNotification(ev state.ContainedNotificationEvent) (objEvent, error) if v == nil { return res, fmt.Errorf("missing '%s' key for %s object type", deletedKey, res.typ) } - stackDeleted := v.Value().([]stackitem.Item) - for i, d := range stackDeleted { - rawDeleted, ok := d.Value().([]byte) - if !ok { - return res, fmt.Errorf("unexpected %d deleted object type: %T", i, d.Value()) - } - res.deletedObjects = append(res.deletedObjects, rawDeleted...) + res.deletedObject, ok = v.Value().([]byte) + if !ok { + return res, fmt.Errorf("unexpected deleted object type: %T", v.Value()) } case object.TypeLock: v = getFromMap(meta, lockedKey) if v == nil { return res, fmt.Errorf("missing '%s' key for %s object type", lockedKey, res.typ) } - stackLocked := v.Value().([]stackitem.Item) - for i, d := range stackLocked { - rawLocked, ok := d.Value().([]byte) - if !ok { - return res, fmt.Errorf("unexpected %d locked object type: %T", i, d.Value()) - } - res.lockedObjects = append(res.deletedObjects, rawLocked...) + res.lockedObject, ok = v.Value().([]byte) + if !ok { + return res, fmt.Errorf("unexpected deleted object type: %T", v.Value()) } case object.TypeLink, object.TypeRegular: default: @@ -442,207 +151,18 @@ func getFromMap(m *stackitem.Map, key string) stackitem.Item { return m.Value().([]stackitem.MapElement)[i].Value } -type cnrEvent struct { - cID cid.ID -} - -func parseCnrNotification(ev state.ContainedNotificationEvent) (cnrEvent, error) { - var res cnrEvent - - arr, ok := ev.Item.Value().([]stackitem.Item) - if !ok { - return res, fmt.Errorf("unexpected notification stack item: %T", ev.Item.Value()) - } - - switch ev.Name { - case cnrDeleteName: - const expectedNotificationArgs = 1 - if len(arr) != expectedNotificationArgs { - return res, fmt.Errorf("unexpected number of items on stack: %d, expected: %d", len(arr), expectedNotificationArgs) - } - case cnrPutName, cnrCrtName, cnrRmName: - const expectedNotificationArgs = 2 - if len(arr) != expectedNotificationArgs { - return res, fmt.Errorf("unexpected number of items on stack: %d, expected: %d", len(arr), expectedNotificationArgs) - } - } - - cID, ok := arr[0].Value().([]byte) - if !ok { - return res, fmt.Errorf("unexpected container ID stack item: %T", arr[0].Value()) - } - if len(cID) != cid.Size { - return res, fmt.Errorf("unexpected container ID len: %d", len(cID)) - } - - return cnrEvent{cID: cid.ID(cID)}, nil -} - -func (m *Meta) dropContainer(cID cid.ID) error { - m.stM.Lock() - defer m.stM.Unlock() - - st, ok := m.storages[cID] - if !ok { - return nil - } - - err := st.drop() - if err != nil { - m.l.Warn("drop container %s: %w", zap.Stringer("cID", cID), zap.Error(err)) - } - - delete(m.storages, cID) - - if len(m.storages) == 0 { - m.cliM.Lock() - m.unsubscribeFromBlocks() - err = m.subscribeForNewContainers() - m.cliM.Unlock() - if err != nil { - return fmt.Errorf("subscribing for new containers: %w", err) - } - } - - return nil -} - -func (m *Meta) addContainer(cID cid.ID) error { - var err error - m.stM.Lock() - defer m.stM.Unlock() - - if len(m.storages) == 0 { - m.cliM.Lock() - - m.blockSubID, err = m.subscribeForBlocks(m.bCh) - if err != nil { - m.cliM.Unlock() - return fmt.Errorf("blocks subscription: %w", err) - } - m.unsubscribeFromNewContainers() - - m.cliM.Unlock() - } - - st, err := storageForContainer(m.l, m.rootPath, cID) - if err != nil { - return fmt.Errorf("open new storage for %s container: %w", cID, err) - } - m.storages[cID] = st - - return nil -} - -func parseEpochNotification(ev *state.ContainedNotificationEvent) (uint64, error) { - const expectedNotificationArgs = 1 - - arr, ok := ev.Item.Value().([]stackitem.Item) - if !ok { - return 0, fmt.Errorf("unexpected notification stack item: %T", ev.Item.Value()) - } - if len(arr) != expectedNotificationArgs { - return 0, fmt.Errorf("unexpected number of items on stack: %d, expected: %d", len(arr), expectedNotificationArgs) - } - - epoch, ok := arr[0].Value().(*big.Int) - if !ok { - return 0, fmt.Errorf("unexpected epoch stack item: %T", arr[0].Value()) - } - - return epoch.Uint64(), nil -} - -func (m *Meta) handleEpochNotification(e uint64) error { - l := m.l.With(zap.Uint64("epoch", e)) - l.Debug("handling new epoch notification") - - cnrsNetwork, err := m.net.List(e) - if err != nil { - return fmt.Errorf("list containers: %w", err) - } - - m.stM.Lock() - - for cID, st := range m.storages { - _, ok := cnrsNetwork[cID] - if !ok { - l.Debug("drop container node does not belong to", zap.Stringer("cid", cID)) - - err = st.drop() - if err != nil { - l.Warn("drop inactual container", zap.Stringer("cID", cID), zap.Error(err)) - } - - delete(m.storages, cID) - } - } - for cID := range cnrsNetwork { - if _, ok := m.storages[cID]; ok { - continue - } - - st, err := storageForContainer(m.l, m.rootPath, cID) - if err != nil { - m.stM.Unlock() - return fmt.Errorf("create storage for container %s: %w", cID, err) - } - - m.storages[cID] = st - } - - m.stM.Unlock() - - m.cliM.Lock() - if len(m.storages) > 0 { - if m.blockSubID == "" { - m.blockSubID, err = m.subscribeForBlocks(m.bCh) - if err != nil { - m.cliM.Unlock() - return fmt.Errorf("blocks subscription: %w", err) - } - } - if m.cnrSubID != "" { - m.unsubscribeFromBlocks() - } - } else { - if m.blockSubID != "" { - m.unsubscribeFromBlocks() - } - - if m.cnrSubID == "" { - err = m.subscribeForNewContainers() - if err != nil { - m.cliM.Unlock() - return fmt.Errorf("containers subscription: %w", err) - } - } - } - m.cliM.Unlock() - - m.stM.RLock() - defer m.stM.RUnlock() - var gcWG sync.WaitGroup - for cID, st := range m.storages { - gcWG.Go(func() { - err := st.handleNewEpoch(e) - if err != nil { - l.Error("handling new epoch", zap.Stringer("cID", cID), zap.Error(err)) - } - }) - } - gcWG.Wait() - - l.Debug("handled new epoch successfully") - - return nil +type storageTask struct { + addr oid.Address + // cached object if object creation was initialized by _this_ storage node, + // and header is already stored in memory, otherwise it is nil + o *object.Object } // NotifyObjectSuccess subscribes channel for object notification chain inclusion. // Channel must be read before subscription is made and writing to it must be // non-blocking. -func (m *Meta) NotifyObjectSuccess(ch chan<- struct{}, addr oid.Address) { - m.notifier.subscribe(addr, ch) +func (m *Meta) NotifyObjectSuccess(ch chan<- struct{}, obj object.Object, h util.Uint256) { + m.notifier.subscribe(obj, ch, h) } // UnsubscribeFromObject unsibscribes from object notification. Should be called diff --git a/pkg/services/meta/notifications_test.go b/pkg/services/meta/notifications_test.go deleted file mode 100644 index 94684c91f2..0000000000 --- a/pkg/services/meta/notifications_test.go +++ /dev/null @@ -1,686 +0,0 @@ -package meta - -import ( - "context" - "crypto/rand" - "errors" - "fmt" - "math/big" - "os" - "path" - "slices" - "sync" - "testing" - "time" - - "github.com/google/uuid" - "github.com/nspcc-dev/bbolt" - "github.com/nspcc-dev/neo-go/pkg/core/block" - "github.com/nspcc-dev/neo-go/pkg/core/mpt" - "github.com/nspcc-dev/neo-go/pkg/core/state" - "github.com/nspcc-dev/neo-go/pkg/core/storage" - "github.com/nspcc-dev/neo-go/pkg/core/transaction" - "github.com/nspcc-dev/neo-go/pkg/neorpc" - "github.com/nspcc-dev/neo-go/pkg/neorpc/result" - "github.com/nspcc-dev/neo-go/pkg/smartcontract" - "github.com/nspcc-dev/neo-go/pkg/util" - "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" - objectcore "github.com/nspcc-dev/neofs-node/pkg/core/object" - meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase" - cid "github.com/nspcc-dev/neofs-sdk-go/container/id" - cidtest "github.com/nspcc-dev/neofs-sdk-go/container/id/test" - "github.com/nspcc-dev/neofs-sdk-go/object" - oid "github.com/nspcc-dev/neofs-sdk-go/object/id" - oidtest "github.com/nspcc-dev/neofs-sdk-go/object/id/test" - objecttest "github.com/nspcc-dev/neofs-sdk-go/object/test" - "github.com/stretchr/testify/require" - "go.uber.org/zap/zaptest" -) - -const ( - testNetworkMagic = 123 - testVUB = 12345 - testObjectSize = 1234567 -) - -type testNetwork struct { - m sync.RWMutex - - resCIDs map[cid.ID]struct{} - resObjects map[oid.Address]object.Object - resErr error -} - -func (t *testNetwork) Epoch() (uint64, error) { - return 123, nil -} - -func cidMap(v []cid.ID) map[cid.ID]struct{} { - var cids = make(map[cid.ID]struct{}, len(v)) - for _, c := range v { - cids[c] = struct{}{} - } - return cids -} - -func (t *testNetwork) setContainers(v []cid.ID) { - var cids = cidMap(v) - t.m.Lock() - t.resCIDs = cids - t.m.Unlock() -} - -func (t *testNetwork) setObjects(v map[oid.Address]object.Object) { - t.m.Lock() - t.resObjects = v - t.m.Unlock() -} - -func (t *testNetwork) Head(_ context.Context, cID cid.ID, oID oid.ID) (object.Object, error) { - t.m.RLock() - defer t.m.RUnlock() - - return t.resObjects[oid.NewAddress(cID, oID)], t.resErr -} - -func (t *testNetwork) IsMineWithMeta(_ cid.ID, _ []byte) (bool, error) { - return true, nil -} - -func (t *testNetwork) List(uint64) (map[cid.ID]struct{}, error) { - t.m.RLock() - defer t.m.RUnlock() - - return t.resCIDs, t.resErr -} - -func testContainers(t *testing.T, num int) []cid.ID { - res := make([]cid.ID, num) - for i := range num { - _, _ = rand.Read(res[i][:]) - } - - return res -} - -func newEpoch(m *Meta, epoch int) { - m.epochEv <- &state.ContainedNotificationEvent{ - NotificationEvent: state.NotificationEvent{ - Name: newEpochName, - Item: stackitem.NewArray([]stackitem.Item{stackitem.Make(epoch)}), - }, - } -} - -func checkDBFiles(t *testing.T, path string, cnrs []cid.ID) { - require.Eventually(t, func() bool { - entries, err := os.ReadDir(path) - if err != nil { - return false - } - - cnrsMap := cidMap(cnrs) - if len(entries) != len(cnrsMap) { - return false - } - for _, e := range entries { - var cID cid.ID - err = cID.DecodeString(e.Name()) - if err != nil { - t.Fatal("unexpected db file name", e.Name()) - } - - if _, ok := cnrsMap[cID]; !ok { - return false - } - - delete(cnrsMap, cID) - } - - return true - }, 5*time.Second, time.Millisecond*100, "expected to find db files") -} - -type testWS struct { - m sync.RWMutex - bCh chan<- *block.Header - notifications []state.ContainedNotificationEvent - err error -} - -func (t *testWS) blockCh() chan<- *block.Header { - t.m.RLock() - defer t.m.RUnlock() - - return t.bCh -} - -func (t *testWS) Unsubscribe(id string) error { - // TODO implement me - panic("not expected for now") -} - -func (t *testWS) swapResults(notifications []state.ContainedNotificationEvent, err error) { - t.m.Lock() - defer t.m.Unlock() - - t.notifications = notifications - t.err = err -} - -func (t *testWS) GetBlockNotifications(blockHash util.Uint256, filters *neorpc.NotificationFilter) (*result.BlockNotifications, error) { - t.m.RLock() - defer t.m.RUnlock() - - return &result.BlockNotifications{ - Application: t.notifications, - }, t.err -} - -func (t *testWS) GetVersion() (*result.Version, error) { - panic("not expected for now") -} - -func (t *testWS) ReceiveHeadersOfAddedBlocks(flt *neorpc.BlockFilter, rcvr chan<- *block.Header) (string, error) { - t.m.Lock() - t.bCh = rcvr - t.m.Unlock() - - return "", nil -} - -func (t *testWS) ReceiveExecutionNotifications(flt *neorpc.NotificationFilter, rcvr chan<- *state.ContainedNotificationEvent) (string, error) { - panic("not expected for now") -} - -func (t *testWS) Close() { - panic("not expected for now") -} - -func (t *testWS) InvokeContractVerify(contract util.Uint160, params []smartcontract.Parameter, signers []transaction.Signer, witnesses ...transaction.Witness) (*result.Invoke, error) { - panic("not called") -} - -func (t *testWS) InvokeFunction(contract util.Uint160, operation string, params []smartcontract.Parameter, signers []transaction.Signer) (*result.Invoke, error) { - panic("not called") -} - -func (t *testWS) InvokeScript(script []byte, signers []transaction.Signer) (*result.Invoke, error) { - panic("not called") -} - -func (t *testWS) TerminateSession(sessionID uuid.UUID) (bool, error) { - panic("not called") -} - -func (t *testWS) TraverseIterator(sessionID, iteratorID uuid.UUID, maxItemsCount int) ([]stackitem.Item, error) { - panic("not called") -} - -func createAndRunTestMeta(t *testing.T, ws wsClient, network NeoFSNetwork) (*Meta, func(), chan struct{}) { - ctx, cancel := context.WithCancel(context.Background()) - m := &Meta{ - l: zaptest.NewLogger(t), - rootPath: t.TempDir(), - magicNumber: 102938475, - bCh: make(chan *block.Header), - cnrPutEv: make(chan *state.ContainedNotificationEvent), - epochEv: make(chan *state.ContainedNotificationEvent), - blockHeadersBuff: make(chan *block.Header, blockBuffSize), - blockEventsBuff: make(chan blockObjEvents, blockBuffSize), - ws: ws, - - // no-op, to be filled by test cases if needed - storages: make(map[cid.ID]*containerStorage), - netmapH: util.Uint160{}, - cnrH: util.Uint160{}, - net: network, - endpoints: []string{}, - timeout: time.Second, - } - - exitCh := make(chan struct{}) - - var wg sync.WaitGroup - - wg.Go(func() { m.flusher(ctx) }) - wg.Go(func() { m.blockHandler(ctx, m.blockHeadersBuff) }) - wg.Go(func() { m.blockStorer(ctx, m.blockEventsBuff) }) - go func() { - _ = m.listenNotifications(ctx) - wg.Wait() - exitCh <- struct{}{} - }() - - return m, cancel, exitCh -} - -// args list consists of [testing.T], [Meta] and the list from -// [objectcore.EncodeReplicationMetaInfo] can be improved at the time [object] -// package will do it. -func checkObject(t *testing.T, m *Meta, cID cid.ID, oID, firstPart, previousPart oid.ID, pSize uint64, typ object.Type, deleted, locked []oid.ID, _ uint64, _ uint32) bool { - getBoth := func(trie *mpt.Trie, st storage.Store, key, expV []byte) bool { - mptV, err := trie.Get(key) - if err != nil { - if errors.Is(err, mpt.ErrNotFound) { - return false - } - t.Fatalf("failed to get oid value from mpt: %v", err) - } - - dbV, err := st.Get(key) - if err != nil { - if errors.Is(err, storage.ErrKeyNotFound) { - return false - } - t.Fatalf("failed to get oid value from db: %v", err) - } - - require.Equal(t, dbV, mptV) - require.Equal(t, dbV, expV) - - return true - } - - getMPT := func(trie *mpt.Trie, st storage.Store, key, expV []byte) bool { - v, err := trie.Get(key) - if err != nil { - if errors.Is(err, mpt.ErrNotFound) { - return false - } - t.Fatalf("failed to get oid value from mpt: %v", err) - } - - require.Equal(t, expV, v) - - return true - } - - m.stM.RLock() - st := m.storages[cID] - m.stM.RUnlock() - - st.m.RLock() - defer st.m.RUnlock() - - commSuffix := oID[:] - - ok := getBoth(st.mpt, st.db, append([]byte{oidIndex}, commSuffix...), []byte{}) - if !ok { - return false - } - - if len(deleted) != 0 { - expVal := make([]byte, 0, oid.Size*(len(deleted))) - for _, d := range deleted { - expVal = append(expVal, d[:]...) - } - - ok = getBoth(st.mpt, st.db, append([]byte{deletedIndex}, commSuffix...), expVal) - if !ok { - return false - } - } - - if len(locked) != 0 { - expVal := make([]byte, 0, oid.Size*(len(locked))) - for _, l := range locked { - expVal = append(expVal, l[:]...) - } - - ok = getBoth(st.mpt, st.db, append([]byte{lockedIndex}, commSuffix...), expVal) - if !ok { - return false - } - } - - var sizeB big.Int - sizeB.SetUint64(pSize) - ok = getMPT(st.mpt, st.db, append([]byte{sizeIndex}, commSuffix...), sizeB.Bytes()) - if !ok { - return false - } - - if firstPart != (oid.ID{}) { - ok = getMPT(st.mpt, st.db, append([]byte{firstPartIndex}, commSuffix...), firstPart[:]) - if !ok { - return false - } - } - - if previousPart != (oid.ID{}) { - ok = getMPT(st.mpt, st.db, append([]byte{previousPartIndex}, commSuffix...), previousPart[:]) - if !ok { - return false - } - } - - if typ != object.TypeRegular { - ok = getMPT(st.mpt, st.db, append([]byte{typeIndex}, commSuffix...), []byte{byte(typ)}) - if !ok { - return false - } - } - - return true -} - -func TestObjectPut(t *testing.T) { - ws := testWS{} - net := testNetwork{} - m, stop, exitCh := createAndRunTestMeta(t, &ws, &net) - t.Cleanup(func() { - stop() - <-exitCh - }) - - testCnrs := testContainers(t, 10) - - var epoch int - net.setContainers(testCnrs) - newEpoch(m, epoch) - - time.Sleep(time.Second) - - t.Run("storages for containers", func(t *testing.T) { - checkDBFiles(t, m.rootPath, testCnrs) - }) - - t.Run("drop storage", func(t *testing.T) { - var newContainers = testCnrs[1:] - net.setContainers(newContainers) - - epoch++ - newEpoch(m, epoch) - checkDBFiles(t, m.rootPath, newContainers) - }) - - t.Run("add storage", func(t *testing.T) { - // add just dropped storage back - epoch++ - net.setContainers(testCnrs) - newEpoch(m, epoch) - checkDBFiles(t, m.rootPath, testCnrs) - }) - - t.Run("put object", func(t *testing.T) { - cID := testCnrs[0] - oID := oidtest.ID() - fPart := oidtest.ID() - pPart := oidtest.ID() - size := uint64(testObjectSize) - typ := object.TypeRegular - - o := objecttest.Object() - o.SetContainerID(cID) - o.SetID(oID) - o.SetFirstID(fPart) - o.SetPreviousID(pPart) - o.SetPayloadSize(size) - o.SetType(typ) - - metaRaw := objectcore.EncodeReplicationMetaInfo(cID, oID, fPart, pPart, size, typ, nil, nil, testVUB, m.magicNumber) - metaStack, err := stackitem.Deserialize(metaRaw) - require.NoError(t, err) - - bCH := ws.blockCh() - - net.setObjects(map[oid.Address]object.Object{oid.NewAddress(cID, oID): o}) - ws.swapResults([]state.ContainedNotificationEvent{{ - NotificationEvent: state.NotificationEvent{ - Name: objPutEvName, - Item: stackitem.NewArray([]stackitem.Item{stackitem.Make(cID[:]), stackitem.Make(oID[:]), metaStack}), - }, - }}, nil) - bCH <- &block.Header{Index: 0} - - require.Eventually(t, func() bool { - return checkObject(t, m, cID, oID, fPart, pPart, size, typ, nil, nil, testVUB, m.magicNumber) - }, 3*time.Second, time.Millisecond*100, "object was not handled properly") - }) - - t.Run("delete object", func(t *testing.T) { - cID := testCnrs[0] - objToDeleteOID := oidtest.ID() - size := uint64(testObjectSize) - - o := objecttest.Object() - o.ResetRelations() - o.SetContainerID(cID) - o.SetID(objToDeleteOID) - o.SetPayloadSize(size) - - metaRaw := objectcore.EncodeReplicationMetaInfo(cID, objToDeleteOID, oid.ID{}, oid.ID{}, size, object.TypeRegular, nil, nil, testVUB, m.magicNumber) - metaStack, err := stackitem.Deserialize(metaRaw) - require.NoError(t, err) - - bCH := ws.blockCh() - net.setObjects(map[oid.Address]object.Object{oid.NewAddress(cID, objToDeleteOID): o}) - ws.swapResults([]state.ContainedNotificationEvent{{ - NotificationEvent: state.NotificationEvent{ - Name: objPutEvName, - Item: stackitem.NewArray([]stackitem.Item{stackitem.Make(cID[:]), stackitem.Make(objToDeleteOID[:]), metaStack}), - }, - }}, nil) - bCH <- &block.Header{Index: 1} - - require.Eventually(t, func() bool { - return checkObject(t, m, cID, objToDeleteOID, oid.ID{}, oid.ID{}, size, object.TypeRegular, nil, nil, testVUB, m.magicNumber) - }, 3*time.Second, time.Millisecond*100, "object was not handled properly before deletion") - - tsCID := cID - tsOID := oidtest.ID() - tsSize := uint64(testObjectSize) - deleted := []oid.ID{objToDeleteOID} - - ts := objecttest.Object() - ts.SetContainerID(tsCID) - ts.SetID(tsOID) - ts.SetPayloadSize(tsSize) - - metaRaw = objectcore.EncodeReplicationMetaInfo(tsCID, tsOID, oid.ID{}, oid.ID{}, tsSize, object.TypeTombstone, deleted, nil, testVUB, m.magicNumber) - metaStack, err = stackitem.Deserialize(metaRaw) - require.NoError(t, err) - - net.setObjects(map[oid.Address]object.Object{oid.NewAddress(tsCID, tsOID): ts}) - ws.swapResults([]state.ContainedNotificationEvent{{ - NotificationEvent: state.NotificationEvent{ - Name: objPutEvName, - Item: stackitem.NewArray([]stackitem.Item{stackitem.Make(tsCID[:]), stackitem.Make(tsOID[:]), metaStack}), - }, - }}, nil) - bCH <- &block.Header{Index: 2} - - require.Eventually(t, func() bool { - m.stM.RLock() - st := m.storages[tsCID] - m.stM.RUnlock() - - st.m.RLock() - defer st.m.RUnlock() - - commSuffix := objToDeleteOID[:] - mptKeys := [][]byte{ - append([]byte{0, oidIndex}, commSuffix...), - append([]byte{0, sizeIndex}, commSuffix...), - append([]byte{0, firstPartIndex}, commSuffix...), - append([]byte{0, previousPartIndex}, commSuffix...), - append([]byte{0, deletedIndex}, commSuffix...), - append([]byte{0, lockedIndex}, commSuffix...), - append([]byte{0, typeIndex}, commSuffix...), - } - - tempM := make(map[string][]byte) - fillObjectIndex(tempM, o, false) - - for _, key := range mptKeys { - _, err = st.mpt.Get(key) - if !errors.Is(err, mpt.ErrNotFound) { - return false - } - } - - for key := range tempM { - _, err = st.db.Get([]byte(key)) - if !errors.Is(err, storage.ErrKeyNotFound) { - return false - } - } - - return true - }, 3*time.Second, time.Millisecond*100, "object was not deleted") - }) -} - -func TestValidation(t *testing.T) { - cID := cidtest.ID() - ctx := context.Background() - l := zaptest.NewLogger(t) - - path.Join(t.TempDir(), "db.db") - s, err := storageForContainer(zaptest.NewLogger(t), path.Join(t.TempDir(), "db.db"), cID) - require.NoError(t, err) - t.Cleanup(func() { - _ = s.drop() - }) - - t.Run("delete non-existent", func(t *testing.T) { - objToDelete := oidtest.ID() - ev := objEvent{ - cID: cID, - oID: oidtest.ID(), - size: big.NewInt(testObjectSize), - network: big.NewInt(testNetworkMagic), - deletedObjects: objToDelete[:], - typ: object.TypeTombstone, - } - - require.ErrorContains(t, isOpAllowed(s.db, ev), "object-to-delete is missing") - }) - - t.Run("lock non-existent", func(t *testing.T) { - objToLock := oidtest.ID() - ev := objEvent{ - cID: cID, - oID: oidtest.ID(), - size: big.NewInt(testObjectSize), - network: big.NewInt(testNetworkMagic), - lockedObjects: objToLock[:], - typ: object.TypeLock, - } - - require.ErrorContains(t, isOpAllowed(s.db, ev), "presence check") - }) - - t.Run("delete locked", func(t *testing.T) { - obj1 := objecttest.Object() - oID1 := obj1.GetID() - obj1.SetContainerID(cID) - obj2 := objecttest.Object() - oID2 := obj2.GetID() - obj2.SetContainerID(cID) - - net := &testNetwork{} - ee := []objEvent{ - { - cID: cID, - oID: oID1, - size: big.NewInt(testObjectSize), - network: big.NewInt(testNetworkMagic), - }, - { - cID: cID, - oID: oID2, - size: big.NewInt(testObjectSize), - network: big.NewInt(testNetworkMagic), - }, - } - net.setObjects(map[oid.Address]object.Object{ - oid.NewAddress(cID, oID1): obj1, - oid.NewAddress(cID, oID2): obj2, - }) - - s.putObjects(ctx, l, 0, ee, net) - - lock := objecttest.Object() - lock.SetContainerID(cID) - lock.SetType(object.TypeLock) - - ee = []objEvent{ - { - cID: cID, - oID: lock.GetID(), - size: big.NewInt(testObjectSize), - network: big.NewInt(testNetworkMagic), - lockedObjects: slices.Concat(oID1[:], oID2[:]), - typ: object.TypeLock, - }, - } - net.setObjects(map[oid.Address]object.Object{ - oid.NewAddress(cID, lock.GetID()): lock, - }) - - s.putObjects(ctx, l, 0, ee, net) - - ts := objecttest.Object() - ts.SetContainerID(cID) - ts.SetType(object.TypeTombstone) - - e := objEvent{ - cID: cID, - oID: ts.GetID(), - size: big.NewInt(testObjectSize), - network: big.NewInt(testNetworkMagic), - deletedObjects: slices.Concat(oID1[:], oID2[:]), - typ: object.TypeTombstone, - } - net.setObjects(map[oid.Address]object.Object{ - oid.NewAddress(cID, ts.GetID()): ts, - }) - - require.ErrorContains(t, isOpAllowed(s.db, e), fmt.Sprintf("is locked by %s", lock.GetID())) - }) -} - -func TestCompatibility(t *testing.T) { - o := objecttest.Object() - o.SetSplitID(nil) // no split info is expected for split V2 era - o.ResetRelations() - - // database from engine's metabases - - db, err := bbolt.Open(path.Join(t.TempDir(), "db.db"), 0600, bbolt.DefaultOptions) - require.NoError(t, err) - t.Cleanup(func() { - _ = db.Close() - }) - - metabaseMap := make(map[string][]byte) - - err = db.Update(func(tx *bbolt.Tx) error { - err = meta.PutMetadataForObject(tx, o, true) - require.NoError(t, err) - - cID := o.GetContainerID() - metaBucketKey := []byte{255} - metaBucketKey = append(metaBucketKey, cID[:]...) - - b := tx.Bucket(metaBucketKey) - return b.ForEach(func(k, v []byte) error { - metabaseMap[string(k)] = v - return nil - }) - }) - require.NoError(t, err) - - // batch for meta-data service - - serviceMap := make(map[string][]byte) - fillObjectIndex(serviceMap, o, false) - - require.Equal(t, len(metabaseMap), len(serviceMap)) - for k := range metabaseMap { - _, found := serviceMap[k] - require.Truef(t, found, "%s key not found: %v", k, []byte(k)) - } -} diff --git a/pkg/services/meta/search.go b/pkg/services/meta/search.go index 847bea7766..40c9a685d6 100644 --- a/pkg/services/meta/search.go +++ b/pkg/services/meta/search.go @@ -1,127 +1,13 @@ package meta import ( - "bytes" - "fmt" - "slices" - - "github.com/nspcc-dev/neo-go/pkg/core/storage" objectcore "github.com/nspcc-dev/neofs-node/pkg/core/object" "github.com/nspcc-dev/neofs-sdk-go/client" cid "github.com/nspcc-dev/neofs-sdk-go/container/id" - oid "github.com/nspcc-dev/neofs-sdk-go/object/id" - "go.uber.org/zap" ) // Search selects up to count container's objects from the given container // matching the specified filters. func (m *Meta) Search(cID cid.ID, fs []objectcore.SearchFilter, attrs []string, cursor *objectcore.SearchCursor, count uint16) ([]client.SearchResultItem, []byte, error) { - m.stM.RLock() - s, ok := m.storages[cID] - m.stM.RUnlock() - - if !ok { - m.l.Debug("skip search request due inactual container", zap.Stringer("cid", cID)) - return nil, nil, nil - } - - return s.search(fs, attrs, cursor, count) -} - -func (s *containerStorage) search(fs []objectcore.SearchFilter, attrs []string, cursor *objectcore.SearchCursor, count uint16) ([]client.SearchResultItem, []byte, error) { - s.m.Lock() - defer s.m.Unlock() - - if len(fs) == 0 { - return searchUnfiltered(s.db, cursor, count) - } - - resHolder := objectcore.SearchResult{Objects: make([]client.SearchResultItem, 0, count)} - handleKV := objectcore.MetaDataKVHandler(&resHolder, &attrGetter{db: s.db}, nil, fs, attrs, cursor, count) - - var rng storage.SeekRange - if cursor != nil { - rng.Prefix = cursor.PrimaryKeysPrefix - rng.Start = bytes.TrimPrefix(cursor.PrimarySeekKey, cursor.PrimaryKeysPrefix) - } - s.db.Seek(rng, func(k, v []byte) bool { - if cursor != nil && bytes.Equal(k, cursor.PrimarySeekKey) { - // points to the last response element, so go next - return true - } - - return handleKV(k, v) - }) - - return resHolder.Objects, resHolder.UpdatedSearchCursor, resHolder.Err -} - -// lock must be taken. -func searchUnfiltered(st storage.Store, cursor *objectcore.SearchCursor, count uint16) ([]client.SearchResultItem, []byte, error) { - res := make([]client.SearchResultItem, 0, count) - var err error - var newCursor []byte - - var rng storage.SeekRange - if cursor != nil { - rng.Prefix = cursor.PrimaryKeysPrefix - rng.Start = bytes.TrimPrefix(cursor.PrimarySeekKey, cursor.PrimaryKeysPrefix) - } else { - rng.Prefix = []byte{oidIndex} - } - - st.Seek(rng, func(k, _ []byte) bool { - if bytes.Equal(k, cursor.PrimarySeekKey) { - return true - } - if len(res) == int(count) { - newCursor = res[len(res)-1].ID[:] - return false - } - - if len(k) != oid.Size+1 { - err = fmt.Errorf("invalid meta bucket key (prefix 0x%X): unexpected object key len %d", k[0], len(k)) - return false - } - res = append(res, client.SearchResultItem{ID: oid.ID(k[1:])}) - - return true - }) - - return res, newCursor, err -} - -type attrGetter struct { - keyBuff []byte - - db storage.Store -} - -func (a *attrGetter) Get(oID []byte, attributeKey string) (attributeValue []byte, err error) { - if len(a.keyBuff) > 0 { - a.keyBuff = a.keyBuff[:0] - } - - a.keyBuff = slices.Grow(a.keyBuff, attrIDFixedLen+len(attributeKey)) - a.keyBuff = append(a.keyBuff, oidToAttrIndex) - a.keyBuff = append(a.keyBuff, oID...) - a.keyBuff = append(a.keyBuff, attributeKey...) - a.keyBuff = append(a.keyBuff, objectcore.MetaAttributeDelimiter...) - - var rng storage.SeekRange - rng.Start = a.keyBuff - a.db.Seek(rng, func(k, v []byte) bool { - if !bytes.HasPrefix(k, a.keyBuff) { - return false - } - if len(k[len(a.keyBuff):]) == 0 { - err = fmt.Errorf("invalid meta bucket key (prefix 0x%X): missing attribute value", k[0]) - return false - } - attributeValue = slices.Clone(k[len(a.keyBuff):]) - - return false - }) - - return + return m.metabase.Search(cID, fs, attrs, cursor, count) } diff --git a/pkg/services/meta/search_test.go b/pkg/services/meta/search_test.go index 8ca4258c0e..16b5539c82 100644 --- a/pkg/services/meta/search_test.go +++ b/pkg/services/meta/search_test.go @@ -1,49 +1,44 @@ package meta import ( + "context" "testing" - objectcore "github.com/nspcc-dev/neofs-node/pkg/core/object" + "github.com/nspcc-dev/neofs-node/pkg/services/sidechain" metatest "github.com/nspcc-dev/neofs-node/pkg/util/meta/test" - "github.com/nspcc-dev/neofs-sdk-go/client" cid "github.com/nspcc-dev/neofs-sdk-go/container/id" "github.com/nspcc-dev/neofs-sdk-go/object" + oid "github.com/nspcc-dev/neofs-sdk-go/object/id" "github.com/stretchr/testify/require" - "go.uber.org/zap" "go.uber.org/zap/zaptest" ) -type searchTestDB struct { - t *testing.T - storages map[cid.ID]*containerStorage - l *zap.Logger +type fixedNameMeta struct { + *Meta } -func (s searchTestDB) Put(obj *object.Object) error { - cID := obj.GetContainerID() - st, ok := s.storages[cID] - if !ok { - var err error +type noopNetwork struct{} - st, err = storageForContainer(s.l, s.t.TempDir(), cID) - require.NoError(s.t, err) - s.t.Cleanup(func() { - _ = st.drop() - }) - - s.storages[cID] = st - } +func (n noopNetwork) IsMineWithMeta(id cid.ID, bytes []byte) (bool, error) { + panic("do not call me") +} - batch := make(map[string][]byte) - fillObjectIndex(batch, *obj, false) - return st.db.PutChangeSet(batch, nil) +func (n noopNetwork) Head(ctx context.Context, cID cid.ID, oID oid.ID) (object.Object, error) { + panic("do not call me") } -func (s searchTestDB) Search(cnr cid.ID, fs []objectcore.SearchFilter, attrs []string, cursor *objectcore.SearchCursor, count uint16) ([]client.SearchResultItem, []byte, error) { - return s.storages[cnr].search(fs, attrs, cursor, count) +func (n *fixedNameMeta) Put(obj *object.Object) error { + return n.PutObject(obj) } func TestMeta_Search(t *testing.T) { - db := searchTestDB{t: t, l: zaptest.NewLogger(t), storages: make(map[cid.ID]*containerStorage)} - metatest.TestSearchObjects(t, db, false) + m, err := New(Parameters{ + Logger: zaptest.NewLogger(t), + Chain: &sidechain.SideChain{}, + Path: t.TempDir(), + Network: noopNetwork{}, + }) + require.NoError(t, err) + + metatest.TestSearchObjects(t, &fixedNameMeta{m}, false) } diff --git a/pkg/services/meta/storage.go b/pkg/services/meta/storage.go new file mode 100644 index 0000000000..b6746b8962 --- /dev/null +++ b/pkg/services/meta/storage.go @@ -0,0 +1,53 @@ +package meta + +import ( + "context" + + "github.com/nspcc-dev/neofs-sdk-go/object" + "go.uber.org/zap" +) + +// PutObject forces [Meta] to index provided object. +func (m *Meta) PutObject(o *object.Object) error { + return m.metabase.Put(o) +} + +func (m *Meta) storager(ctx context.Context, buff <-chan storageTask) { + for { + if len(buff) >= notificationBuffSize-1 { + m.l.Warn("storage task queue buffer has been completely filled") + } + + select { + case <-ctx.Done(): + return + case n := <-buff: + l := m.l.With(zap.Stringer("addr", n.addr)) + + cID := n.addr.Container() + ok, err := m.net.IsMineWithMeta(cID, nil) + if err != nil { + l.Error("failed to check container relation to node", zap.Stringer("cid", cID), zap.Error(err)) + return + } + if !ok { + return + } + + if n.o == nil { + o, err := m.net.Head(ctx, n.addr.Container(), n.addr.Object()) + if err != nil { + l.Error("cannot fetch object header", zap.Error(err)) + continue + } + + n.o = &o + } + + err = m.PutObject(n.o) + if err != nil { + l.Error("failed to store object", zap.Error(err)) + } + } + } +} diff --git a/pkg/services/object/get_test.go b/pkg/services/object/get_test.go index 3aabc8a9ed..92b651f1e7 100644 --- a/pkg/services/object/get_test.go +++ b/pkg/services/object/get_test.go @@ -55,7 +55,7 @@ func TestServer_Get_Local(t *testing.T) { ) handlers := &getOnlyHandler{svc: handler} - srv := New(handlers, 0, nil, fsChain, nil, nil, signer.ECDSAPrivateKey, mtrc, aclChecker, reqInfoExt, nil) + srv := New(handlers, nil, fsChain, nil, nil, signer.ECDSAPrivateKey, mtrc, aclChecker, reqInfoExt, nil) for _, pldLen := range []uint64{ 0, 1, @@ -179,7 +179,7 @@ func TestServer_Get_Remote(t *testing.T) { ) handlers := getOnlyHandler{svc: handler} - srv := New(handlers, 0, nil, fsChain, nil, nil, signer.ECDSAPrivateKey, &mtrc, aclChecker, reqInfoExt, nil) + srv := New(handlers, nil, fsChain, nil, nil, signer.ECDSAPrivateKey, &mtrc, aclChecker, reqInfoExt, nil) t.Run("object", func(t *testing.T) { const payloadLen = 100 << 10 diff --git a/pkg/services/object/head_test.go b/pkg/services/object/head_test.go index 615def197d..bbb43eb176 100644 --- a/pkg/services/object/head_test.go +++ b/pkg/services/object/head_test.go @@ -56,7 +56,7 @@ func TestServer_Head_Local(t *testing.T) { ) handlers := headOnlyHandler{svc: handler} - srv := New(handlers, 0, nil, fsChain, nil, nil, signer.ECDSAPrivateKey, mtrc, aclChecker, reqInfoExt, nil) + srv := New(handlers, nil, fsChain, nil, nil, signer.ECDSAPrivateKey, mtrc, aclChecker, reqInfoExt, nil) assertWithVersion := func(t *testing.T, ver version.Version) *protoobject.HeadResponse { req := newLocalHeadRequest(t, ver, obj.Address(), signer) @@ -136,7 +136,7 @@ func TestServer_Head_Remote(t *testing.T) { ) handlers := headOnlyHandler{svc: handler} - srv := New(handlers, 0, nil, fsChain, nil, nil, signer.ECDSAPrivateKey, mtrc, aclChecker, reqInfoExt, nil) + srv := New(handlers, nil, fsChain, nil, nil, signer.ECDSAPrivateKey, mtrc, aclChecker, reqInfoExt, nil) t.Run("EC part", func(t *testing.T) { nodes := make([]netmap.NodeInfo, 3) @@ -188,7 +188,7 @@ func TestServer_Head_Remote(t *testing.T) { ) handlers := headOnlyHandler{svc: handler} - srv := New(handlers, 0, nil, fsChain, nil, nil, signer.ECDSAPrivateKey, mtrc, aclChecker, reqInfoExt, nil) + srv := New(handlers, nil, fsChain, nil, nil, signer.ECDSAPrivateKey, mtrc, aclChecker, reqInfoExt, nil) t.Run("header", func(t *testing.T) { const payloadLen = 100 << 10 diff --git a/pkg/services/object/put/distributed.go b/pkg/services/object/put/distributed.go index 8ddb99b711..f24fe4c01b 100644 --- a/pkg/services/object/put/distributed.go +++ b/pkg/services/object/put/distributed.go @@ -11,12 +11,13 @@ import ( "sync" "sync/atomic" + "github.com/nspcc-dev/neo-go/pkg/core/transaction" iec "github.com/nspcc-dev/neofs-node/internal/ec" islices "github.com/nspcc-dev/neofs-node/internal/slices" netmapcore "github.com/nspcc-dev/neofs-node/pkg/core/netmap" objectcore "github.com/nspcc-dev/neofs-node/pkg/core/object" chaincontainer "github.com/nspcc-dev/neofs-node/pkg/morph/client/container" - "github.com/nspcc-dev/neofs-node/pkg/services/meta" + meta "github.com/nspcc-dev/neofs-node/pkg/services/meta" svcutil "github.com/nspcc-dev/neofs-node/pkg/services/object/util" "github.com/nspcc-dev/neofs-node/pkg/util" apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status" @@ -29,10 +30,13 @@ import ( ) type metaCollection struct { - objectData []byte + sortedNodes [][]netmap.NodeInfo // by public keys, required for metadata optimization + + dataToSign []byte + metaTransaction *transaction.Transaction signaturesMtx sync.RWMutex - signatures [][][]byte + signatures [][]meta.IndexedSignature } type distributedTarget struct { @@ -40,16 +44,15 @@ type distributedTarget struct { placementIterator placementIterator - obj *object.Object - networkMagicNumber uint32 - fsState netmapcore.StateDetailed + obj *object.Object + fsState netmapcore.StateDetailed cnrClient *chaincontainer.Client metainfoConsistencyAttr string metaSvc *meta.Meta metaSigner neofscrypto.Signer - metaCollection metaCollection + metaCollection *metaCollection containerNodes ContainerNodes localNodeInContainer bool @@ -206,7 +209,7 @@ func (t *distributedTarget) saveObject(obj object.Object, encObj encodedObject) return t.distributeObject(obj, encObj, func(obj object.Object, encObj encodedObject) error { return t.placementIterator.iterateNodesForObject(obj.GetID(), useRepRules, objNodeLists, broadcast, func(node nodeDesc) error { - return t.sendObject(obj, encObj, node, &t.metaCollection) + return t.sendObject(obj, encObj, node, t.metaCollection) }) }) } @@ -229,7 +232,7 @@ func (t *distributedTarget) saveObject(obj object.Object, encObj encodedObject) return nil } - return t.saveECPart(obj, encObj, t.ecPart.RuleIndex, t.ecPart.Index, total, nodes, &t.metaCollection) + return t.saveECPart(obj, encObj, t.ecPart.RuleIndex, t.ecPart.Index, total, nodes, t.metaCollection) } if t.sessionSigner == nil { @@ -398,8 +401,8 @@ nextRule: continue } - if t.localNodeInContainer && t.metainfoConsistencyAttr != "" && t.metaCollection.objectData == nil { - t.metaCollection.objectData = t.encodeObjectMetadata(obj) + if t.localNodeInContainer && t.metainfoConsistencyAttr != "" && t.metaCollection.dataToSign == nil { + t.metaCollection.metaTransaction, t.metaCollection.dataToSign = t.encodeObjectMetadata(obj) } if l == nil { @@ -421,7 +424,7 @@ nextRule: } stored, overloaded, err := t.placementIterator.handleREPRule(l, repProg, ruleIdx, minReps, maxReps, objNodeLists[ruleIdx], func(node nodeDesc) error { - return t.sendObject(obj, encObj, node, &t.metaCollection) + return t.sendObject(obj, encObj, node, t.metaCollection) }) if err != nil { if maxReplicas > 0 { @@ -439,7 +442,7 @@ nextRule: } if len(repRules) > 0 { - err = t.submitMetaCollection(obj.Address(), &t.metaCollection) + err = t.submitMetaCollection(obj, t.metaCollection) if err != nil { return err } @@ -517,7 +520,8 @@ func (t *distributedTarget) resetMetaCollection() { t.metaCollection.signatures[i] = t.metaCollection.signatures[i][:0] } - t.metaCollection.objectData = nil + t.metaCollection.metaTransaction = nil + t.metaCollection.dataToSign = nil } func (t *distributedTarget) distributeObject(obj object.Object, encObj encodedObject, @@ -525,10 +529,10 @@ func (t *distributedTarget) distributeObject(obj object.Object, encObj encodedOb defer t.resetMetaCollection() if t.localNodeInContainer && t.metainfoConsistencyAttr != "" { - t.metaCollection.objectData = t.encodeObjectMetadata(obj) + t.metaCollection.metaTransaction, t.metaCollection.dataToSign = t.encodeObjectMetadata(obj) } - return t.distributeObjectWithMeta(obj, encObj, &t.metaCollection, placementFn) + return t.distributeObjectWithMeta(obj, encObj, t.metaCollection, placementFn) } func (t *distributedTarget) distributeObjectWithMeta(obj object.Object, encObj encodedObject, metaC *metaCollection, @@ -549,10 +553,10 @@ func (t *distributedTarget) distributeObjectWithMeta(obj object.Object, encObj e return err } - return t.submitMetaCollection(obj.Address(), metaC) + return t.submitMetaCollection(obj, metaC) } -func (t *distributedTarget) submitMetaCollection(addr oid.Address, metaC *metaCollection) error { +func (t *distributedTarget) submitMetaCollection(o object.Object, metaC *metaCollection) error { if t.localOnly || !t.localNodeInContainer || t.metainfoConsistencyAttr == "" { return nil } @@ -570,16 +574,24 @@ func (t *distributedTarget) submitMetaCollection(addr oid.Address, metaC *metaCo return nil } - var objAccepted chan struct{} + var ( + objAcceptedCh chan struct{} + addr = o.Address() + ) if await { - objAccepted = make(chan struct{}, 1) - t.metaSvc.NotifyObjectSuccess(objAccepted, addr) + h := t.metaCollection.metaTransaction.Hash() + + var objCopy object.Object + o.CutPayload().CopyTo(&objCopy) + + objAcceptedCh = make(chan struct{}, 1) + t.metaSvc.NotifyObjectSuccess(objAcceptedCh, objCopy, h) } - err := t.cnrClient.SubmitObjectPut(metaC.objectData, metaC.signatures) + err := t.metaSvc.SubmitObjectPut(t.metaCollection.metaTransaction, t.metaCollection.signatures) if err != nil { if await { - t.metaSvc.UnsubscribeFromObject(addr) + t.metaSvc.UnsubscribeFromObject(o.Address()) } return fmt.Errorf("failed to submit %s object meta information: %w", addr, err) } @@ -589,7 +601,7 @@ func (t *distributedTarget) submitMetaCollection(addr oid.Address, metaC *metaCo case <-t.opCtx.Done(): t.metaSvc.UnsubscribeFromObject(addr) return fmt.Errorf("interrupted awaiting for %s object meta information: %w", addr, t.opCtx.Err()) - case <-objAccepted: + case <-objAcceptedCh: } } @@ -598,8 +610,8 @@ func (t *distributedTarget) submitMetaCollection(addr oid.Address, metaC *metaCo return nil } -func (t *distributedTarget) encodeObjectMetadata(obj object.Object) []byte { - currBlock := t.fsState.CurrentBlock() +func (t *distributedTarget) encodeObjectMetadata(obj object.Object) (*transaction.Transaction, []byte) { + currBlock := t.metaSvc.Height() currEpochDuration := t.fsState.CurrentEpochDuration() expectedVUB := (uint64(currBlock)/currEpochDuration + 2) * currEpochDuration @@ -609,19 +621,23 @@ func (t *distributedTarget) encodeObjectMetadata(obj object.Object) []byte { firstObj = obj.GetID() } - var deletedObjs []oid.ID - var lockedObjs []oid.ID + var ( + deletedObj oid.ID + lockedObj oid.ID + ) typ := obj.Type() switch typ { case object.TypeTombstone: - deletedObjs = append(deletedObjs, obj.AssociatedObject()) + deletedObj = obj.AssociatedObject() case object.TypeLock: - lockedObjs = append(lockedObjs, obj.AssociatedObject()) + lockedObj = obj.AssociatedObject() default: } - return objectcore.EncodeReplicationMetaInfo(obj.GetContainerID(), obj.GetID(), firstObj, obj.GetPreviousID(), - obj.PayloadSize(), typ, deletedObjs, lockedObjs, expectedVUB, t.networkMagicNumber) + tx, hash := objectcore.EncodeChainMetaInfo(len(t.containerNodes.PrimaryCounts()), obj.GetContainerID(), obj.GetID(), firstObj, obj.GetPreviousID(), + obj.PayloadSize(), typ, deletedObj, lockedObj, expectedVUB, t.metaSvc.MagicNumber()) + + return tx, hash } func (t *distributedTarget) sendObject(obj object.Object, encObj encodedObject, node nodeDesc, metaC *metaCollection) error { @@ -636,13 +652,21 @@ func (t *distributedTarget) sendObject(obj object.Object, encObj encodedObject, } if t.localNodeInContainer && t.metainfoConsistencyAttr != "" { - sig, err := t.metaSigner.Sign(metaC.objectData) + sig, err := t.metaSigner.Sign(metaC.dataToSign) if err != nil { return fmt.Errorf("failed to sign object metadata: %w", err) } + ind := slices.IndexFunc(t.metaCollection.sortedNodes[node.placementVector], func(info netmap.NodeInfo) bool { + return bytes.Equal(info.PublicKey(), node.info.PublicKey()) + }) + if ind < 0 { + // unexpected at all + return fmt.Errorf("local node is not container's part, placement vector number: %d, public key: %X", node.placementVector, node.info.PublicKey()) + } + metaC.signaturesMtx.Lock() - metaC.signatures[node.placementVector] = append(metaC.signatures[node.placementVector], sig) + metaC.signatures[node.placementVector] = append(metaC.signatures[node.placementVector], meta.IndexedSignature{Index: uint8(ind), Signature: neofscrypto.NewSignature(t.metaSigner.Scheme(), t.metaSigner.Public(), sig)}) metaC.signaturesMtx.Unlock() } @@ -689,12 +713,20 @@ func (t *distributedTarget) sendObject(obj object.Object, encObj encodedObject, continue } - if !sig.Verify(metaC.objectData) { + if !sig.Verify(metaC.dataToSign) { continue } + ind := slices.IndexFunc(t.metaCollection.sortedNodes[node.placementVector], func(info netmap.NodeInfo) bool { + return bytes.Equal(info.PublicKey(), node.info.PublicKey()) + }) + if ind < 0 { + // unexpected at all + return fmt.Errorf("local node is not container's part, placement vector number: %d, public key: %X", node.placementVector, node.info.PublicKey()) + } + metaC.signaturesMtx.Lock() - metaC.signatures[node.placementVector] = append(metaC.signatures[node.placementVector], sig.Value()) + metaC.signatures[node.placementVector] = append(metaC.signatures[node.placementVector], meta.IndexedSignature{Index: uint8(ind), Signature: sig}) metaC.signaturesMtx.Unlock() return nil diff --git a/pkg/services/object/put/ec.go b/pkg/services/object/put/ec.go index b9d4b42425..15379321a3 100644 --- a/pkg/services/object/put/ec.go +++ b/pkg/services/object/put/ec.go @@ -5,6 +5,7 @@ import ( "math" iec "github.com/nspcc-dev/neofs-node/internal/ec" + meta "github.com/nspcc-dev/neofs-node/pkg/services/meta" neofscrypto "github.com/nspcc-dev/neofs-sdk-go/crypto" "github.com/nspcc-dev/neofs-sdk-go/netmap" "github.com/nspcc-dev/neofs-sdk-go/object" @@ -61,9 +62,11 @@ func (t *distributedTarget) formAndSaveObjectForECPart(signer neofscrypto.Signer var metaC *metaCollection if t.localNodeInContainer && t.metainfoConsistencyAttr != "" { + tx, dataToSign := t.encodeObjectMetadata(partObj) metaC = &metaCollection{ - objectData: t.encodeObjectMetadata(partObj), - signatures: make([][][]byte, len(t.containerNodes.PrimaryCounts())+len(t.containerNodes.ECRules())), + metaTransaction: tx, + dataToSign: dataToSign, + signatures: make([][]meta.IndexedSignature, len(t.containerNodes.PrimaryCounts())+len(t.containerNodes.ECRules())), } } diff --git a/pkg/services/object/put/service.go b/pkg/services/object/put/service.go index 65ee10cd06..f618381951 100644 --- a/pkg/services/object/put/service.go +++ b/pkg/services/object/put/service.go @@ -141,8 +141,6 @@ type cfg struct { log *zap.Logger - networkMagic uint32 - cnrClient *chaincontainer.Client metaSvc *meta.Meta @@ -263,12 +261,6 @@ func WithLogger(l *zap.Logger) Option { } } -func WithNetworkMagic(m uint32) Option { - return func(c *cfg) { - c.networkMagic = m - } -} - func WithNNSResolver(resolver session.NNSResolver) Option { return func(c *cfg) { c.nnsResolver = resolver diff --git a/pkg/services/object/put/streamer.go b/pkg/services/object/put/streamer.go index 84e7123ce1..94cd4b9ffd 100644 --- a/pkg/services/object/put/streamer.go +++ b/pkg/services/object/put/streamer.go @@ -1,16 +1,22 @@ package putsvc import ( + "bytes" "context" + "crypto/elliptic" "errors" "fmt" + "slices" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" iec "github.com/nspcc-dev/neofs-node/internal/ec" "github.com/nspcc-dev/neofs-node/pkg/core/client" + meta "github.com/nspcc-dev/neofs-node/pkg/services/meta" "github.com/nspcc-dev/neofs-node/pkg/services/object/internal" apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status" "github.com/nspcc-dev/neofs-sdk-go/container" neofsecdsa "github.com/nspcc-dev/neofs-sdk-go/crypto/ecdsa" + "github.com/nspcc-dev/neofs-sdk-go/netmap" "github.com/nspcc-dev/neofs-sdk-go/object" oid "github.com/nspcc-dev/neofs-sdk-go/object/id" "github.com/nspcc-dev/neofs-sdk-go/user" @@ -249,11 +255,41 @@ func (p *Streamer) newCommonTarget(prm *PutInitPrm) internal.Target { } } + metaAttr := metaAttribute(prm.cnr) + var metaC *metaCollection + if metaAttr != "" { + cnrNodes := prm.containerNodes.Unsorted() + for _, vector := range cnrNodes { + slices.SortFunc(vector, func(a, b netmap.NodeInfo) int { + // comparing compressed public keys without decompression (in most + // cases), taken from neo-go source code + + k1Raw := a.PublicKey() + k2Raw := b.PublicKey() + + // Sort by ECPoint's (X, Y) components: compare X first, and then compare Y. + cmpX := bytes.Compare(k1Raw[1:], k2Raw[1:]) + if cmpX != 0 { + return cmpX + } + // The case when X components are the same is extremely rare, thus we perform + // key deserialization only if needed. No error can occur. + ka, _ := keys.NewPublicKeyFromBytes(k1Raw, elliptic.P256()) + kb, _ := keys.NewPublicKeyFromBytes(k2Raw, elliptic.P256()) + return ka.Y.Cmp(kb.Y) + }) + } + + metaC = &metaCollection{ + sortedNodes: cnrNodes, + signatures: make([][]meta.IndexedSignature, len(prm.containerNodes.PrimaryCounts())+len(prm.containerNodes.ECRules())), + } + } + return &distributedTarget{ - opCtx: p.ctx, - fsState: p.networkState, - networkMagicNumber: p.networkMagic, - metaSvc: p.metaSvc, + opCtx: p.ctx, + fsState: p.networkState, + metaSvc: p.metaSvc, placementIterator: placementIterator{ log: p.log, neoFSNet: p.neoFSNet, @@ -272,10 +308,8 @@ func (p *Streamer) newCommonTarget(prm *PutInitPrm) internal.Target { localNodeSigner: prm.localNodeSigner, sessionSigner: prm.sessionSigner, cnrClient: p.cnrClient, - metainfoConsistencyAttr: metaAttribute(prm.cnr), - metaCollection: metaCollection{ - signatures: make([][][]byte, len(prm.containerNodes.PrimaryCounts())+len(prm.containerNodes.ECRules())), - }, + metainfoConsistencyAttr: metaAttr, + metaCollection: metaC, metaSigner: prm.localSignerRFC6979, localOnly: prm.common.LocalOnly(), initialPolicy: prm.cnr.PlacementPolicy().Initial(), diff --git a/pkg/services/object/server.go b/pkg/services/object/server.go index f1d14ec77e..38f9dee67a 100644 --- a/pkg/services/object/server.go +++ b/pkg/services/object/server.go @@ -201,7 +201,6 @@ type Server struct { meta *metasvc.Meta signer ecdsa.PrivateKey pubKeyBytes []byte - mNumber uint32 metrics MetricCollector aclChecker aclsvc.ACLChecker reqInfoProc ACLInfoExtractor @@ -210,7 +209,7 @@ type Server struct { } // New provides protoobject.ObjectServiceServer for the given parameters. -func New(hs Handlers, magicNumber uint32, sp *ants.Pool, fsChain FSChain, st Storage, metaSvc *metasvc.Meta, signer ecdsa.PrivateKey, m MetricCollector, ac aclsvc.ACLChecker, rp ACLInfoExtractor, cs ClientConstructor) *Server { +func New(hs Handlers, sp *ants.Pool, fsChain FSChain, st Storage, metaSvc *metasvc.Meta, signer ecdsa.PrivateKey, m MetricCollector, ac aclsvc.ACLChecker, rp ACLInfoExtractor, cs ClientConstructor) *Server { return &Server{ handlers: hs, fsChain: fsChain, @@ -218,7 +217,6 @@ func New(hs Handlers, magicNumber uint32, sp *ants.Pool, fsChain FSChain, st Sto meta: metaSvc, signer: signer, pubKeyBytes: (*keys.PublicKey)(&signer.PublicKey).Bytes(), - mNumber: magicNumber, metrics: m, aclChecker: ac, reqInfoProc: rp, @@ -2025,6 +2023,11 @@ func objectFromMessage(gMsg *protoobject.Object) (*object.Object, error) { } func (s *Server) metaInfoSignature(o object.Object) ([]byte, error) { + cnr, err := s.fsChain.Get(o.GetContainerID()) + if err != nil { + return nil, fmt.Errorf("fetching container info: %w", err) + } + firstObj := o.GetFirstID() if o.HasParent() && firstObj.IsZero() { // object itself is the first one @@ -2032,32 +2035,34 @@ func (s *Server) metaInfoSignature(o object.Object) ([]byte, error) { } prevObj := o.GetPreviousID() - var deleted []oid.ID - var locked []oid.ID + var ( + deleted oid.ID + locked oid.ID + ) typ := o.Type() switch typ { case object.TypeTombstone: - deleted = append(deleted, o.AssociatedObject()) + deleted = o.AssociatedObject() case object.TypeLock: - locked = append(locked, o.AssociatedObject()) + locked = o.AssociatedObject() default: } - currentBlock := s.fsChain.CurrentBlock() + currentBlock := s.meta.Height() currentEpochDuration := s.fsChain.CurrentEpochDuration() firstBlock := (uint64(currentBlock)/currentEpochDuration + 1) * currentEpochDuration secondBlock := firstBlock + currentEpochDuration thirdBlock := secondBlock + currentEpochDuration - firstMeta := objectcore.EncodeReplicationMetaInfo(o.GetContainerID(), o.GetID(), firstObj, prevObj, o.PayloadSize(), typ, deleted, locked, firstBlock, s.mNumber) - secondMeta := objectcore.EncodeReplicationMetaInfo(o.GetContainerID(), o.GetID(), firstObj, prevObj, o.PayloadSize(), typ, deleted, locked, secondBlock, s.mNumber) - thirdMeta := objectcore.EncodeReplicationMetaInfo(o.GetContainerID(), o.GetID(), firstObj, prevObj, o.PayloadSize(), typ, deleted, locked, thirdBlock, s.mNumber) + _, firstMeta := objectcore.EncodeChainMetaInfo(len(cnr.PlacementPolicy().Replicas()), o.GetContainerID(), o.GetID(), firstObj, prevObj, o.PayloadSize(), typ, deleted, locked, firstBlock, s.meta.MagicNumber()) + _, secondMeta := objectcore.EncodeChainMetaInfo(len(cnr.PlacementPolicy().Replicas()), o.GetContainerID(), o.GetID(), firstObj, prevObj, o.PayloadSize(), typ, deleted, locked, secondBlock, s.meta.MagicNumber()) + _, thirdMeta := objectcore.EncodeChainMetaInfo(len(cnr.PlacementPolicy().Replicas()), o.GetContainerID(), o.GetID(), firstObj, prevObj, o.PayloadSize(), typ, deleted, locked, thirdBlock, s.meta.MagicNumber()) var firstSig neofscrypto.Signature var secondSig neofscrypto.Signature var thirdSig neofscrypto.Signature signer := neofsecdsa.SignerRFC6979(s.signer) - err := firstSig.Calculate(signer, firstMeta) + err = firstSig.Calculate(signer, firstMeta) if err != nil { return nil, fmt.Errorf("signature failure: %w", err) } diff --git a/pkg/services/object/server_test.go b/pkg/services/object/server_test.go index 9c66ae890d..090f255eac 100644 --- a/pkg/services/object/server_test.go +++ b/pkg/services/object/server_test.go @@ -16,6 +16,7 @@ import ( "time" "github.com/nspcc-dev/neo-go/pkg/core/block" + "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/core/transaction" "github.com/nspcc-dev/neo-go/pkg/neorpc/result" "github.com/nspcc-dev/neo-go/pkg/smartcontract/trigger" @@ -27,6 +28,7 @@ import ( "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/engine" meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard" + metasvc "github.com/nspcc-dev/neofs-node/pkg/services/meta" . "github.com/nspcc-dev/neofs-node/pkg/services/object" v2 "github.com/nspcc-dev/neofs-node/pkg/services/object/acl/v2" deletesvc "github.com/nspcc-dev/neofs-node/pkg/services/object/delete" @@ -54,6 +56,7 @@ import ( "github.com/nspcc-dev/neofs-sdk-go/user" "github.com/panjf2000/ants/v2" "github.com/stretchr/testify/require" + "go.uber.org/zap/zaptest" "google.golang.org/grpc" ) @@ -341,7 +344,7 @@ func TestServer_Replicate(t *testing.T) { var noCallReqProc noCallTestReqInfoExtractor var noCallCs noCallClients sp := newSearchPool(t) - noCallSrv := New(noCallObjSvc, 0, sp, &noCallFSChain, noCallStorage, nil, neofscryptotest.Signer().ECDSAPrivateKey, nopMetrics{}, noCallACLChecker, noCallReqProc, noCallCs) + noCallSrv := New(noCallObjSvc, sp, &noCallFSChain, noCallStorage, nil, neofscryptotest.Signer().ECDSAPrivateKey, nopMetrics{}, noCallACLChecker, noCallReqProc, noCallCs) clientSigner := neofscryptotest.Signer() clientPubKey := neofscrypto.PublicKeyBytes(clientSigner.Public()) serverPubKey := neofscrypto.PublicKeyBytes(neofscryptotest.Signer().Public()) @@ -505,7 +508,7 @@ func TestServer_Replicate(t *testing.T) { t.Run("apply storage policy failure", func(t *testing.T) { fsChain := newTestFSChain(t, serverPubKey, clientPubKey, cnr) - srv := New(noCallObjSvc, 0, sp, fsChain, noCallStorage, nil, neofscryptotest.Signer().ECDSAPrivateKey, nopMetrics{}, noCallACLChecker, noCallReqProc, noCallCs) + srv := New(noCallObjSvc, sp, fsChain, noCallStorage, nil, neofscryptotest.Signer().ECDSAPrivateKey, nopMetrics{}, noCallACLChecker, noCallReqProc, noCallCs) fsChain.cnrErr = errors.New("any error") @@ -517,7 +520,7 @@ func TestServer_Replicate(t *testing.T) { t.Run("client or server mismatches object's storage policy", func(t *testing.T) { fsChain := newTestFSChain(t, serverPubKey, clientPubKey, cnr) - srv := New(noCallObjSvc, 0, sp, fsChain, noCallStorage, nil, neofscryptotest.Signer().ECDSAPrivateKey, nopMetrics{}, noCallACLChecker, noCallReqProc, noCallCs) + srv := New(noCallObjSvc, sp, fsChain, noCallStorage, nil, neofscryptotest.Signer().ECDSAPrivateKey, nopMetrics{}, noCallACLChecker, noCallReqProc, noCallCs) fsChain.serverOutsideCnr = true fsChain.clientOutsideCnr = true @@ -538,7 +541,7 @@ func TestServer_Replicate(t *testing.T) { t.Run("local storage failure", func(t *testing.T) { fsChain := newTestFSChain(t, serverPubKey, clientPubKey, cnr) s := newTestStorage(t, req.Object) - srv := New(noCallObjSvc, 0, sp, fsChain, s, nil, neofscryptotest.Signer().ECDSAPrivateKey, nopMetrics{}, noCallACLChecker, noCallReqProc, noCallCs) + srv := New(noCallObjSvc, sp, fsChain, s, nil, neofscryptotest.Signer().ECDSAPrivateKey, nopMetrics{}, noCallACLChecker, noCallReqProc, noCallCs) s.storeErr = errors.New("any error") @@ -550,11 +553,18 @@ func TestServer_Replicate(t *testing.T) { t.Run("meta information signature", func(t *testing.T) { var mNumber uint32 = 123 + testMetaSvc, err := metasvc.New(metasvc.Parameters{ + Logger: zaptest.NewLogger(t), + Chain: &mockMetadataChain{}, + Path: t.TempDir(), + Network: mockNeofsNetwork{}, + }) + require.NoError(t, err) signer := neofscryptotest.Signer() reqForSignature, o := anyValidRequest(t, clientSigner, cnr, objID) fsChain := newTestFSChain(t, serverPubKey, clientPubKey, cnr) s := newTestStorage(t, reqForSignature.Object) - srv := New(noCallObjSvc, mNumber, sp, fsChain, s, nil, signer.ECDSAPrivateKey, nopMetrics{}, noCallACLChecker, noCallReqProc, noCallCs) + srv := New(noCallObjSvc, sp, fsChain, s, testMetaSvc, signer.ECDSAPrivateKey, nopMetrics{}, noCallACLChecker, noCallReqProc, noCallCs) t.Run("signature not requested", func(t *testing.T) { resp, err := srv.Replicate(context.Background(), reqForSignature) @@ -582,8 +592,8 @@ func TestServer_Replicate(t *testing.T) { require.NoError(t, sig.Unmarshal(sigsRaw[4:4+l])) require.Equal(t, signer.PublicKeyBytes, sig.PublicKeyBytes()) - require.True(t, sig.Verify(objectcore.EncodeReplicationMetaInfo( - o.GetContainerID(), o.GetID(), o.GetFirstID(), o.GetPreviousID(), o.PayloadSize(), o.Type(), nil, nil, + require.True(t, sig.Verify(objectcore.EncodeObjectMetadata( + o.GetContainerID(), o.GetID(), o.GetFirstID(), o.GetPreviousID(), o.PayloadSize(), o.Type(), oid.ID{}, oid.ID{}, uint64((123+1+i)*240), mNumber)), fmt.Sprintf("wrong %d signature", i+1)) sigsRaw = sigsRaw[4+l:] @@ -594,7 +604,7 @@ func TestServer_Replicate(t *testing.T) { t.Run("OK", func(t *testing.T) { fsChain := newTestFSChain(t, serverPubKey, clientPubKey, cnr) s := newTestStorage(t, req.Object) - srv := New(noCallObjSvc, 0, sp, fsChain, s, nil, neofscryptotest.Signer().ECDSAPrivateKey, nopMetrics{}, noCallACLChecker, noCallReqProc, noCallCs) + srv := New(noCallObjSvc, sp, fsChain, s, nil, neofscryptotest.Signer().ECDSAPrivateKey, nopMetrics{}, noCallACLChecker, noCallReqProc, noCallCs) resp, err := srv.Replicate(context.Background(), req) require.NoError(t, err) @@ -660,7 +670,7 @@ func BenchmarkServer_Replicate(b *testing.B) { var fsChain nopFSChain sp := newSearchPool(b) - srv := New(nil, 0, sp, fsChain, nopStorage{}, nil, neofscryptotest.Signer().ECDSAPrivateKey, nopMetrics{}, nopACLChecker{}, nopReqInfoExtractor{}, noCallClients{}) + srv := New(nil, sp, fsChain, nopStorage{}, nil, neofscryptotest.Signer().ECDSAPrivateKey, nopMetrics{}, nopACLChecker{}, nopReqInfoExtractor{}, noCallClients{}) for _, tc := range []struct { name string @@ -738,6 +748,38 @@ func newSimpleStorage(t *testing.T, fsChain FSChain) *engine.StorageEngine { return storage } +type mockNeofsNetwork struct{} + +func (m mockNeofsNetwork) Head(ctx context.Context, id cid.ID, id2 oid.ID) (object.Object, error) { + panic("unimplemented") +} + +func (m mockNeofsNetwork) IsMineWithMeta(id cid.ID, i []byte) (bool, error) { + panic("unimplemented") +} + +type mockMetadataChain struct{} + +func (m mockMetadataChain) Magic() uint32 { + return 456 +} + +func (m mockMetadataChain) Height() uint32 { + return 123 +} + +func (m mockMetadataChain) AddTx(tx *transaction.Transaction) error { + panic("unimplemented") +} + +func (m mockMetadataChain) SubscribeForBlocks(ch chan *block.Header) { + panic("unimplemented") +} + +func (m mockMetadataChain) SubscribeForNotifications(ch chan *state.ContainedNotificationEvent) { + panic("unimplemented") +} + type mockConnections struct { conns map[string]clientcore.MultiAddressClient } diff --git a/pkg/services/sidechain/sidechain.go b/pkg/services/sidechain/sidechain.go new file mode 100644 index 0000000000..6b9eedf3fb --- /dev/null +++ b/pkg/services/sidechain/sidechain.go @@ -0,0 +1,161 @@ +package sidechain + +import ( + "context" + "fmt" + "time" + + "github.com/nspcc-dev/neo-go/pkg/config" + "github.com/nspcc-dev/neo-go/pkg/core" + "github.com/nspcc-dev/neo-go/pkg/core/block" + "github.com/nspcc-dev/neo-go/pkg/core/state" + "github.com/nspcc-dev/neo-go/pkg/core/storage" + "github.com/nspcc-dev/neo-go/pkg/core/transaction" + "github.com/nspcc-dev/neo-go/pkg/network" + "github.com/nspcc-dev/neo-go/pkg/services/rpcsrv" + "github.com/nspcc-dev/neofs-node/pkg/core/metachain" + "go.uber.org/zap" +) + +// SideChain defines side chain that runs independently but with redefined +// native contracts, see [metachain.NewCustomNatives] for details. It must +// be created with [New]. +type SideChain struct { + logger *zap.Logger + storage storage.Store + core *core.Blockchain + netServer *network.Server + rpcServer *rpcsrv.Server + + magicNumber uint32 + + chErr chan error +} + +// New creates [SideChain]. +func New(cfg config.Config, log *zap.Logger, errCh chan error) (*SideChain, error) { + store, err := storage.NewStore(cfg.ApplicationConfiguration.DBConfiguration) + if err != nil { + return &SideChain{}, fmt.Errorf("could not initialize storage: %w", err) + } + defer func() { + if err != nil { + closeErr := store.Close() + if closeErr != nil { + err = fmt.Errorf("%w; also failed to close blockchain storage: %w", err, closeErr) + } + } + }() + + chain, err := core.NewBlockchain(store, cfg.Blockchain(), log.With(zap.String("subcomponent", "core chain")), metachain.NewCustomNatives) + if err != nil { + return &SideChain{}, fmt.Errorf("initializing meta block chain: %w", err) + } + + cfgServer, err := network.NewServerConfig(cfg) + if err != nil { + return nil, fmt.Errorf("compose NeoGo server config from the base one: %w", err) + } + + netServer, err := network.NewServer(cfgServer, chain, chain.GetStateSyncModule(), log.With(zap.String("subcomponent", "network server"))) + if err != nil { + return nil, fmt.Errorf("init NeoGo network server: %w", err) + } + + chErr := make(chan error) + go func() { + for { + err, ok := <-chErr + if !ok { + return + } + errCh <- err + } + }() + + rpcServer := rpcsrv.New(chain, cfg.ApplicationConfiguration.RPC, netServer, nil, log.With(zap.String("subcomponent", "rpc server")), chErr) + netServer.AddService(rpcServer) + + return &SideChain{ + logger: log, + storage: store, + core: chain, + netServer: netServer, + rpcServer: rpcServer, + chErr: chErr, + magicNumber: uint32(cfg.ProtocolConfiguration.Magic), + }, nil +} + +// Run starts [SideChain]. Must be called only on instances created +// with [New]. Blocked until either context is done or side chais is +// in sync with specified seed nodes. +// To cancel this func, [SideChain.Stop] should be called. +func (s *SideChain) Run(ctx context.Context) error { + var err error + defer func() { + // note that we can't rely on the fact that the method never returns an error + // since this may not be forever + if err != nil { + closeErr := s.storage.Close() + if closeErr != nil { + err = fmt.Errorf("%w; also failed to close blockchain storage: %w", err, closeErr) + } + } + }() + + go s.core.Run() + go s.netServer.Start() + + t := time.NewTicker(s.core.GetConfig().Genesis.TimePerBlock) + + for { + s.logger.Info("waiting for synchronization with the blockchain network...") + select { + case <-ctx.Done(): + return fmt.Errorf("await state sync: %w", context.Cause(ctx)) + case <-t.C: + if s.netServer.IsInSync() { + s.logger.Info("blockchain state successfully synchronized") + return nil + } + } + } +} + +// Stop stops the side chain. Must be called only after a successuf [SideChain.Run]. +func (s *SideChain) Stop() { + s.netServer.Shutdown() + s.core.Close() + close(s.chErr) +} + +// Height returns current side chaih block height. +func (s *SideChain) Height() uint32 { + return s.core.HeaderHeight() +} + +// AddTx adds transaction to node's transaction pools. Non-nil return +// value _does not mean_ transaction is included, use +// [SideChain.SubscribeForNotifications] or [SideChain.SubscribeForBlocks] +// to be notified about accepted chain changes. +func (s *SideChain) AddTx(tx *transaction.Transaction) error { + return s.netServer.RelayTxn(tx) +} + +// Magic returns chain's magic number. +func (s *SideChain) Magic() uint32 { + return s.magicNumber +} + +// SubscribeForNotifications subscribes for chain notifications. Channel must +// be read to prevent deadlock. +func (s *SideChain) SubscribeForNotifications(ch chan *state.ContainedNotificationEvent) { + s.core.SubscribeForNotifications(ch) +} + +// SubscribeForBlocks subscribes for chain block headers. Channel must +// be read to prevent deadlock. +func (s *SideChain) SubscribeForBlocks(ch chan *block.Header) { + s.core.SubscribeForHeadersOfAddedBlocks(ch) +}