Skip to content

Commit e6b5371

Browse files
ivan4thfasmat
ivan4th
and
fasmat
committed
sync2: ATX integration (#6448)
## Motivation We need more efficient ATX sync. Co-authored-by: Matthias <[email protected]>
1 parent 282d7b4 commit e6b5371

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

43 files changed

+2070
-1031
lines changed

‎config/mainnet.go

+20
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import (
2424
"github.com/spacemeshos/go-spacemesh/hare4"
2525
"github.com/spacemeshos/go-spacemesh/miner"
2626
"github.com/spacemeshos/go-spacemesh/p2p"
27+
"github.com/spacemeshos/go-spacemesh/sync2"
2728
"github.com/spacemeshos/go-spacemesh/syncer"
2829
"github.com/spacemeshos/go-spacemesh/syncer/atxsync"
2930
"github.com/spacemeshos/go-spacemesh/syncer/malsync"
@@ -77,6 +78,14 @@ func MainnetConfig() Config {
7778

7879
hare4conf := hare4.DefaultConfig()
7980
hare4conf.Enable = false
81+
82+
oldAtxSyncCfg := sync2.DefaultConfig()
83+
oldAtxSyncCfg.MultiPeerReconcilerConfig.SyncInterval = time.Hour
84+
oldAtxSyncCfg.MaxDepth = 16
85+
newAtxSyncCfg := sync2.DefaultConfig()
86+
newAtxSyncCfg.MaxDepth = 21
87+
newAtxSyncCfg.MultiPeerReconcilerConfig.SyncInterval = 5 * time.Minute
88+
8089
return Config{
8190
BaseConfig: BaseConfig{
8291
DataDirParent: defaultDataDir,
@@ -212,6 +221,17 @@ func MainnetConfig() Config {
212221
DisableMeshAgreement: true,
213222
AtxSync: atxsync.DefaultConfig(),
214223
MalSync: malsync.DefaultConfig(),
224+
ReconcSync: syncer.ReconcSyncConfig{
225+
OldAtxSyncCfg: oldAtxSyncCfg,
226+
NewAtxSyncCfg: newAtxSyncCfg,
227+
ParallelLoadLimit: 10,
228+
HardTimeout: 10 * time.Minute,
229+
ServerConfig: fetch.ServerConfig{
230+
Queue: 200,
231+
Requests: 100,
232+
Interval: time.Second,
233+
},
234+
},
215235
},
216236
Recovery: checkpoint.DefaultConfig(),
217237
Cache: datastore.DefaultConfig(),

‎config/presets/testnet.go

+19
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import (
2525
"github.com/spacemeshos/go-spacemesh/hare4"
2626
"github.com/spacemeshos/go-spacemesh/miner"
2727
"github.com/spacemeshos/go-spacemesh/p2p"
28+
"github.com/spacemeshos/go-spacemesh/sync2"
2829
"github.com/spacemeshos/go-spacemesh/syncer"
2930
"github.com/spacemeshos/go-spacemesh/syncer/atxsync"
3031
"github.com/spacemeshos/go-spacemesh/syncer/malsync"
@@ -65,6 +66,13 @@ func testnet() config.Config {
6566
hare4conf := hare4.DefaultConfig()
6667
hare4conf.Enable = false
6768
defaultdir := filepath.Join(home, "spacemesh-testnet", "/")
69+
70+
oldAtxSyncCfg := sync2.DefaultConfig()
71+
oldAtxSyncCfg.MaxDepth = 16
72+
newAtxSyncCfg := sync2.DefaultConfig()
73+
newAtxSyncCfg.MaxDepth = 21
74+
newAtxSyncCfg.MultiPeerReconcilerConfig.SyncInterval = 5 * time.Minute
75+
6876
return config.Config{
6977
Preset: "testnet",
7078
BaseConfig: config.BaseConfig{
@@ -163,6 +171,17 @@ func testnet() config.Config {
163171
OutOfSyncThresholdLayers: 10,
164172
AtxSync: atxsync.DefaultConfig(),
165173
MalSync: malsync.DefaultConfig(),
174+
ReconcSync: syncer.ReconcSyncConfig{
175+
OldAtxSyncCfg: oldAtxSyncCfg,
176+
NewAtxSyncCfg: newAtxSyncCfg,
177+
ParallelLoadLimit: 10,
178+
HardTimeout: time.Minute,
179+
ServerConfig: fetch.ServerConfig{
180+
Queue: 200,
181+
Requests: 100,
182+
Interval: time.Second,
183+
},
184+
},
166185
},
167186
Recovery: checkpoint.DefaultConfig(),
168187
Cache: datastore.DefaultConfig(),

‎fetch/fetch.go

+4-11
Original file line numberDiff line numberDiff line change
@@ -116,7 +116,7 @@ type ServerConfig struct {
116116
Interval time.Duration `mapstructure:"interval"`
117117
}
118118

119-
func (s ServerConfig) toOpts() []server.Opt {
119+
func (s ServerConfig) ToOpts() []server.Opt {
120120
opts := []server.Opt{}
121121
if s.Queue != 0 {
122122
opts = append(opts, server.WithQueueSize(s.Queue))
@@ -270,6 +270,7 @@ func NewFetch(
270270
cdb *datastore.CachedDB,
271271
proposals *store.Store,
272272
host *p2p.Host,
273+
peerCache *peers.Peers,
273274
opts ...Option,
274275
) (*Fetch, error) {
275276
bs := datastore.NewBlobStore(cdb, proposals)
@@ -293,7 +294,7 @@ func NewFetch(
293294
opt(f)
294295
}
295296
f.getAtxsLimiter = semaphore.NewWeighted(f.cfg.GetAtxsConcurrency)
296-
f.peers = peers.New()
297+
f.peers = peerCache
297298
// NOTE(dshulyak) this is to avoid tests refactoring.
298299
// there is one test that covers this part.
299300
if host != nil {
@@ -375,7 +376,7 @@ func (f *Fetch) registerServer(
375376
if f.cfg.EnableServerMetrics {
376377
opts = append(opts, server.WithMetrics())
377378
}
378-
opts = append(opts, f.cfg.getServerConfig(protocol).toOpts()...)
379+
opts = append(opts, f.cfg.getServerConfig(protocol).ToOpts()...)
379380
f.servers[protocol] = server.New(host, protocol, handler, opts...)
380381
}
381382

@@ -1008,14 +1009,6 @@ func (f *Fetch) RegisterPeerHashes(peer p2p.Peer, hashes []types.Hash32) {
10081009
f.hashToPeers.RegisterPeerHashes(peer, hashes)
10091010
}
10101011

1011-
// RegisterPeerHashes registers provided peer for a hash.
1012-
func (f *Fetch) RegisterPeerHash(peer p2p.Peer, hash types.Hash32) {
1013-
if peer == f.host.ID() {
1014-
return
1015-
}
1016-
f.hashToPeers.Add(hash, peer)
1017-
}
1018-
10191012
func (f *Fetch) SelectBestShuffled(n int) []p2p.Peer {
10201013
// shuffle to split the load between peers with good latency.
10211014
// and it avoids sticky behavior, when temporarily faulty peer had good latency in the past.

‎fetch/fetch_test.go

+4
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import (
1818
"github.com/spacemeshos/go-spacemesh/common/types"
1919
"github.com/spacemeshos/go-spacemesh/datastore"
2020
"github.com/spacemeshos/go-spacemesh/fetch/mocks"
21+
"github.com/spacemeshos/go-spacemesh/fetch/peers"
2122
"github.com/spacemeshos/go-spacemesh/p2p"
2223
"github.com/spacemeshos/go-spacemesh/p2p/pubsub"
2324
"github.com/spacemeshos/go-spacemesh/p2p/server"
@@ -87,6 +88,7 @@ func createFetch(tb testing.TB) *testFetch {
8788
cdb,
8889
store.New(),
8990
nil,
91+
peers.New(),
9092
WithContext(context.Background()),
9193
WithConfig(cfg),
9294
WithLogger(lg),
@@ -133,6 +135,7 @@ func TestFetch_Start(t *testing.T) {
133135
cdb,
134136
store.New(),
135137
nil,
138+
peers.New(),
136139
WithContext(context.Background()),
137140
WithConfig(DefaultConfig()),
138141
WithLogger(lg),
@@ -407,6 +410,7 @@ func TestFetch_PeerDroppedWhenMessageResultsInValidationReject(t *testing.T) {
407410
cdb,
408411
store.New(),
409412
h,
413+
peers.New(),
410414
WithContext(ctx),
411415
WithConfig(cfg),
412416
WithLogger(lg),

‎fetch/limiter.go

+4-1
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@ package fetch
22

33
import (
44
"context"
5+
6+
"github.com/spacemeshos/go-spacemesh/common/types"
57
)
68

79
type limiter interface {
@@ -10,7 +12,8 @@ type limiter interface {
1012
}
1113

1214
type getHashesOpts struct {
13-
limiter limiter
15+
limiter limiter
16+
callback func(types.Hash32, error)
1417
}
1518

1619
type noLimit struct{}

‎fetch/mesh_data.go

+30-6
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ func (f *Fetch) GetAtxs(ctx context.Context, ids []types.ATXID, opts ...system.G
3131
return nil
3232
}
3333

34-
options := system.GetAtxOpts{}
34+
var options system.GetAtxOpts
3535
for _, opt := range opts {
3636
opt(&options)
3737
}
@@ -42,10 +42,17 @@ func (f *Fetch) GetAtxs(ctx context.Context, ids []types.ATXID, opts ...system.G
4242
zap.Bool("limiting", !options.LimitingOff),
4343
)
4444
hashes := types.ATXIDsToHashes(ids)
45-
if options.LimitingOff {
46-
return f.getHashes(ctx, hashes, datastore.ATXDB, f.validators.atx.HandleMessage)
45+
handler := f.validators.atx.HandleMessage
46+
var ghOpts []getHashesOpt
47+
if !options.LimitingOff {
48+
ghOpts = append(ghOpts, withLimiter(f.getAtxsLimiter))
4749
}
48-
return f.getHashes(ctx, hashes, datastore.ATXDB, f.validators.atx.HandleMessage, withLimiter(f.getAtxsLimiter))
50+
if options.Callback != nil {
51+
ghOpts = append(ghOpts, withHashCallback(func(hash types.Hash32, err error) {
52+
options.Callback(types.ATXID(hash), err)
53+
}))
54+
}
55+
return f.getHashes(ctx, hashes, datastore.ATXDB, handler, ghOpts...)
4956
}
5057

5158
type dataReceiver func(context.Context, types.Hash32, p2p.Peer, []byte) error
@@ -58,6 +65,12 @@ func withLimiter(l limiter) getHashesOpt {
5865
}
5966
}
6067

68+
func withHashCallback(callback func(types.Hash32, error)) getHashesOpt {
69+
return func(o *getHashesOpts) {
70+
o.callback = callback
71+
}
72+
}
73+
6174
func (f *Fetch) getHashes(
6275
ctx context.Context,
6376
hashes []types.Hash32,
@@ -66,7 +79,8 @@ func (f *Fetch) getHashes(
6679
opts ...getHashesOpt,
6780
) error {
6881
options := getHashesOpts{
69-
limiter: noLimit{},
82+
limiter: noLimit{},
83+
callback: func(types.Hash32, error) {},
7084
}
7185
for _, opt := range opts {
7286
opt(&options)
@@ -83,18 +97,26 @@ func (f *Fetch) getHashes(
8397
for i, hash := range hashes {
8498
if err := options.limiter.Acquire(ctx, 1); err != nil {
8599
pendingMetric.Add(float64(i - len(hashes)))
86-
return fmt.Errorf("acquiring slot to get hash: %w", err)
100+
err = fmt.Errorf("acquiring slot to get hash: %w", err)
101+
for _, h := range hashes[i:] {
102+
options.callback(h, err)
103+
}
104+
return err
87105
}
88106
p, err := f.getHash(ctx, hash, hint, receiver)
89107
if err != nil {
90108
options.limiter.Release(1)
91109
pendingMetric.Add(float64(i - len(hashes)))
110+
for _, h := range hashes[i:] {
111+
options.callback(h, err)
112+
}
92113
return err
93114
}
94115
if p == nil {
95116
// data is available locally
96117
options.limiter.Release(1)
97118
pendingMetric.Add(-1)
119+
options.callback(hash, nil)
98120
continue
99121
}
100122

@@ -103,6 +125,7 @@ func (f *Fetch) getHashes(
103125
case <-ctx.Done():
104126
options.limiter.Release(1)
105127
pendingMetric.Add(-1)
128+
options.callback(hash, ctx.Err())
106129
return ctx.Err()
107130
case <-p.completed:
108131
options.limiter.Release(1)
@@ -118,6 +141,7 @@ func (f *Fetch) getHashes(
118141
bfailure.Add(hash, p.err)
119142
mu.Unlock()
120143
}
144+
options.callback(hash, p.err)
121145
return nil
122146
}
123147
})

‎fetch/mesh_data_test.go

+19-4
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import (
55
"context"
66
"errors"
77
"fmt"
8+
"sync"
89
"testing"
910

1011
p2phost "github.com/libp2p/go-libp2p/core/host"
@@ -20,6 +21,7 @@ import (
2021
"github.com/spacemeshos/go-spacemesh/common/types"
2122
"github.com/spacemeshos/go-spacemesh/datastore"
2223
"github.com/spacemeshos/go-spacemesh/fetch/mocks"
24+
"github.com/spacemeshos/go-spacemesh/fetch/peers"
2325
"github.com/spacemeshos/go-spacemesh/genvm/sdk/wallet"
2426
"github.com/spacemeshos/go-spacemesh/p2p"
2527
"github.com/spacemeshos/go-spacemesh/p2p/peerinfo"
@@ -87,7 +89,7 @@ func startTestLoop(tb testing.TB, f *Fetch, eg *errgroup.Group, stop chan struct
8789
default:
8890
f.mu.Lock()
8991
for h, req := range f.unprocessed {
90-
require.NoError(tb, req.validator(req.ctx, types.Hash32{}, p2p.NoPeer, []byte{}))
92+
require.NoError(tb, req.validator(req.ctx, h, p2p.NoPeer, []byte{}))
9193
close(req.promise.completed)
9294
delete(f.unprocessed, h)
9395
}
@@ -596,7 +598,7 @@ func genATXs(tb testing.TB, num uint32) []*types.ActivationTx {
596598
}
597599

598600
func TestGetATXs(t *testing.T) {
599-
atxs := genATXs(t, 2)
601+
atxs := genATXs(t, 4)
600602
f := createFetch(t)
601603
f.mAtxH.EXPECT().
602604
HandleMessage(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).
@@ -607,10 +609,22 @@ func TestGetATXs(t *testing.T) {
607609
var eg errgroup.Group
608610
startTestLoop(t, f.Fetch, &eg, stop)
609611

610-
atxIDs := types.ToATXIDs(atxs)
611-
require.NoError(t, f.GetAtxs(context.Background(), atxIDs))
612+
atxIDs1 := types.ToATXIDs(atxs[:2])
613+
require.NoError(t, f.GetAtxs(context.Background(), atxIDs1))
614+
615+
atxIDs2 := types.ToATXIDs(atxs[2:])
616+
var recvIDs []types.ATXID
617+
var mtx sync.Mutex
618+
require.NoError(t, f.GetAtxs(context.Background(), atxIDs2,
619+
system.WithATXCallback(func(id types.ATXID, err error) {
620+
mtx.Lock()
621+
defer mtx.Unlock()
622+
require.NoError(t, err)
623+
recvIDs = append(recvIDs, id)
624+
})))
612625
close(stop)
613626
require.NoError(t, eg.Wait())
627+
require.ElementsMatch(t, atxIDs2, recvIDs)
614628
}
615629

616630
func TestGetActiveSet(t *testing.T) {
@@ -1010,6 +1024,7 @@ func Test_GetAtxsLimiting(t *testing.T) {
10101024
host, err := p2p.Upgrade(mesh.Hosts()[0])
10111025
require.NoError(t, err)
10121026
f, err := NewFetch(cdb, store.New(), host,
1027+
peers.New(),
10131028
WithContext(context.Background()),
10141029
withServers(map[string]requester{hashProtocol: client}),
10151030
WithConfig(cfg),

‎fetch/p2p_test.go

+3
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import (
1515
"github.com/spacemeshos/go-spacemesh/codec"
1616
"github.com/spacemeshos/go-spacemesh/common/types"
1717
"github.com/spacemeshos/go-spacemesh/datastore"
18+
"github.com/spacemeshos/go-spacemesh/fetch/peers"
1819
"github.com/spacemeshos/go-spacemesh/p2p"
1920
"github.com/spacemeshos/go-spacemesh/p2p/server"
2021
"github.com/spacemeshos/go-spacemesh/proposals/store"
@@ -132,6 +133,7 @@ func createP2PFetch(
132133
tpf.serverCDB,
133134
tpf.serverPDB,
134135
serverHost,
136+
peers.New(),
135137
WithContext(ctx),
136138
WithConfig(p2pFetchCfg(serverStreaming)),
137139
WithLogger(lg),
@@ -153,6 +155,7 @@ func createP2PFetch(
153155
tpf.clientCDB,
154156
tpf.clientPDB,
155157
clientHost,
158+
peers.New(),
156159
WithContext(ctx),
157160
WithConfig(p2pFetchCfg(clientStreaming)),
158161
WithLogger(lg),

0 commit comments

Comments
 (0)