Skip to content

Commit

Permalink
Remove CachingBlockstore
Browse files Browse the repository at this point in the history
The improvements in the range-export code lead to avoid reading most blocks
twice, as well as to allowing some blocks to be written to disk multiple times.

The cache hit-rate went down from being close to 50% to a maximum of 12% at
the very end.

These numbers do not support the maintenance of the CachingBlockstore, which
increases memory usage and causes in the end a time-penalty on the export (as
proven when testing).
  • Loading branch information
hsanjuan committed Feb 6, 2023
1 parent ea5eb52 commit 436c034
Show file tree
Hide file tree
Showing 5 changed files with 4 additions and 136 deletions.
1 change: 0 additions & 1 deletion api/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -403,7 +403,6 @@ func (m *MsgUuidMapType) UnmarshalJSON(b []byte) error {
type ChainExportConfig struct {
WriteBufferSize int
NumWorkers int
CacheSize int
IncludeMessages bool
IncludeReceipts bool
IncludeStateRoots bool
Expand Down
118 changes: 0 additions & 118 deletions chain/store/cache.go

This file was deleted.

10 changes: 2 additions & 8 deletions chain/store/snapshot.go
Original file line number Diff line number Diff line change
Expand Up @@ -508,8 +508,7 @@ func (cs *ChainStore) ExportRange(
w io.Writer,
head, tail *types.TipSet,
messages, receipts, stateroots bool,
workers int,
cacheSize int) error {
workers int) error {

h := &car.CarHeader{
Roots: head.Cids(),
Expand All @@ -520,11 +519,6 @@ func (cs *ChainStore) ExportRange(
return xerrors.Errorf("failed to write car header: %s", err)
}

cacheStore, err := NewCachingBlockstore(cs.UnionStore(), cacheSize)
if err != nil {
return err
}

start := time.Now()
log.Infow("walking snapshot range",
"head", head.Key(),
Expand All @@ -544,7 +538,7 @@ func (cs *ChainStore) ExportRange(
includeReceipts: receipts,
}

pw, err := newWalkScheduler(ctx, cacheStore, cfg, w)
pw, err := newWalkScheduler(ctx, cs.UnionStore(), cfg, w)
if err != nil {
return err
}
Expand Down
7 changes: 0 additions & 7 deletions cli/chain.go
Original file line number Diff line number Diff line change
Expand Up @@ -1182,11 +1182,6 @@ var ChainExportRangeCmd = &cli.Command{
Usage: "specify the number of workers",
Value: 1,
},
&cli.IntFlag{
Name: "cache-size",
Usage: "specify the size of the cache (in objects) to use while exporting",
Value: 100_000,
},
&cli.IntFlag{
Name: "write-buffer",
Usage: "specify write buffer size",
Expand Down Expand Up @@ -1243,7 +1238,6 @@ var ChainExportRangeCmd = &cli.Command{
if err := api.ChainExportRangeInternal(ctx, head.Key(), tail.Key(), lapi.ChainExportConfig{
WriteBufferSize: cctx.Int("write-buffer"),
NumWorkers: cctx.Int("workers"),
CacheSize: cctx.Int("cache-size"),
IncludeMessages: cctx.Bool("messages"),
IncludeReceipts: cctx.Bool("receipts"),
IncludeStateRoots: cctx.Bool("stateroots"),
Expand All @@ -1256,7 +1250,6 @@ var ChainExportRangeCmd = &cli.Command{
stream, err := api.ChainExportRange(ctx, head.Key(), tail.Key(), lapi.ChainExportConfig{
WriteBufferSize: cctx.Int("write-buffer"),
NumWorkers: cctx.Int("workers"),
CacheSize: cctx.Int("cache-size"),
IncludeMessages: cctx.Bool("messages"),
IncludeReceipts: cctx.Bool("receipts"),
IncludeStateRoots: cctx.Bool("stateroots"),
Expand Down
4 changes: 2 additions & 2 deletions node/impl/full/chain.go
Original file line number Diff line number Diff line change
Expand Up @@ -630,7 +630,7 @@ func (a ChainAPI) ChainExportRangeInternal(ctx context.Context, head, tail types
bw,
headTs, tailTs,
cfg.IncludeMessages, cfg.IncludeReceipts, cfg.IncludeStateRoots,
cfg.NumWorkers, cfg.CacheSize,
cfg.NumWorkers,
); err != nil {
return fmt.Errorf("exporting chain range: %w", err)
}
Expand Down Expand Up @@ -658,7 +658,7 @@ func (a ChainAPI) ChainExportRange(ctx context.Context, head, tail types.TipSetK
headTs,
tailTs,
cfg.IncludeMessages, cfg.IncludeReceipts, cfg.IncludeStateRoots,
cfg.NumWorkers, cfg.CacheSize,
cfg.NumWorkers,
)
bw.Flush() //nolint:errcheck // it is a write to a pipe
w.CloseWithError(err) //nolint:errcheck // it is a pipe
Expand Down

0 comments on commit 436c034

Please sign in to comment.