From 6875527224e4840f4df0032bac4e365d33290b64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 30 Sep 2020 15:17:29 +0200 Subject: [PATCH 01/15] shed: Datastore utils --- cmd/lotus-shed/datastore.go | 193 ++++++++++++++++++++++++++++++++++++ cmd/lotus-shed/main.go | 1 + go.mod | 1 + 3 files changed, 195 insertions(+) create mode 100644 cmd/lotus-shed/datastore.go diff --git a/cmd/lotus-shed/datastore.go b/cmd/lotus-shed/datastore.go new file mode 100644 index 00000000000..dcd774a90a9 --- /dev/null +++ b/cmd/lotus-shed/datastore.go @@ -0,0 +1,193 @@ +package main + +import ( + "encoding/json" + "fmt" + "strings" + + "github.com/ipfs/go-datastore" + dsq "github.com/ipfs/go-datastore/query" + logging "github.com/ipfs/go-log" + "github.com/polydawn/refmt/cbor" + "github.com/urfave/cli/v2" + "golang.org/x/xerrors" + + "github.com/filecoin-project/lotus/node/repo" +) + +var datastoreCmd = &cli.Command{ + Name: "datastore", + Description: "access node datastores directly", + Subcommands: []*cli.Command{ + datastoreListCmd, + datastoreGetCmd, + }, +} + +var datastoreListCmd = &cli.Command{ + Name: "list", + Description: "list datastore keys", + Flags: []cli.Flag{ + &cli.IntFlag{ + Name: "repo-type", + Value: 1, + }, + &cli.BoolFlag{ + Name: "top-level", + Usage: "only print top-level keys", + }, + &cli.StringFlag{ + Name: "get-enc", + Usage: "print values [esc/hex/cbor]", + }, + }, + ArgsUsage: "[namespace prefix]", + Action: func(cctx *cli.Context) error { + logging.SetLogLevel("badger", "ERROR") // nolint:errchec + + r, err := repo.NewFS(cctx.String("repo")) + if err != nil { + return xerrors.Errorf("opening fs repo: %w", err) + } + + exists, err := r.Exists() + if err != nil { + return err + } + if !exists { + return xerrors.Errorf("lotus repo doesn't exist") + } + + lr, err := r.Lock(repo.RepoType(cctx.Int("repo-type"))) + if err != nil { + return err + } + defer lr.Close() //nolint:errcheck + + ds, err := lr.Datastore(datastore.NewKey(cctx.Args().First()).String()) + if err != nil { + return err + } + + genc := cctx.String("get-enc") + + q, err := ds.Query(dsq.Query{ + Prefix: datastore.NewKey(cctx.Args().Get(1)).String(), + KeysOnly: genc == "", + }) + if err != nil { + return xerrors.Errorf("datastore query: %w", err) + } + defer q.Close() //nolint:errcheck + + seen := map[string]struct{}{} + for res := range q.Next() { + s := res.Key + if cctx.Bool("top-level") { + k := datastore.NewKey(datastore.NewKey(s).List()[0]) + if k.Type() != "" { + s = k.Type() + } else { + s = k.String() + } + + _, has := seen[s] + if has { + continue + } + seen[s] = struct{}{} + } + + + s = fmt.Sprintf("%q", s) + s = strings.Trim(s, "\"") + fmt.Println(s) + + if genc != "" { + fmt.Print("\t") + if err := printVal(genc, res.Value); err != nil { + return err + } + } + } + + return nil + }, +} + +var datastoreGetCmd = &cli.Command{ + Name: "get", + Description: "list datastore keys", + Flags: []cli.Flag{ + &cli.IntFlag{ + Name: "repo-type", + Value: 1, + }, + &cli.StringFlag{ + Name: "enc", + Usage: "encoding (esc/hex/cbor)", + Value: "esc", + }, + }, + ArgsUsage: "[namespace key]", + Action: func(cctx *cli.Context) error { + logging.SetLogLevel("badger", "ERROR") // nolint:errchec + + r, err := repo.NewFS(cctx.String("repo")) + if err != nil { + return xerrors.Errorf("opening fs repo: %w", err) + } + + exists, err := r.Exists() + if err != nil { + return err + } + if !exists { + return xerrors.Errorf("lotus repo doesn't exist") + } + + lr, err := r.Lock(repo.RepoType(cctx.Int("repo-type"))) + if err != nil { + return err + } + defer lr.Close() //nolint:errcheck + + ds, err := lr.Datastore(datastore.NewKey(cctx.Args().First()).String()) + if err != nil { + return err + } + + val, err := ds.Get(datastore.NewKey(cctx.Args().Get(1))) + if err != nil { + return xerrors.Errorf("get: %w", err) + } + + return printVal(cctx.String("enc"), val) + }, +} + +func printVal(enc string, val []byte) error { + switch enc { + case "esc": + s := fmt.Sprintf("%q", string(val)) + s = strings.Trim(s, "\"") + fmt.Println(s) + case "hex": + fmt.Printf("%x\n", val) + case "cbor": + var out interface{} + if err := cbor.Unmarshal(cbor.DecodeOptions{}, val, &out); err != nil { + return xerrors.Errorf("unmarshaling cbor: %w", err) + } + s, err := json.Marshal(&out) + if err != nil { + return xerrors.Errorf("remarshaling as json: %w", err) + } + + fmt.Println(string(s)) + default: + return xerrors.New("unknown encoding") + } + + return nil +} diff --git a/cmd/lotus-shed/main.go b/cmd/lotus-shed/main.go index 3864d3014e3..409d8928b47 100644 --- a/cmd/lotus-shed/main.go +++ b/cmd/lotus-shed/main.go @@ -39,6 +39,7 @@ func main() { consensusCmd, serveDealStatsCmd, syncCmd, + datastoreCmd, } app := &cli.App{ diff --git a/go.mod b/go.mod index f745ccf67ce..3f2661d0efa 100644 --- a/go.mod +++ b/go.mod @@ -109,6 +109,7 @@ require ( github.com/multiformats/go-multibase v0.0.3 github.com/multiformats/go-multihash v0.0.14 github.com/opentracing/opentracing-go v1.2.0 + github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a github.com/raulk/clock v1.1.0 github.com/stretchr/testify v1.6.1 github.com/supranational/blst v0.1.1 From d20ebe93b9f46bd2f138e1a3172599c1a9434d00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 30 Sep 2020 17:26:24 +0200 Subject: [PATCH 02/15] shed: gofmt --- cmd/lotus-shed/datastore.go | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/cmd/lotus-shed/datastore.go b/cmd/lotus-shed/datastore.go index dcd774a90a9..5c668e0819a 100644 --- a/cmd/lotus-shed/datastore.go +++ b/cmd/lotus-shed/datastore.go @@ -16,7 +16,7 @@ import ( ) var datastoreCmd = &cli.Command{ - Name: "datastore", + Name: "datastore", Description: "access node datastores directly", Subcommands: []*cli.Command{ datastoreListCmd, @@ -25,19 +25,19 @@ var datastoreCmd = &cli.Command{ } var datastoreListCmd = &cli.Command{ - Name: "list", + Name: "list", Description: "list datastore keys", Flags: []cli.Flag{ &cli.IntFlag{ - Name: "repo-type", + Name: "repo-type", Value: 1, }, &cli.BoolFlag{ - Name: "top-level", + Name: "top-level", Usage: "only print top-level keys", }, &cli.StringFlag{ - Name: "get-enc", + Name: "get-enc", Usage: "print values [esc/hex/cbor]", }, }, @@ -72,8 +72,8 @@ var datastoreListCmd = &cli.Command{ genc := cctx.String("get-enc") q, err := ds.Query(dsq.Query{ - Prefix: datastore.NewKey(cctx.Args().Get(1)).String(), - KeysOnly: genc == "", + Prefix: datastore.NewKey(cctx.Args().Get(1)).String(), + KeysOnly: genc == "", }) if err != nil { return xerrors.Errorf("datastore query: %w", err) @@ -98,7 +98,6 @@ var datastoreListCmd = &cli.Command{ seen[s] = struct{}{} } - s = fmt.Sprintf("%q", s) s = strings.Trim(s, "\"") fmt.Println(s) @@ -116,15 +115,15 @@ var datastoreListCmd = &cli.Command{ } var datastoreGetCmd = &cli.Command{ - Name: "get", + Name: "get", Description: "list datastore keys", Flags: []cli.Flag{ &cli.IntFlag{ - Name: "repo-type", + Name: "repo-type", Value: 1, }, &cli.StringFlag{ - Name: "enc", + Name: "enc", Usage: "encoding (esc/hex/cbor)", Value: "esc", }, From 2dc9a1ee4ed042471784982c4459ffd52dc524f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 1 Oct 2020 13:58:26 +0200 Subject: [PATCH 03/15] lotus-miner backup command --- api/api_storage.go | 6 ++ api/apistruct/struct.go | 6 ++ cmd/lotus-storage-miner/backup.go | 42 ++++++++ cmd/lotus-storage-miner/main.go | 1 + lib/backupds/backupds.go | 165 ++++++++++++++++++++++++++++++ node/impl/storminer.go | 67 +++++++++++- node/modules/storage.go | 8 +- 7 files changed, 292 insertions(+), 3 deletions(-) create mode 100644 cmd/lotus-storage-miner/backup.go create mode 100644 lib/backupds/backupds.go diff --git a/api/api_storage.go b/api/api_storage.go index 82477218197..529224f6ea4 100644 --- a/api/api_storage.go +++ b/api/api_storage.go @@ -101,6 +101,12 @@ type StorageMiner interface { PiecesListCidInfos(ctx context.Context) ([]cid.Cid, error) PiecesGetPieceInfo(ctx context.Context, pieceCid cid.Cid) (*piecestore.PieceInfo, error) PiecesGetCIDInfo(ctx context.Context, payloadCid cid.Cid) (*piecestore.CIDInfo, error) + + // CreateBackup creates node backup onder the specified file name. The + // method requires that the lotus-miner is running with the + // LOTUS_BACKUP_BASE_PATH environment variable set to some path, and that + // the path specified when calling CreateBackup is within the base path + CreateBackup(ctx context.Context, fpath string) error } type SealRes struct { diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index c79d0841c30..6a3d709a7c8 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -319,6 +319,8 @@ type StorageMinerStruct struct { PiecesListCidInfos func(ctx context.Context) ([]cid.Cid, error) `perm:"read"` PiecesGetPieceInfo func(ctx context.Context, pieceCid cid.Cid) (*piecestore.PieceInfo, error) `perm:"read"` PiecesGetCIDInfo func(ctx context.Context, payloadCid cid.Cid) (*piecestore.CIDInfo, error) `perm:"read"` + + CreateBackup func(ctx context.Context, fpath string) error `perm:"admin"` } } @@ -1265,6 +1267,10 @@ func (c *StorageMinerStruct) PiecesGetCIDInfo(ctx context.Context, payloadCid ci return c.Internal.PiecesGetCIDInfo(ctx, payloadCid) } +func (c *StorageMinerStruct) CreateBackup(ctx context.Context, fpath string) error { + return c.Internal.CreateBackup(ctx, fpath) +} + // WorkerStruct func (w *WorkerStruct) Version(ctx context.Context) (build.Version, error) { diff --git a/cmd/lotus-storage-miner/backup.go b/cmd/lotus-storage-miner/backup.go new file mode 100644 index 00000000000..779fd1fb4a6 --- /dev/null +++ b/cmd/lotus-storage-miner/backup.go @@ -0,0 +1,42 @@ +package main + +import ( + "fmt" + + "github.com/urfave/cli/v2" + "golang.org/x/xerrors" + + lcli "github.com/filecoin-project/lotus/cli" +) + +var backupCmd = &cli.Command{ + Name: "backup", + Usage: "Create node metadata backup", + Description: `The backup command writes a copy of node metadata under the specified path + +For security reasons, the daemon must be have LOTUS_BACKUP_BASE_PATH env var set +to a path where backup files are supposed to be saved, and the path specified in +this command must be within this base path`, + ArgsUsage: "[backup file path]", + Flags: []cli.Flag{}, + Action: func(cctx *cli.Context) error { + api, closer, err := lcli.GetStorageMinerAPI(cctx) + if err != nil { + return err + } + defer closer() + + if cctx.Args().Len() != 1 { + return xerrors.Errorf("expected 1 argument") + } + + err = api.CreateBackup(lcli.ReqContext(cctx), cctx.Args().First()) + if err != nil { + return err + } + + fmt.Println("Success") + + return nil + }, +} diff --git a/cmd/lotus-storage-miner/main.go b/cmd/lotus-storage-miner/main.go index cee64f07798..671f75cf0fc 100644 --- a/cmd/lotus-storage-miner/main.go +++ b/cmd/lotus-storage-miner/main.go @@ -35,6 +35,7 @@ func main() { runCmd, stopCmd, configCmd, + backupCmd, lcli.WithCategory("chain", actorCmd), lcli.WithCategory("chain", infoCmd), lcli.WithCategory("market", storageDealsCmd), diff --git a/lib/backupds/backupds.go b/lib/backupds/backupds.go new file mode 100644 index 00000000000..8845eacc7a5 --- /dev/null +++ b/lib/backupds/backupds.go @@ -0,0 +1,165 @@ +package backupds + +import ( + "io" + "sync" + + logging "github.com/ipfs/go-log/v2" + cbg "github.com/whyrusleeping/cbor-gen" + "golang.org/x/xerrors" + + "github.com/ipfs/go-datastore" + "github.com/ipfs/go-datastore/query" +) + +var log = logging.Logger("backupds") + +type Datastore struct { + child datastore.Batching + + backupLk sync.RWMutex +} + +func Wrap(child datastore.Batching) *Datastore { + return &Datastore{ + child: child, + } +} + +// Writes a datastore dump into the provided writer as indefinite length cbor +// array of [key, value] tuples +func (d *Datastore) Backup(out io.Writer) error { + // write indefinite length array header + if _, err := out.Write([]byte{0x9f}); err != nil { + return xerrors.Errorf("writing header: %w", err) + } + + d.backupLk.Lock() + defer d.backupLk.Unlock() + + log.Info("Starting datastore backup") + defer log.Info("Datastore backup done") + + qr, err := d.child.Query(query.Query{}) + if err != nil { + return xerrors.Errorf("query: %w", err) + } + defer func() { + if err := qr.Close(); err != nil { + log.Errorf("query close error: %+v", err) + return + } + }() + + scratch := make([]byte, 9) + + for result := range qr.Next() { + if err := cbg.WriteMajorTypeHeaderBuf(scratch, out, cbg.MajArray, 2); err != nil { + return xerrors.Errorf("writing tuple header: %w", err) + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, out, cbg.MajByteString, uint64(len([]byte(result.Key)))); err != nil { + return xerrors.Errorf("writing key header: %w", err) + } + + if _, err := out.Write([]byte(result.Key)[:]); err != nil { + return xerrors.Errorf("writing key: %w", err) + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, out, cbg.MajByteString, uint64(len(result.Value))); err != nil { + return xerrors.Errorf("writing value header: %w", err) + } + + if _, err := out.Write(result.Value[:]); err != nil { + return xerrors.Errorf("writing value: %w", err) + } + } + + // array break + if _, err := out.Write([]byte{0xff}); err != nil { + return xerrors.Errorf("writing array 'break': %w", err) + } + + return nil +} + +// proxy + +func (d *Datastore) Get(key datastore.Key) (value []byte, err error) { + return d.child.Get(key) +} + +func (d *Datastore) Has(key datastore.Key) (exists bool, err error) { + return d.child.Has(key) +} + +func (d *Datastore) GetSize(key datastore.Key) (size int, err error) { + return d.child.GetSize(key) +} + +func (d *Datastore) Query(q query.Query) (query.Results, error) { + return d.child.Query(q) +} + +func (d *Datastore) Put(key datastore.Key, value []byte) error { + d.backupLk.RLock() + defer d.backupLk.RUnlock() + + return d.child.Put(key, value) +} + +func (d *Datastore) Delete(key datastore.Key) error { + d.backupLk.RLock() + defer d.backupLk.RUnlock() + + return d.child.Delete(key) +} + +func (d *Datastore) Sync(prefix datastore.Key) error { + d.backupLk.RLock() + defer d.backupLk.RUnlock() + + return d.child.Sync(prefix) +} + +func (d *Datastore) Close() error { + d.backupLk.RLock() + defer d.backupLk.RUnlock() + + return d.child.Close() +} + +func (d *Datastore) Batch() (datastore.Batch, error) { + b, err := d.child.Batch() + if err != nil { + return nil, err + } + + return &bbatch{ + b: b, + rlk: d.backupLk.RLocker(), + }, nil +} + +type bbatch struct { + b datastore.Batch + rlk sync.Locker +} + +func (b *bbatch) Put(key datastore.Key, value []byte) error { + return b.b.Put(key, value) +} + +func (b *bbatch) Delete(key datastore.Key) error { + return b.b.Delete(key) +} + +func (b *bbatch) Commit() error { + b.rlk.Lock() + defer b.rlk.Unlock() + + return b.b.Commit() +} + +var _ datastore.Batch = &bbatch{} +var _ datastore.Batching = &Datastore{} diff --git a/node/impl/storminer.go b/node/impl/storminer.go index 5634c140b16..c0cbe0081e8 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -5,21 +5,24 @@ import ( "encoding/json" "net/http" "os" + "path/filepath" "strconv" + "strings" "time" - datatransfer "github.com/filecoin-project/go-data-transfer" - "github.com/filecoin-project/go-state-types/big" "github.com/ipfs/go-cid" "github.com/libp2p/go-libp2p-core/host" + "github.com/mitchellh/go-homedir" "golang.org/x/xerrors" "github.com/filecoin-project/go-address" + datatransfer "github.com/filecoin-project/go-data-transfer" "github.com/filecoin-project/go-fil-markets/piecestore" retrievalmarket "github.com/filecoin-project/go-fil-markets/retrievalmarket" storagemarket "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/go-jsonrpc/auth" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/big" sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage" "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" @@ -31,9 +34,11 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/api/apistruct" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/lib/backupds" "github.com/filecoin-project/lotus/miner" "github.com/filecoin-project/lotus/node/impl/common" "github.com/filecoin-project/lotus/node/modules/dtypes" + "github.com/filecoin-project/lotus/node/repo" "github.com/filecoin-project/lotus/storage" "github.com/filecoin-project/lotus/storage/sectorblocks" ) @@ -56,6 +61,9 @@ type StorageMinerAPI struct { DataTransfer dtypes.ProviderDataTransfer Host host.Host + DS dtypes.MetadataDS + Repo repo.LockedRepo + ConsiderOnlineStorageDealsConfigFunc dtypes.ConsiderOnlineStorageDealsConfigFunc SetConsiderOnlineStorageDealsConfigFunc dtypes.SetConsiderOnlineStorageDealsConfigFunc ConsiderOnlineRetrievalDealsConfigFunc dtypes.ConsiderOnlineRetrievalDealsConfigFunc @@ -516,4 +524,59 @@ func (sm *StorageMinerAPI) PiecesGetCIDInfo(ctx context.Context, payloadCid cid. return &ci, nil } +func (sm *StorageMinerAPI) CreateBackup(ctx context.Context, fpath string) error { + // TODO: Config + bb, ok := os.LookupEnv("LOTUS_BACKUP_BASE_PATH") + if !ok { + return xerrors.Errorf("LOTUS_BACKUP_BASE_PATH env var not set") + } + + bds, ok := sm.DS.(*backupds.Datastore) + if !ok { + return xerrors.Errorf("expected a backup datastore") + } + + bb, err := homedir.Expand(bb) + if err != nil { + return xerrors.Errorf("expanding base path: %w", err) + } + + bb, err = filepath.Abs(bb) + if err != nil { + return xerrors.Errorf("getting absolute base path: %w", err) + } + + fpath, err = homedir.Expand(fpath) + if err != nil { + return xerrors.Errorf("expanding file path: %w", err) + } + + fpath, err = filepath.Abs(fpath) + if err != nil { + return xerrors.Errorf("getting absolute file path: %w", err) + } + + if !strings.HasPrefix(fpath, bb) { + return xerrors.Errorf("backup file name (%s) must be inside base path (%s)", fpath, bb) + } + + out, err := os.OpenFile(fpath, os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return xerrors.Errorf("open %s: %w", fpath, err) + } + + if err := bds.Backup(out); err != nil { + if cerr := out.Close(); cerr != nil { + log.Errorw("error closing backup file while handling backup error", "closeErr", cerr, "backupErr", err) + } + return xerrors.Errorf("backup error: %w", err) + } + + if err := out.Close(); err != nil { + return xerrors.Errorf("closing backup file: %w", err) + } + + return nil +} + var _ api.StorageMiner = &StorageMinerAPI{} diff --git a/node/modules/storage.go b/node/modules/storage.go index 1bdce1d2f41..9c1a18368ce 100644 --- a/node/modules/storage.go +++ b/node/modules/storage.go @@ -6,6 +6,7 @@ import ( "go.uber.org/fx" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/lib/backupds" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/repo" ) @@ -27,5 +28,10 @@ func KeyStore(lr repo.LockedRepo) (types.KeyStore, error) { } func Datastore(r repo.LockedRepo) (dtypes.MetadataDS, error) { - return r.Datastore("/metadata") + mds, err := r.Datastore("/metadata") + if err != nil { + return nil, err + } + + return backupds.Wrap(mds), nil } From c8a3269c4b472133da5d61e3f5b9a17808f7a8c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 1 Oct 2020 14:36:19 +0200 Subject: [PATCH 04/15] shed: utils to read datastore backups --- cmd/lotus-shed/datastore.go | 148 +++++++++++++++++---- lib/backupds/{backupds.go => datastore.go} | 0 lib/backupds/read.go | 52 ++++++++ 3 files changed, 174 insertions(+), 26 deletions(-) rename lib/backupds/{backupds.go => datastore.go} (100%) create mode 100644 lib/backupds/read.go diff --git a/cmd/lotus-shed/datastore.go b/cmd/lotus-shed/datastore.go index 5c668e0819a..eb5db2aa9d3 100644 --- a/cmd/lotus-shed/datastore.go +++ b/cmd/lotus-shed/datastore.go @@ -3,8 +3,10 @@ package main import ( "encoding/json" "fmt" + "os" "strings" + "github.com/docker/go-units" "github.com/ipfs/go-datastore" dsq "github.com/ipfs/go-datastore/query" logging "github.com/ipfs/go-log" @@ -12,6 +14,7 @@ import ( "github.com/urfave/cli/v2" "golang.org/x/xerrors" + "github.com/filecoin-project/lotus/lib/backupds" "github.com/filecoin-project/lotus/node/repo" ) @@ -19,6 +22,7 @@ var datastoreCmd = &cli.Command{ Name: "datastore", Description: "access node datastores directly", Subcommands: []*cli.Command{ + datastoreBackupCmd, datastoreListCmd, datastoreGetCmd, }, @@ -80,33 +84,11 @@ var datastoreListCmd = &cli.Command{ } defer q.Close() //nolint:errcheck - seen := map[string]struct{}{} - for res := range q.Next() { - s := res.Key - if cctx.Bool("top-level") { - k := datastore.NewKey(datastore.NewKey(s).List()[0]) - if k.Type() != "" { - s = k.Type() - } else { - s = k.String() - } - - _, has := seen[s] - if has { - continue - } - seen[s] = struct{}{} - } + printKv := kvPrinter(cctx.Bool("top-level"), genc) - s = fmt.Sprintf("%q", s) - s = strings.Trim(s, "\"") - fmt.Println(s) - - if genc != "" { - fmt.Print("\t") - if err := printVal(genc, res.Value); err != nil { - return err - } + for res := range q.Next() { + if err := printKv(res.Key, res.Value); err != nil { + return err } } @@ -165,6 +147,120 @@ var datastoreGetCmd = &cli.Command{ }, } +var datastoreBackupCmd = &cli.Command{ + Name: "backup", + Description: "manage datastore backups", + Subcommands: []*cli.Command{ + datastoreBackupStatCmd, + datastoreBackupListCmd, + }, +} + +var datastoreBackupStatCmd = &cli.Command{ + Name: "stat", + Description: "validate and print info about datastore backup", + ArgsUsage: "[file]", + Action: func(cctx *cli.Context) error { + if cctx.Args().Len() != 1 { + return xerrors.Errorf("expected 1 argument") + } + + f, err := os.Open(cctx.Args().First()) + if err != nil { + return xerrors.Errorf("opening backup file: %w", err) + } + defer f.Close() // nolint:errcheck + + var keys, kbytes, vbytes uint64 + err = backupds.ReadBackup(f, func(key datastore.Key, value []byte) error { + keys++ + kbytes += uint64(len(key.String())) + vbytes += uint64(len(value)) + return nil + }) + if err != nil { + return err + } + + fmt.Println("Keys: ", keys) + fmt.Println("Key bytes: ", units.BytesSize(float64(kbytes))) + fmt.Println("Value bytes: ", units.BytesSize(float64(vbytes))) + + return err + }, +} + +var datastoreBackupListCmd = &cli.Command{ + Name: "list", + Description: "list data in a backup", + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "top-level", + Usage: "only print top-level keys", + }, + &cli.StringFlag{ + Name: "get-enc", + Usage: "print values [esc/hex/cbor]", + }, + }, + ArgsUsage: "[file]", + Action: func(cctx *cli.Context) error { + if cctx.Args().Len() != 1 { + return xerrors.Errorf("expected 1 argument") + } + + f, err := os.Open(cctx.Args().First()) + if err != nil { + return xerrors.Errorf("opening backup file: %w", err) + } + defer f.Close() // nolint:errcheck + + printKv := kvPrinter(cctx.Bool("top-level"), cctx.String("get-enc")) + err = backupds.ReadBackup(f, func(key datastore.Key, value []byte) error { + return printKv(key.String(), value) + }) + if err != nil { + return err + } + + return err + }, +} + +func kvPrinter(toplevel bool, genc string) func(sk string, value []byte) error { + seen := map[string]struct{}{} + + return func(s string, value []byte) error { + if toplevel { + k := datastore.NewKey(datastore.NewKey(s).List()[0]) + if k.Type() != "" { + s = k.Type() + } else { + s = k.String() + } + + _, has := seen[s] + if has { + return nil + } + seen[s] = struct{}{} + } + + s = fmt.Sprintf("%q", s) + s = strings.Trim(s, "\"") + fmt.Println(s) + + if genc != "" { + fmt.Print("\t") + if err := printVal(genc, value); err != nil { + return err + } + } + + return nil + } +} + func printVal(enc string, val []byte) error { switch enc { case "esc": diff --git a/lib/backupds/backupds.go b/lib/backupds/datastore.go similarity index 100% rename from lib/backupds/backupds.go rename to lib/backupds/datastore.go diff --git a/lib/backupds/read.go b/lib/backupds/read.go new file mode 100644 index 00000000000..e1ae5837e64 --- /dev/null +++ b/lib/backupds/read.go @@ -0,0 +1,52 @@ +package backupds + +import ( + "io" + + "github.com/ipfs/go-datastore" + cbg "github.com/whyrusleeping/cbor-gen" + "golang.org/x/xerrors" +) + +func ReadBackup(r io.Reader, cb func(key datastore.Key, value []byte) error) error { + scratch := make([]byte, 9) + + if _, err := r.Read(scratch[:1]); err != nil { + return xerrors.Errorf("reading array header: %w", err) + } + + if scratch[0] != 0x9f { + return xerrors.Errorf("expected indefinite length array header byte 0x9f, got %x", scratch[0]) + } + + for { + if _, err := r.Read(scratch[:1]); err != nil { + return xerrors.Errorf("reading tuple header: %w", err) + } + + if scratch[0] == 0xff { + break + } + + if scratch[0] != 0x82 { + return xerrors.Errorf("expected array(2) header 0x82, got %x", scratch[0]) + } + + keyb, err := cbg.ReadByteArray(r, 1<<40) + if err != nil { + return xerrors.Errorf("reading key: %w", err) + } + key := datastore.NewKey(string(keyb)) + + value, err := cbg.ReadByteArray(r, 1<<40) + if err != nil { + return xerrors.Errorf("reading value: %w", err) + } + + if err := cb(key, value); err != nil { + return err + } + } + + return nil +} From deac7166b5251f7c503cce3a3b20f1f6fa55efeb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 1 Oct 2020 14:46:07 +0200 Subject: [PATCH 05/15] Support offline backups --- cmd/lotus-shed/datastore.go | 2 +- cmd/lotus-storage-miner/backup.go | 99 +++++++++++++++++++++++++++---- 2 files changed, 87 insertions(+), 14 deletions(-) diff --git a/cmd/lotus-shed/datastore.go b/cmd/lotus-shed/datastore.go index eb5db2aa9d3..62ee6bc6ebf 100644 --- a/cmd/lotus-shed/datastore.go +++ b/cmd/lotus-shed/datastore.go @@ -47,7 +47,7 @@ var datastoreListCmd = &cli.Command{ }, ArgsUsage: "[namespace prefix]", Action: func(cctx *cli.Context) error { - logging.SetLogLevel("badger", "ERROR") // nolint:errchec + logging.SetLogLevel("badger", "ERROR") // nolint:errcheck r, err := repo.NewFS(cctx.String("repo")) if err != nil { diff --git a/cmd/lotus-storage-miner/backup.go b/cmd/lotus-storage-miner/backup.go index 779fd1fb4a6..a1ad360c9de 100644 --- a/cmd/lotus-storage-miner/backup.go +++ b/cmd/lotus-storage-miner/backup.go @@ -2,11 +2,16 @@ package main import ( "fmt" + "os" + logging "github.com/ipfs/go-log/v2" + "github.com/mitchellh/go-homedir" "github.com/urfave/cli/v2" "golang.org/x/xerrors" lcli "github.com/filecoin-project/lotus/cli" + "github.com/filecoin-project/lotus/lib/backupds" + "github.com/filecoin-project/lotus/node/repo" ) var backupCmd = &cli.Command{ @@ -14,29 +19,97 @@ var backupCmd = &cli.Command{ Usage: "Create node metadata backup", Description: `The backup command writes a copy of node metadata under the specified path +Online backups: For security reasons, the daemon must be have LOTUS_BACKUP_BASE_PATH env var set to a path where backup files are supposed to be saved, and the path specified in this command must be within this base path`, + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "offline", + Usage: "create backup without the node running", + }, + }, ArgsUsage: "[backup file path]", - Flags: []cli.Flag{}, Action: func(cctx *cli.Context) error { - api, closer, err := lcli.GetStorageMinerAPI(cctx) - if err != nil { - return err - } - defer closer() - if cctx.Args().Len() != 1 { return xerrors.Errorf("expected 1 argument") } - err = api.CreateBackup(lcli.ReqContext(cctx), cctx.Args().First()) - if err != nil { - return err + if cctx.Bool("offline") { + return offlineBackup(cctx) } - fmt.Println("Success") - - return nil + return onlineBackup(cctx) }, } + +func offlineBackup(cctx *cli.Context) error { + logging.SetLogLevel("badger", "ERROR") // nolint:errcheck + + repoPath := cctx.String(FlagMinerRepo) + r, err := repo.NewFS(repoPath) + if err != nil { + return err + } + + ok, err := r.Exists() + if err != nil { + return err + } + if !ok { + return xerrors.Errorf("repo at '%s' is not initialized", cctx.String(FlagMinerRepo)) + } + + lr, err := r.Lock(repo.StorageMiner) + if err != nil { + return xerrors.Errorf("locking repo: %w", err) + } + defer lr.Close() // nolint:errcheck + + mds, err := lr.Datastore("/metadata") + if err != nil { + return xerrors.Errorf("getting metadata datastore: %w", err) + } + + bds := backupds.Wrap(mds) + + fpath, err := homedir.Expand(cctx.Args().First()) + if err != nil { + return xerrors.Errorf("expanding file path: %w", err) + } + + out, err := os.OpenFile(fpath, os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return xerrors.Errorf("opening backup file %s: %w", fpath, err) + } + + if err := bds.Backup(out); err != nil { + if cerr := out.Close(); cerr != nil { + log.Errorw("error closing backup file while handling backup error", "closeErr", cerr, "backupErr", err) + } + return xerrors.Errorf("backup error: %w", err) + } + + if err := out.Close(); err != nil { + return xerrors.Errorf("closing backup file: %w", err) + } + + return nil +} + +func onlineBackup(cctx *cli.Context) error { + api, closer, err := lcli.GetStorageMinerAPI(cctx) + if err != nil { + return xerrors.Errorf("getting api: %w (if the node isn't running you can use the --offline flag)", err) + } + defer closer() + + err = api.CreateBackup(lcli.ReqContext(cctx), cctx.Args().First()) + if err != nil { + return err + } + + fmt.Println("Success") + + return nil +} From 9b5a0815fc557f41fac8ebccc40721b86292481c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 1 Oct 2020 14:51:37 +0200 Subject: [PATCH 06/15] backup: open datastores in readonly in offline mode --- cmd/lotus-storage-miner/backup.go | 2 +- node/repo/fsrepo.go | 12 ++++++++++++ node/repo/fsrepo_ds.go | 18 +++++++++++------- 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/cmd/lotus-storage-miner/backup.go b/cmd/lotus-storage-miner/backup.go index a1ad360c9de..55604b6fbe0 100644 --- a/cmd/lotus-storage-miner/backup.go +++ b/cmd/lotus-storage-miner/backup.go @@ -60,7 +60,7 @@ func offlineBackup(cctx *cli.Context) error { return xerrors.Errorf("repo at '%s' is not initialized", cctx.String(FlagMinerRepo)) } - lr, err := r.Lock(repo.StorageMiner) + lr, err := r.LockRO(repo.StorageMiner) if err != nil { return xerrors.Errorf("locking repo: %w", err) } diff --git a/node/repo/fsrepo.go b/node/repo/fsrepo.go index a69cdd55d6b..c4571c478b0 100644 --- a/node/repo/fsrepo.go +++ b/node/repo/fsrepo.go @@ -226,11 +226,23 @@ func (fsr *FsRepo) Lock(repoType RepoType) (LockedRepo, error) { }, nil } +// Like Lock, except datastores will work in read-only mode +func (fsr *FsRepo) LockRO(repoType RepoType) (LockedRepo, error) { + lr, err := fsr.Lock(repoType) + if err != nil { + return nil, err + } + + lr.(*fsLockedRepo).readonly = true + return lr, nil +} + type fsLockedRepo struct { path string configPath string repoType RepoType closer io.Closer + readonly bool ds map[string]datastore.Batching dsErr error diff --git a/node/repo/fsrepo_ds.go b/node/repo/fsrepo_ds.go index f4afe0dee8c..aa91d2514d0 100644 --- a/node/repo/fsrepo_ds.go +++ b/node/repo/fsrepo_ds.go @@ -14,7 +14,7 @@ import ( ldbopts "github.com/syndtr/goleveldb/leveldb/opt" ) -type dsCtor func(path string) (datastore.Batching, error) +type dsCtor func(path string, readonly bool) (datastore.Batching, error) var fsDatastores = map[string]dsCtor{ "chain": chainBadgerDs, @@ -26,9 +26,10 @@ var fsDatastores = map[string]dsCtor{ "client": badgerDs, // client specific } -func chainBadgerDs(path string) (datastore.Batching, error) { +func chainBadgerDs(path string, readonly bool) (datastore.Batching, error) { opts := badger.DefaultOptions opts.GcInterval = 0 // disable GC for chain datastore + opts.ReadOnly = readonly opts.Options = dgbadger.DefaultOptions("").WithTruncate(true). WithValueThreshold(1 << 10) @@ -36,23 +37,26 @@ func chainBadgerDs(path string) (datastore.Batching, error) { return badger.NewDatastore(path, &opts) } -func badgerDs(path string) (datastore.Batching, error) { +func badgerDs(path string, readonly bool) (datastore.Batching, error) { opts := badger.DefaultOptions + opts.ReadOnly = readonly + opts.Options = dgbadger.DefaultOptions("").WithTruncate(true). WithValueThreshold(1 << 10) return badger.NewDatastore(path, &opts) } -func levelDs(path string) (datastore.Batching, error) { +func levelDs(path string, readonly bool) (datastore.Batching, error) { return levelds.NewDatastore(path, &levelds.Options{ Compression: ldbopts.NoCompression, NoSync: false, Strict: ldbopts.StrictAll, + ReadOnly: readonly, }) } -func (fsr *fsLockedRepo) openDatastores() (map[string]datastore.Batching, error) { +func (fsr *fsLockedRepo) openDatastores(readonly bool) (map[string]datastore.Batching, error) { if err := os.MkdirAll(fsr.join(fsDatastore), 0755); err != nil { return nil, xerrors.Errorf("mkdir %s: %w", fsr.join(fsDatastore), err) } @@ -63,7 +67,7 @@ func (fsr *fsLockedRepo) openDatastores() (map[string]datastore.Batching, error) prefix := datastore.NewKey(p) // TODO: optimization: don't init datastores we don't need - ds, err := ctor(fsr.join(filepath.Join(fsDatastore, p))) + ds, err := ctor(fsr.join(filepath.Join(fsDatastore, p)), readonly) if err != nil { return nil, xerrors.Errorf("opening datastore %s: %w", prefix, err) } @@ -78,7 +82,7 @@ func (fsr *fsLockedRepo) openDatastores() (map[string]datastore.Batching, error) func (fsr *fsLockedRepo) Datastore(ns string) (datastore.Batching, error) { fsr.dsOnce.Do(func() { - fsr.ds, fsr.dsErr = fsr.openDatastores() + fsr.ds, fsr.dsErr = fsr.openDatastores(fsr.readonly) }) if fsr.dsErr != nil { From d7ec5e36184cbfc3a152d06c26ae2531f1524aee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 1 Oct 2020 16:22:54 +0200 Subject: [PATCH 07/15] lotus-miner init restore --- cmd/lotus-storage-miner/init.go | 3 + cmd/lotus-storage-miner/init_restore.go | 169 ++++++++++++++++++++++++ lib/backupds/read.go | 24 ++++ 3 files changed, 196 insertions(+) create mode 100644 cmd/lotus-storage-miner/init_restore.go diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index 4faf6a25b22..9218faa77e0 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -115,6 +115,9 @@ var initCmd = &cli.Command{ Usage: "select which address to send actor creation message from", }, }, + Subcommands: []*cli.Command{ + initRestoreCmd, + }, Action: func(cctx *cli.Context) error { log.Info("Initializing lotus miner") diff --git a/cmd/lotus-storage-miner/init_restore.go b/cmd/lotus-storage-miner/init_restore.go new file mode 100644 index 00000000000..8c8c6213ecc --- /dev/null +++ b/cmd/lotus-storage-miner/init_restore.go @@ -0,0 +1,169 @@ +package main + +import ( + "os" + + "github.com/docker/go-units" + "github.com/ipfs/go-datastore" + "github.com/libp2p/go-libp2p-core/peer" + "github.com/urfave/cli/v2" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-address" + paramfetch "github.com/filecoin-project/go-paramfetch" + "github.com/filecoin-project/go-state-types/big" + + "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/types" + lcli "github.com/filecoin-project/lotus/cli" + "github.com/filecoin-project/lotus/lib/backupds" + "github.com/filecoin-project/lotus/node/repo" +) + +var initRestoreCmd = &cli.Command{ + Name: "restore", + Usage: "Initialize a lotus miner repo from a backup", + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "nosync", + Usage: "don't check full-node sync status", + }, + // TODO: Config + // TODO: Storage paths + }, + ArgsUsage: "[backupFile]", + Action: func(cctx *cli.Context) error { + log.Info("Initializing lotus miner using a backup") + if cctx.Args().Len() != 1 { + return xerrors.Errorf("expected 1 argument") + } + + log.Info("Trying to connect to full node RPC") + + api, closer, err := lcli.GetFullNodeAPI(cctx) // TODO: consider storing full node address in config + if err != nil { + return err + } + defer closer() + + log.Info("Checking full node version") + + ctx := lcli.ReqContext(cctx) + + v, err := api.Version(ctx) + if err != nil { + return err + } + + if !v.APIVersion.EqMajorMinor(build.FullAPIVersion) { + return xerrors.Errorf("Remote API version didn't match (expected %s, remote %s)", build.FullAPIVersion, v.APIVersion) + } + + if !cctx.Bool("nosync") { + if err := lcli.SyncWait(ctx, api); err != nil { + return xerrors.Errorf("sync wait: %w", err) + } + } + + f, err := os.Open(cctx.Args().First()) + if err != nil { + return xerrors.Errorf("opening backup file: %w", err) + } + defer f.Close() // nolint:errcheck + + log.Info("Checking if repo exists") + + repoPath := cctx.String(FlagMinerRepo) + r, err := repo.NewFS(repoPath) + if err != nil { + return err + } + + ok, err := r.Exists() + if err != nil { + return err + } + if ok { + return xerrors.Errorf("repo at '%s' is already initialized", cctx.String(FlagMinerRepo)) + } + + log.Info("Initializing repo") + + if err := r.Init(repo.StorageMiner); err != nil { + return err + } + + lr, err := r.Lock(repo.StorageMiner) + if err != nil { + return err + } + defer lr.Close() //nolint:errcheck + + log.Info("Restoring metadata backup") + + mds, err := lr.Datastore("/metadata") + if err != nil { + return err + } + + if err := backupds.RestoreInto(f, mds); err != nil { + return xerrors.Errorf("restoring metadata: %w", err) + } + + log.Info("Checking actor metadata") + + abytes, err := mds.Get(datastore.NewKey("miner-address")) + if err != nil { + return xerrors.Errorf("getting actor address from metadata datastore: %w", err) + } + + maddr, err := address.NewFromBytes(abytes) + if err != nil { + return xerrors.Errorf("parsing actor address: %w", err) + } + + log.Info("ACTOR ADDRESS: ", maddr.String()) + + mi, err := api.StateMinerInfo(ctx, maddr, types.EmptyTSK) + if err != nil { + return xerrors.Errorf("getting miner info: %w", err) + } + + log.Info("SECTOR SIZE: ", units.BytesSize(float64(mi.SectorSize))) + + has, err := api.WalletHas(ctx, mi.Worker) + if err != nil { + return xerrors.Errorf("checking worker address: %w", err) + } + + if !has { + return xerrors.Errorf("worker address %s for miner actor %s not present in full node wallet", mi.Worker, maddr) + } + + log.Info("Checking proof parameters") + + if err := paramfetch.GetParams(ctx, build.ParametersJSON(), uint64(mi.SectorSize)); err != nil { + return xerrors.Errorf("fetching proof parameters: %w", err) + } + + log.Info("Initializing libp2p identity") + + p2pSk, err := makeHostKey(lr) + if err != nil { + return xerrors.Errorf("make host key: %w", err) + } + + peerid, err := peer.IDFromPrivateKey(p2pSk) + if err != nil { + return xerrors.Errorf("peer ID from private key: %w", err) + } + + log.Info("Configuring miner actor") + + if err := configureStorageMiner(ctx, api, maddr, peerid, big.Zero()); err != nil { + return err + } + + return nil + }, +} diff --git a/lib/backupds/read.go b/lib/backupds/read.go index e1ae5837e64..d368d1d09a9 100644 --- a/lib/backupds/read.go +++ b/lib/backupds/read.go @@ -50,3 +50,27 @@ func ReadBackup(r io.Reader, cb func(key datastore.Key, value []byte) error) err return nil } + +func RestoreInto(r io.Reader, dest datastore.Batching) error { + batch, err := dest.Batch() + if err != nil { + return xerrors.Errorf("creating batch: %w", err) + } + + err = ReadBackup(r, func(key datastore.Key, value []byte) error { + if err := batch.Put(key, value); err != nil { + return xerrors.Errorf("put key: %w", err) + } + + return nil + }) + if err != nil { + return xerrors.Errorf("reading backup: %w", err) + } + + if err := batch.Commit(); err != nil { + return xerrors.Errorf("committing batch: %w", err) + } + + return nil +} From 44747446a95e5e4e32a473f3291ff0842f787c82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 1 Oct 2020 16:45:02 +0200 Subject: [PATCH 08/15] miner restore: config flag --- cmd/lotus-storage-miner/init_restore.go | 47 ++++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/cmd/lotus-storage-miner/init_restore.go b/cmd/lotus-storage-miner/init_restore.go index 8c8c6213ecc..e1219c039db 100644 --- a/cmd/lotus-storage-miner/init_restore.go +++ b/cmd/lotus-storage-miner/init_restore.go @@ -1,6 +1,8 @@ package main import ( + "github.com/filecoin-project/lotus/node/config" + "github.com/mitchellh/go-homedir" "os" "github.com/docker/go-units" @@ -28,7 +30,10 @@ var initRestoreCmd = &cli.Command{ Name: "nosync", Usage: "don't check full-node sync status", }, - // TODO: Config + &cli.StringFlag{ + Name: "config", + Usage: "config file", + }, // TODO: Storage paths }, ArgsUsage: "[backupFile]", @@ -99,6 +104,46 @@ var initRestoreCmd = &cli.Command{ } defer lr.Close() //nolint:errcheck + if cctx.IsSet("config") { + log.Info("Restoring config") + + cf, err := homedir.Expand(cctx.String("config")) + if err != nil { + return xerrors.Errorf("expanding config path: %w", err) + } + + _, err = os.Stat(cf) + if err != nil { + return xerrors.Errorf("stat config file (%s): %w", cf, err) + } + + var cerr error + err = lr.SetConfig(func(raw interface{}) { + rcfg, ok := raw.(*config.StorageMiner) + if !ok { + cerr = xerrors.New("expected miner config") + return + } + + ff, err := config.FromFile(cf, rcfg) + if err != nil { + cerr = xerrors.Errorf("loading config: %w", err) + return + } + + *rcfg = *ff.(*config.StorageMiner) + }) + if cerr != nil { + return cerr + } + if err != nil { + return xerrors.Errorf("setting config: %w", err) + } + + } else { + log.Warn("--config NOT SET, WILL USE DEFAULT VALUES") + } + log.Info("Restoring metadata backup") mds, err := lr.Datastore("/metadata") From 5c33982f72bece681bd0f3465cd6ff66eae58263 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 1 Oct 2020 16:51:30 +0200 Subject: [PATCH 09/15] miner restore: storage config flag --- cmd/lotus-storage-miner/init_restore.go | 37 +++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/cmd/lotus-storage-miner/init_restore.go b/cmd/lotus-storage-miner/init_restore.go index e1219c039db..560eae33b69 100644 --- a/cmd/lotus-storage-miner/init_restore.go +++ b/cmd/lotus-storage-miner/init_restore.go @@ -1,8 +1,11 @@ package main import ( + "encoding/json" + "github.com/filecoin-project/lotus/extern/sector-storage/stores" "github.com/filecoin-project/lotus/node/config" "github.com/mitchellh/go-homedir" + "io/ioutil" "os" "github.com/docker/go-units" @@ -32,9 +35,12 @@ var initRestoreCmd = &cli.Command{ }, &cli.StringFlag{ Name: "config", - Usage: "config file", + Usage: "config file (config.toml)", + }, + &cli.StringFlag{ + Name: "storage-config", + Usage: "storage paths config (storage.json)", }, - // TODO: Storage paths }, ArgsUsage: "[backupFile]", Action: func(cctx *cli.Context) error { @@ -144,6 +150,33 @@ var initRestoreCmd = &cli.Command{ log.Warn("--config NOT SET, WILL USE DEFAULT VALUES") } + if cctx.IsSet("storage-config") { + log.Info("Restoring storage path config") + + cf, err := homedir.Expand(cctx.String("storage-config")) + if err != nil { + return xerrors.Errorf("expanding storage config path: %w", err) + } + + cfb, err := ioutil.ReadFile(cf) + if err != nil { + return xerrors.Errorf("reading storage config: %w", err) + } + + var cerr error + err = lr.SetStorage(func(scfg *stores.StorageConfig) { + cerr = json.Unmarshal(cfb, scfg) + }) + if cerr != nil { + return xerrors.Errorf("unmarshalling storage config: %w", cerr) + } + if err != nil { + return xerrors.Errorf("setting storage config: %w", err) + } + } else { + log.Warn("--storage-config NOT SET. NO SECTOR PATHS WILL BE CONFIGURED") + } + log.Info("Restoring metadata backup") mds, err := lr.Datastore("/metadata") From e444977891bd17e8809c3ed72105537046bb6a36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 1 Oct 2020 17:14:08 +0200 Subject: [PATCH 10/15] Full node metadata backup --- api/api_full.go | 6 ++ api/apistruct/struct.go | 6 ++ cli/backup.go | 125 ++++++++++++++++++++++++++++++ cmd/lotus-storage-miner/backup.go | 111 ++------------------------ cmd/lotus/backup.go | 14 ++++ cmd/lotus/main.go | 1 + node/impl/backup.go | 67 ++++++++++++++++ node/impl/full.go | 9 +++ node/impl/storminer.go | 61 +-------------- 9 files changed, 235 insertions(+), 165 deletions(-) create mode 100644 cli/backup.go create mode 100644 cmd/lotus/backup.go create mode 100644 node/impl/backup.go diff --git a/api/api_full.go b/api/api_full.go index 767739582e1..acbae19c416 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -475,6 +475,12 @@ type FullNode interface { PaychVoucherAdd(context.Context, address.Address, *paych.SignedVoucher, []byte, types.BigInt) (types.BigInt, error) PaychVoucherList(context.Context, address.Address) ([]*paych.SignedVoucher, error) PaychVoucherSubmit(context.Context, address.Address, *paych.SignedVoucher, []byte, []byte) (cid.Cid, error) + + // CreateBackup creates node backup onder the specified file name. The + // method requires that the lotus daemon is running with the + // LOTUS_BACKUP_BASE_PATH environment variable set to some path, and that + // the path specified when calling CreateBackup is within the base path + CreateBackup(ctx context.Context, fpath string) error } type FileRef struct { diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 6a3d709a7c8..2ae4ac08df5 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -239,6 +239,8 @@ type FullNodeStruct struct { PaychVoucherCreate func(context.Context, address.Address, big.Int, uint64) (*api.VoucherCreateResult, error) `perm:"sign"` PaychVoucherList func(context.Context, address.Address) ([]*paych.SignedVoucher, error) `perm:"write"` PaychVoucherSubmit func(context.Context, address.Address, *paych.SignedVoucher, []byte, []byte) (cid.Cid, error) `perm:"sign"` + + CreateBackup func(ctx context.Context, fpath string) error `perm:"admin"` } } @@ -1027,6 +1029,10 @@ func (c *FullNodeStruct) PaychVoucherSubmit(ctx context.Context, ch address.Addr return c.Internal.PaychVoucherSubmit(ctx, ch, sv, secret, proof) } +func (c *FullNodeStruct) CreateBackup(ctx context.Context, fpath string) error { + return c.Internal.CreateBackup(ctx, fpath) +} + // StorageMinerStruct func (c *StorageMinerStruct) ActorAddress(ctx context.Context) (address.Address, error) { diff --git a/cli/backup.go b/cli/backup.go new file mode 100644 index 00000000000..c748e47c438 --- /dev/null +++ b/cli/backup.go @@ -0,0 +1,125 @@ +package cli + +import ( + "context" + "fmt" + "os" + + logging "github.com/ipfs/go-log/v2" + "github.com/mitchellh/go-homedir" + "github.com/urfave/cli/v2" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-jsonrpc" + + "github.com/filecoin-project/lotus/lib/backupds" + "github.com/filecoin-project/lotus/node/repo" +) + +type BackupAPI interface { + CreateBackup(ctx context.Context, fpath string) error +} + +type BackupApiFn func(ctx *cli.Context) (BackupAPI, jsonrpc.ClientCloser, error) + +func BackupCmd(repoFlag string, rt repo.RepoType, getApi BackupApiFn) *cli.Command { + var offlineBackup = func(cctx *cli.Context) error { + logging.SetLogLevel("badger", "ERROR") // nolint:errcheck + + repoPath := cctx.String(repoFlag) + r, err := repo.NewFS(repoPath) + if err != nil { + return err + } + + ok, err := r.Exists() + if err != nil { + return err + } + if !ok { + return xerrors.Errorf("repo at '%s' is not initialized", cctx.String(repoFlag)) + } + + lr, err := r.LockRO(rt) + if err != nil { + return xerrors.Errorf("locking repo: %w", err) + } + defer lr.Close() // nolint:errcheck + + mds, err := lr.Datastore("/metadata") + if err != nil { + return xerrors.Errorf("getting metadata datastore: %w", err) + } + + bds := backupds.Wrap(mds) + + fpath, err := homedir.Expand(cctx.Args().First()) + if err != nil { + return xerrors.Errorf("expanding file path: %w", err) + } + + out, err := os.OpenFile(fpath, os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return xerrors.Errorf("opening backup file %s: %w", fpath, err) + } + + if err := bds.Backup(out); err != nil { + if cerr := out.Close(); cerr != nil { + log.Errorw("error closing backup file while handling backup error", "closeErr", cerr, "backupErr", err) + } + return xerrors.Errorf("backup error: %w", err) + } + + if err := out.Close(); err != nil { + return xerrors.Errorf("closing backup file: %w", err) + } + + return nil + } + + var onlineBackup = func(cctx *cli.Context) error { + api, closer, err := getApi(cctx) + if err != nil { + return xerrors.Errorf("getting api: %w (if the node isn't running you can use the --offline flag)", err) + } + defer closer() + + err = api.CreateBackup(ReqContext(cctx), cctx.Args().First()) + if err != nil { + return err + } + + fmt.Println("Success") + + return nil + } + + return &cli.Command{ + Name: "backup", + Usage: "Create node metadata backup", + Description: `The backup command writes a copy of node metadata under the specified path + +Online backups: +For security reasons, the daemon must be have LOTUS_BACKUP_BASE_PATH env var set +to a path where backup files are supposed to be saved, and the path specified in +this command must be within this base path`, + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "offline", + Usage: "create backup without the node running", + }, + }, + ArgsUsage: "[backup file path]", + Action: func(cctx *cli.Context) error { + if cctx.Args().Len() != 1 { + return xerrors.Errorf("expected 1 argument") + } + + if cctx.Bool("offline") { + return offlineBackup(cctx) + } + + return onlineBackup(cctx) + }, + } +} diff --git a/cmd/lotus-storage-miner/backup.go b/cmd/lotus-storage-miner/backup.go index 55604b6fbe0..cf8c9f9125c 100644 --- a/cmd/lotus-storage-miner/backup.go +++ b/cmd/lotus-storage-miner/backup.go @@ -1,115 +1,14 @@ package main import ( - "fmt" - "os" - - logging "github.com/ipfs/go-log/v2" - "github.com/mitchellh/go-homedir" "github.com/urfave/cli/v2" - "golang.org/x/xerrors" + + "github.com/filecoin-project/go-jsonrpc" lcli "github.com/filecoin-project/lotus/cli" - "github.com/filecoin-project/lotus/lib/backupds" "github.com/filecoin-project/lotus/node/repo" ) -var backupCmd = &cli.Command{ - Name: "backup", - Usage: "Create node metadata backup", - Description: `The backup command writes a copy of node metadata under the specified path - -Online backups: -For security reasons, the daemon must be have LOTUS_BACKUP_BASE_PATH env var set -to a path where backup files are supposed to be saved, and the path specified in -this command must be within this base path`, - Flags: []cli.Flag{ - &cli.BoolFlag{ - Name: "offline", - Usage: "create backup without the node running", - }, - }, - ArgsUsage: "[backup file path]", - Action: func(cctx *cli.Context) error { - if cctx.Args().Len() != 1 { - return xerrors.Errorf("expected 1 argument") - } - - if cctx.Bool("offline") { - return offlineBackup(cctx) - } - - return onlineBackup(cctx) - }, -} - -func offlineBackup(cctx *cli.Context) error { - logging.SetLogLevel("badger", "ERROR") // nolint:errcheck - - repoPath := cctx.String(FlagMinerRepo) - r, err := repo.NewFS(repoPath) - if err != nil { - return err - } - - ok, err := r.Exists() - if err != nil { - return err - } - if !ok { - return xerrors.Errorf("repo at '%s' is not initialized", cctx.String(FlagMinerRepo)) - } - - lr, err := r.LockRO(repo.StorageMiner) - if err != nil { - return xerrors.Errorf("locking repo: %w", err) - } - defer lr.Close() // nolint:errcheck - - mds, err := lr.Datastore("/metadata") - if err != nil { - return xerrors.Errorf("getting metadata datastore: %w", err) - } - - bds := backupds.Wrap(mds) - - fpath, err := homedir.Expand(cctx.Args().First()) - if err != nil { - return xerrors.Errorf("expanding file path: %w", err) - } - - out, err := os.OpenFile(fpath, os.O_CREATE|os.O_WRONLY, 0644) - if err != nil { - return xerrors.Errorf("opening backup file %s: %w", fpath, err) - } - - if err := bds.Backup(out); err != nil { - if cerr := out.Close(); cerr != nil { - log.Errorw("error closing backup file while handling backup error", "closeErr", cerr, "backupErr", err) - } - return xerrors.Errorf("backup error: %w", err) - } - - if err := out.Close(); err != nil { - return xerrors.Errorf("closing backup file: %w", err) - } - - return nil -} - -func onlineBackup(cctx *cli.Context) error { - api, closer, err := lcli.GetStorageMinerAPI(cctx) - if err != nil { - return xerrors.Errorf("getting api: %w (if the node isn't running you can use the --offline flag)", err) - } - defer closer() - - err = api.CreateBackup(lcli.ReqContext(cctx), cctx.Args().First()) - if err != nil { - return err - } - - fmt.Println("Success") - - return nil -} +var backupCmd = lcli.BackupCmd(FlagMinerRepo, repo.StorageMiner, func(cctx *cli.Context) (lcli.BackupAPI, jsonrpc.ClientCloser, error) { + return lcli.GetStorageMinerAPI(cctx) +}) diff --git a/cmd/lotus/backup.go b/cmd/lotus/backup.go new file mode 100644 index 00000000000..aec0000c90d --- /dev/null +++ b/cmd/lotus/backup.go @@ -0,0 +1,14 @@ +package main + +import ( + "github.com/urfave/cli/v2" + + "github.com/filecoin-project/go-jsonrpc" + + lcli "github.com/filecoin-project/lotus/cli" + "github.com/filecoin-project/lotus/node/repo" +) + +var backupCmd = lcli.BackupCmd("repo", repo.FullNode, func(cctx *cli.Context) (lcli.BackupAPI, jsonrpc.ClientCloser, error) { + return lcli.GetFullNodeAPI(cctx) +}) diff --git a/cmd/lotus/main.go b/cmd/lotus/main.go index ee95ad64b44..eb97045eeb1 100644 --- a/cmd/lotus/main.go +++ b/cmd/lotus/main.go @@ -22,6 +22,7 @@ func main() { local := []*cli.Command{ DaemonCmd, + backupCmd, } if AdvanceBlockCmd != nil { local = append(local, AdvanceBlockCmd) diff --git a/node/impl/backup.go b/node/impl/backup.go new file mode 100644 index 00000000000..10f673a4bfe --- /dev/null +++ b/node/impl/backup.go @@ -0,0 +1,67 @@ +package impl + +import ( + "os" + "path/filepath" + "strings" + + "github.com/mitchellh/go-homedir" + "golang.org/x/xerrors" + + "github.com/filecoin-project/lotus/lib/backupds" + "github.com/filecoin-project/lotus/node/modules/dtypes" +) + +func backup(mds dtypes.MetadataDS, fpath string) error { + bb, ok := os.LookupEnv("LOTUS_BACKUP_BASE_PATH") + if !ok { + return xerrors.Errorf("LOTUS_BACKUP_BASE_PATH env var not set") + } + + bds, ok := mds.(*backupds.Datastore) + if !ok { + return xerrors.Errorf("expected a backup datastore") + } + + bb, err := homedir.Expand(bb) + if err != nil { + return xerrors.Errorf("expanding base path: %w", err) + } + + bb, err = filepath.Abs(bb) + if err != nil { + return xerrors.Errorf("getting absolute base path: %w", err) + } + + fpath, err = homedir.Expand(fpath) + if err != nil { + return xerrors.Errorf("expanding file path: %w", err) + } + + fpath, err = filepath.Abs(fpath) + if err != nil { + return xerrors.Errorf("getting absolute file path: %w", err) + } + + if !strings.HasPrefix(fpath, bb) { + return xerrors.Errorf("backup file name (%s) must be inside base path (%s)", fpath, bb) + } + + out, err := os.OpenFile(fpath, os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return xerrors.Errorf("open %s: %w", fpath, err) + } + + if err := bds.Backup(out); err != nil { + if cerr := out.Close(); cerr != nil { + log.Errorw("error closing backup file while handling backup error", "closeErr", cerr, "backupErr", err) + } + return xerrors.Errorf("backup error: %w", err) + } + + if err := out.Close(); err != nil { + return xerrors.Errorf("closing backup file: %w", err) + } + + return nil +} diff --git a/node/impl/full.go b/node/impl/full.go index feeb2974572..add40917c84 100644 --- a/node/impl/full.go +++ b/node/impl/full.go @@ -1,6 +1,8 @@ package impl import ( + "context" + logging "github.com/ipfs/go-log/v2" "github.com/filecoin-project/lotus/api" @@ -9,6 +11,7 @@ import ( "github.com/filecoin-project/lotus/node/impl/full" "github.com/filecoin-project/lotus/node/impl/market" "github.com/filecoin-project/lotus/node/impl/paych" + "github.com/filecoin-project/lotus/node/modules/dtypes" ) var log = logging.Logger("node") @@ -26,6 +29,12 @@ type FullNodeAPI struct { full.WalletAPI full.SyncAPI full.BeaconAPI + + DS dtypes.MetadataDS +} + +func (n *FullNodeAPI) CreateBackup(ctx context.Context, fpath string) error { + return backup(n.DS, fpath) } var _ api.FullNode = &FullNodeAPI{} diff --git a/node/impl/storminer.go b/node/impl/storminer.go index c0cbe0081e8..6090e8a58c4 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -5,14 +5,11 @@ import ( "encoding/json" "net/http" "os" - "path/filepath" "strconv" - "strings" "time" "github.com/ipfs/go-cid" "github.com/libp2p/go-libp2p-core/host" - "github.com/mitchellh/go-homedir" "golang.org/x/xerrors" "github.com/filecoin-project/go-address" @@ -34,11 +31,9 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/api/apistruct" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/lib/backupds" "github.com/filecoin-project/lotus/miner" "github.com/filecoin-project/lotus/node/impl/common" "github.com/filecoin-project/lotus/node/modules/dtypes" - "github.com/filecoin-project/lotus/node/repo" "github.com/filecoin-project/lotus/storage" "github.com/filecoin-project/lotus/storage/sectorblocks" ) @@ -61,8 +56,7 @@ type StorageMinerAPI struct { DataTransfer dtypes.ProviderDataTransfer Host host.Host - DS dtypes.MetadataDS - Repo repo.LockedRepo + DS dtypes.MetadataDS ConsiderOnlineStorageDealsConfigFunc dtypes.ConsiderOnlineStorageDealsConfigFunc SetConsiderOnlineStorageDealsConfigFunc dtypes.SetConsiderOnlineStorageDealsConfigFunc @@ -525,58 +519,7 @@ func (sm *StorageMinerAPI) PiecesGetCIDInfo(ctx context.Context, payloadCid cid. } func (sm *StorageMinerAPI) CreateBackup(ctx context.Context, fpath string) error { - // TODO: Config - bb, ok := os.LookupEnv("LOTUS_BACKUP_BASE_PATH") - if !ok { - return xerrors.Errorf("LOTUS_BACKUP_BASE_PATH env var not set") - } - - bds, ok := sm.DS.(*backupds.Datastore) - if !ok { - return xerrors.Errorf("expected a backup datastore") - } - - bb, err := homedir.Expand(bb) - if err != nil { - return xerrors.Errorf("expanding base path: %w", err) - } - - bb, err = filepath.Abs(bb) - if err != nil { - return xerrors.Errorf("getting absolute base path: %w", err) - } - - fpath, err = homedir.Expand(fpath) - if err != nil { - return xerrors.Errorf("expanding file path: %w", err) - } - - fpath, err = filepath.Abs(fpath) - if err != nil { - return xerrors.Errorf("getting absolute file path: %w", err) - } - - if !strings.HasPrefix(fpath, bb) { - return xerrors.Errorf("backup file name (%s) must be inside base path (%s)", fpath, bb) - } - - out, err := os.OpenFile(fpath, os.O_CREATE|os.O_WRONLY, 0644) - if err != nil { - return xerrors.Errorf("open %s: %w", fpath, err) - } - - if err := bds.Backup(out); err != nil { - if cerr := out.Close(); cerr != nil { - log.Errorw("error closing backup file while handling backup error", "closeErr", cerr, "backupErr", err) - } - return xerrors.Errorf("backup error: %w", err) - } - - if err := out.Close(); err != nil { - return xerrors.Errorf("closing backup file: %w", err) - } - - return nil + return backup(sm.DS, fpath) } var _ api.StorageMiner = &StorageMinerAPI{} From 738efe3f9a53f6231fb36f0c09adf53deb7c4a4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 1 Oct 2020 17:18:34 +0200 Subject: [PATCH 11/15] miner restore: Add progerss bar for metadata import --- cmd/lotus-storage-miner/init_restore.go | 32 +++++++++++++++++++++---- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/cmd/lotus-storage-miner/init_restore.go b/cmd/lotus-storage-miner/init_restore.go index 560eae33b69..cfb13be31af 100644 --- a/cmd/lotus-storage-miner/init_restore.go +++ b/cmd/lotus-storage-miner/init_restore.go @@ -2,17 +2,16 @@ package main import ( "encoding/json" - "github.com/filecoin-project/lotus/extern/sector-storage/stores" - "github.com/filecoin-project/lotus/node/config" - "github.com/mitchellh/go-homedir" "io/ioutil" "os" "github.com/docker/go-units" "github.com/ipfs/go-datastore" "github.com/libp2p/go-libp2p-core/peer" + "github.com/mitchellh/go-homedir" "github.com/urfave/cli/v2" "golang.org/x/xerrors" + "gopkg.in/cheggaaa/pb.v1" "github.com/filecoin-project/go-address" paramfetch "github.com/filecoin-project/go-paramfetch" @@ -21,7 +20,9 @@ import ( "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/types" lcli "github.com/filecoin-project/lotus/cli" + "github.com/filecoin-project/lotus/extern/sector-storage/stores" "github.com/filecoin-project/lotus/lib/backupds" + "github.com/filecoin-project/lotus/node/config" "github.com/filecoin-project/lotus/node/repo" ) @@ -76,7 +77,17 @@ var initRestoreCmd = &cli.Command{ } } - f, err := os.Open(cctx.Args().First()) + bf, err := homedir.Expand(cctx.Args().First()) + if err != nil { + return xerrors.Errorf("expand backup file path: %w", err) + } + + st, err := os.Stat(bf) + if err != nil { + return xerrors.Errorf("stat backup file (%s): %w", bf, err) + } + + f, err := os.Open(bf) if err != nil { return xerrors.Errorf("opening backup file: %w", err) } @@ -184,7 +195,18 @@ var initRestoreCmd = &cli.Command{ return err } - if err := backupds.RestoreInto(f, mds); err != nil { + bar := pb.New64(st.Size()) + br := bar.NewProxyReader(f) + bar.ShowTimeLeft = true + bar.ShowPercent = true + bar.ShowSpeed = true + bar.Units = pb.U_BYTES + + bar.Start() + err = backupds.RestoreInto(br, mds) + bar.Finish() + + if err != nil { return xerrors.Errorf("restoring metadata: %w", err) } From ec7282ff090107f105285e99aec78e8532d25f83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 1 Oct 2020 17:51:56 +0200 Subject: [PATCH 12/15] miner restore: Resolve worker key for wallet check --- cmd/lotus-storage-miner/init_restore.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cmd/lotus-storage-miner/init_restore.go b/cmd/lotus-storage-miner/init_restore.go index cfb13be31af..bdbb99fe0a3 100644 --- a/cmd/lotus-storage-miner/init_restore.go +++ b/cmd/lotus-storage-miner/init_restore.go @@ -231,7 +231,12 @@ var initRestoreCmd = &cli.Command{ log.Info("SECTOR SIZE: ", units.BytesSize(float64(mi.SectorSize))) - has, err := api.WalletHas(ctx, mi.Worker) + wk, err := api.StateAccountKey(ctx, mi.Worker, types.EmptyTSK) + if err != nil { + return xerrors.Errorf("resolving worker key: %w", err) + } + + has, err := api.WalletHas(ctx, wk) if err != nil { return xerrors.Errorf("checking worker address: %w", err) } From 15885ad535e7330fd8ad6ae8def9a5a8b5c8480f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 1 Oct 2020 18:00:24 +0200 Subject: [PATCH 13/15] docsgen --- documentation/en/api-methods.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index 2b28816f7a2..f74a975b0d4 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -50,6 +50,8 @@ * [ClientRetrieveTryRestartInsufficientFunds](#ClientRetrieveTryRestartInsufficientFunds) * [ClientRetrieveWithEvents](#ClientRetrieveWithEvents) * [ClientStartDeal](#ClientStartDeal) +* [Create](#Create) + * [CreateBackup](#CreateBackup) * [Gas](#Gas) * [GasEstimateFeeCap](#GasEstimateFeeCap) * [GasEstimateGasLimit](#GasEstimateGasLimit) @@ -1278,6 +1280,27 @@ Inputs: Response: `null` +## Create + + +### CreateBackup +CreateBackup creates node backup onder the specified file name. The +method requires that the lotus daemon is running with the +LOTUS_BACKUP_BASE_PATH environment variable set to some path, and that +the path specified when calling CreateBackup is within the base path + + +Perms: admin + +Inputs: +```json +[ + "string value" +] +``` + +Response: `{}` + ## Gas From 70d88f226e16e925b02bfa945720684f215e6341 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 6 Oct 2020 01:33:51 +0200 Subject: [PATCH 14/15] Add usage to repo-type in shed datastore --- cmd/lotus-shed/datastore.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/lotus-shed/datastore.go b/cmd/lotus-shed/datastore.go index 62ee6bc6ebf..c6bac6815bf 100644 --- a/cmd/lotus-shed/datastore.go +++ b/cmd/lotus-shed/datastore.go @@ -34,6 +34,7 @@ var datastoreListCmd = &cli.Command{ Flags: []cli.Flag{ &cli.IntFlag{ Name: "repo-type", + Usage: "node type (1 - full, 2 - storage, 3 - worker)", Value: 1, }, &cli.BoolFlag{ @@ -102,6 +103,7 @@ var datastoreGetCmd = &cli.Command{ Flags: []cli.Flag{ &cli.IntFlag{ Name: "repo-type", + Usage: "node type (1 - full, 2 - storage, 3 - worker)", Value: 1, }, &cli.StringFlag{ From 8cdf07899953021b4237e847248b8d39517d0867 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 6 Oct 2020 01:50:43 +0200 Subject: [PATCH 15/15] backup: append checksum --- lib/backupds/datastore.go | 98 ++++++++++++++++++++++++--------------- lib/backupds/read.go | 30 ++++++++++-- 2 files changed, 88 insertions(+), 40 deletions(-) diff --git a/lib/backupds/datastore.go b/lib/backupds/datastore.go index 8845eacc7a5..1555577f346 100644 --- a/lib/backupds/datastore.go +++ b/lib/backupds/datastore.go @@ -1,6 +1,7 @@ package backupds import ( + "crypto/sha256" "io" "sync" @@ -26,58 +27,81 @@ func Wrap(child datastore.Batching) *Datastore { } } -// Writes a datastore dump into the provided writer as indefinite length cbor -// array of [key, value] tuples +// Writes a datastore dump into the provided writer as +// [array(*) of [key, value] tuples, checksum] func (d *Datastore) Backup(out io.Writer) error { - // write indefinite length array header - if _, err := out.Write([]byte{0x9f}); err != nil { - return xerrors.Errorf("writing header: %w", err) - } + scratch := make([]byte, 9) - d.backupLk.Lock() - defer d.backupLk.Unlock() + if err := cbg.WriteMajorTypeHeaderBuf(scratch, out, cbg.MajArray, 2); err != nil { + return xerrors.Errorf("writing tuple header: %w", err) + } - log.Info("Starting datastore backup") - defer log.Info("Datastore backup done") + hasher := sha256.New() + hout := io.MultiWriter(hasher, out) - qr, err := d.child.Query(query.Query{}) - if err != nil { - return xerrors.Errorf("query: %w", err) - } - defer func() { - if err := qr.Close(); err != nil { - log.Errorf("query close error: %+v", err) - return + // write KVs + { + // write indefinite length array header + if _, err := hout.Write([]byte{0x9f}); err != nil { + return xerrors.Errorf("writing header: %w", err) } - }() - scratch := make([]byte, 9) + d.backupLk.Lock() + defer d.backupLk.Unlock() - for result := range qr.Next() { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, out, cbg.MajArray, 2); err != nil { - return xerrors.Errorf("writing tuple header: %w", err) - } + log.Info("Starting datastore backup") + defer log.Info("Datastore backup done") - if err := cbg.WriteMajorTypeHeaderBuf(scratch, out, cbg.MajByteString, uint64(len([]byte(result.Key)))); err != nil { - return xerrors.Errorf("writing key header: %w", err) + qr, err := d.child.Query(query.Query{}) + if err != nil { + return xerrors.Errorf("query: %w", err) } - - if _, err := out.Write([]byte(result.Key)[:]); err != nil { - return xerrors.Errorf("writing key: %w", err) + defer func() { + if err := qr.Close(); err != nil { + log.Errorf("query close error: %+v", err) + return + } + }() + + for result := range qr.Next() { + if err := cbg.WriteMajorTypeHeaderBuf(scratch, hout, cbg.MajArray, 2); err != nil { + return xerrors.Errorf("writing tuple header: %w", err) + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, hout, cbg.MajByteString, uint64(len([]byte(result.Key)))); err != nil { + return xerrors.Errorf("writing key header: %w", err) + } + + if _, err := hout.Write([]byte(result.Key)[:]); err != nil { + return xerrors.Errorf("writing key: %w", err) + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, hout, cbg.MajByteString, uint64(len(result.Value))); err != nil { + return xerrors.Errorf("writing value header: %w", err) + } + + if _, err := hout.Write(result.Value[:]); err != nil { + return xerrors.Errorf("writing value: %w", err) + } } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, out, cbg.MajByteString, uint64(len(result.Value))); err != nil { - return xerrors.Errorf("writing value header: %w", err) + // array break + if _, err := hout.Write([]byte{0xff}); err != nil { + return xerrors.Errorf("writing array 'break': %w", err) } + } - if _, err := out.Write(result.Value[:]); err != nil { - return xerrors.Errorf("writing value: %w", err) + // Write the checksum + { + sum := hasher.Sum(nil) + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, hout, cbg.MajByteString, uint64(len(sum))); err != nil { + return xerrors.Errorf("writing checksum header: %w", err) } - } - // array break - if _, err := out.Write([]byte{0xff}); err != nil { - return xerrors.Errorf("writing array 'break': %w", err) + if _, err := hout.Write(sum[:]); err != nil { + return xerrors.Errorf("writing checksum: %w", err) + } } return nil diff --git a/lib/backupds/read.go b/lib/backupds/read.go index d368d1d09a9..f9a4336374c 100644 --- a/lib/backupds/read.go +++ b/lib/backupds/read.go @@ -1,6 +1,8 @@ package backupds import ( + "bytes" + "crypto/sha256" "io" "github.com/ipfs/go-datastore" @@ -15,12 +17,23 @@ func ReadBackup(r io.Reader, cb func(key datastore.Key, value []byte) error) err return xerrors.Errorf("reading array header: %w", err) } + if scratch[0] != 0x82 { + return xerrors.Errorf("expected array(2) header byte 0x82, got %x", scratch[0]) + } + + hasher := sha256.New() + hr := io.TeeReader(r, hasher) + + if _, err := hr.Read(scratch[:1]); err != nil { + return xerrors.Errorf("reading array header: %w", err) + } + if scratch[0] != 0x9f { return xerrors.Errorf("expected indefinite length array header byte 0x9f, got %x", scratch[0]) } for { - if _, err := r.Read(scratch[:1]); err != nil { + if _, err := hr.Read(scratch[:1]); err != nil { return xerrors.Errorf("reading tuple header: %w", err) } @@ -32,13 +45,13 @@ func ReadBackup(r io.Reader, cb func(key datastore.Key, value []byte) error) err return xerrors.Errorf("expected array(2) header 0x82, got %x", scratch[0]) } - keyb, err := cbg.ReadByteArray(r, 1<<40) + keyb, err := cbg.ReadByteArray(hr, 1<<40) if err != nil { return xerrors.Errorf("reading key: %w", err) } key := datastore.NewKey(string(keyb)) - value, err := cbg.ReadByteArray(r, 1<<40) + value, err := cbg.ReadByteArray(hr, 1<<40) if err != nil { return xerrors.Errorf("reading value: %w", err) } @@ -48,6 +61,17 @@ func ReadBackup(r io.Reader, cb func(key datastore.Key, value []byte) error) err } } + sum := hasher.Sum(nil) + + expSum, err := cbg.ReadByteArray(r, 32) + if err != nil { + return xerrors.Errorf("reading expected checksum: %w", err) + } + + if !bytes.Equal(sum, expSum) { + return xerrors.Errorf("checksum didn't match; expected %x, got %x", expSum, sum) + } + return nil }