Skip to content

Commit

Permalink
fix(cmd): useful errors in dag import (#9945)
Browse files Browse the repository at this point in the history
* fix: useful errors during dag import

Most of the time the error is either a bitflip in one of blocks,
or a truncation of car stream.

This allows user to understand what happened
and at which place in the car stream, making debug more humane.

* fix: correct message when root pin failed

this also correctly exits CLI commands with code 1 (was silent
false-positive 0 before)
  • Loading branch information
lidel authored Jun 14, 2023
1 parent b8da86e commit f5f6b66
Show file tree
Hide file tree
Showing 2 changed files with 22 additions and 5 deletions.
2 changes: 1 addition & 1 deletion core/commands/dag/dag.go
Original file line number Diff line number Diff line change
Expand Up @@ -237,7 +237,7 @@ Specification of CAR formats: https://ipld.io/specs/transport/car/
}

if event.Root.PinErrorMsg != "" {
event.Root.PinErrorMsg = fmt.Sprintf("FAILED: %s", event.Root.PinErrorMsg)
return fmt.Errorf("pinning root %q FAILED: %s", enc.Encode(event.Root.Cid), event.Root.PinErrorMsg)
} else {
event.Root.PinErrorMsg = "success"
}
Expand Down
25 changes: 21 additions & 4 deletions core/commands/dag/import.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,13 @@ package dagcmd

import (
"errors"
"fmt"
"io"

"github.com/ipfs/boxo/coreiface/options"
"github.com/ipfs/boxo/files"
gocarv2 "github.com/ipfs/boxo/ipld/car/v2"
blocks "github.com/ipfs/go-block-format"
cid "github.com/ipfs/go-cid"
cmds "github.com/ipfs/go-ipfs-cmds"
ipld "github.com/ipfs/go-ipld-format"
Expand Down Expand Up @@ -58,6 +60,18 @@ func dagImport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment
roots := cid.NewSet()
var blockCount, blockBytesCount uint64

// remember last valid block and provide a meaningful error message
// when a truncated/mangled CAR is being imported
importError := func(previous blocks.Block, current blocks.Block, err error) error {
if current != nil {
return fmt.Errorf("import failed at block %q: %w", current.Cid(), err)
}
if previous != nil {
return fmt.Errorf("import failed after block %q: %w", previous.Cid(), err)
}
return fmt.Errorf("import failed: %w", err)
}

it := req.Files.Entries()
for it.Next() {
file := files.FileFromEntry(it)
Expand All @@ -75,6 +89,8 @@ func dagImport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment
// this won't/can't help with not running out of handles
defer file.Close()

var previous blocks.Block

car, err := gocarv2.NewBlockReader(file)
if err != nil {
return err
Expand All @@ -87,25 +103,26 @@ func dagImport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment
for {
block, err := car.Next()
if err != nil && err != io.EOF {
return err
return importError(previous, block, err)
} else if block == nil {
break
}
if err := cmdutils.CheckBlockSize(req, uint64(len(block.RawData()))); err != nil {
return err
return importError(previous, block, err)
}

// the double-decode is suboptimal, but we need it for batching
nd, err := blockDecoder.DecodeNode(req.Context, block)
if err != nil {
return err
return importError(previous, block, err)
}

if err := batch.Add(req.Context, nd); err != nil {
return err
return importError(previous, block, err)
}
blockCount++
blockBytesCount += uint64(len(block.RawData()))
previous = block
}
return nil
}()
Expand Down

0 comments on commit f5f6b66

Please sign in to comment.