Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update bootstrappers #89

Merged
merged 4 commits into from
Jan 10, 2020
Merged

Update bootstrappers #89

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
7376e6c
feat: update bootstrappers
dirkmc Dec 17, 2019
6d6b79c
fix: rework 7 to 8 logic
dirkmc Dec 20, 2019
8ae7849
fix: don't break /dns/ipfs-mydomain.com addresses in 7 to 8 migration
dirkmc Jan 7, 2020
b650735
fix: add (very) old bootstrap node to 7 to 8 migration
dirkmc Jan 7, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ install:
test: test_go sharness

test_go:
go test ./ipfs-5-to-6/... # go test ./... fails see #66
go test ./ipfs-5-to-6/... ./ipfs-7-to-8/... # go test ./... fails see #66

sharness:
make -C sharness
Expand Down
11 changes: 11 additions & 0 deletions ipfs-7-to-8/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
package main

import (
migrate "github.com/ipfs/fs-repo-migrations/go-migrate"
mg7 "github.com/ipfs/fs-repo-migrations/ipfs-7-to-8/migration"
)

func main() {
m := mg7.Migration{}
migrate.Main(&m)
}
177 changes: 177 additions & 0 deletions ipfs-7-to-8/migration/config_conv.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,177 @@
package mg7

import (
"encoding/json"
"fmt"
"io"
"io/ioutil"
"os"
"strings"

"github.com/ipfs/go-cid"
dirkmc marked this conversation as resolved.
Show resolved Hide resolved
)

var (
dnsAddr = "bootstrap.libp2p.io"
dnsBootstrapPeers = []string{
"QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN",
"QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa",
"QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb",
"QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt",
}
smallKeyBootstrapPeers = []string{
"QmSoLnSGccFuZQJzRadHn95W2CrSFmZuTdDWP8HXaHca9z",
"QmSoLueR4xBeUbY9WZ9xGUUxunbKWcrNFTDAadQJmocnWm",
"QmSoLMeWqB7YGVLJN3pNLQpmmEk35v6wYtsMGLzSr5QBU3",
"QmSoLju6m7xTh3DuokvT3886QRYqxAzb1kShaanJgW36yx",
"QmSoLPppuBtQSGwKDZT2M73ULpjvfd3aZ6ha4oFGL1KrGM",
"QmSoLSafTMBsPKadTEgaXctDQVcqN88CNLHXMkTNwMKPnu",
"QmSoLV4Bbm51jM9C4gDYZQ9Cy3U6aXMJDAbzgu2fzaDs64",
"QmSoLer265NRgSp2LA3dPaeykiS1J6DifTC88f5uVQKNAd",
}
oldBootstrapAddrs = []string{
"/ip4/104.131.131.82/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ",
"/ip4/104.236.179.241/tcp/4001/p2p/QmSoLPppuBtQSGwKDZT2M73ULpjvfd3aZ6ha4oFGL1KrGM",
"/ip4/128.199.219.111/tcp/4001/p2p/QmSoLSafTMBsPKadTEgaXctDQVcqN88CNLHXMkTNwMKPnu",
"/ip4/104.236.76.40/tcp/4001/p2p/QmSoLV4Bbm51jM9C4gDYZQ9Cy3U6aXMJDAbzgu2fzaDs64",
"/ip4/178.62.158.247/tcp/4001/p2p/QmSoLer265NRgSp2LA3dPaeykiS1J6DifTC88f5uVQKNAd",
"/ip6/2604:a880:1:20::203:d001/tcp/4001/p2p/QmSoLPppuBtQSGwKDZT2M73ULpjvfd3aZ6ha4oFGL1KrGM",
"/ip6/2400:6180:0:d0::151:6001/tcp/4001/p2p/QmSoLSafTMBsPKadTEgaXctDQVcqN88CNLHXMkTNwMKPnu",
"/ip6/2604:a880:800:10::4a:5001/tcp/4001/p2p/QmSoLV4Bbm51jM9C4gDYZQ9Cy3U6aXMJDAbzgu2fzaDs64",
"/ip6/2a03:b0c0:0:1010::23:1001/tcp/4001/p2p/QmSoLer265NRgSp2LA3dPaeykiS1J6DifTC88f5uVQKNAd",
}
)

// convFunc does an inplace conversion of the "Bootstrap"
// configuration from one version to another
type convFunc func([]string) []string

// convertFile converts a config file from one version to another, the
// converted config is stored in
func convertFile(orig string, new string, convFunc convFunc) error {
in, err := os.Open(orig)
if err != nil {
return err
}
out, err := os.Create(new)
if err != nil {
return err
}
return convert(in, out, convFunc)
}

// convert converts the config from one version to another, returns
// the converted config as a map[string]interface{}
func convert(in io.Reader, out io.Writer, convFunc convFunc) error {
data, err := ioutil.ReadAll(in)
if err != nil {
return err
}
confMap := make(map[string]interface{})
dirkmc marked this conversation as resolved.
Show resolved Hide resolved
if err = json.Unmarshal(data, &confMap); err != nil {
return err
}
bootstrapi, _ := confMap["Bootstrap"].([]interface{})
if bootstrapi == nil {
bootstrapi, _ := confMap["bootstrap"].([]interface{})
dirkmc marked this conversation as resolved.
Show resolved Hide resolved
if bootstrapi == nil {
return fmt.Errorf("Bootstrap field missing or of the wrong type")
}
}
bootstrap := make([]string, len(bootstrapi))
for i := range bootstrapi {
bootstrap[i] = bootstrapi[i].(string)
}
res := convFunc(bootstrap)
confMap["Bootstrap"] = res
fixed, err := json.MarshalIndent(confMap, "", " ")
if err != nil {
return err
}
out.Write(fixed)
out.Write([]byte("\n"))
return nil
}

func ver7to8(bootstrap []string) []string {
dirkmc marked this conversation as resolved.
Show resolved Hide resolved
// Make sure the dnsaddrs bootstrap peers are included
var res []string
for _, p := range dnsBootstrapPeers {
res = append(res, fmt.Sprintf("/dnsaddr/%s/p2p/%s", dnsAddr, p))
}

// Filter out peers that we added above, or that have an ID known to belong
// to a peer with a small key
for _, addr := range bootstrap {
dirkmc marked this conversation as resolved.
Show resolved Hide resolved
if ok, err := isDNSBootstrapPeer(addr); !ok && err == nil {
if ok, err = isSmallKeyPeer(addr); !ok && err == nil {
// Replace /ipfs with /p2p
addr = strings.Replace(addr, "/ipfs", "/p2p", -1)
dirkmc marked this conversation as resolved.
Show resolved Hide resolved
res = append(res, addr)
}
}
}

return res
}

func ver8to7(bootstrap []string) []string {
// Make sure the old addresses are included
res := append([]string{}, oldBootstrapAddrs...)

oldPeerIDs := make(map[string]struct{})
for _, addr := range oldBootstrapAddrs {
pid, err := getAddrPeerID(addr)
if err != nil {
panic(err)
}
oldPeerIDs[pid] = struct{}{}
}

// Filter out old addresses added above, and addresses with the new DNS
// addresses
for _, btAddr := range bootstrap {
pid, err := getAddrPeerID(btAddr)
if err == nil {
if _, ok := oldPeerIDs[pid]; !ok && !strings.Contains(btAddr, dnsAddr) {
dirkmc marked this conversation as resolved.
Show resolved Hide resolved
res = append(res, btAddr)
}
}
}

return res
}

func getAddrPeerID(addr string) (string, error) {
// eg /ip4/104.131.131.82/tcp/4001/ipfs/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ
parts := strings.Split(addr, "/")
if len(parts) == 0 {
return "", fmt.Errorf("Could not parse peer ID from addr '%s'", addr)
}
last := parts[len(parts)-1]
if _, err := cid.Decode(last); err != nil {
return "", fmt.Errorf("Could not parse peer ID from addr '%s'", addr)
}
return last, nil
}

func addrPeerIDInList(peers []string, addr string) (bool, error) {
addrID, err := getAddrPeerID(addr)
if err != nil {
return false, err
}
for _, p := range peers {
if p == addrID {
return true, nil
}
}
return false, nil
}

func isDNSBootstrapPeer(addr string) (bool, error) {
return addrPeerIDInList(dnsBootstrapPeers, addr)
}

func isSmallKeyPeer(addr string) (bool, error) {
return addrPeerIDInList(smallKeyBootstrapPeers, addr)
}
120 changes: 120 additions & 0 deletions ipfs-7-to-8/migration/config_conv_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
package mg7

import (
"bytes"
"encoding/json"
"fmt"
"strings"
"testing"
)

func arrayMatch(a []interface{}, b []string) bool {
if len(a) != len(b) {
return false
}
am := make(map[string]struct{})
for _, i := range a {
am[i.(string)] = struct{}{}
}
for _, i := range b {
if _, ok := am[i]; !ok {
return false
}
}
return true
}

func matchesExpected(t *testing.T, res []byte, exp []string) bool {
confMap := make(map[string]interface{})
if err := json.Unmarshal(res, &confMap); err != nil {
t.Fatal(err)
}

return arrayMatch(confMap["Bootstrap"].([]interface{}), exp)
}

func TestOldToNew(t *testing.T) {
in := strings.NewReader(oldConfig)
out := new(bytes.Buffer)
if err := convert(in, out, ver7to8); err != nil {
t.Fatal(err)
}

if !matchesExpected(t, out.Bytes(), expectedMigrateForward) {
t.Fatal(fmt.Errorf("Converted does not match expected result\n%s\n%s\n", out.String(), expectedMigrateForward))
}
}

func TestNewToOld(t *testing.T) {
in := strings.NewReader(newConfig)
out := new(bytes.Buffer)
if err := convert(in, out, ver8to7); err != nil {
t.Fatal(err)
}

if !matchesExpected(t, out.Bytes(), expectedMigrateBackward) {
t.Fatal(fmt.Errorf("Converted does not match expected result\n%s\n%s\n", out.String(), expectedMigrateBackward))
}
}

var oldConfig = `{
"Some": {
"Other": "Config"
},
"Bootstrap": [
"/ip4/104.131.131.82/tcp/4001/ipfs/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ",
"/ip4/104.236.176.52/tcp/4001/ipfs/QmSoLnSGccFuZQJzRadHn95W2CrSFmZuTdDWP8HXaHca9z",
"/ip4/104.236.179.241/tcp/4001/ipfs/QmSoLPppuBtQSGwKDZT2M73ULpjvfd3aZ6ha4oFGL1KrGM",
"/ip4/162.243.248.213/tcp/4001/ipfs/QmSoLueR4xBeUbY9WZ9xGUUxunbKWcrNFTDAadQJmocnWm",
"/ip4/128.199.219.111/tcp/4001/ipfs/QmSoLSafTMBsPKadTEgaXctDQVcqN88CNLHXMkTNwMKPnu",
"/ip4/104.236.76.40/tcp/4001/ipfs/QmSoLV4Bbm51jM9C4gDYZQ9Cy3U6aXMJDAbzgu2fzaDs64",
"/ip4/178.62.158.247/tcp/4001/ipfs/QmSoLer265NRgSp2LA3dPaeykiS1J6DifTC88f5uVQKNAd",
"/ip4/178.62.61.185/tcp/4001/ipfs/QmSoLMeWqB7YGVLJN3pNLQpmmEk35v6wYtsMGLzSr5QBU3",
"/ip4/104.236.151.122/tcp/4001/ipfs/QmSoLju6m7xTh3DuokvT3886QRYqxAzb1kShaanJgW36yx",
"/ip6/2604:a880:1:20::1f9:9001/tcp/4001/ipfs/QmSoLnSGccFuZQJzRadHn95W2CrSFmZuTdDWP8HXaHca9z",
"/ip6/2604:a880:1:20::203:d001/tcp/4001/ipfs/QmSoLPppuBtQSGwKDZT2M73ULpjvfd3aZ6ha4oFGL1KrGM",
"/ip6/2604:a880:0:1010::23:d001/tcp/4001/ipfs/QmSoLueR4xBeUbY9WZ9xGUUxunbKWcrNFTDAadQJmocnWm",
"/ip6/2400:6180:0:d0::151:6001/tcp/4001/ipfs/QmSoLSafTMBsPKadTEgaXctDQVcqN88CNLHXMkTNwMKPnu",
"/ip6/2604:a880:800:10::4a:5001/tcp/4001/ipfs/QmSoLV4Bbm51jM9C4gDYZQ9Cy3U6aXMJDAbzgu2fzaDs64",
"/ip6/2a03:b0c0:0:1010::23:1001/tcp/4001/ipfs/QmSoLer265NRgSp2LA3dPaeykiS1J6DifTC88f5uVQKNAd",
"/ip6/2a03:b0c0:1:d0::e7:1/tcp/4001/ipfs/QmSoLMeWqB7YGVLJN3pNLQpmmEk35v6wYtsMGLzSr5QBU3",
"/ip6/2604:a880:1:20::1d9:6001/tcp/4001/ipfs/QmSoLju6m7xTh3DuokvT3886QRYqxAzb1kShaanJgW36yx"
]
}
`

var newConfig = `{
"Some": {
"Other": "Config"
},
"Bootstrap": [
"/dnsaddr/bootstrap.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN",
"/dnsaddr/bootstrap.libp2p.io/p2p/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa",
"/dnsaddr/bootstrap.libp2p.io/p2p/QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb",
"/dnsaddr/bootstrap.libp2p.io/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt",
"/ip4/104.131.131.82/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ",
"/ip4/104.131.131.83/tcp/4001/p2p/QmcafeMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLaaaa"
]
}
`

var expectedMigrateForward = []string{
"/dnsaddr/bootstrap.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN",
"/dnsaddr/bootstrap.libp2p.io/p2p/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa",
"/dnsaddr/bootstrap.libp2p.io/p2p/QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb",
"/dnsaddr/bootstrap.libp2p.io/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt",
"/ip4/104.131.131.82/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ",
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@dirkmc is it intentional that you are looking only for the .82 host here, but above ( line 96 ) there are 2 "old style" hosts?

Approving, as the rest looks good and the above is likely intentional

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good question - yes, in the migrate forward we move from oldConfig to expectedMigrateForward. In the migrate backwards we move from newConfig to expectedMigrateBackward for which I wanted a test that includes one peer that is "known" (in the list of peers with acceptably large keys) and one that is not in that list, to make sure both get through.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I updated the var names to make it a little clearer what's going on, and added some more explicit tests

}

var expectedMigrateBackward = []string{
"/ip4/104.131.131.83/tcp/4001/p2p/QmcafeMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLaaaa",
"/ip4/104.131.131.82/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ",
"/ip4/104.236.179.241/tcp/4001/p2p/QmSoLPppuBtQSGwKDZT2M73ULpjvfd3aZ6ha4oFGL1KrGM",
"/ip4/128.199.219.111/tcp/4001/p2p/QmSoLSafTMBsPKadTEgaXctDQVcqN88CNLHXMkTNwMKPnu",
"/ip4/104.236.76.40/tcp/4001/p2p/QmSoLV4Bbm51jM9C4gDYZQ9Cy3U6aXMJDAbzgu2fzaDs64",
"/ip4/178.62.158.247/tcp/4001/p2p/QmSoLer265NRgSp2LA3dPaeykiS1J6DifTC88f5uVQKNAd",
"/ip6/2604:a880:1:20::203:d001/tcp/4001/p2p/QmSoLPppuBtQSGwKDZT2M73ULpjvfd3aZ6ha4oFGL1KrGM",
"/ip6/2400:6180:0:d0::151:6001/tcp/4001/p2p/QmSoLSafTMBsPKadTEgaXctDQVcqN88CNLHXMkTNwMKPnu",
"/ip6/2604:a880:800:10::4a:5001/tcp/4001/p2p/QmSoLV4Bbm51jM9C4gDYZQ9Cy3U6aXMJDAbzgu2fzaDs64",
"/ip6/2a03:b0c0:0:1010::23:1001/tcp/4001/p2p/QmSoLer265NRgSp2LA3dPaeykiS1J6DifTC88f5uVQKNAd",
}
Loading