Merge pull request #10563 from ipfs/release-v0.32.0

Release v0.32.0

.github/workflows/docker-image.yml

@@ -19,6 +19,7 @@ on:
   push:
     branches:
       - 'master'
+      - 'staging'
       - 'bifrost-*'
     tags:
       - 'v*'

.github/workflows/gotest.yml

@@ -45,7 +45,7 @@ jobs:
           make -j "$PARALLEL" test/unit/gotest.junit.xml &&
             [[ ! $(jq -s -c 'map(select(.Action == "fail")) | .[]' test/unit/gotest.json) ]]
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@6d798873df2b1b8e5846dba6fb86631229fbcb17 # v4.4.0
+        uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4.6.0
         if: failure() || success()
         with:
           name: unittests

.github/workflows/sharness.yml

@@ -55,7 +55,7 @@ jobs:
           # increasing parallelism beyond 10 doesn't speed up the tests much
           PARALLEL: ${{ github.repository == 'ipfs/kubo' && 10 || 3 }}
       - name: Upload coverage report
-        uses: codecov/codecov-action@6d798873df2b1b8e5846dba6fb86631229fbcb17 # v4.4.0
+        uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4.6.0
         if: failure() || success()
         with:
           name: sharness

CHANGELOG.md

@@ -1,5 +1,6 @@
 # Kubo Changelogs
 
+- [v0.32](docs/changelogs/v0.32.md)
 - [v0.31](docs/changelogs/v0.31.md)
 - [v0.30](docs/changelogs/v0.30.md)
 - [v0.29](docs/changelogs/v0.29.md)

bin/get-docker-tags.sh

@@ -18,7 +18,7 @@ set -euo pipefail
 if [[ $# -lt 1 ]] ; then
   echo 'At least 1 arg required.'
   echo 'Usage:'
-  echo './push-docker-tags.sh <build number> [git commit sha1] [git branch name] [git tag name]'
+  echo './get-docker-tags.sh <build number> [git commit sha1] [git branch name] [git tag name]'
   exit 1
 fi
 
@@ -50,9 +50,9 @@ elif [[ $GIT_BRANCH =~ ^bifrost-.* ]]; then
   branch=$(echo "$GIT_BRANCH" | tr '/' '-' | tr --delete --complement '[:alnum:]-')
   echoImageName "${branch}-${BUILD_NUM}-${GIT_SHA1_SHORT}"
 
-elif [ "$GIT_BRANCH" = "master" ]; then
-  echoImageName "master-${BUILD_NUM}-${GIT_SHA1_SHORT}"
-  echoImageName "master-latest"
+elif [ "$GIT_BRANCH" = "master" ] || [ "$GIT_BRANCH" = "staging" ]; then
+  echoImageName "${GIT_BRANCH}-${BUILD_NUM}-${GIT_SHA1_SHORT}"
+  echoImageName "${GIT_BRANCH}-latest"
 
 else
   echo "Nothing to do. No docker tag defined for branch: $GIT_BRANCH, tag: $GIT_TAG"

bin/push-docker-tags.sh

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
-
+#
+# TODO: this script is legacy, use get-docker-tags.sh instead.
+#
 # push-docker-tags.sh
 #
 # Run from ci to tag images based on the current branch or tag name.
@@ -68,9 +70,9 @@ elif [[ $GIT_BRANCH =~ ^bifrost-.* ]]; then
   branch=$(echo "$GIT_BRANCH" | tr '/' '-' | tr --delete --complement '[:alnum:]-')
   pushTag "${branch}-${BUILD_NUM}-${GIT_SHA1_SHORT}"
 
-elif [ "$GIT_BRANCH" = "master" ]; then
-  pushTag "master-${BUILD_NUM}-${GIT_SHA1_SHORT}"
-  pushTag "master-latest"
+elif [ "$GIT_BRANCH" = "master" ] || [ "$GIT_BRANCH" = "staging" ]; then
+  pushTag "${GIT_BRANCH}-${BUILD_NUM}-${GIT_SHA1_SHORT}"
+  pushTag "${GIT_BRANCH}-latest"
 
 else
   echo "Nothing to do. No docker tag defined for branch: $GIT_BRANCH, tag: $GIT_TAG"

client/rpc/api.go

@@ -20,10 +20,10 @@ import (
 	ipfs "github.com/ipfs/kubo"
 	iface "github.com/ipfs/kubo/core/coreiface"
 	caopts "github.com/ipfs/kubo/core/coreiface/options"
+	"github.com/ipfs/kubo/misc/fsutil"
 	dagpb "github.com/ipld/go-codec-dagpb"
 	_ "github.com/ipld/go-ipld-prime/codec/dagcbor"
 	"github.com/ipld/go-ipld-prime/node/basicnode"
-	"github.com/mitchellh/go-homedir"
 	ma "github.com/multiformats/go-multiaddr"
 	manet "github.com/multiformats/go-multiaddr/net"
 )
@@ -82,7 +82,7 @@ func NewPathApi(ipfspath string) (*HttpApi, error) {
 
 // ApiAddr reads api file in specified ipfs path.
 func ApiAddr(ipfspath string) (ma.Multiaddr, error) {
-	baseDir, err := homedir.Expand(ipfspath)
+	baseDir, err := fsutil.ExpandHome(ipfspath)
 	if err != nil {
 		return nil, err
 	}

cmd/ipfswatch/main.go

@@ -16,12 +16,12 @@ import (
 	core "github.com/ipfs/kubo/core"
 	coreapi "github.com/ipfs/kubo/core/coreapi"
 	corehttp "github.com/ipfs/kubo/core/corehttp"
+	"github.com/ipfs/kubo/misc/fsutil"
 	fsrepo "github.com/ipfs/kubo/repo/fsrepo"
 
 	fsnotify "github.com/fsnotify/fsnotify"
 	"github.com/ipfs/boxo/files"
 	process "github.com/jbenet/goprocess"
-	homedir "github.com/mitchellh/go-homedir"
 )
 
 var (
@@ -57,7 +57,7 @@ func run(ipfsPath, watchPath string) error {
 	proc := process.WithParent(process.Background())
 	log.Printf("running IPFSWatch on '%s' using repo at '%s'...", watchPath, ipfsPath)
 
-	ipfsPath, err := homedir.Expand(ipfsPath)
+	ipfsPath, err := fsutil.ExpandHome(ipfsPath)
 	if err != nil {
 		return err
 	}

config/autotls.go

@@ -0,0 +1,30 @@
+package config
+
+import p2pforge "github.com/ipshipyard/p2p-forge/client"
+
+// AutoTLS includes optional configuration of p2p-forge client of service
+// for obtaining a domain and TLS certificate to improve connectivity for web
+// browser clients. More: https://github.com/ipshipyard/p2p-forge#readme
+type AutoTLS struct {
+	// Enables the p2p-forge feature
+	Enabled Flag `json:",omitempty"`
+
+	// Optional override of the parent domain that will be used
+	DomainSuffix *OptionalString `json:",omitempty"`
+
+	// Optional override of HTTP API that acts as ACME DNS-01 Challenge broker
+	RegistrationEndpoint *OptionalString `json:",omitempty"`
+
+	// Optional Authorization token, used with private/test instances of p2p-forge
+	RegistrationToken *OptionalString `json:",omitempty"`
+
+	// Optional override of CA ACME API used by p2p-forge system
+	CAEndpoint *OptionalString `json:",omitempty"`
+}
+
+const (
+	DefaultAutoTLSEnabled       = false // experimental, opt-in for now (https://github.com/ipfs/kubo/pull/10521)
+	DefaultDomainSuffix         = p2pforge.DefaultForgeDomain
+	DefaultRegistrationEndpoint = p2pforge.DefaultForgeEndpoint
+	DefaultCAEndpoint           = p2pforge.DefaultCAEndpoint
+)

config/config.go

@@ -9,7 +9,7 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/mitchellh/go-homedir"
+	"github.com/ipfs/kubo/misc/fsutil"
 )
 
 // Config is used to load ipfs config files.
@@ -26,6 +26,7 @@ type Config struct {
 	API       API       // local node's API settings
 	Swarm     SwarmConfig
 	AutoNAT   AutoNATConfig
+	AutoTLS   AutoTLS
 	Pubsub    PubsubConfig
 	Peering   Peering
 	DNS       DNS
@@ -58,7 +59,7 @@ func PathRoot() (string, error) {
 	dir := os.Getenv(EnvDir)
 	var err error
 	if len(dir) == 0 {
-		dir, err = homedir.Expand(DefaultPathRoot)
+		dir, err = fsutil.ExpandHome(DefaultPathRoot)
 	}
 	return dir, err
 }

config/swarm.go

@@ -65,8 +65,6 @@ type RelayService struct {
 	// BufferSize is the size of the relayed connection buffers.
 	BufferSize *OptionalInteger `json:",omitempty"`
 
-	// MaxReservationsPerPeer is the maximum number of reservations originating from the same peer.
-	MaxReservationsPerPeer *OptionalInteger `json:",omitempty"`
 	// MaxReservationsPerIP is the maximum number of reservations originating from the same IP address.
 	MaxReservationsPerIP *OptionalInteger `json:",omitempty"`
 	// MaxReservationsPerASN is the maximum number of reservations origination from the same ASN.

core/corehttp/webui.go

@@ -1,11 +1,13 @@
 package corehttp
 
 // WebUI version confirmed to work with this Kubo version
-const WebUIPath = "/ipfs/bafybeif6abowqcavbkz243biyh7pde7ick5kkwwytrh7pd2hkbtuqysjxy" // v4.3.2
+const WebUIPath = "/ipfs/bafybeibgic2ex3fvzkinhy6k6aqyv3zy2o7bkbsmrzvzka24xetv7eeadm" // v4.4.0
 
 // WebUIPaths is a list of all past webUI paths.
 var WebUIPaths = []string{
 	WebUIPath,
+	"/ipfs/bafybeid4uxz7klxcu3ffsnmn64r7ihvysamlj4ohl5h2orjsffuegcpaeq",
+	"/ipfs/bafybeif6abowqcavbkz243biyh7pde7ick5kkwwytrh7pd2hkbtuqysjxy",
 	"/ipfs/bafybeihatzsgposbr3hrngo42yckdyqcc56yean2rynnwpzxstvdlphxf4",
 	"/ipfs/bafybeigggyffcf6yfhx5irtwzx3cgnk6n3dwylkvcpckzhqqrigsxowjwe",
 	"/ipfs/bafybeidf7cpkwsjkq6xs3r6fbbxghbugilx3jtezbza7gua3k5wjixpmba",

core/node/groups.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"strings"
 	"time"
 
 	"github.com/dustin/go-humanize"
@@ -113,6 +114,7 @@ func LibP2P(bcfg *BuildCfg, cfg *config.Config, userResourceOverrides rcmgr.Part
 	enableRelayTransport := cfg.Swarm.Transports.Network.Relay.WithDefault(true) // nolint
 	enableRelayService := cfg.Swarm.RelayService.Enabled.WithDefault(enableRelayTransport)
 	enableRelayClient := cfg.Swarm.RelayClient.Enabled.WithDefault(enableRelayTransport)
+	enableAutoTLS := cfg.AutoTLS.Enabled.WithDefault(config.DefaultAutoTLSEnabled)
 
 	// Log error when relay subsystem could not be initialized due to missing dependency
 	if !enableRelayTransport {
@@ -123,6 +125,23 @@ func LibP2P(bcfg *BuildCfg, cfg *config.Config, userResourceOverrides rcmgr.Part
 			logger.Fatal("Failed to enable `Swarm.RelayClient`, it requires `Swarm.Transports.Network.Relay` to be true.")
 		}
 	}
+	if enableAutoTLS {
+		if !cfg.Swarm.Transports.Network.Websocket.WithDefault(true) {
+			logger.Fatal("Invalid configuration: AutoTLS.Enabled=true requires Swarm.Transports.Network.Websocket to be true as well.")
+		}
+
+		wssWildcard := fmt.Sprintf("/tls/sni/*.%s/ws", cfg.AutoTLS.DomainSuffix.WithDefault(config.DefaultDomainSuffix))
+		wssWildcardPresent := false
+		for _, listener := range cfg.Addresses.Swarm {
+			if strings.Contains(listener, wssWildcard) {
+				wssWildcardPresent = true
+				break
+			}
+		}
+		if !wssWildcardPresent {
+			logger.Fatal(fmt.Sprintf("Invalid configuration: AutoTLS.Enabled=true requires a catch-all Addresses.Swarm listener ending with %q to be present, see https://github.com/ipfs/kubo/blob/master/docs/config.md#autotls", wssWildcard))
+		}
+	}
 
 	// Gather all the options
 	opts := fx.Options(
@@ -133,6 +152,8 @@ func LibP2P(bcfg *BuildCfg, cfg *config.Config, userResourceOverrides rcmgr.Part
 
 		// Services (resource management)
 		fx.Provide(libp2p.ResourceManager(bcfg.Repo.Path(), cfg.Swarm, userResourceOverrides)),
+		maybeProvide(libp2p.P2PForgeCertMgr(bcfg.Repo.Path(), cfg.AutoTLS), enableAutoTLS),
+		maybeInvoke(libp2p.StartP2PAutoTLS, enableAutoTLS),
 		fx.Provide(libp2p.AddrFilters(cfg.Swarm.AddrFilters)),
 		fx.Provide(libp2p.AddrsFactory(cfg.Addresses.Announce, cfg.Addresses.AppendAnnounce, cfg.Addresses.NoAnnounce)),
 		fx.Provide(libp2p.SmuxTransport(cfg.Swarm.Transports)),

core/node/libp2p/addrs.go

@@ -1,12 +1,23 @@
 package libp2p
 
 import (
+	"context"
 	"fmt"
+	"os"
+	"path/filepath"
 
+	logging "github.com/ipfs/go-log"
+	version "github.com/ipfs/kubo"
+	"github.com/ipfs/kubo/config"
+	p2pforge "github.com/ipshipyard/p2p-forge/client"
 	"github.com/libp2p/go-libp2p"
+	"github.com/libp2p/go-libp2p/core/host"
 	p2pbhost "github.com/libp2p/go-libp2p/p2p/host/basic"
 	ma "github.com/multiformats/go-multiaddr"
 	mamask "github.com/whyrusleeping/multiaddr-filter"
+
+	"github.com/caddyserver/certmagic"
+	"go.uber.org/fx"
 )
 
 func AddrFilters(filters []string) func() (*ma.Filters, Libp2pOpts, error) {
@@ -87,12 +98,26 @@ func makeAddrsFactory(announce []string, appendAnnouce []string, noAnnounce []st
 	}, nil
 }
 
-func AddrsFactory(announce []string, appendAnnouce []string, noAnnounce []string) func() (opts Libp2pOpts, err error) {
-	return func() (opts Libp2pOpts, err error) {
-		addrsFactory, err := makeAddrsFactory(announce, appendAnnouce, noAnnounce)
+func AddrsFactory(announce []string, appendAnnouce []string, noAnnounce []string) interface{} {
+	return func(params struct {
+		fx.In
+		ForgeMgr *p2pforge.P2PForgeCertMgr `optional:"true"`
+	},
+	) (opts Libp2pOpts, err error) {
+		var addrsFactory p2pbhost.AddrsFactory
+		announceAddrsFactory, err := makeAddrsFactory(announce, appendAnnouce, noAnnounce)
 		if err != nil {
 			return opts, err
 		}
+		if params.ForgeMgr == nil {
+			addrsFactory = announceAddrsFactory
+		} else {
+			addrsFactory = func(multiaddrs []ma.Multiaddr) []ma.Multiaddr {
+				forgeProcessing := params.ForgeMgr.AddressFactory()(multiaddrs)
+				annouceProcessing := announceAddrsFactory(forgeProcessing)
+				return annouceProcessing
+			}
+		}
 		opts.Opts = append(opts.Opts, libp2p.AddrsFactory(addrsFactory))
 		return
 	}
@@ -107,3 +132,46 @@ func ListenOn(addresses []string) interface{} {
 		}
 	}
 }
+
+func P2PForgeCertMgr(repoPath string, cfg config.AutoTLS) interface{} {
+	return func() (*p2pforge.P2PForgeCertMgr, error) {
+		storagePath := filepath.Join(repoPath, "p2p-forge-certs")
+
+		forgeLogger := logging.Logger("autotls").Desugar()
+
+		// TODO: this should not be necessary, but we do it to help tracking
+		// down any race conditions causing
+		// https://github.com/ipshipyard/p2p-forge/issues/8
+		certmagic.Default.Logger = forgeLogger.Named("default_fixme")
+		certmagic.DefaultACME.Logger = forgeLogger.Named("default_acme_client_fixme")
+
+		certStorage := &certmagic.FileStorage{Path: storagePath}
+		certMgr, err := p2pforge.NewP2PForgeCertMgr(
+			p2pforge.WithLogger(forgeLogger.Sugar()),
+			p2pforge.WithForgeDomain(cfg.DomainSuffix.WithDefault(config.DefaultDomainSuffix)),
+			p2pforge.WithForgeRegistrationEndpoint(cfg.RegistrationEndpoint.WithDefault(config.DefaultRegistrationEndpoint)),
+			p2pforge.WithCAEndpoint(cfg.CAEndpoint.WithDefault(config.DefaultCAEndpoint)),
+			p2pforge.WithForgeAuth(cfg.RegistrationToken.WithDefault(os.Getenv(p2pforge.ForgeAuthEnv))),
+			p2pforge.WithUserAgent(version.GetUserAgentVersion()),
+			p2pforge.WithCertificateStorage(certStorage),
+		)
+		if err != nil {
+			return nil, err
+		}
+
+		return certMgr, nil
+	}
+}
+
+func StartP2PAutoTLS(lc fx.Lifecycle, certMgr *p2pforge.P2PForgeCertMgr, h host.Host) {
+	lc.Append(fx.Hook{
+		OnStart: func(ctx context.Context) error {
+			certMgr.ProvideHost(h)
+			return certMgr.Start()
+		},
+		OnStop: func(ctx context.Context) error {
+			certMgr.Stop()
+			return nil
+		},
+	})
+}

core/node/libp2p/dns.go

@@ -2,10 +2,11 @@ package libp2p
 
 import (
 	"github.com/libp2p/go-libp2p"
+	"github.com/libp2p/go-libp2p/p2p/net/swarm"
 	madns "github.com/multiformats/go-multiaddr-dns"
 )
 
 func MultiaddrResolver(rslv *madns.Resolver) (opts Libp2pOpts, err error) {
-	opts.Opts = append(opts.Opts, libp2p.MultiaddrResolver(rslv))
+	opts.Opts = append(opts.Opts, libp2p.MultiaddrResolver(swarm.ResolverFromMaDNS{Resolver: rslv}))
 	return opts, nil
 }

core/node/libp2p/relay.go

@@ -33,13 +33,12 @@ func RelayService(enable bool, relayOpts config.RelayService) func() (opts Libp2
 					Data:     relayOpts.ConnectionDataLimit.WithDefault(def.Limit.Data),
 					Duration: relayOpts.ConnectionDurationLimit.WithDefault(def.Limit.Duration),
 				},
-				MaxCircuits:            int(relayOpts.MaxCircuits.WithDefault(int64(def.MaxCircuits))),
-				BufferSize:             int(relayOpts.BufferSize.WithDefault(int64(def.BufferSize))),
-				ReservationTTL:         relayOpts.ReservationTTL.WithDefault(def.ReservationTTL),
-				MaxReservations:        int(relayOpts.MaxReservations.WithDefault(int64(def.MaxReservations))),
-				MaxReservationsPerIP:   int(relayOpts.MaxReservationsPerIP.WithDefault(int64(def.MaxReservationsPerIP))),
-				MaxReservationsPerPeer: int(relayOpts.MaxReservationsPerPeer.WithDefault(int64(def.MaxReservationsPerPeer))),
-				MaxReservationsPerASN:  int(relayOpts.MaxReservationsPerASN.WithDefault(int64(def.MaxReservationsPerASN))),
+				MaxCircuits:           int(relayOpts.MaxCircuits.WithDefault(int64(def.MaxCircuits))),
+				BufferSize:            int(relayOpts.BufferSize.WithDefault(int64(def.BufferSize))),
+				ReservationTTL:        relayOpts.ReservationTTL.WithDefault(def.ReservationTTL),
+				MaxReservations:       int(relayOpts.MaxReservations.WithDefault(int64(def.MaxReservations))),
+				MaxReservationsPerIP:  int(relayOpts.MaxReservationsPerIP.WithDefault(int64(def.MaxReservationsPerIP))),
+				MaxReservationsPerASN: int(relayOpts.MaxReservationsPerASN.WithDefault(int64(def.MaxReservationsPerASN))),
 			})))
 		}
 		return