core/node: prioritize announcing pin roots

core/node/provider.go

@@ -129,11 +129,13 @@ func OnlineProviders(useStrategicProviding bool, reprovideStrategy string, repro
 	var keyProvider fx.Option
 	switch reprovideStrategy {
 	case "all", "":
-		keyProvider = fx.Provide(provider.NewBlockstoreProvider)
+		keyProvider = fx.Provide(newProvidingStrategy(false, false))
 	case "roots":
-		keyProvider = fx.Provide(pinnedProviderStrategy(true))
+		keyProvider = fx.Provide(newProvidingStrategy(true, true))
 	case "pinned":
-		keyProvider = fx.Provide(pinnedProviderStrategy(false))
+		keyProvider = fx.Provide(newProvidingStrategy(true, false))
+	case "flat":
+		keyProvider = fx.Provide(provider.NewBlockstoreProvider)
 	default:
 		return fx.Error(fmt.Errorf("unknown reprovider strategy %q", reprovideStrategy))
 	}
@@ -149,13 +151,25 @@ func OfflineProviders() fx.Option {
 	return fx.Provide(provider.NewNoopProvider)
 }
 
-func pinnedProviderStrategy(onlyRoots bool) interface{} {
+func newProvidingStrategy(onlyPinned, onlyRoots bool) interface{} {
 	type input struct {
 		fx.In
 		Pinner      pin.Pinner
+		Blockstore  blockstore.Blockstore
 		IPLDFetcher fetcher.Factory `name:"ipldFetcher"`
 	}
 	return func(in input) provider.KeyChanFunc {
-		return provider.NewPinnedProvider(onlyRoots, in.Pinner, in.IPLDFetcher)
+		if onlyRoots {
+			return provider.NewPinnedProvider(true, in.Pinner, in.IPLDFetcher)
+		}
+
+		if onlyPinned {
+			return provider.NewPinnedProvider(false, in.Pinner, in.IPLDFetcher)
+		}
+
+		return provider.NewPrioritizedProvider(
+			provider.NewPinnedProvider(true, in.Pinner, in.IPLDFetcher),
+			provider.NewBlockstoreProvider(in.Blockstore),
+		)
 	}
 }

docs/changelogs/v0.28.md

@@ -10,6 +10,7 @@
   - [Gateway: `/api/v0` is removed](#gateway-apiv0-is-removed)
   - [Removed deprecated Object API commands](#removed-deprecated-object-api-commands)
   - [No longer publishes loopback and private addresses on DHT](#no-longer-publishes-loopback-and-private-addresses-on-dht)
+  - [Pin roots are now prioritized when announcing](#pin-roots-are-now-prioritized-when-announcing)
 - [📝 Changelog](#-changelog)
 - [👨‍👩‍👧‍👦 Contributors](#-contributors)
 
@@ -35,6 +36,10 @@ Kubo no longer keeps track of loopback and private addresses on the LAN and WAN
 
 To support testing scenarios where multiple Kubo instances run on the same machine, [`Routing.LoopbackAddressesOnLanDHT`](https://github.com/ipfs/kubo/blob/master/docs/config.md#routingloopbackaddressesonlandht) is set to `true` when the `test` profile is applied.
 
+#### Pin roots are now prioritized when announcing
+
+The root CIDs of pinned content are now prioritized when announcing to the Amino DHT with [`Reprovider.Strategy`](https://github.com/ipfs/kubo/blob/master/docs/config.md#reproviderstrategy) set to `all` (default) or `pinned`, making the important CIDs accessible faster.
+
 ### 📝 Changelog
 
 ### 👨‍👩‍👧‍👦 Contributors

docs/config.md

@@ -1503,7 +1503,9 @@ Type: `optionalDuration` (unset for the default)
 Tells reprovider what should be announced. Valid strategies are:
 
 - `"all"` - announce all CIDs of stored blocks
+  - Order: root blocks of direct and recursive pins are announced first, then the rest of blockstore
 - `"pinned"` - only announce pinned CIDs recursively (both roots and child blocks)
+  - Order: root blocks of direct and recursive pins are announced first, then the child blocks of recursive pins
 - `"roots"` - only announce the root block of explicitly pinned CIDs
   - **⚠️  BE CAREFUL:** node with `roots` strategy will not announce child blocks.
     It makes sense only for use cases where the entire DAG is fetched in full,
@@ -1512,6 +1514,7 @@ Tells reprovider what should be announced. Valid strategies are:
     providers for the missing block in the middle of a file, unless the peer
     happens to already be connected to a provider and ask for child CID over
     bitswap.
+- `"flat"` - same as `all`, announce all CIDs of stored blocks, but without prioritizing anything
 
 Default: `"all"`
 

docs/examples/kubo-as-a-library/go.mod

@@ -9,7 +9,7 @@ toolchain go1.22.0
 replace github.com/ipfs/kubo => ./../../..
 
 require (
-	github.com/ipfs/boxo v0.18.0
+	github.com/ipfs/boxo v0.18.1-0.20240409062800-ec207931045d
 	github.com/ipfs/kubo v0.0.0-00010101000000-000000000000
 	github.com/libp2p/go-libp2p v0.33.2
 	github.com/multiformats/go-multiaddr v0.12.3

docs/examples/kubo-as-a-library/go.sum

@@ -266,8 +266,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c h1:7Uy
 github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c/go.mod h1:6EekK/jo+TynwSE/ZOiOJd4eEvRXoavEC3vquKtv4yI=
 github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
 github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
-github.com/ipfs/boxo v0.18.0 h1:MOL9/AgoV3e7jlVMInicaSdbgralfqSsbkc31dZ9tmw=
-github.com/ipfs/boxo v0.18.0/go.mod h1:pIZgTWdm3k3pLF9Uq6MB8JEcW07UDwNJjlXW1HELW80=
+github.com/ipfs/boxo v0.18.1-0.20240409062800-ec207931045d h1:4y8xHp4ZDUgnwXK3a146K/sEYq6BSO/nA46DOLMVp5k=
+github.com/ipfs/boxo v0.18.1-0.20240409062800-ec207931045d/go.mod h1:V5gJzbIMwKEXrg3IdvAxIdF7UPgU4RsXmNGS8MQ/0D4=
 github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
 github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
 github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk=

go.mod

@@ -17,7 +17,7 @@ require (
 	github.com/hashicorp/go-multierror v1.1.1
 	github.com/ipfs-shipyard/nopfs v0.0.12
 	github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c
-	github.com/ipfs/boxo v0.18.0
+	github.com/ipfs/boxo v0.18.1-0.20240409062800-ec207931045d
 	github.com/ipfs/go-block-format v0.2.0
 	github.com/ipfs/go-cid v0.4.1
 	github.com/ipfs/go-cidutil v0.1.0

go.sum

@@ -329,8 +329,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c h1:7Uy
 github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c/go.mod h1:6EekK/jo+TynwSE/ZOiOJd4eEvRXoavEC3vquKtv4yI=
 github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
 github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
-github.com/ipfs/boxo v0.18.0 h1:MOL9/AgoV3e7jlVMInicaSdbgralfqSsbkc31dZ9tmw=
-github.com/ipfs/boxo v0.18.0/go.mod h1:pIZgTWdm3k3pLF9Uq6MB8JEcW07UDwNJjlXW1HELW80=
+github.com/ipfs/boxo v0.18.1-0.20240409062800-ec207931045d h1:4y8xHp4ZDUgnwXK3a146K/sEYq6BSO/nA46DOLMVp5k=
+github.com/ipfs/boxo v0.18.1-0.20240409062800-ec207931045d/go.mod h1:V5gJzbIMwKEXrg3IdvAxIdF7UPgU4RsXmNGS8MQ/0D4=
 github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
 github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
 github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ=

test/cli/provider_test.go

@@ -0,0 +1,165 @@
+package cli
+
+import (
+	"bytes"
+	"testing"
+	"time"
+
+	"github.com/ipfs/kubo/test/cli/harness"
+	"github.com/ipfs/kubo/test/cli/testutils"
+	"github.com/stretchr/testify/require"
+)
+
+func TestProvider(t *testing.T) {
+	t.Parallel()
+
+	initNodes := func(t *testing.T, n int, fn func(n *harness.Node)) harness.Nodes {
+		nodes := harness.NewT(t).NewNodes(n).Init()
+		nodes.ForEachPar(fn)
+		return nodes.StartDaemons().Connect()
+	}
+
+	expectNoProviders := func(t *testing.T, cid string, nodes ...*harness.Node) {
+		for _, node := range nodes {
+			res := node.IPFS("routing", "findprovs", "-n=1", cid)
+			require.Empty(t, res.Stdout.String())
+		}
+	}
+
+	expectProviders := func(t *testing.T, cid, expectedProvider string, nodes ...*harness.Node) {
+		for _, node := range nodes {
+			res := node.IPFS("routing", "findprovs", "-n=1", cid)
+			require.Equal(t, expectedProvider, res.Stdout.Trimmed())
+		}
+	}
+
+	t.Run("Basic Providing", func(t *testing.T) {
+		t.Parallel()
+
+		nodes := initNodes(t, 2, func(n *harness.Node) {
+			n.SetIPFSConfig("Experimental.StrategicProviding", false)
+		})
+		defer nodes.StopDaemons()
+
+		cid := nodes[0].IPFSAddStr(time.Now().String())
+		expectProviders(t, cid, nodes[0].PeerID().String(), nodes[1:]...)
+	})
+
+	t.Run("Basic Strategic Providing", func(t *testing.T) {
+		t.Parallel()
+
+		nodes := initNodes(t, 2, func(n *harness.Node) {
+			n.SetIPFSConfig("Experimental.StrategicProviding", true)
+		})
+		defer nodes.StopDaemons()
+
+		cid := nodes[0].IPFSAddStr(time.Now().String())
+		expectNoProviders(t, cid, nodes[1:]...)
+	})
+
+	t.Run("Reprovides with 'all' strategy", func(t *testing.T) {
+		t.Parallel()
+
+		nodes := initNodes(t, 2, func(n *harness.Node) {
+			n.SetIPFSConfig("Reprovider.Strategy", "all")
+		})
+		defer nodes.StopDaemons()
+
+		cid := nodes[0].IPFSAddStr(time.Now().String(), "--local")
+
+		expectNoProviders(t, cid, nodes[1:]...)
+
+		nodes[0].IPFS("bitswap", "reprovide")
+
+		expectProviders(t, cid, nodes[0].PeerID().String(), nodes[1:]...)
+	})
+
+	t.Run("Reprovides with 'flat' strategy", func(t *testing.T) {
+		t.Parallel()
+
+		nodes := initNodes(t, 2, func(n *harness.Node) {
+			n.SetIPFSConfig("Reprovider.Strategy", "flat")
+		})
+		defer nodes.StopDaemons()
+
+		cid := nodes[0].IPFSAddStr(time.Now().String(), "--local")
+
+		expectNoProviders(t, cid, nodes[1:]...)
+
+		nodes[0].IPFS("bitswap", "reprovide")
+
+		expectProviders(t, cid, nodes[0].PeerID().String(), nodes[1:]...)
+	})
+
+	t.Run("Reprovides with 'pinned' strategy", func(t *testing.T) {
+		t.Parallel()
+
+		foo := testutils.RandomBytes(1000)
+		bar := testutils.RandomBytes(1000)
+
+		nodes := initNodes(t, 2, func(n *harness.Node) {
+			n.SetIPFSConfig("Reprovider.Strategy", "pinned")
+		})
+		defer nodes.StopDaemons()
+
+		cidFoo := nodes[0].IPFSAdd(bytes.NewReader(foo), "--offline", "--pin=false")
+		cidBar := nodes[0].IPFSAdd(bytes.NewReader(bar), "--offline", "--pin=false")
+		cidBarDir := nodes[0].IPFSAdd(bytes.NewReader(bar), "-Q", "--offline", "-w")
+
+		expectNoProviders(t, cidFoo, nodes[1:]...)
+		expectNoProviders(t, cidBar, nodes[1:]...)
+		expectNoProviders(t, cidBarDir, nodes[1:]...)
+
+		nodes[0].IPFS("bitswap", "reprovide")
+
+		expectNoProviders(t, cidFoo, nodes[1:]...)
+		expectProviders(t, cidBar, nodes[0].PeerID().String(), nodes[1:]...)
+		expectProviders(t, cidBarDir, nodes[0].PeerID().String(), nodes[1:]...)
+	})
+
+	t.Run("Reprovides with 'roots' strategy", func(t *testing.T) {
+		t.Parallel()
+
+		foo := testutils.RandomBytes(1000)
+		bar := testutils.RandomBytes(1000)
+		baz := testutils.RandomBytes(1000)
+
+		nodes := initNodes(t, 2, func(n *harness.Node) {
+			n.SetIPFSConfig("Reprovider.Strategy", "roots")
+		})
+		defer nodes.StopDaemons()
+
+		cidFoo := nodes[0].IPFSAdd(bytes.NewReader(foo), "--offline", "--pin=false")
+		cidBar := nodes[0].IPFSAdd(bytes.NewReader(bar), "--offline", "--pin=false")
+		cidBaz := nodes[0].IPFSAdd(bytes.NewReader(baz), "--offline")
+		cidBarDir := nodes[0].IPFSAdd(bytes.NewReader(bar), "-Q", "--offline", "-w")
+
+		expectNoProviders(t, cidFoo, nodes[1:]...)
+		expectNoProviders(t, cidBar, nodes[1:]...)
+		expectNoProviders(t, cidBarDir, nodes[1:]...)
+
+		nodes[0].IPFS("bitswap", "reprovide")
+
+		expectNoProviders(t, cidFoo, nodes[1:]...)
+		expectNoProviders(t, cidBar, nodes[1:]...)
+		expectProviders(t, cidBaz, nodes[0].PeerID().String(), nodes[1:]...)
+		expectProviders(t, cidBarDir, nodes[0].PeerID().String(), nodes[1:]...)
+	})
+
+	t.Run("Providing works without ticking", func(t *testing.T) {
+		t.Parallel()
+
+		nodes := initNodes(t, 2, func(n *harness.Node) {
+			n.SetIPFSConfig("Reprovider.Interval", "0")
+		})
+		defer nodes.StopDaemons()
+
+		cid := nodes[0].IPFSAddStr(time.Now().String(), "--offline")
+
+		expectNoProviders(t, cid, nodes[1:]...)
+
+		nodes[0].IPFS("bitswap", "reprovide")
+
+		expectProviders(t, cid, nodes[0].PeerID().String(), nodes[1:]...)
+	})
+}

test/dependencies/go.mod

@@ -105,7 +105,7 @@ require (
 	github.com/hexops/gotextdiff v1.0.3 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/ipfs/bbloom v0.0.4 // indirect
-	github.com/ipfs/boxo v0.18.0 // indirect
+	github.com/ipfs/boxo v0.18.1-0.20240409062800-ec207931045d // indirect
 	github.com/ipfs/go-block-format v0.2.0 // indirect
 	github.com/ipfs/go-cid v0.4.1 // indirect
 	github.com/ipfs/go-datastore v0.6.0 // indirect

test/dependencies/go.sum

@@ -362,8 +362,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
 github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
-github.com/ipfs/boxo v0.18.0 h1:MOL9/AgoV3e7jlVMInicaSdbgralfqSsbkc31dZ9tmw=
-github.com/ipfs/boxo v0.18.0/go.mod h1:pIZgTWdm3k3pLF9Uq6MB8JEcW07UDwNJjlXW1HELW80=
+github.com/ipfs/boxo v0.18.1-0.20240409062800-ec207931045d h1:4y8xHp4ZDUgnwXK3a146K/sEYq6BSO/nA46DOLMVp5k=
+github.com/ipfs/boxo v0.18.1-0.20240409062800-ec207931045d/go.mod h1:V5gJzbIMwKEXrg3IdvAxIdF7UPgU4RsXmNGS8MQ/0D4=
 github.com/ipfs/go-block-format v0.2.0 h1:ZqrkxBA2ICbDRbK8KJs/u0O3dlp6gmAuuXUJNiW1Ycs=
 github.com/ipfs/go-block-format v0.2.0/go.mod h1:+jpL11nFx5A/SPpsoBn6Bzkra/zaArfSmsknbPMYgzM=
 github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s=