feat: migrate mfs root to datastore (#126)

We've been storing the mfs root in the repo's 'root' datastore since
forever which means it's a block on the filesystem.

It should have been stored in the datastore instead so add migration 11
which moves the key to the datastore.

BREAKING CHANGE: adds a new migration, should go out as a major

migrations/index.js

@@ -22,5 +22,6 @@ module.exports = [
   Object.assign({version: 7}, emptyMigration),
   require('./migration-8'),
   require('./migration-9'),
-  require('./migration-10')
+  require('./migration-10'),
+  require('./migration-11')
 ]

migrations/migration-10/index.js

@@ -3,8 +3,8 @@
 const {
   findLevelJs
 } = require('../../src/utils')
-const fromString = require('uint8arrays/from-string')
-const toString = require('uint8arrays/to-string')
+const { fromString } = require('uint8arrays/from-string')
+const { toString } = require('uint8arrays/to-string')
 
 /**
  * @typedef {import('../../src/types').Migration} Migration

migrations/migration-11/index.js

@@ -0,0 +1,53 @@
+'use strict'
+
+const { Key } = require('interface-datastore')
+
+const MFS_ROOT_KEY = new Key('/local/filesroot')
+
+/**
+ * @param {import('../../src/types').Backends} backends
+ * @param {import('../../src/types').MigrationProgressCallback} onProgress
+ */
+async function storeMfsRootInDatastore (backends, onProgress = () => {}) {
+  onProgress(100, 'Migrating MFS root to repo datastore')
+
+  await backends.root.open()
+  await backends.datastore.open()
+
+  const root = await backends.root.get(MFS_ROOT_KEY)
+  await backends.datastore.put(MFS_ROOT_KEY, root)
+  await backends.root.delete(MFS_ROOT_KEY)
+
+  await backends.datastore.close()
+  await backends.root.close()
+
+  onProgress(100, 'Stored MFS root in repo datastore')
+}
+
+/**
+ * @param {import('../../src/types').Backends} backends
+ * @param {import('../../src/types').MigrationProgressCallback} onProgress
+ */
+ async function storeMfsRootInRoot (backends, onProgress = () => {}) {
+  onProgress(100, 'Migrating MFS root to repo root datastore')
+
+  await backends.root.open()
+  await backends.datastore.open()
+
+  const root = await backends.datastore.get(MFS_ROOT_KEY)
+  await backends.root.put(MFS_ROOT_KEY, root)
+  await backends.datastore.delete(MFS_ROOT_KEY)
+
+  await backends.datastore.close()
+  await backends.root.close()
+
+  onProgress(100, 'Stored MFS root in repo root datastore')
+}
+
+/** @type {import('../../src/types').Migration} */
+module.exports = {
+  version: 11,
+  description: 'Store mfs root in the datastore',
+  migrate: storeMfsRootInDatastore,
+  revert: storeMfsRootInRoot
+}

migrations/migration-9/pin-set.js

@@ -14,10 +14,10 @@ const fnv1a = require('fnv1a')
 const varint = require('varint')
 const dagPb = require('@ipld/dag-pb')
 const { DEFAULT_FANOUT, MAX_ITEMS, EMPTY_KEY } = require('./utils')
-const uint8ArrayConcat = require('uint8arrays/concat')
-const uint8ArrayCompare = require('uint8arrays/compare')
-const uint8ArrayToString = require('uint8arrays/to-string')
-const uint8ArrayFromString = require('uint8arrays/from-string')
+const { concat: uint8ArrayConcat } = require('uint8arrays/concat')
+const { compare: uint8ArrayCompare } = require('uint8arrays/compare')
+const { toString: uint8ArrayToString } = require('uint8arrays/to-string')
+const { fromString: uint8ArrayFromString } = require('uint8arrays/from-string')
 const { sha256 } = require('multiformats/hashes/sha2')
 
 /**

package.json

@@ -56,14 +56,14 @@
     "multiformats": "^9.0.0",
     "proper-lockfile": "^4.1.1",
     "protobufjs": "^6.10.2",
-    "uint8arrays": "^2.0.5",
+    "uint8arrays": "^3.0.0",
     "varint": "^6.0.0"
   },
   "devDependencies": {
     "@ipld/car": "^3.0.0",
     "@types/debug": "^4.1.5",
     "@types/varint": "^6.0.0",
-    "aegir": "^34.1.0",
+    "aegir": "^35.0.1",
     "assert": "^2.0.0",
     "aws-sdk": "^2.884.0",
     "blockstore-datastore-adapter": "1.0.0",

src/repo/version.js

@@ -3,8 +3,8 @@
 const repoInit = require('./init')
 const { MissingRepoOptionsError, NotInitializedRepoError } = require('../errors')
 const { VERSION_KEY } = require('../utils')
-const uint8ArrayFromString = require('uint8arrays/from-string')
-const uint8ArrayToString = require('uint8arrays/to-string')
+const { fromString: uint8ArrayFromString } = require('uint8arrays/from-string')
+const { toString: uint8ArrayToString } = require('uint8arrays/to-string')
 
 /**
  * Function that has responsibility to retrieve version of repo from its root datastore's instance.

test/init-test.js

@@ -4,7 +4,7 @@
 const { expect } = require('aegir/utils/chai')
 const { CONFIG_KEY, VERSION_KEY } = require('../src/utils')
 const repoInit = require('../src/repo/init')
-const uint8ArrayFromString = require('uint8arrays/from-string')
+const { fromString: uint8ArrayFromString } = require('uint8arrays/from-string')
 
 module.exports = (setup, cleanup) => {
   let dir

test/migrations/index.js

@@ -4,4 +4,5 @@ module.exports = (setup, cleanup) => {
   require('./migration-8-test')(setup, cleanup)
   require('./migration-9-test')(setup, cleanup)
   require('./migration-10-test')(setup, cleanup)
+  require('./migration-11-test')(setup, cleanup)
 }

test/migrations/migration-10-test.js

@@ -8,8 +8,8 @@ const { BlockstoreAdapter } = require('interface-blockstore')
 
 const migration = require('../../migrations/migration-10')
 const Key = require('interface-datastore').Key
-const fromString = require('uint8arrays/from-string')
-const equals = require('uint8arrays/equals')
+const { fromString } = require('uint8arrays/from-string')
+const { equals } = require('uint8arrays/equals')
 const Level5 = require('level-5')
 const Level6 = require('level-6')
 

test/migrations/migration-11-test.js

@@ -0,0 +1,69 @@
+/* eslint-env mocha */
+/* eslint-disable max-nested-callbacks */
+'use strict'
+
+const { expect } = require('aegir/utils/chai')
+const { CID } = require('multiformats/cid')
+const migration = require('../../migrations/migration-11')
+const { Key } = require('interface-datastore')
+
+const MFS_ROOT_KEY = new Key('/local/filesroot')
+const MFS_ROOT = CID.parse('Qmc42sn2WBHYeAShU3nx8mYkhKVq4sRLapawTaGh4XH4iE')
+
+module.exports = (setup, cleanup) => {
+  describe('migration 11', function () {
+    this.timeout(240 * 1000)
+    let dir
+    let backends
+
+    beforeEach(async () => {
+      ({ dir, backends } = await setup())
+    })
+
+    afterEach(async () => {
+      await cleanup(dir)
+    })
+
+    describe('forwards', () => {
+      beforeEach(async () => {
+        await backends.root.open()
+        await backends.root.put(MFS_ROOT_KEY, MFS_ROOT.bytes)
+        await backends.root.close()
+      })
+
+      it('should migrate MFS root forward', async () => {
+        await migration.migrate(backends, () => {})
+
+        await backends.root.open()
+        await backends.datastore.open()
+
+        await expect(backends.root.has(MFS_ROOT_KEY)).to.eventually.be.false()
+        await expect(backends.datastore.has(MFS_ROOT_KEY)).to.eventually.be.true()
+
+        await backends.datastore.close()
+        await backends.root.close()
+      })
+    })
+
+    describe('backwards', () => {
+      beforeEach(async () => {
+        await backends.datastore.open()
+        await backends.datastore.put(MFS_ROOT_KEY, MFS_ROOT.bytes)
+        await backends.datastore.close()
+      })
+
+      it('should migrate MFS root backward', async () => {
+        await migration.revert(backends, () => {})
+
+        await backends.root.open()
+        await backends.datastore.open()
+
+        await expect(backends.root.has(MFS_ROOT_KEY)).to.eventually.be.true()
+        await expect(backends.datastore.has(MFS_ROOT_KEY)).to.eventually.be.false()
+
+        await backends.datastore.close()
+        await backends.root.close()
+      })
+    })
+  })
+}

test/node.js

@@ -34,7 +34,7 @@ const CONFIGURATIONS = [{
           new NextToLast(2))
       ),
       datastore: new DatastoreLevel(`${prefix}/datastore`),
-      keys: new DatastoreLevel(`${prefix}/keys`),
+      keys: new DatastoreFS(`${prefix}/keys`),
       pins: new DatastoreLevel(`${prefix}/pins`)
     }
   }
@@ -50,7 +50,7 @@ const CONFIGURATIONS = [{
         })
       ),
       datastore: new DatastoreLevel(`${prefix}/datastore`),
-      keys: new DatastoreLevel(`${prefix}/keys`),
+      keys: new DatastoreFS(`${prefix}/keys`),
       pins: new DatastoreLevel(`${prefix}/pins`)
     }
   }

test/test-migrations/migration-2/index.js

@@ -2,7 +2,7 @@
 
 const Key = require('interface-datastore').Key
 const _set = require('just-safe-set')
-const uint8ArrayFromString = require('uint8arrays/from-string')
+const { fromString: uint8ArrayFromString } = require('uint8arrays/from-string')
 
 const CONFIG_KEY = new Key('config')
 const NEW_API_ADDRESS = '/ip6/::/tcp/5001'

test/version-test.js

@@ -4,7 +4,7 @@
 const { expect } = require('aegir/utils/chai')
 const { VERSION_KEY, CONFIG_KEY } = require('../src/utils')
 const version = require('../src/repo/version')
-const uint8ArrayFromString = require('uint8arrays/from-string')
+const { fromString: uint8ArrayFromString } = require('uint8arrays/from-string')
 const errors = require('../src/errors')
 
 // When new versioning mechanism is introduced in new version don't forget to update