feat: add pinned storage to user account API (#1044)

packages/api/test/fixtures/init-data.sql

@@ -53,14 +53,17 @@ VALUES (1, 'bafkreigpimx5kl6thyfysh2witvbo5nexvu3q3uc3y65rj5sr5czcc7wae', 'bafkr
 
 -- /status route data testing
 INSERT INTO content (cid, dag_size, inserted_at)
-VALUES ('bafybeifnfkzjeohjf2dch2iqqpef3bfjylwxlcjws2msvdfyze5bvdprfm', 101, '2021-07-14T19:27:14.934572Z'),
-       ('bafybeica6klnrhlrbx6z24icefykpbwyypouglnypvnwb5esdm6yzcie3q', 101, '2021-07-14T19:27:14.934572Z'),
-       ('bafybeiaiipiibr7aletbbrzmpklw4l5go6sodl22xs6qtcqo3lqogfogy4', 101, '2021-07-14T19:27:14.934572Z');
+VALUES ('bafybeifnfkzjeohjf2dch2iqqpef3bfjylwxlcjws2msvdfyze5bvdprfm', 1000, '2021-07-14T19:27:14.934572Z'),
+       ('bafybeica6klnrhlrbx6z24icefykpbwyypouglnypvnwb5esdm6yzcie3q', 2000, '2021-07-14T19:27:14.934572Z'),
+       ('bafybeiaiipiibr7aletbbrzmpklw4l5go6sodl22xs6qtcqo3lqogfogy4', 30000, '2021-07-14T19:27:14.934572Z');
 
-INSERT INTO upload (user_id, content_cid, source_cid, type, inserted_at, updated_at)
-VALUES (3, 'bafybeifnfkzjeohjf2dch2iqqpef3bfjylwxlcjws2msvdfyze5bvdprfm', 'bafybeifnfkzjeohjf2dch2iqqpef3bfjylwxlcjws2msvdfyze5bvdprfm', 'Car', '2021-07-14T19:27:14.934572Z', '2021-07-14T19:27:14.934572Z'),
-       (3, 'bafybeica6klnrhlrbx6z24icefykpbwyypouglnypvnwb5esdm6yzcie3q', 'bafybeica6klnrhlrbx6z24icefykpbwyypouglnypvnwb5esdm6yzcie3q', 'Car', '2021-07-14T19:27:14.934572Z', '2021-07-14T19:27:14.934572Z'),
-       (3, 'bafybeiaiipiibr7aletbbrzmpklw4l5go6sodl22xs6qtcqo3lqogfogy4', 'bafybeiaiipiibr7aletbbrzmpklw4l5go6sodl22xs6qtcqo3lqogfogy4', 'Car', '2021-07-14T19:27:14.934572Z', '2021-07-14T19:27:14.934572Z');
+INSERT INTO upload (user_id, content_cid, source_cid, type, name, inserted_at, updated_at)
+VALUES (3, 'bafybeifnfkzjeohjf2dch2iqqpef3bfjylwxlcjws2msvdfyze5bvdprfm', 'bafybeifnfkzjeohjf2dch2iqqpef3bfjylwxlcjws2msvdfyze5bvdprfm', 'Car', 'image.jpeg', '2021-07-14T19:27:14.934572Z', '2021-07-14T19:27:14.934572Z'),
+       (3, 'bafybeica6klnrhlrbx6z24icefykpbwyypouglnypvnwb5esdm6yzcie3q', 'bafybeica6klnrhlrbx6z24icefykpbwyypouglnypvnwb5esdm6yzcie3q', 'Car', 'image.webp', '2021-07-14T19:27:14.934572Z', '2021-07-14T19:27:14.934572Z'),
+       (3, 'bafybeiaiipiibr7aletbbrzmpklw4l5go6sodl22xs6qtcqo3lqogfogy4', 'bafybeiaiipiibr7aletbbrzmpklw4l5go6sodl22xs6qtcqo3lqogfogy4', 'Car', 'image.jpg', '2021-07-14T19:27:14.934572Z', '2021-07-14T19:27:14.934572Z'),
+       (1, 'bafybeica6klnrhlrbx6z24icefykpbwyypouglnypvnwb5esdm6yzcie3q', 'bafybeica6klnrhlrbx6z24icefykpbwyypouglnypvnwb5esdm6yzcie3q', 'Car', 'image.webp', '2021-07-14T19:27:14.934572Z', '2021-07-14T19:27:14.934572Z'),
+       (1, 'bafybeiaiipiibr7aletbbrzmpklw4l5go6sodl22xs6qtcqo3lqogfogy4', 'bafybeiaiipiibr7aletbbrzmpklw4l5go6sodl22xs6qtcqo3lqogfogy4', 'Car', 'image.jpg', '2021-07-14T19:27:14.934572Z', '2021-07-14T19:27:14.934572Z')
+       ;
 
 INSERT INTO pin_location (peer_id, peer_name, ipfs_peer_id, region)
 VALUES ('12D3KooEL1Jc', 'who?', '12D3KooWR1Js', 'where?'),
@@ -102,16 +105,16 @@ VALUES  (4, 'HasPsaAccess', true, 'test', null),
         (6, 'HasAccountRestriction', true, 'Revoked access', null),
         (7, 'HasAccountRestriction', true, 'Revoked access', null);
 
-INSERT INTO content (cid)
-VALUES  ('bafybeid46f7zggioxjm5p2ze2l6s6wbqvoo4gzbdzfjtdosthmfyxdign4'),
-        ('bafybeig7yvw6a4uhio4pmg5gahyd2xumowkfljdukad7pmdsv5uk5zcseu'),
-        ('bafybeia45bscvzxngto555xsel4gwoclb5fxd7zpxige7rl3maoleznswu'),
-        ('bafybeidw7pc6nvm7u4rfhpctac4qgtpmwxapw4duugvsl3ppivvzibdlgy'),
-        ('bafybeidrzt6t4k25qjeasydgi3fyh6ejos5x4d6tk2pdzxkb66bkomezy4'),
-        ('bafybeifsrhq2qtkcgjt4gzi7rkafrv2gaai24ptt6rohe2ebqzydkz47sm'),
-        ('bafybeiaqu6ijhfhwzjipwesbqf4myz6uczyigahib5joqbo5jw2xmjczfa'),
-        ('bafybeifsrhq2qtkcgjt4gzi7rkafrv2gaai24ptt6rohe2ebqzydkz47s5'),
-        ('bafybeidqts3rbwkprggjojbvcxy4jzpgzgcvs4a73y3gx2jjxphjeerbcy');
+INSERT INTO content (cid, dag_size)
+VALUES  ('bafybeid46f7zggioxjm5p2ze2l6s6wbqvoo4gzbdzfjtdosthmfyxdign4', 10000),
+        ('bafybeig7yvw6a4uhio4pmg5gahyd2xumowkfljdukad7pmdsv5uk5zcseu', 20000),
+        ('bafybeia45bscvzxngto555xsel4gwoclb5fxd7zpxige7rl3maoleznswu', 30000),
+        ('bafybeidw7pc6nvm7u4rfhpctac4qgtpmwxapw4duugvsl3ppivvzibdlgy', 100),
+        ('bafybeidrzt6t4k25qjeasydgi3fyh6ejos5x4d6tk2pdzxkb66bkomezy4', 200),
+        ('bafybeifsrhq2qtkcgjt4gzi7rkafrv2gaai24ptt6rohe2ebqzydkz47sm', 400),
+        ('bafybeiaqu6ijhfhwzjipwesbqf4myz6uczyigahib5joqbo5jw2xmjczfa', 500000),
+        ('bafybeidqts3rbwkprggjojbvcxy4jzpgzgcvs4a73y3gx2jjxphjeerbcy', 200000),
+        ('bafybeifsrhq2qtkcgjt4gzi7rkafrv2gaai24ptt6rohe2ebqzydkz47s5', 200000);
 
 INSERT INTO pin (status, content_cid, pin_location_id, inserted_at, updated_at)
 VALUES
@@ -133,4 +136,6 @@ VALUES ('ab62cf3c-c98d-494b-a756-b3a3fb6ddcab', 4, 'bafybeid46f7zggioxjm5p2ze2l6
        ('5c7e7885-7f68-462d-bdfb-3f0abfb367b5', 4, 'bafybeifsrhq2qtkcgjt4gzi7rkafrv2gaai24ptt6rohe2ebqzydkz47sm', 'bafybeifsrhq2qtkcgjt4gzi7rkafrv2gaai24ptt6rohe2ebqzydkz47sm', 'Image.jpg', null, null, '2021-07-20T19:27:14.934572Z', '2021-07-14T19:27:14.934572Z'),
        ('5c7e7885-7f68-462d-bdfb-3f0abfb367b6', 4, 'bafybeifsrhq2qtkcgjt4gzi7rkafrv2gaai24ptt6rohe2ebqzydkz47s5', 'bafybeifsrhq2qtkcgjt4gzi7rkafrv2gaai24ptt6rohe2ebqzydkz47sm', 'Image.jpg', null, null, '2021-07-20T19:27:14.934572Z', '2021-07-14T19:27:14.934572Z'),
        ('3a19e48d-d6db-4f36-b686-fb8bc37c9d48', 2, 'bafybeiaqu6ijhfhwzjipwesbqf4myz6uczyigahib5joqbo5jw2xmjczfa', 'bafybeiaqu6ijhfhwzjipwesbqf4myz6uczyigahib5joqbo5jw2xmjczfa', 'Image.jpg', null, null, '2021-07-20T19:27:14.934572Z', '2021-07-14T19:27:14.934572Z'),
-       ('9be23b92-918e-44b8-98f4-6043c346fb4e', 2, 'bafybeidqts3rbwkprggjojbvcxy4jzpgzgcvs4a73y3gx2jjxphjeerbcy', 'bafybeidqts3rbwkprggjojbvcxy4jzpgzgcvs4a73y3gx2jjxphjeerbcy', 'Image.jpg', null, null, '2021-07-14T19:27:14.934572Z', '2021-07-14T19:27:14.934572Z');
+       ('9be23b92-918e-44b8-98f4-6043c346fb4e', 2, 'bafybeidqts3rbwkprggjojbvcxy4jzpgzgcvs4a73y3gx2jjxphjeerbcy', 'bafybeidqts3rbwkprggjojbvcxy4jzpgzgcvs4a73y3gx2jjxphjeerbcy', 'Image.jpg', null, null, '2021-07-14T19:27:14.934572Z', '2021-07-14T19:27:14.934572Z'),
+       ('aa62cf3c-c98d-494b-a756-b3a3fb6ddcab', 1, 'bafybeid46f7zggioxjm5p2ze2l6s6wbqvoo4gzbdzfjtdosthmfyxdign4', 'bafybeid46f7zggioxjm5p2ze2l6s6wbqvoo4gzbdzfjtdosthmfyxdign4', 'ReportDoc.pdf', '["/ip6/2606:4700:60::6/tcp/4009/p2p/QmYyQSo1c1Ym7orWxLYvCrM2EmxFTANf8wXmmE7DWjhx5N", "/ip4/172.65.0.13/tcp/4009/p2p/QmYyQSo1c1Ym7orWxLYvCrM2EmxFTANf8wXmmE7DWjhx4N"]', null, '2021-07-14T19:27:14.934572Z', '2021-07-14T19:27:14.934572Z')
+       ;

packages/api/test/fixtures/pgrest/get-user-uploads.js

@@ -1,4 +1,55 @@
 export default [
+  {
+    _id: '7',
+    cid: 'bafybeica6klnrhlrbx6z24icefykpbwyypouglnypvnwb5esdm6yzcie3q',
+    created: '2021-07-14T19:27:14.934572+00:00',
+    dagSize: 2000,
+    deals: [
+      {
+        activation: '2021-09-11T14:11:00+00:00',
+        created: '2021-09-09T16:30:52.252233+00:00',
+        dataCid: 'bafybeiek5gau46j4dxoyty27qtirb3iuoq7aax4l3xt25mfk2igyt35bmf',
+        dataModelSelector: 'Links/0/Links',
+        dealId: 2424133,
+        expiration: '2023-02-03T14:11:00+00:00',
+        pieceCid: 'bafybeidymjmzqihaz7oeiod3zyolzgcxwbum4b4nvo4aublm6quh6zb5af',
+        status: 'Queued',
+        storageProvider: 'f0678914',
+        updated: '2021-09-10T00:45:50.408956+00:00'
+      }
+    ],
+    name: 'image.webp',
+    pins: [
+      {
+        peerId: '12D3KooWR1Js',
+        peerName: 'who?',
+        region: 'where?',
+        status: 'Pinned',
+        updated: '2021-07-14T19:27:14.934572+00:00'
+      }
+    ],
+    type: 'Car',
+    updated: '2021-07-14T19:27:14.934572+00:00'
+  },
+  {
+    _id: '8',
+    cid: 'bafybeiaiipiibr7aletbbrzmpklw4l5go6sodl22xs6qtcqo3lqogfogy4',
+    created: '2021-07-14T19:27:14.934572+00:00',
+    dagSize: 30000,
+    deals: [],
+    name: 'image.jpg',
+    pins: [
+      {
+        peerId: '12D3KooWR1Js',
+        peerName: 'who?',
+        region: 'where?',
+        status: 'Pinned',
+        updated: '2021-07-14T19:27:14.934572+00:00'
+      }
+    ],
+    type: 'Car',
+    updated: '2021-07-14T19:27:14.934572+00:00'
+  },
   {
     _id: '1',
     name: 'Upload at 2021-07-09T16:20:32.658Z',

packages/api/test/fixtures/pgrest/status-with-active-deal.js

@@ -1,7 +1,7 @@
 export default {
   cid: 'bafybeifnfkzjeohjf2dch2iqqpef3bfjylwxlcjws2msvdfyze5bvdprfm',
   created: '2021-07-14T19:27:14.934572+00:00',
-  dagSize: 101,
+  dagSize: 1000,
   pins: [
     {
       peerId: '12D3KooWR1Js',

packages/api/test/fixtures/pgrest/status-with-no-deal.js

@@ -1,7 +1,7 @@
 export default {
   cid: 'bafybeiaiipiibr7aletbbrzmpklw4l5go6sodl22xs6qtcqo3lqogfogy4',
   created: '2021-07-14T19:27:14.934572+00:00',
-  dagSize: 101,
+  dagSize: 30000,
   pins: [
     {
       peerId: '12D3KooWR1Js',

packages/api/test/fixtures/pgrest/status-with-queued-deal.js

@@ -1,7 +1,7 @@
 export default {
   cid: 'bafybeica6klnrhlrbx6z24icefykpbwyypouglnypvnwb5esdm6yzcie3q',
   created: '2021-07-14T19:27:14.934572+00:00',
-  dagSize: 101,
+  dagSize: 2000,
   pins: [
     {
       peerId: '12D3KooWR1Js',

packages/api/test/user.spec.js

@@ -28,7 +28,8 @@ describe('GET /user/account', () => {
     })
     assert(res.ok)
     const data = await res.json()
-    assert.strictEqual(data.usedStorage, 0)
+    assert.strictEqual(data.usedStorage.uploaded, 32000)
+    assert.strictEqual(data.usedStorage.pinned, 10000)
   })
 })
 

packages/db/db-client-types.ts

@@ -330,3 +330,8 @@ export type ListPsaPinRequestResults = {
 export type NameItem = {
   record: definitions['name']['record']
 }
+
+export type UsedStorage = {
+  uploaded: number,
+  pinned: number
+}

packages/db/index.d.ts

@@ -38,7 +38,7 @@ export class DBClient {
   getMetricsValue (key: string): Promise<{ total: number }>
   upsertUser (user: UpsertUserInput): Promise<UpsertUserOutput>
   getUser (issuer: string): Promise<UserOutput>
-  getUsedStorage (userId: number): Promise<number>
+  getUsedStorage (userId: number): Promise<{ uploaded: number, pinned: number }>
   createUpload (data: CreateUploadInput): Promise<CreateUploadOutput>
   getUpload (cid: string, userId: number): Promise<UploadItemOutput>
   listUploads (userId: number, opts?: ListUploadsOptions): Promise<UploadItemOutput[]>

packages/db/index.js

@@ -1,7 +1,7 @@
 import { PostgrestClient } from '@supabase/postgrest-js'
 
 import {
-  normalizeUpload, normalizeContent, normalizePins, normalizeDeals, normalizePsaPinRequest
+  normalizeUpload, normalizeContent, normalizePins, normalizeDeals, normalizePsaPinRequest, parseTextToNumber
 } from './utils.js'
 import { ConstraintError, DBError } from './errors.js'
 
@@ -190,20 +190,23 @@ export class DBClient {
   }
 
   /**
-   * Get used storage in bytes.
+   * Get used storage in bytes, both uploaded and pinned.
    *
    * @param {number} userId
-   * @returns {Promise<number>}
+   * @returns {Promise<import('./db-client-types').UsedStorage>}
    */
   async getUsedStorage (userId) {
-    /** @type {{ data: string, error: PostgrestError }} */
+    /** @type {{ data: { uploaded: string, pinned: string }, error: PostgrestError }} */
     const { data, error } = await this._client.rpc('user_used_storage', { query_user_id: userId }).single()
 
     if (error) {
       throw new DBError(error)
     }
 
-    return data || 0 // No uploads for the user
+    return {
+      uploaded: parseTextToNumber(data.uploaded),
+      pinned: parseTextToNumber(data.pinned)
+    }
   }
 
   /**

packages/db/postgres/functions.sql

@@ -269,17 +269,40 @@ BEGIN
 END
 $$;
 
-CREATE OR REPLACE FUNCTION user_used_storage(query_user_id BIGINT) RETURNS TEXT
-  LANGUAGE plpgsql
+CREATE TYPE used_storage AS (uploaded TEXT, pinned TEXT);
+
+CREATE OR REPLACE FUNCTION user_used_storage(query_user_id BIGINT) 
+RETURNS used_storage
+LANGUAGE plpgsql
 AS
 $$
+DECLARE used_storage used_storage;
 BEGIN
-  return(
-    select sum(c.dag_size)
-    from upload u
-    join content c on c.cid = u.content_cid
-    where u.user_id = query_user_id and u.deleted_at is null
-  )::TEXT;
+  SELECT COALESCE(SUM(c.dag_size), 0)
+  INTO used_storage.uploaded::TEXT
+  FROM upload u
+  JOIN content c ON c.cid = u.content_cid
+  WHERE u.user_id = query_user_id::BIGINT
+  AND u.deleted_at is null;
+
+  SELECT COALESCE((
+    SELECT SUM(dag_size) 
+    FROM (
+      SELECT  psa_pr.content_cid,
+              c.dag_size
+      FROM psa_pin_request psa_pr
+      JOIN content c ON c.cid = psa_pr.content_cid
+      JOIN pin p ON p.content_cid = psa_pr.content_cid
+      JOIN auth_key a ON a.id = psa_pr.auth_key_id
+      WHERE a.user_id = query_user_id::BIGINT
+      AND psa_pr.deleted_at is null
+      AND p.status = 'Pinned'
+      GROUP BY psa_pr.content_cid,
+              c.dag_size
+    ) AS pinned_content), 0)
+  INTO used_storage.pinned::TEXT;
+
+  return used_storage;
 END
 $$;
 

packages/db/postgres/migrations/006-user-used-storage.sql

@@ -0,0 +1,38 @@
+CREATE TYPE used_storage AS (uploaded TEXT, pinned TEXT);
+
+DROP FUNCTION user_used_storage(bigint);
+
+CREATE OR REPLACE FUNCTION user_used_storage(query_user_id BIGINT) 
+RETURNS used_storage
+LANGUAGE plpgsql
+AS
+$$
+DECLARE used_storage used_storage;
+BEGIN
+  SELECT COALESCE(SUM(c.dag_size), 0)
+  INTO used_storage.uploaded::TEXT
+  FROM upload u
+  JOIN content c ON c.cid = u.content_cid
+  WHERE u.user_id = query_user_id::BIGINT
+  AND u.deleted_at is null;
+
+  SELECT COALESCE((
+    SELECT SUM(dag_size) 
+    FROM (
+      SELECT  psa_pr.content_cid,
+              c.dag_size
+      FROM psa_pin_request psa_pr
+      JOIN content c ON c.cid = psa_pr.content_cid
+      JOIN pin p ON p.content_cid = psa_pr.content_cid
+      JOIN auth_key a ON a.id = psa_pr.auth_key_id
+      WHERE a.user_id = query_user_id::BIGINT
+      AND psa_pr.deleted_at is null
+      AND p.status = 'Pinned'
+      GROUP BY psa_pr.content_cid,
+              c.dag_size
+    ) AS pinned_content), 0)
+  INTO used_storage.pinned::TEXT;
+
+  return used_storage;
+END
+$$;

packages/db/test/pinning.spec.js

@@ -60,6 +60,7 @@ describe('Pin Request', () => {
 
   const meta = { key: 'value' }
   const origins = ['origin1', 'origin2']
+  const dagSize1 = 200
 
   const normalizedCids = cids.map(cid => normalizeCid(cid))
 
@@ -74,13 +75,29 @@ describe('Pin Request', () => {
       }
     },
     {
-      status: 'Pinning',
+      status: 'Pinned',
       location: {
         peerId: '12D3KooWFe387JFDpgNEVCP5ARut7gRkX7YuJCXMStpkq714ziK7',
         peerName: 'web3-storage-sv16',
         ipfsPeerId: '12D3KooWR19qPPiZH4khepNjS3CLXiB7AbrbAD4ZcDjN1UjGUNE3',
         region: 'region'
       }
+    },
+    {
+      status: 'Pinned',
+      location: {
+        peerId: '12D3KooWFe387JFDpgNEVCP5ARut7gRkX7YuJCXMStpkq714ziK8',
+        peerName: 'web3-storage-sv17',
+        region: 'region'
+      }
+    },
+    {
+      status: 'Pinned',
+      location: {
+        peerId: '12D3KooWFe387JFDpgNEVCP5ARut7gRkX7YuJCXMStpkq714ziK9',
+        peerName: 'web3-storage-sv18',
+        region: 'region'
+      }
     }
   ]
 
@@ -102,6 +119,7 @@ describe('Pin Request', () => {
     aPinRequestInput = {
       sourceCid: cids[0],
       contentCid: normalizedCids[0],
+      dagSize: dagSize1,
       meta,
       origins,
       pins,
@@ -151,9 +169,27 @@ describe('Pin Request', () => {
 
     it('returns the right pins', async () => {
       // Only checking statuses for simplicity
-      const statuses = aPinRequestOutput.pins
-        .map((p) => p.status)
-      assert.deepStrictEqual(statuses, [pins[0].status, pins[1].status])
+      const statuses = aPinRequestOutput.pins.map((p) => p.status)
+      const expected = pins.map((p) => p.status)
+      assert.deepStrictEqual(statuses, expected, 'pin statuses match')
+    })
+
+    it('sums pinned size for unique CIDs in used storage', async () => {
+      let usedStorage = await client.getUsedStorage(user._id)
+      assert.strictEqual(usedStorage.pinned, dagSize1, 'used storage for pinned')
+
+      await client.createPsaPinRequest({
+        sourceCid: cids[1],
+        contentCid: normalizedCids[1],
+        dagSize: dagSize1,
+        meta,
+        origins,
+        pins,
+        authKey
+      })
+
+      usedStorage = await client.getUsedStorage(user._id)
+      assert.strictEqual(usedStorage.pinned, dagSize1 * 2, 'used storage for pinned')
     })
   })
 
@@ -177,9 +213,9 @@ describe('Pin Request', () => {
 
     it('returns the right pins', async () => {
       // Only checking statuses for simplicity
-      const statuses = savedPinRequest.pins
-        .map((p) => p.status)
-      assert.deepStrictEqual(statuses, [pins[0].status, pins[1].status])
+      const statuses = savedPinRequest.pins.map((p) => p.status)
+      const expected = pins.map((p) => p.status)
+      assert.deepStrictEqual(statuses, expected, 'pin statuses match')
     })
 
     it('throws if does not exists', async () => {
@@ -310,6 +346,7 @@ describe('Pin Request', () => {
           authKey: authKeyPinList,
           sourceCid: sourceCid,
           contentCid: normalizedCid,
+          dagSize: 10,
           pins: item.pins || pinnedPins,
           meta: item.meta
         })