feat: send email notifications for storage quota usage (#1273)

This includes:
- a general email component in cron package for sending emails
- notifications to web3.storage users when they get to specific thresholds
- notifications to web3.storage admins when users go over their quota


Co-authored-by: Gary Homewood <[email protected]>
Co-authored-by: Paolo <[email protected]>
Co-authored-by: Oli Evans <[email protected]>
Co-authored-by: francois-potato <[email protected]>

.github/workflows/cron-storage-limits.yaml

@@ -0,0 +1,41 @@
+name: Storage Limit Email Notifications
+
+on:
+  schedule:
+    - cron: '0 5 * * *'
+  # Including 'workflow_dispatch' here allows the job to be triggered manually,
+  # as well as on the schedule.
+  workflow_dispatch:
+
+jobs:
+  send-notifications:
+    name: Send notifications
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        # TODO: Update this to include production once it's been successfully
+        # tested on staging
+        # env: ['staging', 'production']
+        env: ['staging']
+    timeout-minutes: 100
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Setup node
+        uses: actions/setup-node@v2
+        with:
+          node-version: 16
+
+      - name: Install dependencies
+        uses: bahmutov/npm-install@v1
+
+      - name: Run job
+        env:
+          DEBUG: '*'
+          ENV: ${{ matrix.env }}
+          PROD_PG_REST_JWT: ${{ secrets.PROD_PG_REST_JWT }}
+          STAGING_PG_REST_JWT: ${{ secrets.STAGING_PG_REST_JWT }}
+          PROD_PG_REST_URL: ${{ secrets.PROD_PG_REST_URL }}
+          STAGING_PG_REST_URL: ${{ secrets.STAGING_PG_REST_URL }}
+          MAILCHIMP_API_KEY: ${{ secrets.MAILCHIMP_API_KEY }}
+        run: npm run start:storage -w packages/cron

packages/api/src/user.js

@@ -110,7 +110,7 @@ export async function userTokensPost (request, env) {
  * @param {import('./env').Env} env
  */
 export async function userAccountGet (request, env) {
-  const usedStorage = await env.db.getUsedStorage(request.auth.user._id)
+  const usedStorage = await env.db.getStorageUsed(Number(request.auth.user._id))
 
   return new JSONResponse({
     usedStorage

packages/cron/package.json

@@ -10,6 +10,7 @@
     "start:metrics": "NODE_TLS_REJECT_UNAUTHORIZED=0 node src/bin/metrics.js",
     "start:pins": "node src/bin/pins.js",
     "start:dagcargo:sizes": "NODE_TLS_REJECT_UNAUTHORIZED=0 node src/bin/dagcargo-sizes.js",
+    "start:storage": "node src/bin/storage.js",
     "test": "npm-run-all -p -r test:e2e",
     "test:e2e": "mocha --require ./test/hooks.js test/*.spec.js --timeout 5000"
   },
@@ -32,6 +33,7 @@
     "@types/node": "^16.3.1",
     "execa": "^5.1.1",
     "mocha": "^8.3.2",
-    "npm-run-all": "^4.1.5"
+    "npm-run-all": "^4.1.5",
+    "sinon": "^13.0.2"
   }
 }

packages/cron/src/bin/storage.js

@@ -0,0 +1,15 @@
+#!/usr/bin/env node
+
+import { getDBClient } from '../lib/utils.js'
+import { EmailService, EMAIL_PROVIDERS } from '../lib/email/service.js'
+import { checkStorageUsed } from '../jobs/storage.js'
+import { envConfig } from '../lib/env.js'
+
+async function main () {
+  const db = getDBClient(process.env)
+  const emailService = new EmailService({ db, provider: EMAIL_PROVIDERS.dummy })
+  await checkStorageUsed({ db, emailService })
+}
+
+envConfig()
+main()

packages/cron/src/jobs/storage.js

@@ -0,0 +1,106 @@
+import debug from 'debug'
+import { EMAIL_TYPE } from '@web3-storage/db'
+
+const log = debug('storage:checkStorageUsed')
+
+const STORAGE_QUOTA_EMAILS = [
+  {
+    emailType: EMAIL_TYPE.User100PercentStorage,
+    fromPercent: 100,
+    // 1 day approx, with flexibility for cron irregularity
+    secondsSinceLastSent: 60 * 60 * 23
+  },
+  {
+    emailType: EMAIL_TYPE.User90PercentStorage,
+    fromPercent: 90,
+    toPercent: 100,
+    // 1 day approx, with flexibility for cron irregularity
+    secondsSinceLastSent: 60 * 60 * 23
+  },
+  {
+    emailType: EMAIL_TYPE.User85PercentStorage,
+    fromPercent: 85,
+    toPercent: 90,
+    // 1 week approx, with flexibility for cron irregularity
+    secondsSinceLastSent: 60 * 60 * 24 * 7 - (60 * 60)
+  },
+  {
+    emailType: EMAIL_TYPE.User80PercentStorage,
+    fromPercent: 80,
+    toPercent: 85,
+    // 1 week approx, with flexibility for cron irregularity
+    secondsSinceLastSent: 60 * 60 * 24 * 7 - (60 * 60)
+  },
+  {
+    emailType: EMAIL_TYPE.User75PercentStorage,
+    fromPercent: 75,
+    toPercent: 80,
+    // 1 week approx, with flexibility for cron irregularity
+    secondsSinceLastSent: 60 * 60 * 24 * 7 - (60 * 60)
+  }
+]
+
+/**
+ * Get users with storage quota usage in percentage range and email them as
+ * appropriate when approaching their storage quota limit.
+ * @param {{
+ *  db: import('@web3-storage/db').DBClient
+ *  emailService: import('../lib/email/service').EmailService
+ * }} config
+ */
+export async function checkStorageUsed ({ db, emailService }) {
+  if (!log.enabled) {
+    console.log('ℹ️ Enable logging by setting DEBUG=storage:checkStorageUsed')
+  }
+
+  log('🗄 Checking users storage quotas')
+
+  for (const email of STORAGE_QUOTA_EMAILS) {
+    const users = await db.getUsersByStorageUsed({
+      fromPercent: email.fromPercent,
+      ...(email.toPercent && { toPercent: email.toPercent })
+    })
+
+    if (users.length) {
+      if (email.emailType === EMAIL_TYPE.User100PercentStorage) {
+        const adminUser = await db.getUserByEmail('[email protected]')
+        const toAdmin = {
+          _id: Number(adminUser._id),
+          email: adminUser.email,
+          name: adminUser.name
+        }
+
+        const emailSent = await emailService.sendEmail(toAdmin, EMAIL_TYPE.AdminStorageExceeded, {
+          secondsSinceLastSent: email.secondsSinceLastSent,
+          templateVars: { users }
+        })
+
+        if (emailSent) {
+          log('📧 Sent a list of users exceeding their quotas to admin')
+        }
+      }
+
+      for (const user of users) {
+        const to = {
+          _id: Number(user.id),
+          email: user.email,
+          name: user.name
+        }
+
+        const emailSent = await emailService.sendEmail(to, email.emailType, {
+          ...(email.secondsSinceLastSent && { secondsSinceLastSent: email.secondsSinceLastSent })
+        })
+
+        if (emailSent) {
+          if (email.emailType === EMAIL_TYPE.User100PercentStorage) {
+            log(`📧 Sent a quota exceeded email to ${user.name}: ${user.percentStorageUsed}% of quota used`)
+          } else {
+            log(`📧 Sent an email to ${user.name}: ${user.percentStorageUsed}% of quota used`)
+          }
+        }
+      }
+    }
+  }
+
+  log('✅ Done')
+}

packages/cron/src/lib/email/README.md

@@ -0,0 +1,73 @@
+Email
+=====
+
+This folder contains an `EmailService` for sending emails to users.
+It has a dependency on the `@web3-storage/db` package for storing records of sent emails in the DB,
+and it is aware of the concept of a "user", but it is designed to be generic enough that it can
+be used for other parts of the web3.storage site if necessary; it's not specific to the 'cron'
+package.
+
+The service is designed such that all other parts of the application can be agnostic to which email
+provider we're using to do the actual sending, and we could in theory switch email provider in
+future without affecting any other parts of the application. Hence the workings of the
+"email provider" are kept separate to the `EmailService`. For now we're using Mailchimp/Mandrill.
+
+The code is aware of the different "email types" which we send, in order that it can validate they
+are being sent with the correct variables and so that it can track their sending.
+
+
+Adding new email types
+----------------------
+
+To add a new email:
+* Create a new subclass of `EmailType` in `types.js` to provide the necessary subject and any variable formatting.
+* Add the new email type to `EMAIL_TYPE` in `packages/db/constants.js`. It name should match that of the class in `types.js`.
+* Add the new template using the instructions below.
+
+
+Adding or editing templates
+---------------------------
+
+Creating a new template, or editing an existing one, is a slightly involved process.
+Mailchimp provides a visual template editor, which is good insofar as it saves you the pain of having to write your own HTML emails from scratch,
+but the templates that you create in Mailchimp can't be used directly for transactional emails because
+(1) they need to be exported to Mandrill first, and
+(2) they need some manual adaptations and fixes in order to fully work.
+
+To add or edit a template:
+
+* If you're editing an existing template and you want to keep the same template ID (so that you don't have to update it in the code), then you'll need to first delete the existing template from Mandrill so that its ID isn't already taken. But note that doing so will cause any attempts to send that email to break until you replace the template.
+* Go to the [templates page in Mailchimp](https://us5.admin.mailchimp.com/templates/) and create/edit your template.
+* Export the template from Mailchimp to Mandrill using the "Send to Mandrill" option.
+* Go to the [templates page in Mandrill](https://mandrillapp.com/templates)
+* Click on the template to edit the HTML, and make the following changes/fixes:
+  - Add `background: linear-gradient` to `#bodyTable` (see below), the Mailchimp editor restricts you to a hex code only.
+  - Edit the `#bodyCell` style from `padding: 30px` to `padding:30px` (this doesn't seem to be editable in the Mailchimp editor).
+  - Swap the Mailchimp placeholders such as `*|SUBJECT|*` and `<!--*|IF:MC_PREVIEW_TEXT|*-->` for their Handlebars equivalents, e.g. `{{SUBJECT}}` or `{{#if MC_PREVIEW_TEXT}}`. Mailchimp doesn't support Handlebars, but Mandrill does, and Handlebars supports logic (if/else/for/etc), which the Mailchimp tags don't; so we have to convert.
+  - If the template uses any `{{#each users}}...{{/each}}` tags then you'll need to fix their placement so that they're actually wrapping the HTML fragment which we want to repeat. Mailchimp moves these tags when you save the template, which breaks them.
+  - If you're editing the `AdminStorageExceeded` email, add `id="usersQuotaTable"` to the `<table>` containing the list of users, and then add the CSS below into the `<style>` section somewhere.
+* If you're adding a new template, or if you didn't delete the old template first, then get the new template ID/slug from the URL and add/update it in the `MailchimpEmailProvider.templates` object.
+* Ensure that any variables which you've used or changed in the template are correctly specified in the corresponding `EmailType` subclass.
+
+```css
+body,#bodyTable{
+  background: linear-gradient(to top left, #d169db, rgba(199, 166, 251, 0), #5da9e7), linear-gradient(to top right, #5a69da, rgba(199, 166, 251, 0), #d4a8e7) rgba(199, 166, 251, 1);
+}
+```
+
+For the `AdminStorageExceeded` email:
+```css
+#usersQuotaTable {
+  border-spacing: 5px;
+  border-collapse: separate;
+  border-color: transparent;
+}
+```
+
+### A note about CSS
+
+It would be nice to be able to use the `background-clip: text` style to get the colour gradient text
+which is used on the web3.storage site.
+But unfortunately, Chrome currently (April 2022) supports the `-webkit-` prefixed version of
+`background-clip: text`, and [Gmail only allows](https://developers.google.com/gmail/design/reference/supported_css)
+the unprefixed version. So between the two we can't (yet) support it.

packages/cron/src/lib/email/errors.js

@@ -0,0 +1,2 @@
+export class EmailSendError extends Error {
+}

packages/cron/src/lib/email/providers/dummy.js

@@ -0,0 +1,8 @@
+/**
+ * An email provider that does nothing.
+ */
+export default class DummyEmailProvider {
+  async sendEmail (...args) {
+    return Promise.resolve(`${Math.random()}`)
+  }
+}