feat: handle web storage DOMExceptions (#148)

.github/workflows/ci.yml

@@ -2,6 +2,7 @@ name: Code CI
 
 on:
   push:
+  pull_request:
 
 jobs:
   build:
@@ -11,9 +12,6 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v2
-        with:
-          ref: ${{ github.head_ref }}
-          fetch-depth: 0
 
       - name: Setup node and restore cached dependencies
         uses: actions/setup-node@v2

docs/pages/storages.md

@@ -85,6 +85,19 @@ const withoutPrefix = buildWebStorage(localStorage);
 const withPrefix = buildWebStorage(localStorage, 'axios-cache:');
 ```
 
+### Browser quota
+
+From `v0.9.0`, the web storage is able to detect and evict entries if the browser's quota is
+reached.
+
+The eviction is done by the following algorithm:
+
+1. Saves an value and got an error. (Probably quota exceeded)
+2. Evicts all expired keys that cannot enter the `stale` state.
+3. If it fails again, evicts the oldest key.
+4. Repeat step 4 and 5 until the object could be saved or the storage is empty.
+5. If the storage is empty, ignores the key and don't save it. _(Probably because only this key is greater than the whole quota)_
+
 ## Creating your own storage
 
 There's no mystery implementing a custom storage. You can create your own storage by using

src/storage/build.ts

@@ -1,17 +1,39 @@
 import { Header } from '../header/headers';
 import type { MaybePromise } from '../util/types';
-import type { AxiosStorage, StaleStorageValue, StorageValue } from './types';
+import type {
+  AxiosStorage,
+  CachedStorageValue,
+  StaleStorageValue,
+  StorageValue
+} from './types';
 
 const storage = Symbol();
 
 /** Returns true if the provided object was created from {@link buildStorage} function. */
 export const isStorage = (obj: unknown): obj is AxiosStorage =>
   !!obj && !!(obj as Record<symbol, number>)[storage];
 
+/** Returns true if this storage is expired, but it has sufficient properties to stale. */
+export function canStale(value: CachedStorageValue): boolean {
+  const headers = value.data.headers;
+  return (
+    Header.ETag in headers ||
+    Header.LastModified in headers ||
+    Header.XAxiosCacheEtag in headers ||
+    Header.XAxiosCacheStaleIfError in headers ||
+    Header.XAxiosCacheLastModified in headers
+  );
+}
+
+/** Checks if the provided cache is expired. You should also check if the cache {@link canStale} */
+export function isExpired(value: CachedStorageValue): boolean {
+  return value.createdAt + value.ttl <= Date.now();
+}
+
 export type BuildStorage = Omit<AxiosStorage, 'get'> & {
   /**
    * Returns the value for the given key. This method does not have to make checks for
-   * cache invalidation or etc. It just return what was previous saved, if present.
+   * cache invalidation or anything. It just returns what was previous saved, if present.
    */
   find: (key: string) => MaybePromise<StorageValue | undefined>;
 };
@@ -49,25 +71,18 @@ export function buildStorage({ set, find, remove }: BuildStorage): AxiosStorage
       if (
         // Not cached or fresh value
         value.state !== 'cached' ||
-        value.createdAt + value.ttl > Date.now()
+        !isExpired(value)
       ) {
         return value;
       }
 
-      if (
-        value.data.headers &&
-        // Any header below allows the response to stale
-        (Header.ETag in value.data.headers ||
-          Header.LastModified in value.data.headers ||
-          Header.XAxiosCacheEtag in value.data.headers ||
-          Header.XAxiosCacheStaleIfError in value.data.headers ||
-          Header.XAxiosCacheLastModified in value.data.headers)
-      ) {
+      if (canStale(value)) {
         const stale: StaleStorageValue = {
           state: 'stale',
           createdAt: value.createdAt,
           data: value.data
         };
+
         await set(key, stale);
         return stale;
       }

src/storage/web-api.ts

@@ -1,5 +1,5 @@
-import type { StorageValue } from '..';
-import { buildStorage } from './build';
+import { buildStorage, canStale, isExpired } from './build';
+import type { StorageValue } from './types';
 
 /**
  * Creates a simple storage. You can persist his data by using `sessionStorage` or
@@ -17,6 +17,7 @@ import { buildStorage } from './build';
  * const fromMyStorage = buildWebStorage(myStorage);
  * ```
  *
+ * @param storage The type of web storage to use. localStorage or sessionStorage.
  * @param prefix The prefix to index the storage. Useful to prevent collision between
  *   multiple places using the same storage.
  */
@@ -26,11 +27,55 @@ export function buildWebStorage(storage: Storage, prefix = '') {
       const json = storage.getItem(prefix + key);
       return json ? (JSON.parse(json) as StorageValue) : undefined;
     },
-    set: (key, value) => {
-      storage.setItem(prefix + key, JSON.stringify(value));
-    },
+
     remove: (key) => {
       storage.removeItem(prefix + key);
+    },
+
+    set: (key, value) => {
+      const save = () => storage.setItem(prefix + key, JSON.stringify(value));
+
+      try {
+        return save();
+      } catch (error) {
+        const allValues: [string, StorageValue][] = Object.entries(
+          storage as Record<string, string>
+        )
+          .filter(([key]) => key.startsWith(prefix) && storage.getItem(key))
+          .map(([key, val]) => [key, JSON.parse(val) as StorageValue]);
+
+        // Remove all expired values
+        for (const [prefixedKey, value] of allValues) {
+          if (value.state === 'cached' && isExpired(value) && !canStale(value)) {
+            storage.removeItem(prefixedKey);
+          }
+        }
+
+        // Try save again after removing expired values
+        try {
+          return save();
+        } catch (_) {
+          // Storage still full, try removing the oldest value until it can be saved
+
+          // Descending sort by createdAt
+          const sortedItems = allValues.sort(
+            ([, valueA], [, valueB]) => (valueA.createdAt || 0) - (valueB.createdAt || 0)
+          );
+
+          for (const [prefixedKey] of sortedItems) {
+            storage.removeItem(prefixedKey);
+
+            try {
+              return save();
+            } catch (_) {
+              // This key didn't free all the required space
+            }
+          }
+        }
+
+        // Clear the cache for the specified key
+        storage.removeItem(prefix + key);
+      }
     }
   });
 }

test/storage/quota.ts

@@ -0,0 +1,134 @@
+import { buildWebStorage } from '../../src/storage/web-api';
+import { mockAxios } from '../mocks/axios';
+import { EMPTY_RESPONSE } from '../utils';
+
+export function testStorageQuota(name: string, Storage: () => Storage): void {
+  // Jest quota, in browsers this quota can be different but that isn't a problem.
+  const MAXIMUM_LIMIT = 5_000_000;
+
+  it(`tests ${name} has storage limit`, () => {
+    const storage = Storage();
+
+    expect(storage).toBeDefined();
+
+    expect(() => {
+      storage.setItem('key', '0'.repeat(MAXIMUM_LIMIT * 0.9));
+    }).not.toThrowError();
+
+    expect(() => {
+      storage.setItem('key', '0'.repeat(MAXIMUM_LIMIT));
+    }).toThrowError();
+  });
+
+  it(`tests buildWebStorage(${name}) function`, () => {
+    const webStorage = buildWebStorage(Storage());
+
+    expect(typeof webStorage.get).toBe('function');
+    expect(typeof webStorage.set).toBe('function');
+    expect(typeof webStorage.remove).toBe('function');
+  });
+
+  it(`tests ${name} with gigant values`, async () => {
+    const axios = mockAxios({ storage: buildWebStorage(Storage()) });
+
+    // Does not throw error
+    await axios.storage.set('key', {
+      state: 'cached',
+      createdAt: Date.now(),
+      ttl: 60_000,
+      data: { ...EMPTY_RESPONSE, data: '0'.repeat(MAXIMUM_LIMIT) }
+    });
+
+    // Too big for this storage save
+    expect(await axios.storage.get('key')).toStrictEqual({ state: 'empty' });
+  });
+
+  it(`tests ${name} evicts oldest first`, async () => {
+    const axios = mockAxios({ storage: buildWebStorage(Storage()) });
+
+    // Fills the storage with 5 keys
+    for (const i of [1, 2, 3, 4, 5]) {
+      await axios.storage.set(`dummy-${i}`, {
+        state: 'loading',
+        previous: 'empty'
+      });
+
+      await axios.storage.set(`key-${i}`, {
+        state: 'cached',
+        createdAt: Date.now(),
+        ttl: 60_000,
+        data: {
+          ...EMPTY_RESPONSE,
+          data: '0'.repeat(MAXIMUM_LIMIT * 0.2) // 20% each
+        }
+      });
+    }
+
+    await axios.storage.set('key-initial', {
+      state: 'cached',
+      createdAt: Date.now(),
+      ttl: 60_000,
+      data: {
+        ...EMPTY_RESPONSE,
+        data: '0'.repeat(MAXIMUM_LIMIT * 0.9) // 90%
+      }
+    });
+
+    const initial = await axios.storage.get('key-initial');
+
+    // Key was defined
+    expect(initial.state).toBe('cached');
+
+    // Has evicted all 1-5 keys
+    for (const i of [1, 2, 3, 4]) {
+      const { state } = await axios.storage.get(`key-${i}`);
+      expect(state).toBe('empty');
+    }
+  });
+
+  it(`expects ${name} remove expired keys`, async () => {
+    const axios = mockAxios({ storage: buildWebStorage(Storage()) });
+
+    const year2k = new Date(2000, 1, 1);
+
+    // Fills the storage with 5 keys
+    // Each 10K ms newer than the previous one
+    for (const i of [1, 2, 3, 4, 5]) {
+      await axios.storage.set(`dummy-${i}`, {
+        state: 'loading',
+        previous: 'empty'
+      });
+
+      await axios.storage.set(`expired-${i}`, {
+        state: 'cached',
+        createdAt: year2k.getTime(),
+        ttl: i * 10_000,
+        data: {
+          ...EMPTY_RESPONSE,
+          data: '0'.repeat(MAXIMUM_LIMIT * 0.2) // 20% each
+        }
+      });
+    }
+
+    await axios.storage.set('non-expired', {
+      state: 'cached',
+      createdAt: Date.now(), // today
+      ttl: 10_000,
+      data: {
+        ...EMPTY_RESPONSE,
+        data: '0'.repeat(MAXIMUM_LIMIT * 0.9) // 90%
+      }
+    });
+
+    const initial = await axios.storage.get('non-expired');
+
+    // Key was defined
+    expect(initial.state).toBe('cached');
+
+    // Has evicted all 1-5 keys
+    for (const i of [1, 2, 3, 4]) {
+      const { state } = await axios.storage.get(`expired-${i}`);
+      expect(state).toBe('empty');
+    }
+  });
+}

test/storage/web.test.ts

@@ -1,9 +1,22 @@
 /** @jest-environment jsdom */
 
 import { buildWebStorage } from '../../src/storage/web-api';
+import { testStorageQuota } from './quota';
 import { testStorage } from './storages';
 
 describe('tests web storages', () => {
-  testStorage('local-storage', () => buildWebStorage(sessionStorage));
+  testStorage('local-storage', () => buildWebStorage(localStorage));
   testStorage('session-storage', () => buildWebStorage(sessionStorage));
+
+  testStorageQuota('local-storage', () => {
+    // Clear previous values
+    localStorage.clear();
+    return localStorage;
+  });
+
+  testStorageQuota('session-storage', () => {
+    // Clear previous values
+    sessionStorage.clear();
+    return sessionStorage;
+  });
 });