From 6db89530c14d7cb80b32e37ca3301a6f700dc346 Mon Sep 17 00:00:00 2001 From: Christos Kopanos Date: Mon, 21 Feb 2022 19:10:32 +0200 Subject: [PATCH] feat: handle web storage DOMExceptions (#148) --- .github/workflows/ci.yml | 4 +- docs/pages/storages.md | 13 ++++ src/storage/build.ts | 39 ++++++++---- src/storage/web-api.ts | 55 ++++++++++++++-- test/storage/quota.ts | 134 +++++++++++++++++++++++++++++++++++++++ test/storage/web.test.ts | 15 ++++- 6 files changed, 239 insertions(+), 21 deletions(-) create mode 100644 test/storage/quota.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 695c9bc5..8b8fa9fc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,6 +2,7 @@ name: Code CI on: push: + pull_request: jobs: build: @@ -11,9 +12,6 @@ jobs: steps: - name: Checkout uses: actions/checkout@v2 - with: - ref: ${{ github.head_ref }} - fetch-depth: 0 - name: Setup node and restore cached dependencies uses: actions/setup-node@v2 diff --git a/docs/pages/storages.md b/docs/pages/storages.md index 2856b60f..55156070 100644 --- a/docs/pages/storages.md +++ b/docs/pages/storages.md @@ -85,6 +85,19 @@ const withoutPrefix = buildWebStorage(localStorage); const withPrefix = buildWebStorage(localStorage, 'axios-cache:'); ``` +### Browser quota + +From `v0.9.0`, the web storage is able to detect and evict entries if the browser's quota is +reached. + +The eviction is done by the following algorithm: + +1. Saves an value and got an error. (Probably quota exceeded) +2. Evicts all expired keys that cannot enter the `stale` state. +3. If it fails again, evicts the oldest key. +4. Repeat step 4 and 5 until the object could be saved or the storage is empty. +5. If the storage is empty, ignores the key and don't save it. _(Probably because only this key is greater than the whole quota)_ + ## Creating your own storage There's no mystery implementing a custom storage. You can create your own storage by using diff --git a/src/storage/build.ts b/src/storage/build.ts index 31401dc3..8127190c 100644 --- a/src/storage/build.ts +++ b/src/storage/build.ts @@ -1,6 +1,11 @@ import { Header } from '../header/headers'; import type { MaybePromise } from '../util/types'; -import type { AxiosStorage, StaleStorageValue, StorageValue } from './types'; +import type { + AxiosStorage, + CachedStorageValue, + StaleStorageValue, + StorageValue +} from './types'; const storage = Symbol(); @@ -8,10 +13,27 @@ const storage = Symbol(); export const isStorage = (obj: unknown): obj is AxiosStorage => !!obj && !!(obj as Record)[storage]; +/** Returns true if this storage is expired, but it has sufficient properties to stale. */ +export function canStale(value: CachedStorageValue): boolean { + const headers = value.data.headers; + return ( + Header.ETag in headers || + Header.LastModified in headers || + Header.XAxiosCacheEtag in headers || + Header.XAxiosCacheStaleIfError in headers || + Header.XAxiosCacheLastModified in headers + ); +} + +/** Checks if the provided cache is expired. You should also check if the cache {@link canStale} */ +export function isExpired(value: CachedStorageValue): boolean { + return value.createdAt + value.ttl <= Date.now(); +} + export type BuildStorage = Omit & { /** * Returns the value for the given key. This method does not have to make checks for - * cache invalidation or etc. It just return what was previous saved, if present. + * cache invalidation or anything. It just returns what was previous saved, if present. */ find: (key: string) => MaybePromise; }; @@ -49,25 +71,18 @@ export function buildStorage({ set, find, remove }: BuildStorage): AxiosStorage if ( // Not cached or fresh value value.state !== 'cached' || - value.createdAt + value.ttl > Date.now() + !isExpired(value) ) { return value; } - if ( - value.data.headers && - // Any header below allows the response to stale - (Header.ETag in value.data.headers || - Header.LastModified in value.data.headers || - Header.XAxiosCacheEtag in value.data.headers || - Header.XAxiosCacheStaleIfError in value.data.headers || - Header.XAxiosCacheLastModified in value.data.headers) - ) { + if (canStale(value)) { const stale: StaleStorageValue = { state: 'stale', createdAt: value.createdAt, data: value.data }; + await set(key, stale); return stale; } diff --git a/src/storage/web-api.ts b/src/storage/web-api.ts index a421534b..baf6abff 100644 --- a/src/storage/web-api.ts +++ b/src/storage/web-api.ts @@ -1,5 +1,5 @@ -import type { StorageValue } from '..'; -import { buildStorage } from './build'; +import { buildStorage, canStale, isExpired } from './build'; +import type { StorageValue } from './types'; /** * Creates a simple storage. You can persist his data by using `sessionStorage` or @@ -17,6 +17,7 @@ import { buildStorage } from './build'; * const fromMyStorage = buildWebStorage(myStorage); * ``` * + * @param storage The type of web storage to use. localStorage or sessionStorage. * @param prefix The prefix to index the storage. Useful to prevent collision between * multiple places using the same storage. */ @@ -26,11 +27,55 @@ export function buildWebStorage(storage: Storage, prefix = '') { const json = storage.getItem(prefix + key); return json ? (JSON.parse(json) as StorageValue) : undefined; }, - set: (key, value) => { - storage.setItem(prefix + key, JSON.stringify(value)); - }, + remove: (key) => { storage.removeItem(prefix + key); + }, + + set: (key, value) => { + const save = () => storage.setItem(prefix + key, JSON.stringify(value)); + + try { + return save(); + } catch (error) { + const allValues: [string, StorageValue][] = Object.entries( + storage as Record + ) + .filter(([key]) => key.startsWith(prefix) && storage.getItem(key)) + .map(([key, val]) => [key, JSON.parse(val) as StorageValue]); + + // Remove all expired values + for (const [prefixedKey, value] of allValues) { + if (value.state === 'cached' && isExpired(value) && !canStale(value)) { + storage.removeItem(prefixedKey); + } + } + + // Try save again after removing expired values + try { + return save(); + } catch (_) { + // Storage still full, try removing the oldest value until it can be saved + + // Descending sort by createdAt + const sortedItems = allValues.sort( + ([, valueA], [, valueB]) => (valueA.createdAt || 0) - (valueB.createdAt || 0) + ); + + for (const [prefixedKey] of sortedItems) { + storage.removeItem(prefixedKey); + + try { + return save(); + } catch (_) { + // This key didn't free all the required space + } + } + } + + // Clear the cache for the specified key + storage.removeItem(prefix + key); + } } }); } diff --git a/test/storage/quota.ts b/test/storage/quota.ts new file mode 100644 index 00000000..a8d91923 --- /dev/null +++ b/test/storage/quota.ts @@ -0,0 +1,134 @@ +import { buildWebStorage } from '../../src/storage/web-api'; +import { mockAxios } from '../mocks/axios'; +import { EMPTY_RESPONSE } from '../utils'; + +export function testStorageQuota(name: string, Storage: () => Storage): void { + // Jest quota, in browsers this quota can be different but that isn't a problem. + const MAXIMUM_LIMIT = 5_000_000; + + it(`tests ${name} has storage limit`, () => { + const storage = Storage(); + + expect(storage).toBeDefined(); + + expect(() => { + storage.setItem('key', '0'.repeat(MAXIMUM_LIMIT * 0.9)); + }).not.toThrowError(); + + expect(() => { + storage.setItem('key', '0'.repeat(MAXIMUM_LIMIT)); + }).toThrowError(); + }); + + it(`tests buildWebStorage(${name}) function`, () => { + const webStorage = buildWebStorage(Storage()); + + expect(typeof webStorage.get).toBe('function'); + expect(typeof webStorage.set).toBe('function'); + expect(typeof webStorage.remove).toBe('function'); + }); + + it(`tests ${name} with gigant values`, async () => { + const axios = mockAxios({ storage: buildWebStorage(Storage()) }); + + // Does not throw error + await axios.storage.set('key', { + state: 'cached', + createdAt: Date.now(), + ttl: 60_000, + data: { ...EMPTY_RESPONSE, data: '0'.repeat(MAXIMUM_LIMIT) } + }); + + // Too big for this storage save + expect(await axios.storage.get('key')).toStrictEqual({ state: 'empty' }); + }); + + it(`tests ${name} evicts oldest first`, async () => { + const axios = mockAxios({ storage: buildWebStorage(Storage()) }); + + // Fills the storage with 5 keys + for (const i of [1, 2, 3, 4, 5]) { + await axios.storage.set(`dummy-${i}`, { + state: 'loading', + previous: 'empty' + }); + + await axios.storage.set(`key-${i}`, { + state: 'cached', + createdAt: Date.now(), + ttl: 60_000, + data: { + ...EMPTY_RESPONSE, + data: '0'.repeat(MAXIMUM_LIMIT * 0.2) // 20% each + } + }); + } + + await axios.storage.set('key-initial', { + state: 'cached', + createdAt: Date.now(), + ttl: 60_000, + data: { + ...EMPTY_RESPONSE, + data: '0'.repeat(MAXIMUM_LIMIT * 0.9) // 90% + } + }); + + const initial = await axios.storage.get('key-initial'); + + // Key was defined + expect(initial.state).toBe('cached'); + + // Has evicted all 1-5 keys + for (const i of [1, 2, 3, 4]) { + const { state } = await axios.storage.get(`key-${i}`); + expect(state).toBe('empty'); + } + }); + + it(`expects ${name} remove expired keys`, async () => { + const axios = mockAxios({ storage: buildWebStorage(Storage()) }); + + const year2k = new Date(2000, 1, 1); + + // Fills the storage with 5 keys + // Each 10K ms newer than the previous one + for (const i of [1, 2, 3, 4, 5]) { + await axios.storage.set(`dummy-${i}`, { + state: 'loading', + previous: 'empty' + }); + + await axios.storage.set(`expired-${i}`, { + state: 'cached', + createdAt: year2k.getTime(), + ttl: i * 10_000, + data: { + ...EMPTY_RESPONSE, + data: '0'.repeat(MAXIMUM_LIMIT * 0.2) // 20% each + } + }); + } + + await axios.storage.set('non-expired', { + state: 'cached', + createdAt: Date.now(), // today + ttl: 10_000, + data: { + ...EMPTY_RESPONSE, + data: '0'.repeat(MAXIMUM_LIMIT * 0.9) // 90% + } + }); + + const initial = await axios.storage.get('non-expired'); + + // Key was defined + expect(initial.state).toBe('cached'); + + // Has evicted all 1-5 keys + for (const i of [1, 2, 3, 4]) { + const { state } = await axios.storage.get(`expired-${i}`); + expect(state).toBe('empty'); + } + }); +} diff --git a/test/storage/web.test.ts b/test/storage/web.test.ts index 8b13d880..81d4540b 100644 --- a/test/storage/web.test.ts +++ b/test/storage/web.test.ts @@ -1,9 +1,22 @@ /** @jest-environment jsdom */ import { buildWebStorage } from '../../src/storage/web-api'; +import { testStorageQuota } from './quota'; import { testStorage } from './storages'; describe('tests web storages', () => { - testStorage('local-storage', () => buildWebStorage(sessionStorage)); + testStorage('local-storage', () => buildWebStorage(localStorage)); testStorage('session-storage', () => buildWebStorage(sessionStorage)); + + testStorageQuota('local-storage', () => { + // Clear previous values + localStorage.clear(); + return localStorage; + }); + + testStorageQuota('session-storage', () => { + // Clear previous values + sessionStorage.clear(); + return sessionStorage; + }); });