From d3f76857a1d139aa15646fe96dd0ef5d8a791fbe Mon Sep 17 00:00:00 2001 From: Andris Reinman Date: Thu, 4 Jul 2024 13:38:40 +0300 Subject: [PATCH] fix(api): Added support for paging cursors --- lib/api-client/gmail-client.js | 19 +++++++++-- lib/mailbox.js | 60 +++++++++++++++++++++++++++++++++- lib/schemas.js | 8 ++--- workers/api.js | 33 ++++++++++++++++--- 4 files changed, 105 insertions(+), 15 deletions(-) diff --git a/lib/api-client/gmail-client.js b/lib/api-client/gmail-client.js index e3ea7710d..336183bad 100644 --- a/lib/api-client/gmail-client.js +++ b/lib/api-client/gmail-client.js @@ -101,6 +101,7 @@ class PageCursor { if (cursorType && this.type !== cursorType) { let error = new Error('Invalid cursor'); error.code = 'InvalidCursorType'; + error.statusCode = 400; throw error; } } @@ -139,7 +140,7 @@ class PageCursor { } prevPageCursor() { - if (this.cursorList.length <= 1) { + if (this.cursorList.length < 1) { return null; } @@ -359,6 +360,14 @@ class GmailClient extends BaseClient { await this.prepare(); + let page = Number(query.page) || 0; + if (page > 0) { + let error = new Error('Invalid page number. Only paging cursors are allowed for Gmail accounts.'); + error.code = 'InvalidInput'; + error.statusCode = 400; + throw error; + } + let pageSize = Math.abs(Number(query.pageSize) || 20); let requestQuery = { maxResults: pageSize @@ -1274,11 +1283,13 @@ class GmailClient extends BaseClient { if (this.cachedLabels && !force && now <= this.cachedLabelsTime + 3600 * 1000) { return this.cachedLabels; } - let cachedLabels; + try { - cachedLabels = await this.request(`${GMAIL_API_BASE}/gmail/v1/users/me/labels`); + let cachedLabels = await this.request(`${GMAIL_API_BASE}/gmail/v1/users/me/labels`); this.cachedLabels = cachedLabels?.labels; this.cachedLabelsTime = now; + + return this.cachedLabels; } catch (err) { if (this.cachedLabels) { return this.cachedLabels; @@ -1613,6 +1624,7 @@ class GmailClient extends BaseClient { if (disabledKey in search) { let error = new Error(`Unsupported search term "${disabledKey}"`); error.code = 'UnsupportedSearchTerm'; + error.statusCode = 400; throw error; } } @@ -1646,6 +1658,7 @@ class GmailClient extends BaseClient { default: { let error = new Error(`Unsupported search header "${headerKey}"`); error.code = 'UnsupportedSearchTerm'; + error.statusCode = 400; throw error; } } diff --git a/lib/mailbox.js b/lib/mailbox.js index 64f0efe2c..3424ae940 100644 --- a/lib/mailbox.js +++ b/lib/mailbox.js @@ -2669,7 +2669,16 @@ class Mailbox { async listMessages(options, allowSecondary) { options = options || {}; + let page = Number(options.page) || 0; + + if (options.cursor) { + let cursorPage = this.decodeCursorStr(options.cursor); + if (typeof cursorPage === 'number' && cursorPage >= 0) { + page = cursorPage; + } + } + let pageSize = Math.abs(Number(options.pageSize) || 20); const connectionClient = await this.connection.getImapConnection(allowSecondary); @@ -2698,10 +2707,15 @@ class Mailbox { let messages = []; let seqMax, seqMin, range; + let nextPageCursor = page < pages - 1 ? this.encodeCursorString(page + 1) : null; + let prevPageCursor = page > 0 ? this.encodeCursorString(Math.min(page - 1, pages - 1)) : null; + if (!messageCount || page >= pages) { return { page, pages, + nextPageCursor, + prevPageCursor, messages }; } @@ -2762,7 +2776,9 @@ class Mailbox { total: messageCount, page, pages, - // list newer first. servers like yahoo do not return ordered list, so we need to order manually + nextPageCursor, + prevPageCursor, + // List newer entries first. Servers like yahoo do not return ordered list, so we need to order manually messages: messages.sort((a, b) => b.uid - a.uid) }; } finally { @@ -2884,6 +2900,48 @@ class Mailbox { return false; } + + decodeCursorStr(cursorStr) { + let type = 'imap'; + + if (cursorStr) { + let splitPos = cursorStr.indexOf('_'); + if (splitPos >= 0) { + let cursorType = cursorStr.substring(0, splitPos); + cursorStr = cursorStr.substring(splitPos + 1); + if (cursorType && type !== cursorType) { + let error = new Error('Invalid cursor'); + error.code = 'InvalidCursorType'; + throw error; + } + } + + try { + let { page: cursorPage } = JSON.parse(Buffer.from(cursorStr, 'base64url')); + if (typeof cursorPage === 'number' && cursorPage >= 0) { + return cursorPage; + } + } catch (err) { + this.logger.error({ msg: 'Cursor parsing error', cursorStr, err }); + + let error = new Error('Invalid paging cursor'); + error.code = 'InvalidCursorValue'; + error.statusCode = 400; + throw error; + } + } + + return null; + } + + encodeCursorString(cursorPage) { + if (typeof cursorPage !== 'number' || cursorPage < 0) { + return null; + } + cursorPage = cursorPage || 0; + let type = 'imap'; + return `${type}_${Buffer.from(JSON.stringify({ page: cursorPage })).toString('base64url')}`; + } } module.exports.Mailbox = Mailbox; diff --git a/lib/schemas.js b/lib/schemas.js index e834d95bf..1d1f00242 100644 --- a/lib/schemas.js +++ b/lib/schemas.js @@ -995,7 +995,7 @@ const documentStoreSchema = Joi.boolean() const searchSchema = Joi.object({ seq: Joi.string() .max(8 * 1024) - .description('Sequence number range. Not allowed with `documentStore`.'), + .description('Sequence number range. Only for IMAP accounts.'), answered: Joi.boolean().truthy('Y', 'true', '1').falsy('N', 'false', 0).description('Check if message is answered or not').label('AnsweredFlag'), deleted: Joi.boolean() @@ -1039,11 +1039,7 @@ const searchSchema = Joi.object({ .description('UID range') .label('UIDRange'), - modseq: Joi.number() - .integer() - .min(0) - .description('Matches messages with modseq higher than value. Not allowed with `documentStore`.') - .label('ModseqLarger'), + modseq: Joi.number().integer().min(0).description('Matches messages with modseq higher than value. Only for IMAP accounts.').label('ModseqLarger'), before: Joi.date().description('Matches messages received before date').label('EnvelopeBefore'), since: Joi.date().description('Matches messages received after date').label('EnvelopeSince'), diff --git a/workers/api.js b/workers/api.js index 83b28f4de..b3b5029bf 100644 --- a/workers/api.js +++ b/workers/api.js @@ -3543,7 +3543,7 @@ When making API calls remember that requests against the same account are queued .min(0) .max(1024 * 1024 * 1024) .example(5 * 1025 * 1024) - .description('Max length of text content. This setting is ignored if `documentStore` is `true`.'), + .description('Max length of text content'), textType: Joi.string() .lowercase() .valid('html', 'plain', '*') @@ -4308,7 +4308,7 @@ When making API calls remember that requests against the same account are queued .min(0) .max(1024 * 1024 * 1024) .example(MAX_ATTACHMENT_SIZE) - .description('Max length of text content. This setting is ignored if `documentStore` is `true`.'), + .description('Max length of text content'), textType: Joi.string() .lowercase() .valid('html', 'plain', '*') @@ -4393,14 +4393,25 @@ When making API calls remember that requests against the same account are queued query: Joi.object({ path: Joi.string().required().example('INBOX').description('Mailbox folder path').label('Path'), + + cursor: Joi.string() + .trim() + .empty('') + .max(1024 * 1024) + .example('imap_kcQIji3UobDDTxc') + .description('Paging cursor from `nextPageCursor` or `prevPageCursor` value') + .label('PageCursor'), page: Joi.number() .integer() .min(0) .max(1024 * 1024) .default(0) .example(0) - .description('Page number (zero indexed, so use 0 for first page)') + .description( + 'Page number (zero indexed, so use 0 for first page). Only supported for IMAP accounts. Deprecated, use paging cursor instead.' + ) .label('PageNumber'), + pageSize: Joi.number().integer().min(1).max(1000).default(20).example(20).description('How many entries per page').label('PageSize'), documentStore: documentStoreSchema.default(false) }).label('MessageQuery') @@ -4496,14 +4507,26 @@ When making API calls remember that requests against the same account are queued otherwise: Joi.required() }) .example('INBOX') - .description('Mailbox folder path. Not required if `documentStore` is `true`'), + .description('Mailbox folder path'), + + cursor: Joi.string() + .trim() + .empty('') + .max(1024 * 1024) + .example('imap_kcQIji3UobDDTxc') + .description('Paging cursor from `nextPageCursor` or `prevPageCursor` value') + .label('PageCursor'), page: Joi.number() .integer() .min(0) .max(1024 * 1024) .default(0) .example(0) - .description('Page number (zero indexed, so use 0 for first page)'), + .description( + 'Page number (zero indexed, so use 0 for first page). Only supported for IMAP accounts. Deprecated, use paging cursor instead.' + ) + .label('PageNumber'), + pageSize: Joi.number().integer().min(1).max(1000).default(20).example(20).description('How many entries per page'), documentStore: documentStoreSchema.default(false), exposeQuery: Joi.boolean()