From 1d6df05e5a95b45eabda368bee8f93471fc21377 Mon Sep 17 00:00:00 2001 From: Andris Reinman Date: Fri, 11 Oct 2024 13:36:34 +0300 Subject: [PATCH] feat(imap): Added new IMAP indexing option: 'fast' --- lib/email-client/imap-client.js | 3 + lib/email-client/imap/mailbox.js | 611 ++++++++++-------- .../imap-core/lib/imap-connection.js | 4 +- lib/imapproxy/imap-core/lib/imap-server.js | 4 +- lib/routes-ui.js | 46 +- lib/schemas.js | 19 +- views/config/service.hbs | 33 + workers/imap.js | 4 + 8 files changed, 463 insertions(+), 261 deletions(-) diff --git a/lib/email-client/imap-client.js b/lib/email-client/imap-client.js index 69385e6e0..00747cbfc 100644 --- a/lib/email-client/imap-client.js +++ b/lib/email-client/imap-client.js @@ -69,6 +69,7 @@ async function metricsMeta(meta, logger, key, method, ...args) { class IMAPClient extends BaseClient { constructor(account, options) { + options = options || {}; super(account, options); this.isClosing = false; @@ -114,6 +115,8 @@ class IMAPClient extends BaseClient { this.connectionCount = 0; this.connections = new Set(); + this.imapIndexer = options?.imapIndexer; + this.state = 'connecting'; } diff --git a/lib/email-client/imap/mailbox.js b/lib/email-client/imap/mailbox.js index 47b65d341..d4872bf77 100644 --- a/lib/email-client/imap/mailbox.js +++ b/lib/email-client/imap/mailbox.js @@ -79,6 +79,8 @@ class Mailbox { path: this.path }); + this.imapIndexer = connection.imapIndexer; + this.isGmail = connection.isGmail; this.isAllMail = this.isGmail && this.listingEntry.specialUse === '\\All'; @@ -437,6 +439,11 @@ class Mailbox { async onExpunge(event) { this.logEvent('Untagged EXPUNGE', event); + if (this.imapIndexer !== 'full') { + // ignore as we can not compare this value against the index + return null; + } + let deletedEntry; if (event.seq) { @@ -456,6 +463,11 @@ class Mailbox { async onFlags(event) { this.logEvent('Untagged FETCH', event); + if (this.imapIndexer !== 'full') { + // ignore as we can not compare this value against the index + return null; + } + let storedMessage = await this.entryListGet(event.uid || event.seq, { uid: !!event.uid }); let changes; @@ -483,203 +495,6 @@ class Mailbox { return mailboxStatus.messages !== storedStatus.messages; } - async partialSync(storedStatus) { - storedStatus = storedStatus || (await this.getStoredStatus()); - let mailboxStatus = this.getMailboxStatus(); - - let lock = await this.getMailboxLock(null, { description: 'Partial sync' }); - this.connection.syncing = true; - this.syncing = true; - try { - if (!this.connection.imapClient) { - throw new Error('IMAP connection not available'); - } - - let fields = { uid: true, flags: true, modseq: true, emailId: true, labels: true, internalDate: true }; - let range = '1:*'; - let opts = { - uid: true - }; - - if (this.connection.imapClient.enabled.has('CONDSTORE') && storedStatus.highestModseq) { - opts.changedSince = storedStatus.highestModseq; - } else if (storedStatus.uidNext) { - range = `${storedStatus.uidNext}:*`; - } - - if (mailboxStatus.messages) { - // only fetch messages if there is some - let fetchCompleted = false; - let fetchRetryCount = 0; - let imapClient = this.connection.imapClient; - while (!fetchCompleted) { - try { - // only fetch messages if there is some - for await (let messageData of imapClient.fetch(range, fields, opts)) { - if (!messageData || !messageData.uid) { - //TODO: support partial responses - this.logger.debug({ msg: 'Partial FETCH response', code: 'partial_fetch', query: { range, fields, opts } }); - continue; - } - - // ignore Recent flag - messageData.flags.delete('\\Recent'); - - let storedMessage = await this.entryListGet(messageData.uid, { uid: true }); - - let changes; - if (!storedMessage) { - // new! - let seq = await this.entryListSet(messageData); - if (seq) { - await this.connection.redis.zadd( - this.getNotificationsKey(), - messageData.uid, - JSON.stringify({ - uid: messageData.uid, - flags: messageData.flags, - internalDate: - (messageData.internalDate && - typeof messageData.internalDate.toISOString === 'function' && - messageData.internalDate.toISOString()) || - null - }) - ); - } - } else if ((changes = compareExisting(storedMessage.entry, messageData))) { - let seq = await this.entryListSet(messageData); - if (seq) { - await this.processChanges(messageData, changes); - } - } - } - try { - // clear failure flag - await this.connection.redis.hdel(this.connection.getAccountKey(), 'syncError'); - } catch (err) { - // ignore - } - fetchCompleted = true; - } catch (err) { - try { - // set failure flag - await this.connection.redis.hSetExists( - this.connection.getAccountKey(), - 'syncError', - JSON.stringify({ - path: this.path, - time: new Date().toISOString(), - error: { - error: err.message, - responseStatus: err.responseStatus, - responseText: err.responseText - } - }) - ); - } catch (err) { - // ignore - } - - // retry - if (!imapClient.usable) { - // nothing to do here, connection closed - this.logger.error({ msg: `FETCH failed, connection already closed, not retrying` }); - return; - } - - const fetchRetryDelay = calculateFetchBackoff(++fetchRetryCount); - this.logger.error({ msg: `FETCH failed, retrying in ${Math.round(fetchRetryDelay / 1000)}s` }); - await new Promise(r => setTimeout(r, fetchRetryDelay)); - - if (!imapClient.usable) { - // nothing to do here, connection closed - this.logger.error({ msg: `FETCH failed, connection already closed, not retrying` }); - return; - } - } - } - } - - await this.updateStoredStatus(this.getMailboxStatus()); - - let messageFetchOptions = {}; - - let documentStoreEnabled = await settings.get('documentStoreEnabled'); - - let notifyText = await settings.get('notifyText'); - if (documentStoreEnabled || notifyText) { - messageFetchOptions.textType = '*'; - let notifyTextSize = await settings.get('notifyTextSize'); - - if (documentStoreEnabled && notifyTextSize) { - notifyTextSize = Math.max(notifyTextSize, 1024 * 1024); - } - - if (notifyTextSize) { - messageFetchOptions.maxBytes = notifyTextSize; - } - } - - let notifyHeaders = (await settings.get('notifyHeaders')) || []; - if (documentStoreEnabled || notifyHeaders.length) { - messageFetchOptions.headers = notifyHeaders.includes('*') || documentStoreEnabled ? true : notifyHeaders.length ? notifyHeaders : false; - } - - // also request autoresponse headers - if (messageFetchOptions.headers !== true) { - let fetchHeaders = new Set(messageFetchOptions.headers || []); - - fetchHeaders.add('x-autoreply'); - fetchHeaders.add('x-autorespond'); - fetchHeaders.add('auto-submitted'); - fetchHeaders.add('precedence'); - - fetchHeaders.add('in-reply-to'); - fetchHeaders.add('references'); - - fetchHeaders.add('content-type'); - - messageFetchOptions.fetchHeaders = Array.from(fetchHeaders); - } - - // can't process messages before fetch() has finished - - let queuedEntry; - let hadUpdates = false; - while ((queuedEntry = await this.connection.redis.zpopmin(this.getNotificationsKey(), 1)) && queuedEntry.length) { - hadUpdates = true; - - let [messageData, uid] = queuedEntry; - uid = Number(uid); - try { - messageData = JSON.parse(messageData); - if (typeof messageData.internalDate === 'string') { - messageData.internalDate = new Date(messageData.internalDate); - } - } catch (err) { - continue; - } - - let canSync = documentStoreEnabled && (!this.connection.syncFrom || messageData.internalDate >= this.connection.syncFrom); - - if (this.connection.notifyFrom && messageData.internalDate < this.connection.notifyFrom && !canSync) { - // skip too old messages - continue; - } - - await this.processNew(messageData, messageFetchOptions, canSync, storedStatus); - } - - if (hadUpdates) { - await this.markUpdated(); - } - } finally { - lock.release(); - this.connection.syncing = false; - this.syncing = false; - } - } - async processDeleted(messageData) { this.logger.debug({ msg: 'Deleted', uid: messageData.uid }); @@ -1684,6 +1499,296 @@ class Mailbox { } async fullSync() { + switch (this.imapIndexer) { + case 'fast': + return this.runFastSync(); + case 'full': + default: + return this.runFullSync(); + } + } + + async partialSync(storedStatus) { + switch (this.imapIndexer) { + case 'fast': + return this.runFastSync(storedStatus); + case 'full': + default: + return this.runPartialSync(storedStatus); + } + } + + // TODO: do not list all messages on initial sync if notifyFrom is not a past date + async runFastSync(storedStatus) { + storedStatus = storedStatus || (await this.getStoredStatus()); + let mailboxStatus = this.getMailboxStatus(); + + let lock = await this.getMailboxLock(null, { description: 'Fast sync' }); + this.connection.syncing = true; + this.syncing = true; + try { + if (!this.connection.imapClient) { + throw new Error('IMAP connection not available'); + } + + let knownUidNext = typeof storedStatus.uidNext === 'number' ? storedStatus.uidNext || 1 : 1; + + if (knownUidNext && mailboxStatus.messages) { + // detected new emails + let fields = { uid: true, flags: true, modseq: true, emailId: true, labels: true, internalDate: true }; + + let imapClient = this.connection.imapClient; + + // If we have not yet scanned this folder, then start by finding the earliest matching email + if (typeof storedStatus.uidNext !== 'number' && this.connection.notifyFrom && this.connection.notifyFrom < new Date()) { + let matchingMessages = await imapClient.search({ since: this.connection.notifyFrom }, { uid: true }); + if (matchingMessages) { + let earliestUid = matchingMessages[0]; + if (earliestUid) { + knownUidNext = earliestUid; + } else if (mailboxStatus.uidNext) { + // no match, start from newest + knownUidNext = mailboxStatus.uidNext; + } + } + } + + let range = `${knownUidNext}:*`; + let opts = { + uid: true + }; + + // only fetch messages if there are some + let fetchCompleted = false; + let fetchRetryCount = 0; + + while (!fetchCompleted) { + try { + // only fetch messages if there are some + + let messages = []; + + for await (let messageData of imapClient.fetch(range, fields, opts)) { + if (!messageData || !messageData.uid) { + //TODO: support partial responses + this.logger.debug({ msg: 'Partial FETCH response', code: 'partial_fetch', query: { range, fields, opts } }); + continue; + } + + // ignore Recent flag + messageData.flags.delete('\\Recent'); + + messages.push(messageData); + } + // ensure that messages are sorted by UID + messages = messages.sort((a, b) => a.uid - b.uid); + + for (let messageData of messages) { + let updated = await this.connection.redis.hUpdateBigger(this.getMailboxKey(), 'uidNext', messageData.uid + 1, messageData.uid + 1); + + if (updated) { + // new email! + await this.connection.redis.zadd( + this.getNotificationsKey(), + messageData.uid, + JSON.stringify({ + uid: messageData.uid, + flags: messageData.flags, + internalDate: + (messageData.internalDate && + typeof messageData.internalDate.toISOString === 'function' && + messageData.internalDate.toISOString()) || + null + }) + ); + } + } + + try { + // clear failure flag + await this.connection.redis.hdel(this.connection.getAccountKey(), 'syncError'); + } catch (err) { + // ignore + } + fetchCompleted = true; + } catch (err) { + try { + // set failure flag + await this.connection.redis.hSetExists( + this.connection.getAccountKey(), + 'syncError', + JSON.stringify({ + path: this.path, + time: new Date().toISOString(), + error: { + error: err.message, + responseStatus: err.responseStatus, + responseText: err.responseText + } + }) + ); + } catch (err) { + // ignore + } + + // retry + if (!imapClient.usable) { + // nothing to do here, connection closed + this.logger.error({ msg: `FETCH failed, connection already closed, not retrying`, err }); + return; + } + + const fetchRetryDelay = calculateFetchBackoff(++fetchRetryCount); + this.logger.error({ msg: `FETCH failed, retrying in ${Math.round(fetchRetryDelay / 1000)}s`, err }); + await new Promise(r => setTimeout(r, fetchRetryDelay)); + + if (!imapClient.usable) { + // nothing to do here, connection closed + this.logger.error({ msg: `FETCH failed, connection already closed, not retrying`, err }); + return; + } + } + } + } + + await this.updateStoredStatus(this.getMailboxStatus()); + + await this.publishSyncedEvents(storedStatus); + } finally { + lock.release(); + this.connection.syncing = false; + this.syncing = false; + } + } + + async runPartialSync(storedStatus) { + storedStatus = storedStatus || (await this.getStoredStatus()); + let mailboxStatus = this.getMailboxStatus(); + + let lock = await this.getMailboxLock(null, { description: 'Partial sync' }); + this.connection.syncing = true; + this.syncing = true; + try { + if (!this.connection.imapClient) { + throw new Error('IMAP connection not available'); + } + + let fields = { uid: true, flags: true, modseq: true, emailId: true, labels: true, internalDate: true }; + let range = '1:*'; + let opts = { + uid: true + }; + + if (this.connection.imapClient.enabled.has('CONDSTORE') && storedStatus.highestModseq) { + opts.changedSince = storedStatus.highestModseq; + } else if (storedStatus.uidNext) { + range = `${storedStatus.uidNext}:*`; + } + + if (mailboxStatus.messages) { + // only fetch messages if there are some + let fetchCompleted = false; + let fetchRetryCount = 0; + let imapClient = this.connection.imapClient; + while (!fetchCompleted) { + try { + // only fetch messages if there are some + for await (let messageData of imapClient.fetch(range, fields, opts)) { + if (!messageData || !messageData.uid) { + //TODO: support partial responses + this.logger.debug({ msg: 'Partial FETCH response', code: 'partial_fetch', query: { range, fields, opts } }); + continue; + } + + // ignore Recent flag + messageData.flags.delete('\\Recent'); + + let storedMessage = await this.entryListGet(messageData.uid, { uid: true }); + + let changes; + if (!storedMessage) { + // new! + let seq = await this.entryListSet(messageData); + if (seq) { + await this.connection.redis.zadd( + this.getNotificationsKey(), + messageData.uid, + JSON.stringify({ + uid: messageData.uid, + flags: messageData.flags, + internalDate: + (messageData.internalDate && + typeof messageData.internalDate.toISOString === 'function' && + messageData.internalDate.toISOString()) || + null + }) + ); + } + } else if ((changes = compareExisting(storedMessage.entry, messageData))) { + let seq = await this.entryListSet(messageData); + if (seq) { + await this.processChanges(messageData, changes); + } + } + } + try { + // clear failure flag + await this.connection.redis.hdel(this.connection.getAccountKey(), 'syncError'); + } catch (err) { + // ignore + } + fetchCompleted = true; + } catch (err) { + try { + // set failure flag + await this.connection.redis.hSetExists( + this.connection.getAccountKey(), + 'syncError', + JSON.stringify({ + path: this.path, + time: new Date().toISOString(), + error: { + error: err.message, + responseStatus: err.responseStatus, + responseText: err.responseText + } + }) + ); + } catch (err) { + // ignore + } + + // retry + if (!imapClient.usable) { + // nothing to do here, connection closed + this.logger.error({ msg: `FETCH failed, connection already closed, not retrying` }); + return; + } + + const fetchRetryDelay = calculateFetchBackoff(++fetchRetryCount); + this.logger.error({ msg: `FETCH failed, retrying in ${Math.round(fetchRetryDelay / 1000)}s` }); + await new Promise(r => setTimeout(r, fetchRetryDelay)); + + if (!imapClient.usable) { + // nothing to do here, connection closed + this.logger.error({ msg: `FETCH failed, connection already closed, not retrying` }); + return; + } + } + } + } + + await this.updateStoredStatus(this.getMailboxStatus()); + + await this.publishSyncedEvents(storedStatus); + } finally { + lock.release(); + this.connection.syncing = false; + this.syncing = false; + } + } + + async runFullSync() { let fields = { uid: true, flags: true, modseq: true, emailId: true, labels: true, internalDate: true }; let opts = {}; @@ -1880,81 +1985,83 @@ class Mailbox { await this.updateStoredStatus(status); let storedStatus = await this.getStoredStatus(); - let messageFetchOptions = {}; + await this.publishSyncedEvents(storedStatus); + } finally { + this.connection.syncing = false; + this.syncing = false; + lock.release(); + } + } - let documentStoreEnabled = await settings.get('documentStoreEnabled'); + async publishSyncedEvents(storedStatus) { + let messageFetchOptions = {}; - let notifyText = await settings.get('notifyText'); - if (documentStoreEnabled || notifyText) { - messageFetchOptions.textType = '*'; - let notifyTextSize = await settings.get('notifyTextSize'); + let documentStoreEnabled = await settings.get('documentStoreEnabled'); - if (documentStoreEnabled && notifyTextSize) { - notifyTextSize = Math.max(notifyTextSize, 1024 * 1024); - } + let notifyText = await settings.get('notifyText'); + if (documentStoreEnabled || notifyText) { + messageFetchOptions.textType = '*'; + let notifyTextSize = await settings.get('notifyTextSize'); - if (notifyTextSize) { - messageFetchOptions.maxBytes = notifyTextSize; - } + if (documentStoreEnabled && notifyTextSize) { + notifyTextSize = Math.max(notifyTextSize, 1024 * 1024); } - let notifyHeaders = (await settings.get('notifyHeaders')) || []; - if (documentStoreEnabled || notifyHeaders.length) { - messageFetchOptions.headers = notifyHeaders.includes('*') || documentStoreEnabled ? true : notifyHeaders.length ? notifyHeaders : false; + if (notifyTextSize) { + messageFetchOptions.maxBytes = notifyTextSize; } + } - // also request autoresponse headers - if (messageFetchOptions.headers !== true) { - let fetchHeaders = new Set(messageFetchOptions.headers || []); + let notifyHeaders = (await settings.get('notifyHeaders')) || []; + if (documentStoreEnabled || notifyHeaders.length) { + messageFetchOptions.headers = notifyHeaders.includes('*') || documentStoreEnabled ? true : notifyHeaders.length ? notifyHeaders : false; + } - fetchHeaders.add('x-autoreply'); - fetchHeaders.add('x-autorespond'); - fetchHeaders.add('auto-submitted'); - fetchHeaders.add('precedence'); + // also request autoresponse headers + if (messageFetchOptions.headers !== true) { + let fetchHeaders = new Set(messageFetchOptions.headers || []); - fetchHeaders.add('in-reply-to'); - fetchHeaders.add('references'); + fetchHeaders.add('x-autoreply'); + fetchHeaders.add('x-autorespond'); + fetchHeaders.add('auto-submitted'); + fetchHeaders.add('precedence'); - fetchHeaders.add('content-type'); + fetchHeaders.add('in-reply-to'); + fetchHeaders.add('references'); - messageFetchOptions.fetchHeaders = Array.from(fetchHeaders); - } + fetchHeaders.add('content-type'); - // have to call after fetch is finished + messageFetchOptions.fetchHeaders = Array.from(fetchHeaders); + } - let queuedEntry; - let hadUpdates = false; - while ((queuedEntry = await this.connection.redis.zpopmin(this.getNotificationsKey(), 1)) && queuedEntry.length) { - hadUpdates = true; + let queuedEntry; + let hadUpdates = false; + while ((queuedEntry = await this.connection.redis.zpopmin(this.getNotificationsKey(), 1)) && queuedEntry.length) { + hadUpdates = true; - let [messageData, uid] = queuedEntry; - uid = Number(uid); - try { - messageData = JSON.parse(messageData); - if (typeof messageData.internalDate === 'string') { - messageData.internalDate = new Date(messageData.internalDate); - } - } catch (err) { - continue; + let [messageData, uid] = queuedEntry; + uid = Number(uid); + try { + messageData = JSON.parse(messageData); + if (typeof messageData.internalDate === 'string') { + messageData.internalDate = new Date(messageData.internalDate); } + } catch (err) { + continue; + } - let canSync = documentStoreEnabled && (!this.connection.syncFrom || messageData.internalDate >= this.connection.syncFrom); + let canSync = documentStoreEnabled && (!this.connection.syncFrom || messageData.internalDate >= this.connection.syncFrom); - if (this.connection.notifyFrom && messageData.internalDate < this.connection.notifyFrom && !canSync) { - // skip too old messages - continue; - } - - await this.processNew(messageData, messageFetchOptions, canSync, storedStatus); + if (this.connection.notifyFrom && messageData.internalDate < this.connection.notifyFrom && !canSync) { + // skip too old messages + continue; } - if (hadUpdates) { - await this.markUpdated(); - } - } finally { - this.connection.syncing = false; - this.syncing = false; - lock.release(); + await this.processNew(messageData, messageFetchOptions, canSync, storedStatus); + } + + if (hadUpdates) { + await this.markUpdated(); } } diff --git a/lib/imapproxy/imap-core/lib/imap-connection.js b/lib/imapproxy/imap-core/lib/imap-connection.js index 23b338a94..9f33a34bf 100644 --- a/lib/imapproxy/imap-core/lib/imap-connection.js +++ b/lib/imapproxy/imap-core/lib/imap-connection.js @@ -21,7 +21,7 @@ const SOCKET_TIMEOUT = 5 * 60 * 1000; * @param {Object} server Server instance * @param {Object} socket Socket instance */ -class IMAPClient extends EventEmitter { +class IMAPConnection extends EventEmitter { constructor(server, socket, options) { super(); @@ -926,4 +926,4 @@ class IMAPClient extends EventEmitter { } // Expose to the world -module.exports.IMAPClient = IMAPClient; +module.exports.IMAPClient = IMAPConnection; diff --git a/lib/imapproxy/imap-core/lib/imap-server.js b/lib/imapproxy/imap-core/lib/imap-server.js index 5be7aa0c1..02f50809f 100644 --- a/lib/imapproxy/imap-core/lib/imap-server.js +++ b/lib/imapproxy/imap-core/lib/imap-server.js @@ -3,7 +3,7 @@ const net = require('net'); const tls = require('tls'); const crypto = require('crypto'); -const IMAPClient = require('./imap-connection').IMAPClient; +const { IMAPConnection } = require('./imap-connection'); const tlsOptions = require('./tls-options'); const EventEmitter = require('events').EventEmitter; const shared = require('nodemailer/lib/shared'); @@ -82,7 +82,7 @@ class IMAPServer extends EventEmitter { } connect(socket, socketOptions) { - let connection = new IMAPClient(this, socket, socketOptions); + let connection = new IMAPConnection(this, socket, socketOptions); connection.loggelf = message => this.loggelf(message); this.connections.add(connection); connection.on('error', this._onError.bind(this)); diff --git a/lib/routes-ui.js b/lib/routes-ui.js index 2d1b5bf7d..e819f6567 100644 --- a/lib/routes-ui.js +++ b/lib/routes-ui.js @@ -123,6 +123,17 @@ const OPEN_AI_MODELS = [ } ]; +const IMAP_INDEXERS = [ + { + id: 'full', + name: 'Full (Default): Builds a comprehensive index that detects new, deleted, and updated emails. This method is slower and uses more storage in Redis.' + }, + { + id: 'fast', + name: 'Fast: Quickly detects newly received emails with minimal storage usage in Redis. It does not detect updated or deleted emails.' + } +]; + const AZURE_CLOUDS = [ { id: 'global', @@ -1207,6 +1218,9 @@ function applyRoutes(server, call) { serviceSecret: (await settings.get('serviceSecret')) || null, queueKeep: (await settings.get('queueKeep')) || 0, deliveryAttempts: await settings.get('deliveryAttempts'), + + imapIndexer: (await settings.get('imapIndexer')) || 'full', + templateHeader: (await settings.get('templateHeader')) || '', templateHtmlHead: (await settings.get('templateHtmlHead')) || '', scriptEnv: (await settings.get('scriptEnv')) || '', @@ -1252,6 +1266,13 @@ function applyRoutes(server, call) { selected: entry.tzCode === values.timezone })), + imapIndexers: structuredClone(IMAP_INDEXERS).map(entry => { + if (entry.id === values.imapIndexer) { + entry.selected = true; + } + return entry; + }), + adminAccessLimit: ADMIN_ACCESS_ADDRESSES && ADMIN_ACCESS_ADDRESSES.length, values @@ -1283,7 +1304,8 @@ function applyRoutes(server, call) { ignoreMailCertErrors: request.payload.ignoreMailCertErrors, locale: request.payload.locale, timezone: request.payload.timezone, - deliveryAttempts: request.payload.deliveryAttempts + deliveryAttempts: request.payload.deliveryAttempts, + imapIndexer: request.payload.imapIndexer }; if (request.payload.serviceUrl) { @@ -1292,6 +1314,13 @@ function applyRoutes(server, call) { } for (let key of Object.keys(data)) { + if (key === 'imapIndexer') { + let existingValue = await settings.get(key); + if ((existingValue && existingValue !== data[key]) || (!existingValue && data[key] !== 'full')) { + await request.flash({ type: 'warning', message: `You may need to restart EmailEngine for indexing changes to take effect` }); + } + } + await settings.set(key, data[key]); } @@ -1317,6 +1346,13 @@ function applyRoutes(server, call) { selected: entry.tzCode === request.payload.timezone })), + imapIndexers: structuredClone(IMAP_INDEXERS).map(entry => { + if (entry.id === request.payload.imapIndexer) { + entry.selected = true; + } + return entry; + }), + adminAccessLimit: ADMIN_ACCESS_ADDRESSES && ADMIN_ACCESS_ADDRESSES.length }, { @@ -1363,6 +1399,13 @@ function applyRoutes(server, call) { selected: entry.tzCode === request.payload.timezone })), + imapIndexers: structuredClone(IMAP_INDEXERS).map(entry => { + if (entry.id === request.payload.imapIndexer) { + entry.selected = true; + } + return entry; + }), + adminAccessLimit: ADMIN_ACCESS_ADDRESSES && ADMIN_ACCESS_ADDRESSES.length, errors @@ -1379,6 +1422,7 @@ function applyRoutes(server, call) { serviceSecret: settingsSchema.serviceSecret, queueKeep: settingsSchema.queueKeep.default(0), deliveryAttempts: settingsSchema.deliveryAttempts.default(DEFAULT_DELIVERY_ATTEMPTS), + imapIndexer: settingsSchema.imapIndexer.default('full'), templateHeader: settingsSchema.templateHeader.default(''), templateHtmlHead: settingsSchema.templateHtmlHead.default(''), scriptEnv: settingsSchema.scriptEnv.default(''), diff --git a/lib/schemas.js b/lib/schemas.js index 69a1a91da..4e4eb5a10 100644 --- a/lib/schemas.js +++ b/lib/schemas.js @@ -80,22 +80,33 @@ const settingsSchema = { trackClicks: Joi.boolean() .truthy('Y', 'true', '1', 'on') .falsy('N', 'false', 0, '') - .description('If true, then rewrite html links in sent emails to track clicks'), + .description('Rewrites HTML links in sent emails to track link clicks when set to true'), trackOpens: Joi.boolean() .truthy('Y', 'true', '1', 'on') .falsy('N', 'false', 0, '') - .description('If true, then add an open tracking beacon image to email html'), + .description('Inserts a tracking beacon image in HTML emails to monitor email opens when set to true'), + + imapIndexer: Joi.string() + .empty('') + .trim() + .valid('full', 'fast') + .example('full') + .description( + 'Sets the indexing method for IMAP accounts. Choose "full" to build a complete index that tracks deleted and updated emails, or "fast" to only detect newly received emails.' + ), resolveGmailCategories: Joi.boolean() .truthy('Y', 'true', '1', 'on') .falsy('N', 'false', 0, '') - .description('If true, then resolve the category tab for incoming emails'), + .description( + 'Enables Gmail category tab detection for incoming emails when set to true. This setting applies only to Gmail IMAP accounts, as Gmail API accounts automatically include category information.' + ), ignoreMailCertErrors: Joi.boolean() .truthy('Y', 'true', '1', 'on') .falsy('N', 'false', 0, '') - .description('If true, then allow insecure certificates for IMAP/SMTP'), + .description('Allows connections with insecure or untrusted certificates when set to true. Applies to both IMAP and SMTP connections.'), generateEmailSummary: Joi.boolean() .truthy('Y', 'true', '1', 'on') diff --git a/views/config/service.hbs b/views/config/service.hbs index 8d83a6bec..5c52cf5d1 100644 --- a/views/config/service.hbs +++ b/views/config/service.hbs @@ -285,6 +285,39 @@ +
+
+
IMAP processing settings
+
+
+
+ +
[imapIndexer]
+ + + + + + + + {{#if errors.imapIndexer}} + {{errors.imapIndexer}} + {{/if}} + Specifies the default indexing method for IMAP accounts. Select + "full" to build a comprehensive index that tracks deleted and updated emails, or + "fast" to detect only newly received emails. You can also customize this setting in each + account's settings to use different indexing methods for different accounts. +
+ +
+
+
Templates
diff --git a/workers/imap.js b/workers/imap.js index 9aa3375b3..81cabd800 100644 --- a/workers/imap.js +++ b/workers/imap.js @@ -203,6 +203,8 @@ class ConnectionHandler { } if (!accountObject.connection) { + let imapIndexer = typeof accountData.imap?.imapIndexer === 'boolean' ? accountData.imap?.indexer : (await settings.get('imapIndexer')) || 'full'; + accountObject.connection = new IMAPClient(account, { runIndex, @@ -211,6 +213,8 @@ class ConnectionHandler { accountLogger, secret, + imapIndexer, + notifyQueue, submitQueue, documentsQueue,