feat(imap): Added new IMAP indexing option: 'fast'

lib/email-client/imap-client.js

@@ -69,6 +69,7 @@ async function metricsMeta(meta, logger, key, method, ...args) {
 
 class IMAPClient extends BaseClient {
     constructor(account, options) {
+        options = options || {};
         super(account, options);
 
         this.isClosing = false;
@@ -114,6 +115,8 @@ class IMAPClient extends BaseClient {
         this.connectionCount = 0;
         this.connections = new Set();
 
+        this.imapIndexer = options?.imapIndexer;
+
         this.state = 'connecting';
     }
 

lib/imapproxy/imap-core/lib/imap-connection.js

@@ -21,7 +21,7 @@ const SOCKET_TIMEOUT = 5 * 60 * 1000;
  * @param {Object} server Server instance
  * @param {Object} socket Socket instance
  */
-class IMAPClient extends EventEmitter {
+class IMAPConnection extends EventEmitter {
     constructor(server, socket, options) {
         super();
 
@@ -926,4 +926,4 @@ class IMAPClient extends EventEmitter {
 }
 
 // Expose to the world
-module.exports.IMAPClient = IMAPClient;
+module.exports.IMAPClient = IMAPConnection;

lib/imapproxy/imap-core/lib/imap-server.js

@@ -3,7 +3,7 @@
 const net = require('net');
 const tls = require('tls');
 const crypto = require('crypto');
-const IMAPClient = require('./imap-connection').IMAPClient;
+const { IMAPConnection } = require('./imap-connection');
 const tlsOptions = require('./tls-options');
 const EventEmitter = require('events').EventEmitter;
 const shared = require('nodemailer/lib/shared');
@@ -82,7 +82,7 @@ class IMAPServer extends EventEmitter {
     }
 
     connect(socket, socketOptions) {
-        let connection = new IMAPClient(this, socket, socketOptions);
+        let connection = new IMAPConnection(this, socket, socketOptions);
         connection.loggelf = message => this.loggelf(message);
         this.connections.add(connection);
         connection.on('error', this._onError.bind(this));

lib/routes-ui.js

@@ -123,6 +123,17 @@ const OPEN_AI_MODELS = [
     }
 ];
 
+const IMAP_INDEXERS = [
+    {
+        id: 'full',
+        name: 'Full (Default): Builds a comprehensive index that detects new, deleted, and updated emails. This method is slower and uses more storage in Redis.'
+    },
+    {
+        id: 'fast',
+        name: 'Fast: Quickly detects newly received emails with minimal storage usage in Redis. It does not detect updated or deleted emails.'
+    }
+];
+
 const AZURE_CLOUDS = [
     {
         id: 'global',
@@ -1207,6 +1218,9 @@ function applyRoutes(server, call) {
                 serviceSecret: (await settings.get('serviceSecret')) || null,
                 queueKeep: (await settings.get('queueKeep')) || 0,
                 deliveryAttempts: await settings.get('deliveryAttempts'),
+
+                imapIndexer: (await settings.get('imapIndexer')) || 'full',
+
                 templateHeader: (await settings.get('templateHeader')) || '',
                 templateHtmlHead: (await settings.get('templateHtmlHead')) || '',
                 scriptEnv: (await settings.get('scriptEnv')) || '',
@@ -1252,6 +1266,13 @@ function applyRoutes(server, call) {
                         selected: entry.tzCode === values.timezone
                     })),
 
+                    imapIndexers: structuredClone(IMAP_INDEXERS).map(entry => {
+                        if (entry.id === values.imapIndexer) {
+                            entry.selected = true;
+                        }
+                        return entry;
+                    }),
+
                     adminAccessLimit: ADMIN_ACCESS_ADDRESSES && ADMIN_ACCESS_ADDRESSES.length,
 
                     values
@@ -1283,7 +1304,8 @@ function applyRoutes(server, call) {
                     ignoreMailCertErrors: request.payload.ignoreMailCertErrors,
                     locale: request.payload.locale,
                     timezone: request.payload.timezone,
-                    deliveryAttempts: request.payload.deliveryAttempts
+                    deliveryAttempts: request.payload.deliveryAttempts,
+                    imapIndexer: request.payload.imapIndexer
                 };
 
                 if (request.payload.serviceUrl) {
@@ -1292,6 +1314,13 @@ function applyRoutes(server, call) {
                 }
 
                 for (let key of Object.keys(data)) {
+                    if (key === 'imapIndexer') {
+                        let existingValue = await settings.get(key);
+                        if ((existingValue && existingValue !== data[key]) || (!existingValue && data[key] !== 'full')) {
+                            await request.flash({ type: 'warning', message: `You may need to restart EmailEngine for indexing changes to take effect` });
+                        }
+                    }
+
                     await settings.set(key, data[key]);
                 }
 
@@ -1317,6 +1346,13 @@ function applyRoutes(server, call) {
                             selected: entry.tzCode === request.payload.timezone
                         })),
 
+                        imapIndexers: structuredClone(IMAP_INDEXERS).map(entry => {
+                            if (entry.id === request.payload.imapIndexer) {
+                                entry.selected = true;
+                            }
+                            return entry;
+                        }),
+
                         adminAccessLimit: ADMIN_ACCESS_ADDRESSES && ADMIN_ACCESS_ADDRESSES.length
                     },
                     {
@@ -1363,6 +1399,13 @@ function applyRoutes(server, call) {
                                     selected: entry.tzCode === request.payload.timezone
                                 })),
 
+                                imapIndexers: structuredClone(IMAP_INDEXERS).map(entry => {
+                                    if (entry.id === request.payload.imapIndexer) {
+                                        entry.selected = true;
+                                    }
+                                    return entry;
+                                }),
+
                                 adminAccessLimit: ADMIN_ACCESS_ADDRESSES && ADMIN_ACCESS_ADDRESSES.length,
 
                                 errors
@@ -1379,6 +1422,7 @@ function applyRoutes(server, call) {
                     serviceSecret: settingsSchema.serviceSecret,
                     queueKeep: settingsSchema.queueKeep.default(0),
                     deliveryAttempts: settingsSchema.deliveryAttempts.default(DEFAULT_DELIVERY_ATTEMPTS),
+                    imapIndexer: settingsSchema.imapIndexer.default('full'),
                     templateHeader: settingsSchema.templateHeader.default(''),
                     templateHtmlHead: settingsSchema.templateHtmlHead.default(''),
                     scriptEnv: settingsSchema.scriptEnv.default(''),

lib/schemas.js

@@ -80,22 +80,33 @@ const settingsSchema = {
     trackClicks: Joi.boolean()
         .truthy('Y', 'true', '1', 'on')
         .falsy('N', 'false', 0, '')
-        .description('If true, then rewrite html links in sent emails to track clicks'),
+        .description('Rewrites HTML links in sent emails to track link clicks when set to true'),
 
     trackOpens: Joi.boolean()
         .truthy('Y', 'true', '1', 'on')
         .falsy('N', 'false', 0, '')
-        .description('If true, then add an open tracking beacon image to email html'),
+        .description('Inserts a tracking beacon image in HTML emails to monitor email opens when set to true'),
+
+    imapIndexer: Joi.string()
+        .empty('')
+        .trim()
+        .valid('full', 'fast')
+        .example('full')
+        .description(
+            'Sets the indexing method for IMAP accounts. Choose "full" to build a complete index that tracks deleted and updated emails, or "fast" to only detect newly received emails.'
+        ),
 
     resolveGmailCategories: Joi.boolean()
         .truthy('Y', 'true', '1', 'on')
         .falsy('N', 'false', 0, '')
-        .description('If true, then resolve the category tab for incoming emails'),
+        .description(
+            'Enables Gmail category tab detection for incoming emails when set to true. This setting applies only to Gmail IMAP accounts, as Gmail API accounts automatically include category information.'
+        ),
 
     ignoreMailCertErrors: Joi.boolean()
         .truthy('Y', 'true', '1', 'on')
         .falsy('N', 'false', 0, '')
-        .description('If true, then allow insecure certificates for IMAP/SMTP'),
+        .description('Allows connections with insecure or untrusted certificates when set to true. Applies to both IMAP and SMTP connections.'),
 
     generateEmailSummary: Joi.boolean()
         .truthy('Y', 'true', '1', 'on')

views/config/service.hbs

@@ -285,6 +285,39 @@
         </div>
     </div>
 
+    <div id="submission_settings" class="card mb-4">
+        <div class="card-header py-3">
+            <h6 class="m-0 font-weight-bold text-primary">IMAP processing settings</h6>
+        </div>
+        <div class="card-body">
+            <div class="form-group">
+
+                <div class="text-muted float-right code-link">[<a href="/admin/swagger#/Settings/postV1Settings"
+                        target="_blank" rel="noopener noreferrer">imapIndexer</a>]</div>
+
+                <label for="settingsImapIndexer">Indexing Method for IMAP Accounts</label>
+
+
+                <select id="settingsImapIndexer" class="custom-select custom-select-sm" name="imapIndexer" {{#if
+                    errors.imapIndexer}}is-invalid{{/if}}>
+                    {{#each imapIndexers}}
+                    <option value="{{id}}" {{#if selected}}selected{{/if}}>{{name}}</option>
+                    {{/each}}
+                </select>
+
+
+                {{#if errors.imapIndexer}}
+                <span class="invalid-feedback">{{errors.imapIndexer}}</span>
+                {{/if}}
+                <small class="form-text text-muted">Specifies the default indexing method for IMAP accounts. Select
+                    <em>"full"</em> to build a comprehensive index that tracks deleted and updated emails, or
+                    <em>"fast"</em> to detect only newly received emails. You can also customize this setting in each
+                    account's settings to use different indexing methods for different accounts.</small>
+            </div>
+
+        </div>
+    </div>
+
     <div id="templates_settings" class="card mb-4">
         <div class="card-header py-3 d-flex flex-row align-items-center justify-content-between">
             <h6 class="m-0 font-weight-bold text-primary">Templates</h6>

workers/imap.js

@@ -203,6 +203,8 @@ class ConnectionHandler {
         }
 
         if (!accountObject.connection) {
+            let imapIndexer = typeof accountData.imap?.imapIndexer === 'boolean' ? accountData.imap?.indexer : (await settings.get('imapIndexer')) || 'full';
+
             accountObject.connection = new IMAPClient(account, {
                 runIndex,
 
@@ -211,6 +213,8 @@ class ConnectionHandler {
                 accountLogger,
                 secret,
 
+                imapIndexer,
+
                 notifyQueue,
                 submitQueue,
                 documentsQueue,