fix(ms-graph-api): Upload message as a JSON structure instead of EML …

…with MS Graph API in order to set meta info like flags

.ncurc.js

@@ -9,6 +9,7 @@ module.exports = {
         'js-beautify',
         'ical.js',
         '@elastic/elasticsearch',
+        'escape-string-regexp',
 
         // api changes, check and fix
         'eslint',

lib/email-client/base-client.js

@@ -742,7 +742,9 @@ class BaseClient {
         return true;
     }
 
-    async prepareRawMessage(data, connectionOptions) {
+    async prepareRawMessage(data, options, connectionOptions) {
+        options = options || {};
+
         data.disableFileAccess = true;
         data.disableUrlAccess = true;
         data.boundaryPrefix = MIME_BOUNDARY_PREFIX;
@@ -1018,9 +1020,9 @@ class BaseClient {
             data.date = new Date(data.internalDate);
         }
 
-        let { raw, messageId } = await getRawEmail(data);
+        let { raw, emailObject, messageId } = await getRawEmail(data, null, options);
 
-        return { raw, messageId, documentStoreUsed, referencedMessage };
+        return { raw, emailObject, messageId, documentStoreUsed, referencedMessage };
     }
 
     async queueMessageEntry(data, meta, licenseInfo, connectionOptions) {

lib/email-client/imap-client.js

@@ -1798,7 +1798,7 @@ class IMAPClient extends BaseClient {
 
         const connectionClient = await this.getImapConnection(connectionOptions, 'uploadMessage');
 
-        let { raw, messageId, documentStoreUsed, referencedMessage } = await this.prepareRawMessage(data, { connectionClient });
+        let { raw, messageId, documentStoreUsed, referencedMessage } = await this.prepareRawMessage(data, null, { connectionClient });
 
         // Upload message to selected folder
         try {

lib/email-client/outlook-client.js

@@ -11,6 +11,8 @@ const { emitChangeEvent } = require('../tools');
 const { mimeHtml } = require('@postalsys/email-text-tools');
 const crypto = require('crypto');
 const { Gateway } = require('../gateway');
+const { detectMimeType, detectExtension } = require('nodemailer/lib/mime-funcs/mime-types');
+const escapeStringRegexp = require('escape-string-regexp');
 
 const {
     REDIS_PREFIX,
@@ -1274,6 +1276,151 @@ class OutlookClient extends BaseClient {
         return response;
     }
 
+    convertMessageToUploadObject(emailObject) {
+        let messageUploadObj = {};
+
+        for (let key of Object.keys(emailObject)) {
+            switch (key) {
+                case 'from':
+                    messageUploadObj.from = {
+                        emailAddress: {
+                            address: emailObject.from.address
+                        }
+                    };
+                    if (emailObject.from.name) {
+                        messageUploadObj.from.emailAddress.name = emailObject.from.name;
+                    }
+                    break;
+
+                case 'to':
+                case 'cc':
+                case 'bcc': {
+                    let entryKey = `${key}Recipients`;
+                    messageUploadObj[entryKey] = [];
+                    for (let addressEntry of emailObject[key] || []) {
+                        let addressObj = {
+                            emailAddress: {
+                                address: addressEntry.address
+                            }
+                        };
+                        if (addressEntry.name) {
+                            addressObj.emailAddress.name = addressEntry.name;
+                        }
+                        messageUploadObj[entryKey].push(addressObj);
+                    }
+                    break;
+                }
+
+                case 'subject':
+                    messageUploadObj[key] = emailObject[key];
+                    break;
+
+                case 'headers': {
+                    messageUploadObj.internetMessageHeaders = messageUploadObj.internetMessageHeaders || [];
+                    for (let header of Object.keys(emailObject.headers)) {
+                        messageUploadObj.internetMessageHeaders.push({
+                            name: header.toLowerCase(),
+                            value: emailObject.headers[header]
+                        });
+                    }
+                    break;
+                }
+
+                case 'headerLines': {
+                    messageUploadObj.internetMessageHeaders = messageUploadObj.internetMessageHeaders || [];
+                    for (let i = emailObject.headerLines.length - 1; i >= 0; i--) {
+                        let header = emailObject.headerLines[i];
+                        if (
+                            [
+                                'date',
+                                'content-transfer-encoding',
+                                'from',
+                                'to',
+                                'cc',
+                                'bcc',
+                                'subject',
+                                'mime-version',
+                                'content-type',
+                                'content-disposition',
+                                'message-id',
+                                'content-id'
+                            ].includes(header.key) ||
+                            // MS Graph API only allows up to 5 custom headers
+                            messageUploadObj.internetMessageHeaders.length >= 5
+                        ) {
+                            continue;
+                        }
+
+                        let name = header.key;
+                        let value = header.value ? header.substring(header.value.indexOf(':') + 1).trim() : '';
+                        if (name && value) {
+                            messageUploadObj.internetMessageHeaders.unshift({
+                                name,
+                                value
+                            });
+                        }
+                    }
+                    break;
+                }
+
+                case 'messageId':
+                    messageUploadObj.internetMessageId = emailObject.messageId;
+                    break;
+
+                case 'attachments': {
+                    messageUploadObj.attachments = [];
+                    let attachmentCounter = 0;
+                    for (let attachment of emailObject.attachments) {
+                        let attachmentEntry = {
+                            '@odata.type': '#microsoft.graph.fileAttachment'
+                        };
+                        if (attachment.filename) {
+                            attachmentEntry.name = attachment.filename;
+                        } else {
+                            // generate a filename based on contentType as name is a required value
+                            let ext = detectExtension(attachment.contentType);
+                            attachmentEntry.name = `attachment_${++attachmentCounter}.${ext}`;
+                        }
+
+                        attachmentEntry.contentType = attachment.contentType || detectMimeType(attachment.filename) || 'application/octet-stream';
+                        attachmentEntry.contentBytes = attachment.content;
+                        if (attachment.cid) {
+                            // make sure that cid links to not use <content-id> format, otherwise this will be replaced
+                            attachmentEntry.contentId = attachment.cid.replace(/^[\s<]*|[\s>]*$/g, '');
+                            if (emailObject.html?.indexOf(attachmentEntry.contentId) >= 0) {
+                                attachmentEntry.isInline = true;
+                                emailObject.html.replace(new RegExp(`cid:<${attachment.cid}>`, 'g'), `cid:${attachment.cid}`);
+                            }
+                        }
+                        if (attachment.contentDisposition === 'inline') {
+                            attachmentEntry.isInline = true;
+                        }
+                        messageUploadObj.attachments.push(attachmentEntry);
+                    }
+                    break;
+                }
+            }
+        }
+
+        if (emailObject.html) {
+            messageUploadObj.body = {
+                contentType: 'html',
+                content: emailObject.html
+            };
+        } else if (emailObject.text) {
+            messageUploadObj.body = {
+                contentType: 'text',
+                content: emailObject.text
+            };
+        }
+
+        if (messageUploadObj.internetMessageHeaders && !messageUploadObj.internetMessageHeaders.length) {
+            delete messageUploadObj.internetMessageHeaders;
+        }
+
+        return messageUploadObj;
+    }
+
     async uploadMessage(data) {
         await this.prepare();
 
@@ -1290,17 +1437,44 @@ class OutlookClient extends BaseClient {
             throw error;
         }
 
-        let { raw, messageId, referencedMessage, documentStoreUsed } = await this.prepareRawMessage(data);
-        if (raw?.buffer) {
-            // convert from a Uint8Array to a Buffer
-            raw = Buffer.from(raw);
+        let { emailObject, messageId, referencedMessage, documentStoreUsed } = await this.prepareRawMessage(data, {
+            returnObject: true
+        });
+
+        let messageUploadObj = this.convertMessageToUploadObject(emailObject);
+
+        messageUploadObj.singleValueExtendedProperties = [];
+
+        // https://learn.microsoft.com/en-us/office/client-developer/outlook/mapi/pidtagmessageflags-canonical-property
+        // PR_MESSAGE_FLAGS
+        if (data.flags) {
+            let flagValue = 0;
+
+            for (let flag of data.flags || []) {
+                switch (flag) {
+                    case '\\Seen': // mfRead, MSGFLAG_READ
+                        flagValue |= 0x0001; // eslint-disable-line no-bitwise
+                        break;
+                    case '\\Draft': // mfUnsent, MSGFLAG_UNSENT
+                        flagValue |= 0x0008; // eslint-disable-line no-bitwise
+                        break;
+                }
+            }
+
+            messageUploadObj.singleValueExtendedProperties.push({ id: 'Integer 0x0E07', value: flagValue.toString(10) });
+        } else {
+            messageUploadObj.singleValueExtendedProperties.push({ id: 'Integer 0x0E07', value: '0' });
+        }
+
+        // https://learn.microsoft.com/en-us/office/client-developer/outlook/mapi/pidtagmessagedeliverytime-canonical-property
+        //PR_MESSAGE_DELIVERY_TIME
+        if (data.internalDate) {
+            messageUploadObj.singleValueExtendedProperties.push({ id: 'SystemTime 0x0E06', value: data.internalDate.toISOString() });
         }
 
         let messageData;
         try {
-            messageData = await this.request(`/${this.oauth2UserPath}/messages`, 'post', Buffer.from(raw.toString('base64')), {
-                contentType: 'text/plain'
-            });
+            messageData = await this.request(`/${this.oauth2UserPath}/mailFolders/${targetFolder.id}/messages`, 'post', messageUploadObj);
         } catch (err) {
             switch (err.oauthRequest?.status) {
                 case 400: {
@@ -1323,29 +1497,10 @@ class OutlookClient extends BaseClient {
             }
         }
 
-        if (messageData && messageData.parentFolderId !== targetFolder.id) {
-            // move from Drafts to actual folder
-            try {
-                messageData = await this.request(`/${this.oauth2UserPath}/messages/${messageData.id}/move`, 'post', {
-                    destinationId: targetFolder.id
-                });
-                if (!messageData) {
-                    throw new Error('Failed to move message to target folder');
-                }
-            } catch (err) {
-                this.logger.error({
-                    msg: 'Failed to move message to target folder',
-                    messageId,
-                    err
-                });
-                throw err;
-            }
-        }
-
         let response = {
             message: messageData?.id,
             path: targetFolder.pathName,
-            messageId
+            messageId: messageData?.internetMessageId || messageId
         };
 
         if (data.reference && data.reference.message) {
@@ -2727,6 +2882,8 @@ class OutlookClient extends BaseClient {
 
         return encodedToken;
     }
+
+    async convertRawToMessage(raw) {}
 }
 
 module.exports = { OutlookClient };