Skip to content

Commit

Permalink
fix(gmail-api): Log API requests to user log
Browse files Browse the repository at this point in the history
  • Loading branch information
@andris9
andris9 committed Jul 21, 2024
1 parent cd12547 commit f00f864
Show file tree
Hide file tree
Showing 6 changed files with 144 additions and 87 deletions.
88 changes: 86 additions & 2 deletions lib/api-client/base-client.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,17 @@ const uuid = require('uuid');
const { addTrackers } = require('../add-trackers');
const { getRawEmail } = require('../get-raw-email');
const { getTemplate } = require('@postalsys/templates');
const { deepEqual } = require('assert');

const { getSignedFormDataSync, getServiceSecret, convertDataUrisToAtachments, genBaseBoundary, getByteSize, readEnvValue } = require('../tools');
const {
getSignedFormDataSync,
getServiceSecret,
convertDataUrisToAtachments,
genBaseBoundary,
getByteSize,
readEnvValue,
emitChangeEvent
} = require('../tools');

const {
ACCOUNT_INITIALIZED_NOTIFY,
Expand Down Expand Up @@ -71,7 +80,8 @@ class BaseClient {
this.flowProducer = this.options.flowProducer;

this.call = this.options.call;
this.logger = this.options.logger || logger;

this.logger = this.getLogger();

this.secret = this.options.secret;

Expand Down Expand Up @@ -143,6 +153,44 @@ class BaseClient {
return 'connected';
}

getLogger() {
this.mainLogger =
this.options.logger ||
logger.child({
component: 'connection-client',
account: this.account,
cid: this.cid
});

let synteticLogger = {};
let levels = ['trace', 'debug', 'info', 'warn', 'error', 'fatal'];
for (let level of levels) {
synteticLogger[level] = (...args) => {
this.mainLogger[level](...args);

if (this.accountLogger.enabled && args && args[0] && typeof args[0] === 'object') {
let entry = Object.assign({ level, t: Date.now(), cid: this.cid }, args[0]);
if (entry.err && typeof entry.err === 'object') {
let err = entry.err;
entry.err = {
stack: err.stack
};
// enumerable error fields
Object.keys(err).forEach(key => {
entry.err[key] = err[key];
});
}

this.accountLogger.log(entry);
}
};
}

synteticLogger.child = opts => this.mainLogger.child(opts);

return synteticLogger;
}

async setStateVal() {
let [[e1], [e2], [e3, prevVal], [e4, incrVal], [e5, stateVal]] = await this.redis
.multi()
Expand All @@ -165,6 +213,42 @@ class BaseClient {
}
}

async setErrorState(event, data) {
let prevLastErrorState = await this.redis.hget(this.getAccountKey(), 'lastErrorState');
if (prevLastErrorState) {
try {
prevLastErrorState = JSON.parse(prevLastErrorState);
} catch (err) {
// ignore
}
}

this.state = event;
await this.setStateVal();

await this.redis.hSetExists(this.getAccountKey(), 'lastErrorState', JSON.stringify(data));

await emitChangeEvent(this.logger, this.account, 'state', event, { error: data });

if (data && Object.keys(data).length && prevLastErrorState) {
// we have an error object, let's see if the error hasn't changed

if (data.serverResponseCode && data.serverResponseCode === prevLastErrorState.serverResponseCode) {
return false;
}

try {
deepEqual(data, prevLastErrorState);
// nothing changed
return false;
} catch (err) {
// seems different, can emit
}
}

return true;
}

async notify(mailbox, event, data, extraOpts) {
extraOpts = extraOpts || {};
const { skipWebhook, canSync = true } = extraOpts;
Expand Down
26 changes: 22 additions & 4 deletions lib/api-client/gmail-client.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,8 @@ const {
MAX_INLINE_ATTACHMENT_SIZE,
MESSAGE_NEW_NOTIFY,
EMAIL_BOUNCE_NOTIFY,
EMAIL_COMPLAINT_NOTIFY
EMAIL_COMPLAINT_NOTIFY,
AUTH_ERROR_NOTIFY
} = require('../consts');

const GMAIL_API_BASE = 'https://gmail.googleapis.com';
Expand Down Expand Up @@ -232,11 +233,28 @@ class GmailClient extends BaseClient {

async init() {
await this.getAccount();
await this.getClient(true);

let accountData = await this.accountObject.loadAccountData(this.account, false);

await this.renewWatch(accountData);

let profileRes = await this.request(`${GMAIL_API_BASE}/gmail/v1/users/me/profile`);
let profileRes;
try {
profileRes = await this.request(`${GMAIL_API_BASE}/gmail/v1/users/me/profile`);
} catch (err) {
this.state = 'authenticationError';
await this.setStateVal();

err.authenticationFailed = true;
await this.notify(false, AUTH_ERROR_NOTIFY, {
response: err.oauthRequest?.response?.message || err.response,
serverResponseCode: 'ApiRequestError'
});

throw err;
}

let historyId = Number(profileRes?.historyId) || null;
if (historyId && accountData.googleHistoryId && historyId > accountData.googleHistoryId) {
// changes detected
Expand Down Expand Up @@ -1214,8 +1232,8 @@ class GmailClient extends BaseClient {
return tokenData.accessToken;
}

async getClient() {
if (this.oAuth2Client) {
async getClient(force) {
if (this.oAuth2Client && !force) {
return this.oAuth2Client;
}
let accountData = await this.accountObject.loadAccountData(this.account, false);
Expand Down
77 changes: 0 additions & 77 deletions lib/imap-connection.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ const msgpack = require('msgpack5')();
const nodemailer = require('nodemailer');
const util = require('util');
const { removeBcc } = require('./get-raw-email');
const { deepEqual } = require('assert');
const socks = require('socks');
const { Gateway } = require('./gateway');

Expand Down Expand Up @@ -75,8 +74,6 @@ class IMAPConnection extends BaseClient {
this.isClosing = false;
this.isClosed = false;

this.logger = this.getLogger();

this.imapConfig = {
// Set emitLogs to true if you want to get all the log entries as objects from the IMAP module
logger: this.mainLogger.child({
Expand Down Expand Up @@ -1213,42 +1210,6 @@ class IMAPConnection extends BaseClient {
await this.reconnect();
}

async setErrorState(event, data) {
let prevLastErrorState = await this.redis.hget(this.getAccountKey(), 'lastErrorState');
if (prevLastErrorState) {
try {
prevLastErrorState = JSON.parse(prevLastErrorState);
} catch (err) {
// ignore
}
}

this.state = event;
await this.setStateVal();

await this.redis.hSetExists(this.getAccountKey(), 'lastErrorState', JSON.stringify(data));

await emitChangeEvent(this.logger, this.account, 'state', event, { error: data });

if (data && Object.keys(data).length && prevLastErrorState) {
// we have an error object, let's see if the error hasn't changed

if (data.serverResponseCode && data.serverResponseCode === prevLastErrorState.serverResponseCode) {
return false;
}

try {
deepEqual(data, prevLastErrorState);
// nothing changed
return false;
} catch (err) {
// seems different, can emit
}
}

return true;
}

async delete() {
if (this.isClosed || this.isClosing) {
return;
Expand Down Expand Up @@ -2605,44 +2566,6 @@ class IMAPConnection extends BaseClient {
}
}

getLogger() {
this.mainLogger =
this.options.logger ||
logger.child({
component: 'imap-client',
account: this.account,
cid: this.cid
});

let synteticLogger = {};
let levels = ['trace', 'debug', 'info', 'warn', 'error', 'fatal'];
for (let level of levels) {
synteticLogger[level] = (...args) => {
this.mainLogger[level](...args);

if (this.accountLogger.enabled && args && args[0] && typeof args[0] === 'object') {
let entry = Object.assign({ level, t: Date.now(), cid: this.cid }, args[0]);
if (entry.err && typeof entry.err === 'object') {
let err = entry.err;
entry.err = {
stack: err.stack
};
// enumerable error fields
Object.keys(err).forEach(key => {
entry.err[key] = err[key];
});
}

this.accountLogger.log(entry);
}
};
}

synteticLogger.child = opts => this.mainLogger.child(opts);

return synteticLogger;
}

async pause() {
if (this.paused) {
return false;
Expand Down
32 changes: 31 additions & 1 deletion lib/oauth/gmail.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -441,8 +441,12 @@ class GmailOauth {
url = parsedUrl.href;
}

let startTime = Date.now();

let res = await fetchCmd(url, reqData);

let reqTime = Date.now() - startTime;

if (!res.ok) {
let err = new Error('OAuth2 request failed');
err.oauthRequest = {
Expand All @@ -454,8 +458,10 @@ class GmailOauth {
serviceClient: this.serviceClient,
googleProjectId: this.googleProjectId,
serviceClientEmail: this.serviceClientEmail,
scopes: this.scopes
scopes: this.scopes,
reqTime
};

try {
err.oauthRequest.response = await res.json();

Expand All @@ -471,10 +477,34 @@ class GmailOauth {
}
} catch (err) {
// ignore
} finally {
if (this.logger) {
this.logger.error(Object.assign({ msg: 'API request failed' }, err.oauthRequest));
}
}

throw err;
}

if (this.logger) {
this.logger.info(
Object.assign(
{ msg: 'API request completed' },
{
url,
method,
provider: this.provider,
status: res.status,
clientId: this.clientId,
serviceClient: this.serviceClient,
googleProjectId: this.googleProjectId,
serviceClientEmail: this.serviceClientEmail,
reqTime
}
)
);
}

// clear potential auth flag
await this.setFlag();

Expand Down
5 changes: 3 additions & 2 deletions lib/oauth/pubsub/google.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,19 +93,20 @@ class PubSubInstance {
try {
let start = Date.now();

logger.debug({ msg: 'Pulling subscription messages', source: 'google', app: this.app });
let pullRes = await this.client.request(accessToken, pullUrl, 'POST', { returnImmediately: false, maxMessages: 100 });
if (this.stopped) {
// ignore if stopped
return;
}

let reqTime = Date.now() - start;

logger.debug({
msg: 'Pulled subscription messages',
source: 'google',
app: this.app,
messages: pullRes?.receivedMessages?.length || 0,
delay: Date.now() - start
reqTime
});

for (let receivedMessage of pullRes?.receivedMessages || []) {
Expand Down
3 changes: 2 additions & 1 deletion workers/imap.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -169,7 +169,8 @@ class ConnectionHandler {

call: msg => this.call(msg)
});
accountData.state = 'connected';
accountData.state = 'connecting';
accountObject.logger = accountObject.connection.logger;
}
}

Expand Down

0 comments on commit f00f864

Please sign in to comment.