Skip to content

Commit

Permalink
fix(http-requests): Use undici RetryAgent for HTTP request network er…
Browse files Browse the repository at this point in the history 
…rors and 429 rate limiting, removed custom 429 handler
  • Loading branch information
@andris9
andris9 committed Jan 8, 2025
1 parent 1174a29 commit bafcd1c
Show file tree
Hide file tree
Showing 17 changed files with 157 additions and 181 deletions.
1 change: 0 additions & 1 deletion .ncurc.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ module.exports = {
'js-beautify',
'ical.js',
'@elastic/elasticsearch',
'escape-string-regexp',

// api changes, check and fix
'eslint',
Expand Down
13 changes: 6 additions & 7 deletions lib/autodetect-imap-settings.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,8 @@ const packageData = require('../package.json');
const parseXml = util.promisify(parseXmlCb);
const { gt } = require('./translations');

const { FETCH_TIMEOUT } = require('./consts');
const { fetch: fetchCmd, Agent } = require('undici');
const fetchAgent = new Agent({ connect: { timeout: FETCH_TIMEOUT } });
const { fetch: fetchCmd } = require('undici');
const { retryAgent } = require('./tools');

const RESOLV_TIMEOUT = 5 * 1000;

Expand Down Expand Up @@ -175,7 +174,7 @@ async function resolveUsingMozillaDirectory(email, domain, source) {
headers: {
'User-Agent': `${packageData.name}/${packageData.version} (+${packageData.homepage})`
},
dispatcher: fetchAgent
dispatcher: retryAgent
});

if (!res.ok) {
Expand All @@ -200,7 +199,7 @@ async function resolveUsingAutoconfig(email, domain, source) {
headers: {
'User-Agent': `${packageData.name}/${packageData.version} (+${packageData.homepage})`
},
dispatcher: fetchAgent
dispatcher: retryAgent
});

if (!res.ok) {
Expand All @@ -225,7 +224,7 @@ async function resolveUsingWellKnown(email, domain, source) {
headers: {
'User-Agent': `${packageData.name}/${packageData.version} (+${packageData.homepage})`
},
dispatcher: fetchAgent
dispatcher: retryAgent
});

if (!res.ok) {
Expand Down Expand Up @@ -549,7 +548,7 @@ async function resolveUsingAutodiscovery(email, domain, source) {
'Content-type': 'application/xml'
},
body,
dispatcher: fetchAgent
dispatcher: retryAgent
});

if (!res.ok) {
Expand Down
3 changes: 2 additions & 1 deletion lib/consts.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,8 @@ module.exports = {
ciphers: (process.env.EENGINE_TLS_CIPHERS || '').toString().trim() || 'DEFAULT@SECLEVEL=0'
},

FETCH_TIMEOUT: Number((process.env.EENGINE_FETCH_TIMEOUT || '').toString().trim()) || 90 * 1000,
URL_FETCH_TIMEOUT: Number((process.env.EENGINE_FETCH_TIMEOUT || '').toString().trim()) || 90 * 1000,
URL_FETCH_RETRY_MAX: 5,

// hard limit for subscript execution (does not include waiting for promises)
SUBSCRIPT_RUNTIME_TIMEOUT: 30 * 1000,
Expand Down
3 changes: 0 additions & 3 deletions lib/email-client/outlook-client.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@ const { mimeHtml } = require('@postalsys/email-text-tools');
const crypto = require('crypto');
const { Gateway } = require('../gateway');
const { detectMimeType, detectExtension } = require('nodemailer/lib/mime-funcs/mime-types');
const escapeStringRegexp = require('escape-string-regexp');

const {
REDIS_PREFIX,
Expand Down Expand Up @@ -2882,8 +2881,6 @@ class OutlookClient extends BaseClient {

return encodedToken;
}

async convertRawToMessage(raw) {}
}

module.exports = { OutlookClient };
13 changes: 5 additions & 8 deletions lib/oauth/gmail.js
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,10 @@
'use strict';

const packageData = require('../../package.json');
const { formatPartialSecretKey, structuredClone } = require('../tools');
const { formatPartialSecretKey, structuredClone, retryAgent } = require('../tools');
const crypto = require('crypto');

const { FETCH_TIMEOUT } = require('../consts');

const { fetch: fetchCmd, Agent } = require('undici');
const fetchAgent = new Agent({ connect: { timeout: FETCH_TIMEOUT } });
const { fetch: fetchCmd } = require('undici');

const GMAIL_SCOPES = {
imap: ['https://mail.google.com/'],
Expand Down Expand Up @@ -239,7 +236,7 @@ class GmailOauth {
let res = await fetchCmd(
requestUrl,
Object.assign(fetchOpts, {
dispatcher: fetchAgent
dispatcher: retryAgent
})
);

Expand Down Expand Up @@ -349,7 +346,7 @@ class GmailOauth {
let res = await fetchCmd(
requestUrl,
Object.assign(fetchOpts, {
dispatcher: fetchAgent
dispatcher: retryAgent
})
);

Expand Down Expand Up @@ -431,7 +428,7 @@ class GmailOauth {
Authorization: `Bearer ${accessToken}`,
'User-Agent': `${packageData.name}/${packageData.version} (+${packageData.homepage})`
},
dispatcher: fetchAgent
dispatcher: retryAgent
};

if (payload && method !== 'get') {
Expand Down
14 changes: 5 additions & 9 deletions lib/oauth/mail-ru.js
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,8 @@
'use strict';

const packageData = require('../../package.json');
const { fetch: fetchCmd, Agent } = require('undici');
const { formatPartialSecretKey, structuredClone } = require('../tools');

const { FETCH_TIMEOUT } = require('../consts');

const fetchAgent = new Agent({ connect: { timeout: FETCH_TIMEOUT } });
const { fetch: fetchCmd } = require('undici');
const { formatPartialSecretKey, structuredClone, retryAgent } = require('../tools');

const MAIL_RU_SCOPES = ['userinfo', 'mail.imap'];

Expand Down Expand Up @@ -116,7 +112,7 @@ class MailRuOauth {
Authorization: `Basic ${Buffer.from(`${this.clientId}:${this.clientSecret}`).toString('base64')}`
},
body: tokenRequest.body,
dispatcher: fetchAgent
dispatcher: retryAgent
});

let responseJson;
Expand Down Expand Up @@ -197,7 +193,7 @@ class MailRuOauth {
'User-Agent': `${packageData.name}/${packageData.version} (+${packageData.homepage})`
},
body: url.searchParams,
dispatcher: fetchAgent
dispatcher: retryAgent
});

let responseJson;
Expand Down Expand Up @@ -263,7 +259,7 @@ class MailRuOauth {
headers: {
'User-Agent': `${packageData.name}/${packageData.version} (+${packageData.homepage})`
},
dispatcher: fetchAgent
dispatcher: retryAgent
};

if (payload) {
Expand Down
35 changes: 6 additions & 29 deletions lib/oauth/outlook.js
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,9 @@
'use strict';

const packageData = require('../../package.json');
const { formatPartialSecretKey, structuredClone } = require('../tools');
const { formatPartialSecretKey, structuredClone, retryAgent } = require('../tools');

const { FETCH_TIMEOUT } = require('../consts');
const MAX_THROTTLING_RETRIES = 5;

const { fetch: fetchCmd, Agent } = require('undici');
const fetchAgent = new Agent({ connect: { timeout: FETCH_TIMEOUT } });
const { fetch: fetchCmd } = require('undici');

const outlookScopes = cloud => {
switch (cloud) {
Expand Down Expand Up @@ -246,7 +242,7 @@ class OutlookOauth {
'User-Agent': `${packageData.name}/${packageData.version} (+${packageData.homepage})`
},
body: tokenRequest.body,
dispatcher: fetchAgent
dispatcher: retryAgent
});

let responseJson;
Expand Down Expand Up @@ -338,7 +334,7 @@ class OutlookOauth {
'User-Agent': `${packageData.name}/${packageData.version} (+${packageData.homepage})`
},
body: url.searchParams,
dispatcher: fetchAgent
dispatcher: retryAgent
});

let responseJson;
Expand Down Expand Up @@ -417,7 +413,7 @@ class OutlookOauth {
Authorization: `Bearer ${accessToken}`,
'User-Agent': `${packageData.name}/${packageData.version} (+${packageData.homepage})`
},
dispatcher: fetchAgent
dispatcher: retryAgent
};

if (options.headers) {
Expand Down Expand Up @@ -447,26 +443,7 @@ class OutlookOauth {
let retryCount = 0;

let startTime = Date.now();

let res;

while (++retryCount < MAX_THROTTLING_RETRIES) {
res = await fetchCmd(url, reqData);

let retryAfter;
if (res.status === 429 && res.headers.has('retry-after')) {
retryAfter = !isNaN(res.headers.has('retry-after')) ? Number(res.headers.has('retry-after')) * 1000 : undefined;
if (retryAfter && retryAfter > 0) {
if (this.logger) {
this.logger.error({ msg: 'API request was throttled, retrying', reqData, retryCount, retryAfter });
}
await new Promise(r => setTimeout(r, retryAfter));
continue;
}
}
break;
}

let res = await fetchCmd(url, reqData);
let reqTime = Date.now() - startTime;

if (!res.ok) {
Expand Down
15 changes: 7 additions & 8 deletions lib/routes-ui.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,8 @@ const {
getDuration,
parseSignedFormData,
updatePublicInterfaces,
hasEnvValue
hasEnvValue,
retryAgent
} = require('./tools');
const packageData = require('../package.json');
const he = require('he');
Expand Down Expand Up @@ -77,7 +78,6 @@ const {
DEFAULT_PAGE_SIZE,
REDIS_PREFIX,
TOTP_WINDOW_SIZE,
FETCH_TIMEOUT,
DEFAULT_DELIVERY_ATTEMPTS,
DEFAULT_MAX_BODY_SIZE,
DEFAULT_MAX_PAYLOAD_TIMEOUT,
Expand All @@ -102,8 +102,7 @@ const ADMIN_ACCESS_ADDRESSES = hasEnvValue('EENGINE_ADMIN_ACCESS_ADDRESSES')
.filter(v => v)
: null;

const { fetch: fetchCmd, Agent } = require('undici');
const fetchAgent = new Agent({ connect: { timeout: FETCH_TIMEOUT } });
const { fetch: fetchCmd } = require('undici');

const LICENSE_HOST = 'https://postalsys.com';
const SMTP_TEST_HOST = 'https://api.nodemailer.com';
Expand Down Expand Up @@ -2187,7 +2186,7 @@ return true;`
}
}),
headers,
dispatcher: fetchAgent
dispatcher: retryAgent
});
duration = Date.now() - start;
} catch (err) {
Expand Down Expand Up @@ -5487,7 +5486,7 @@ return payload;`)
url: (await settings.get('serviceUrl')) || ''
}),
headers,
dispatcher: fetchAgent
dispatcher: retryAgent
});

if (!res.ok) {
Expand Down Expand Up @@ -9313,7 +9312,7 @@ Token: ${JSON.stringify(request.params.token)}`
requestor: '@postalsys/emailengine-app'
}),
headers,
dispatcher: fetchAgent
dispatcher: retryAgent
});

if (!res.ok) {
Expand Down Expand Up @@ -9424,7 +9423,7 @@ ${now}`,
let res = await fetchCmd(`${SMTP_TEST_HOST}/test-address/${user}`, {
method: 'get',
headers,
dispatcher: fetchAgent
dispatcher: retryAgent
});

if (!res.ok) {
Expand Down
8 changes: 4 additions & 4 deletions lib/sub-script.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,11 @@ const crypto = require('crypto');
const vm = require('vm');
const logger = require('./logger');
const settings = require('./settings');
const { retryAgent } = require('./tools');

const { FETCH_TIMEOUT, SUBSCRIPT_RUNTIME_TIMEOUT } = require('./consts');
const { SUBSCRIPT_RUNTIME_TIMEOUT } = require('./consts');

const { fetch: fetchCmd, Agent } = require('undici');
const fetchAgent = new Agent({ connect: { timeout: FETCH_TIMEOUT } });
const { fetch: fetchCmd } = require('undici');

const fnCache = new Map();

Expand Down Expand Up @@ -46,7 +46,7 @@ const wrappedFetch = (...args) => {
opts = args[1];
}

return fetchCmd(args[0], Object.assign({}, opts, { dispatcher: fetchAgent }));
return fetchCmd(args[0], Object.assign({}, opts, { dispatcher: retryAgent }));
};

class SubScript {
Expand Down
28 changes: 22 additions & 6 deletions lib/tools.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,16 @@ const { randomUUID: uuid } = require('crypto');
const mimeTypes = require('nodemailer/lib/mime-funcs/mime-types');
const { v3: murmurhash } = require('murmurhash');
const { compare: compareVersions, validate: validateVersion } = require('compare-versions');
const { REDIS_PREFIX, TLS_DEFAULTS, FETCH_TIMEOUT, MAX_FORM_TTL, FETCH_RETRY_INTERVAL, FETCH_RETRY_EXPONENTIAL, FETCH_RETRY_MAX } = require('./consts');
const {
REDIS_PREFIX,
TLS_DEFAULTS,
URL_FETCH_TIMEOUT,
MAX_FORM_TTL,
FETCH_RETRY_INTERVAL,
FETCH_RETRY_EXPONENTIAL,
FETCH_RETRY_MAX,
URL_FETCH_RETRY_MAX
} = require('./consts');
const ipaddr = require('ipaddr.js');
const bullmqPackage = require('bullmq/package.json');
const v8 = require('node:v8');
Expand All @@ -40,8 +49,13 @@ const googleCrawlerRanges = require('../data/google-crawlers.json');
const { resolvePublicInterfaces } = require('pubface');
const { gt } = require('./translations');

const { fetch: fetchCmd, Agent } = require('undici');
const fetchAgent = new Agent({ connect: { timeout: FETCH_TIMEOUT } });
const { fetch: fetchCmd, Agent, RetryAgent } = require('undici');
const fetchAgent = new Agent({ connect: { timeout: URL_FETCH_TIMEOUT } });
const retryAgent = new RetryAgent(fetchAgent, {
maxRetries: URL_FETCH_RETRY_MAX,
methods: ['GET', 'PUT', 'HEAD', 'OPTIONS', 'DELETE', 'POST'],
statusCodes: [429] // do not retry 5xx errors
});

const regexCache = new Map();

Expand Down Expand Up @@ -330,7 +344,7 @@ module.exports = {
parsed.searchParams.set('account', account);
parsed.searchParams.set('proto', proto);

let authResponse = await fetchCmd(parsed.toString(), { method: 'GET', headers, dispatcher: fetchAgent });
let authResponse = await fetchCmd(parsed.toString(), { method: 'GET', headers, dispatcher: retryAgent });
if (!authResponse.ok) {
throw new Error(`Invalid response: ${authResponse.status} ${authResponse.statusText}`);
}
Expand Down Expand Up @@ -1267,7 +1281,7 @@ MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEV3QUiYsp13nD9suD1/ZkEXnuMoSg
'User-Agent': `${packageData.name}/${packageData.version} (+${packageData.homepage})`
};

let releaseResponse = await fetchCmd(releaseUrl, { method: 'GET', headers, dispatcher: fetchAgent });
let releaseResponse = await fetchCmd(releaseUrl, { method: 'GET', headers, dispatcher: retryAgent });
if (!releaseResponse.ok) {
let err = new Error(`Failed loading release data`);
err.response = {
Expand Down Expand Up @@ -1851,7 +1865,9 @@ MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEV3QUiYsp13nD9suD1/ZkEXnuMoSg
}

return account;
}
},

retryAgent
};

function msgpackDecode(buf) {
Expand Down
Loading

0 comments on commit bafcd1c

Please sign in to comment.