diff --git a/DEPENDENCIES.md b/DEPENDENCIES.md index fddb46dd29b05..d1a273352ec05 100644 --- a/DEPENDENCIES.md +++ b/DEPENDENCIES.md @@ -606,6 +606,7 @@ graph LR; npm-registry-fetch-->minizlib; npm-registry-fetch-->npm-package-arg; npm-registry-fetch-->proc-log; + npmcli-agent-->agent-base; npmcli-agent-->http-proxy-agent; npmcli-agent-->https-proxy-agent; npmcli-agent-->lru-cache; diff --git a/node_modules/@npmcli/agent/lib/agents.js b/node_modules/@npmcli/agent/lib/agents.js index 7d32768817c18..15aa8e8764b4d 100644 --- a/node_modules/@npmcli/agent/lib/agents.js +++ b/node_modules/@npmcli/agent/lib/agents.js @@ -1,199 +1,202 @@ 'use strict' -const http = require('http') -const https = require('https') const net = require('net') const tls = require('tls') const { once } = require('events') -const { createTimeout, abortRace, urlify, appendPort, cacheAgent } = require('./util') +const timers = require('timers/promises') const { normalizeOptions, cacheOptions } = require('./options') -const { getProxy, getProxyType, proxyCache } = require('./proxy.js') +const { getProxy, getProxyAgent, proxyCache } = require('./proxy.js') const Errors = require('./errors.js') +const { Agent: AgentBase } = require('agent-base') -const createAgent = (base, name) => { - const SECURE = base === https - const SOCKET_TYPE = SECURE ? tls : net +module.exports = class Agent extends AgentBase { + #options + #timeouts + #proxy + #noProxy + #ProxyAgent - const agent = class extends base.Agent { - #options - #timeouts - #proxy - #socket + constructor (options = {}) { + const { timeouts, proxy, noProxy, ...normalizedOptions } = normalizeOptions(options) - constructor (_options) { - const { timeouts, proxy, noProxy, ...options } = normalizeOptions(_options) + super(normalizedOptions) - super(options) + this.#options = normalizedOptions + this.#timeouts = timeouts - this.#options = options - this.#timeouts = timeouts - this.#proxy = proxy ? { proxies: getProxyType(proxy), proxy: urlify(proxy), noProxy } : null + if (proxy) { + this.#proxy = new URL(proxy) + this.#noProxy = noProxy + this.#ProxyAgent = getProxyAgent(proxy) } + } - get proxy () { - return this.#proxy ? { url: this.#proxy.proxy } : {} - } + get proxy () { + return this.#proxy ? { url: this.#proxy } : {} + } - #getProxy (options) { - const proxy = this.#proxy - ? getProxy(appendPort(`${options.protocol}//${options.host}`, options.port), this.#proxy) - : null + #getProxy (options) { + if (!this.#proxy) { + return + } - if (!proxy) { - return - } + const proxy = getProxy(`${options.protocol}//${options.host}:${options.port}`, { + proxy: this.#proxy, + noProxy: this.#noProxy, + }) - return cacheAgent({ - key: cacheOptions({ - ...options, - ...this.#options, - secure: SECURE, - timeouts: this.#timeouts, - proxy, - }), - cache: proxyCache, - secure: SECURE, - proxies: this.#proxy.proxies, - }, proxy, this.#options) + if (!proxy) { + return } - #setKeepAlive (socket) { - socket.setKeepAlive(this.keepAlive, this.keepAliveMsecs) - socket.setNoDelay(this.keepAlive) - } + const cacheKey = cacheOptions({ + ...options, + ...this.#options, + timeouts: this.#timeouts, + proxy, + }) - #setIdleTimeout (socket, options) { - if (this.#timeouts.idle) { - socket.setTimeout(this.#timeouts.idle, () => { - socket.destroy(new Errors.IdleTimeoutError(options)) - }) - } + if (proxyCache.has(cacheKey)) { + return proxyCache.get(cacheKey) } - async #proxyConnect (proxy, request, options) { - // socks-proxy-agent accepts a dns lookup function - options.lookup ??= this.#options.lookup - - // all the proxy agents use this secureEndpoint option to determine - // if the proxy should connect over tls or not. we can set it based - // on if the HttpAgent or HttpsAgent is used. - options.secureEndpoint = SECURE + let ProxyAgent = this.#ProxyAgent + if (Array.isArray(ProxyAgent)) { + ProxyAgent = options.secureEndpoint ? ProxyAgent[1] : ProxyAgent[0] + } - const socket = await abortRace([ - (ac) => createTimeout(this.#timeouts.connection, ac).catch(() => { - throw new Errors.ConnectionTimeoutError(options) - }), - (ac) => proxy.connect(request, options).then((s) => { - this.#setKeepAlive(s) + const proxyAgent = new ProxyAgent(proxy, this.#options) + proxyCache.set(cacheKey, proxyAgent) - const connectEvent = SECURE ? 'secureConnect' : 'connect' - const connectingEvent = SECURE ? 'secureConnecting' : 'connecting' + return proxyAgent + } - if (!s[connectingEvent]) { - return s + // takes an array of promises and races them against the connection timeout + // which will throw the necessary error if it is hit. This will return the + // result of the promise race. + async #timeoutConnection ({ promises, options, timeout }, ac = new AbortController()) { + if (timeout) { + const connectionTimeout = timers.setTimeout(timeout, null, { signal: ac.signal }) + .then(() => { + throw new Errors.ConnectionTimeoutError(`${options.host}:${options.port}`) + }).catch((err) => { + if (err.name === 'AbortError') { + return } + throw err + }) + promises.push(connectionTimeout) + } - return abortRace([ - () => once(s, 'error', ac).then((err) => { - throw err - }), - () => once(s, connectEvent, ac).then(() => s), - ], ac) - }), - ]) - - this.#setIdleTimeout(socket, options) - - return socket + let result + try { + result = await Promise.race(promises) + ac.abort() + } catch (err) { + ac.abort() + throw err } + return result + } - async connect (request, options) { - const proxy = this.#getProxy(options) - if (proxy) { - return this.#proxyConnect(proxy, request, options) + async connect (request, options) { + // if the connection does not have its own lookup function + // set, then use the one from our options + options.lookup ??= this.#options.lookup + + let socket + let timeout = this.#timeouts.connection + + const proxy = this.#getProxy(options) + if (proxy) { + // some of the proxies will wait for the socket to fully connect before + // returning so we have to await this while also racing it against the + // connection timeout. + const start = Date.now() + socket = await this.#timeoutConnection({ + options, + timeout, + promises: [proxy.connect(request, options)], + }) + // see how much time proxy.connect took and subtract it from + // the timeout + if (timeout) { + timeout = timeout - (Date.now() - start) } + } else { + socket = (options.secureEndpoint ? tls : net).connect(options) + } - const socket = SOCKET_TYPE.connect(options) + socket.setKeepAlive(this.keepAlive, this.keepAliveMsecs) + socket.setNoDelay(this.keepAlive) - this.#setKeepAlive(socket) + const abortController = new AbortController() + const { signal } = abortController - await abortRace([ - (s) => createTimeout(this.#timeouts.connection, s).catch(() => { - throw new Errors.ConnectionTimeoutError(options) - }), - (s) => once(socket, 'error', s).then((err) => { - throw err - }), - (s) => once(socket, 'connect', s), - ]) + const connectPromise = socket[options.secureEndpoint ? 'secureConnecting' : 'connecting'] + ? once(socket, options.secureEndpoint ? 'secureConnect' : 'connect', { signal }) + : Promise.resolve() - this.#setIdleTimeout(socket, options) + await this.#timeoutConnection({ + options, + timeout, + promises: [ + connectPromise, + once(socket, 'error', { signal }).then((err) => { + throw err[0] + }), + ], + }, abortController) - return socket + if (this.#timeouts.idle) { + socket.setTimeout(this.#timeouts.idle, () => { + socket.destroy(new Errors.IdleTimeoutError(`${options.host}:${options.port}`)) + }) } - addRequest (request, options) { - const proxy = this.#getProxy(options) - // it would be better to call proxy.addRequest here but this causes the - // http-proxy-agent to call its super.addRequest which causes the request - // to be added to the agent twice. since we only support 3 agents - // currently (see the required agents in proxy.js) we have manually - // checked that the only public methods we need to call are called in the - // next block. this could change in the future and presumably we would get - // failing tests until we have properly called the necessary methods on - // each of our proxy agents - if (proxy?.setRequestProps) { - proxy.setRequestProps(request, options) - } - - request.setHeader('connection', this.keepAlive ? 'keep-alive' : 'close') - - const responseTimeout = createTimeout(this.#timeouts.response) - if (responseTimeout) { - request.once('finish', () => { - responseTimeout.start(() => { - request.destroy(new Errors.ResponseTimeoutError(request, this.proxy?.url)) - }) - }) - request.once('response', () => { - responseTimeout.clear() - }) - } - - const transferTimeout = createTimeout(this.#timeouts.transfer) - if (transferTimeout) { - request.once('response', (res) => { - transferTimeout.start(() => { - res.destroy(new Errors.TransferTimeoutError(request, this.proxy?.url)) - }) - res.once('close', () => { - transferTimeout.clear() - }) - }) - } + return socket + } - return super.addRequest(request, options) + addRequest (request, options) { + const proxy = this.#getProxy(options) + // it would be better to call proxy.addRequest here but this causes the + // http-proxy-agent to call its super.addRequest which causes the request + // to be added to the agent twice. since we only support 3 agents + // currently (see the required agents in proxy.js) we have manually + // checked that the only public methods we need to call are called in the + // next block. this could change in the future and presumably we would get + // failing tests until we have properly called the necessary methods on + // each of our proxy agents + if (proxy?.setRequestProps) { + proxy.setRequestProps(request, options) } - createSocket (req, options, cb) { - return Promise.resolve() - .then(() => this.connect(req, options)) - .then((socket) => { - this.#socket = socket - return super.createSocket(req, options, cb) - }, cb) + request.setHeader('connection', this.keepAlive ? 'keep-alive' : 'close') + + if (this.#timeouts.response) { + let responseTimeout + request.once('finish', () => { + setTimeout(() => { + request.destroy(new Errors.ResponseTimeoutError(request, this.#proxy)) + }, this.#timeouts.response) + }) + request.once('response', () => { + clearTimeout(responseTimeout) + }) } - createConnection () { - return this.#socket + if (this.#timeouts.transfer) { + let transferTimeout + request.once('response', (res) => { + setTimeout(() => { + res.destroy(new Errors.TransferTimeoutError(request, this.#proxy)) + }, this.#timeouts.transfer) + res.once('close', () => { + clearTimeout(transferTimeout) + }) + }) } - } - Object.defineProperty(agent, 'name', { value: name }) - return agent -} - -module.exports = { - HttpAgent: createAgent(http, 'HttpAgent'), - HttpsAgent: createAgent(https, 'HttpsAgent'), + return super.addRequest(request, options) + } } diff --git a/node_modules/@npmcli/agent/lib/errors.js b/node_modules/@npmcli/agent/lib/errors.js index f41b4a065d713..70475aec8eb35 100644 --- a/node_modules/@npmcli/agent/lib/errors.js +++ b/node_modules/@npmcli/agent/lib/errors.js @@ -1,7 +1,5 @@ 'use strict' -const { appendPort } = require('./util') - class InvalidProxyProtocolError extends Error { constructor (url) { super(`Invalid protocol \`${url.protocol}\` connecting to proxy \`${url.host}\``) @@ -11,8 +9,7 @@ class InvalidProxyProtocolError extends Error { } class ConnectionTimeoutError extends Error { - constructor ({ host, port }) { - host = appendPort(host, port) + constructor (host) { super(`Timeout connecting to host \`${host}\``) this.code = 'ECONNECTIONTIMEOUT' this.host = host @@ -20,8 +17,7 @@ class ConnectionTimeoutError extends Error { } class IdleTimeoutError extends Error { - constructor ({ host, port }) { - host = appendPort(host, port) + constructor (host) { super(`Idle timeout reached for host \`${host}\``) this.code = 'EIDLETIMEOUT' this.host = host diff --git a/node_modules/@npmcli/agent/lib/index.js b/node_modules/@npmcli/agent/lib/index.js index 2cd69390ea77e..b33d6eaef07a2 100644 --- a/node_modules/@npmcli/agent/lib/index.js +++ b/node_modules/@npmcli/agent/lib/index.js @@ -1,38 +1,48 @@ 'use strict' const { LRUCache } = require('lru-cache') -const { urlify, cacheAgent } = require('./util') const { normalizeOptions, cacheOptions } = require('./options') const { getProxy, proxyCache } = require('./proxy.js') const dns = require('./dns.js') -const { HttpAgent, HttpsAgent } = require('./agents.js') +const Agent = require('./agents.js') const agentCache = new LRUCache({ max: 20 }) -const getAgent = (url, { agent: _agent, proxy: _proxy, noProxy, ..._options } = {}) => { +const getAgent = (url, { agent, proxy, noProxy, ...options } = {}) => { // false has meaning so this can't be a simple truthiness check - if (_agent != null) { - return _agent + if (agent != null) { + return agent } - url = urlify(url) + url = new URL(url) - const secure = url.protocol === 'https:' - const proxy = getProxy(url, { proxy: _proxy, noProxy }) - const options = { ...normalizeOptions(_options), proxy } + const proxyForUrl = getProxy(url, { proxy, noProxy }) + const normalizedOptions = { + ...normalizeOptions(options), + proxy: proxyForUrl, + } + + const cacheKey = cacheOptions({ + ...normalizedOptions, + secureEndpoint: url.protocol === 'https:', + }) + + if (agentCache.has(cacheKey)) { + return agentCache.get(cacheKey) + } + + const newAgent = new Agent(normalizedOptions) + agentCache.set(cacheKey, newAgent) - return cacheAgent({ - key: cacheOptions({ ...options, secure }), - cache: agentCache, - secure, - proxies: [HttpAgent, HttpsAgent], - }, options) + return newAgent } module.exports = { getAgent, - HttpAgent, - HttpsAgent, + Agent, + // these are exported for backwards compatability + HttpAgent: Agent, + HttpsAgent: Agent, cache: { proxy: proxyCache, agent: agentCache, diff --git a/node_modules/@npmcli/agent/lib/options.js b/node_modules/@npmcli/agent/lib/options.js index cd87c09d6a25a..0bf53f725f084 100644 --- a/node_modules/@npmcli/agent/lib/options.js +++ b/node_modules/@npmcli/agent/lib/options.js @@ -1,7 +1,6 @@ 'use strict' const dns = require('./dns') -const { createKey } = require('./util') const normalizeOptions = (opts) => { const family = parseInt(opts.family ?? '0', 10) @@ -41,33 +40,46 @@ const normalizeOptions = (opts) => { return normalized } -const cacheOptions = (options) => { - const { secure } = options - return createKey({ - secure: !!secure, - // socket connect options - family: options.family, - hints: options.hints, - localAddress: options.localAddress, - // tls specific connect options - strictSsl: secure ? !!options.rejectUnauthorized : false, - ca: secure ? options.ca : null, - cert: secure ? options.cert : null, - key: secure ? options.key : null, - // http agent options - keepAlive: options.keepAlive, - keepAliveMsecs: options.keepAliveMsecs, - maxSockets: options.maxSockets, - maxTotalSockets: options.maxTotalSockets, - maxFreeSockets: options.maxFreeSockets, - scheduling: options.scheduling, - // timeout options - timeouts: options.timeouts, - // proxy - proxy: options.proxy, - }) +const createKey = (obj) => { + let key = '' + const sorted = Object.entries(obj).sort((a, b) => a[0] - b[0]) + for (let [k, v] of sorted) { + if (v == null) { + v = 'null' + } else if (v instanceof URL) { + v = v.toString() + } else if (typeof v === 'object') { + v = createKey(v) + } + key += `${k}:${v}:` + } + return key } +const cacheOptions = ({ secureEndpoint, ...options }) => createKey({ + secureEndpoint: !!secureEndpoint, + // socket connect options + family: options.family, + hints: options.hints, + localAddress: options.localAddress, + // tls specific connect options + strictSsl: secureEndpoint ? !!options.rejectUnauthorized : false, + ca: secureEndpoint ? options.ca : null, + cert: secureEndpoint ? options.cert : null, + key: secureEndpoint ? options.key : null, + // http agent options + keepAlive: options.keepAlive, + keepAliveMsecs: options.keepAliveMsecs, + maxSockets: options.maxSockets, + maxTotalSockets: options.maxTotalSockets, + maxFreeSockets: options.maxFreeSockets, + scheduling: options.scheduling, + // timeout options + timeouts: options.timeouts, + // proxy + proxy: options.proxy, +}) + module.exports = { normalizeOptions, cacheOptions, diff --git a/node_modules/@npmcli/agent/lib/proxy.js b/node_modules/@npmcli/agent/lib/proxy.js index babedad45ff99..6272e929e57bc 100644 --- a/node_modules/@npmcli/agent/lib/proxy.js +++ b/node_modules/@npmcli/agent/lib/proxy.js @@ -5,30 +5,27 @@ const { HttpsProxyAgent } = require('https-proxy-agent') const { SocksProxyAgent } = require('socks-proxy-agent') const { LRUCache } = require('lru-cache') const { InvalidProxyProtocolError } = require('./errors.js') -const { urlify } = require('./util.js') const PROXY_CACHE = new LRUCache({ max: 20 }) -const PROXY_ENV = (() => { - const keys = new Set(['https_proxy', 'http_proxy', 'proxy', 'no_proxy']) - const values = {} - for (let [key, value] of Object.entries(process.env)) { - key = key.toLowerCase() - if (keys.has(key)) { - values[key] = value - } - } - return values -})() - const SOCKS_PROTOCOLS = new Set(SocksProxyAgent.protocols) -const getProxyType = (url) => { - url = urlify(url) +const PROXY_ENV_KEYS = new Set(['https_proxy', 'http_proxy', 'proxy', 'no_proxy']) + +const PROXY_ENV = Object.entries(process.env).reduce((acc, [key, value]) => { + key = key.toLowerCase() + if (PROXY_ENV_KEYS.has(key)) { + acc[key] = value + } + return acc +}, {}) + +const getProxyAgent = (url) => { + url = new URL(url) const protocol = url.protocol.slice(0, -1) if (SOCKS_PROTOCOLS.has(protocol)) { - return [SocksProxyAgent] + return SocksProxyAgent } if (protocol === 'https' || protocol === 'http') { return [HttpProxyAgent, HttpsProxyAgent] @@ -64,25 +61,28 @@ const isNoProxy = (url, noProxy) => { }) } -const getProxy = (url, { - proxy = PROXY_ENV.https_proxy, - noProxy = PROXY_ENV.no_proxy, -}) => { - url = urlify(url) +const getProxy = (url, { proxy, noProxy }) => { + url = new URL(url) + + if (!proxy) { + proxy = url.protocol === 'https:' + ? PROXY_ENV.https_proxy + : PROXY_ENV.https_proxy || PROXY_ENV.http_proxy || PROXY_ENV.proxy + } - if (!proxy && url.protocol !== 'https:') { - proxy = PROXY_ENV.http_proxy || PROXY_ENV.proxy + if (!noProxy) { + noProxy = PROXY_ENV.no_proxy } if (!proxy || isNoProxy(url, noProxy)) { return null } - return urlify(proxy) + return new URL(proxy) } module.exports = { - getProxyType, + getProxyAgent, getProxy, proxyCache: PROXY_CACHE, } diff --git a/node_modules/@npmcli/agent/lib/util.js b/node_modules/@npmcli/agent/lib/util.js deleted file mode 100644 index 6d42a2e202c1f..0000000000000 --- a/node_modules/@npmcli/agent/lib/util.js +++ /dev/null @@ -1,84 +0,0 @@ -'use strict' - -const timers = require('timers/promises') - -const createKey = (obj) => { - let key = '' - const sorted = Object.entries(obj).sort((a, b) => a[0] - b[0]) - for (let [k, v] of sorted) { - if (v == null) { - v = 'null' - } else if (v instanceof URL) { - v = v.toString() - } else if (typeof v === 'object') { - v = createKey(v) - } - key += `${k}:${v}:` - } - return key -} - -const createTimeout = (delay, signal) => { - if (!delay) { - return signal ? new Promise(() => {}) : null - } - - if (!signal) { - let timeout - return { - start: (cb) => (timeout = setTimeout(cb, delay)), - clear: () => clearTimeout(timeout), - } - } - - return timers.setTimeout(delay, null, signal) - .then(() => { - throw new Error() - }).catch((err) => { - if (err.name === 'AbortError') { - return - } - throw err - }) -} - -const abortRace = async (promises, ac = new AbortController()) => { - let res - try { - res = await Promise.race(promises.map((p) => p(ac))) - ac.abort() - } catch (err) { - ac.abort() - throw err - } - return res -} - -const urlify = (url) => typeof url === 'string' ? new URL(url) : url - -const appendPort = (host, port) => { - // istanbul ignore next - if (port) { - host += `:${port}` - } - return host -} - -const cacheAgent = ({ key, cache, secure, proxies }, ...args) => { - if (cache.has(key)) { - return cache.get(key) - } - const Ctor = (secure ? proxies[1] : proxies[0]) ?? proxies[0] - const agent = new Ctor(...args) - cache.set(key, agent) - return agent -} - -module.exports = { - createKey, - createTimeout, - abortRace, - urlify, - cacheAgent, - appendPort, -} diff --git a/node_modules/@npmcli/agent/package.json b/node_modules/@npmcli/agent/package.json index 32379b39b5b56..7d3d6802947d9 100644 --- a/node_modules/@npmcli/agent/package.json +++ b/node_modules/@npmcli/agent/package.json @@ -1,6 +1,6 @@ { "name": "@npmcli/agent", - "version": "2.1.1", + "version": "2.2.0", "description": "the http/https agent used by the npm cli", "main": "lib/index.js", "scripts": { @@ -28,17 +28,11 @@ }, "templateOSS": { "//@npmcli/template-oss": "This file is partially managed by @npmcli/template-oss. Edits may be overwritten.", - "version": "4.18.0", - "publish": "true", - "ciVersions": [ - "16.14.0", - "16.x", - "18.0.0", - "18.x" - ], - "npmSpec": "next-9" + "version": "4.19.0", + "publish": "true" }, "dependencies": { + "agent-base": "^7.1.0", "http-proxy-agent": "^7.0.0", "https-proxy-agent": "^7.0.1", "lru-cache": "^10.0.1", @@ -46,10 +40,11 @@ }, "devDependencies": { "@npmcli/eslint-config": "^4.0.0", - "@npmcli/template-oss": "4.18.0", + "@npmcli/template-oss": "4.19.0", "minipass-fetch": "^3.0.3", "nock": "^13.2.7", - "simple-socks": "^2.2.2", + "semver": "^7.5.4", + "simple-socks": "^3.1.0", "tap": "^16.3.0" }, "repository": { diff --git a/package-lock.json b/package-lock.json index 7efc38df8a7f9..da884e128107b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2864,11 +2864,12 @@ } }, "node_modules/@npmcli/agent": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/@npmcli/agent/-/agent-2.1.1.tgz", - "integrity": "sha512-6RlbiOAi6L6uUYF4/CDEkDZQnKw0XDsFJVrEpnib8rAx2WRMOsUyAdgnvDpX/fdkDWxtqE+NHwF465llI2wR0g==", + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@npmcli/agent/-/agent-2.2.0.tgz", + "integrity": "sha512-2yThA1Es98orMkpSLVqlDZAMPK3jHJhifP2gnNUdk1754uZ8yI5c+ulCoVG+WlntQA6MzhrURMXjSd9Z7dJ2/Q==", "inBundle": true, "dependencies": { + "agent-base": "^7.1.0", "http-proxy-agent": "^7.0.0", "https-proxy-agent": "^7.0.1", "lru-cache": "^10.0.1",