diff --git a/README.md b/README.md index 61930c8..c92505c 100644 --- a/README.md +++ b/README.md @@ -31,13 +31,15 @@ yarn add import-http --dev ## Usage +### Webpack + In your `webpack.config.js`: ```js -const ImportHttpPlugin = require('import-http') +const ImportHttpWebpackPlugin = require('import-http/webpack') module.exports = { - plugins: [new ImportHttpPlugin()] + plugins: [new ImportHttpWebpackPlugin()] } ``` @@ -52,16 +54,26 @@ console.log(React, Vue) Run webpack and it just works. +### Rollup + +In your `rollup.config.js`: + +```js +export default { + plugins: [require('import-http/rollup')()] +} +``` + ## Caching Resources will be fetched at the very first build, then the response will be cached in `~/.import-http` dir. You can use the `reload` option to invalidate cache: ```js -const ImportHttpPlugin = require('import-http') +const ImportHttpWebpackPlugin = require('import-http/webpack') module.exports = { plugins: [ - new ImportHttpPlugin({ + new ImportHttpWebpackPlugin({ reload: process.env.RELOAD }) ] diff --git a/example/webpack.config.js b/example/webpack.config.js index 06e9d03..06b5a60 100644 --- a/example/webpack.config.js +++ b/example/webpack.config.js @@ -1,4 +1,4 @@ -const ImportHttpPlugin = require('..') +const ImportHttpPlugin = require('../webpack') module.exports = { mode: 'development', diff --git a/lib/constants.js b/lib/constants.js new file mode 100644 index 0000000..ebaca43 --- /dev/null +++ b/lib/constants.js @@ -0,0 +1,4 @@ +const os = require('os') +const path = require('path') + +exports.CACHE_DIR = path.join(os.homedir(), '.import-http') diff --git a/lib/fetch.js b/lib/fetch.js new file mode 100644 index 0000000..97f79b9 --- /dev/null +++ b/lib/fetch.js @@ -0,0 +1,11 @@ +const fetch = require('node-fetch') + +module.exports = async (url, opts) => { + console.log(`Downloading ${url}...`) + const res = await fetch(url, opts) + if (!res.ok) { + console.error(`Failed to download ${url}...`) + throw new Error(res.statusText) + } + return res +} diff --git a/lib/index.js b/lib/index.js index 3883538..cc9991a 100644 --- a/lib/index.js +++ b/lib/index.js @@ -1,117 +1,8 @@ -const os = require('os') -const path = require('path') -const { parse: parseURL, resolve: resolveURL } = require('url') -const fetch = require('node-fetch') -const builtinModules = require('builtin-modules') -const { pathExists, outputFile } = require('./utils') -const createHttpCache = require('./http-cache') - -const RESOLVER_ID = 'import-http-resolver' - -const HTTP_RE = /^https?:\/\// - -class ImportHttpPlugin { - constructor(options = {}) { - this.reload = options.reload - this.cacheDir = options.cacheDir - ? path.resolve(options.cacheDir) - : path.join(os.homedir(), '.import-http') - this.depsDir = path.join(this.cacheDir, 'deps') - this.httpCache = createHttpCache(path.join(this.cacheDir, 'requests.json')) - // Map - this.fileModuleCache = new Map() - } - - getFilePathFromURL(url) { - const { host, pathname, protocol } = parseURL(url) - // Where should this url be in the disk - return path.join(this.depsDir, protocol.replace(':', ''), host, pathname) - } - - apply(compiler) { - compiler.resolverFactory.hooks.resolver - .for('normal') - .tap(RESOLVER_ID, resolver => { - const resolvedHook = resolver.ensureHook(`resolve`) - - resolver - .getHook(`described-resolve`) - .tapAsync( - RESOLVER_ID, - async (requestContext, resolveContext, callback) => { - let id = requestContext.request - const { issuer } = requestContext.context - - // if the issuer is a URL (cached in deps dir) - // resolve the url - if ( - !HTTP_RE.test(id) && - issuer && - this.fileModuleCache.has(issuer) && - !this.fileModuleCache.has(id) - ) { - if (/^\./.test(id)) { - // Relative file - // Excluded something like ./node_modules/webpack/buildin/global.js - if (!/node_modules/.test(id)) { - id = resolveURL(this.fileModuleCache.get(issuer), id) - } - } else if (!builtinModules.includes(id)) { - // Propably an npm package - id = `https://unpkg.com/${id}` - } - } - - if (!HTTP_RE.test(id)) { - return callback() - } - - const canResolve = (request, url) => { - this.fileModuleCache.set(request, url) - resolver.doResolve( - resolvedHook, - Object.assign({}, requestContext, { - request - }), - null, - resolveContext, - callback - ) - } - - try { - // Let's get the actual URL - const url = await this.httpCache.get(id) - let file - - if (url && !this.reload) { - file = this.getFilePathFromURL(url) - if (await pathExists(file)) { - return canResolve(file, url) - } - } - - // We have never requested this before - console.log(`Downloading ${id}...`) - const res = await fetch(id) - if (!res.ok) { - console.error(`Failed to download ${id}...`) - throw new Error(res.statusText) - } - file = this.getFilePathFromURL(res.url) - await this.httpCache.set(id, res.url) - const content = await res - .buffer() - .then(buff => buff.toString('utf8')) - await outputFile(file, content, 'utf8') - canResolve(file, res.url) - } catch (error) { - callback(error) - } - } - ) - }) +module.exports = { + get WebpackPlugin() { + return require('./webpack') + }, + get rollupPlugin() { + return require('./rollup') } } - -module.exports = ImportHttpPlugin diff --git a/lib/rollup.js b/lib/rollup.js new file mode 100644 index 0000000..090da0a --- /dev/null +++ b/lib/rollup.js @@ -0,0 +1,77 @@ +const path = require('path') +const { parse: parseURL } = require('url') +const { + outputFile, + pathExists, + readFile, + HTTP_RE, + resolveURL +} = require('./utils') +const createHttpCache = require('./http-cache') +const { CACHE_DIR } = require('./constants') +const fetch = require('./fetch') + +const PREFIX = '\0' + +const removePrefix = id => id && id.replace(/^\0/, '') + +const shouldLoad = id => { + return HTTP_RE.test(removePrefix(id)) +} + +module.exports = ({ reload, cacheDir = CACHE_DIR } = {}) => { + const DEPS_DIR = path.join(cacheDir, 'deps') + const HTTP_CACHE_FILE = path.join(cacheDir, 'requests.json') + + const getFilePathFromURL = url => { + const { host, pathname, protocol } = parseURL(url) + // Where should this url be in the disk + return path.join(DEPS_DIR, protocol.replace(':', ''), host, pathname) + } + + const httpCache = createHttpCache(HTTP_CACHE_FILE) + + return { + name: 'http', + + async resolveId(importee, importer) { + // We're importing from URL + if (HTTP_RE.test(importee)) { + return PREFIX + importee + } + + // We're importing a file but the importer the a URL + // Then we should do + if (importer) { + importer = importer.replace(/^\0/, '') + if (HTTP_RE.test(importer)) { + return resolveURL(importer, importee) + } + } + }, + + async load(id) { + if (!shouldLoad(id)) return + + id = removePrefix(id) + const url = await httpCache.get(id) + let file + + if (url && !reload) { + file = getFilePathFromURL(url) + if (await pathExists(file)) { + return readFile(file, 'utf8') + } + } + + // We have never requested this before + const res = await fetch(url) + file = getFilePathFromURL(res.url) + await httpCache.set(id, res.url) + const content = await res.buffer().then(buff => buff.toString('utf8')) + await outputFile(file, content, 'utf8') + + return content + } + } +} diff --git a/lib/utils.js b/lib/utils.js index 9a289d7..22172e0 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -1,6 +1,8 @@ const path = require('path') const util = require('util') const fs = require('fs') +const url = require('url') +const builtinModules = require('builtin-modules') const mkdir = util.promisify(require('mkdirp')) @@ -18,3 +20,19 @@ exports.pathExists = fp => resolve(!err) }) }) + +exports.HTTP_RE = /^https?:\/\// + +exports.resolveURL = (issuer, id) => { + if (/^\./.test(id)) { + // Relative file + // Excluded something like ./node_modules/webpack/buildin/global.js + if (!/node_modules/.test(id)) { + id = url.resolve(issuer, id) + } + } else if (!builtinModules.includes(id)) { + // Propably an npm package + id = `https://unpkg.com/${id}` + } + return id +} diff --git a/lib/webpack.js b/lib/webpack.js new file mode 100644 index 0000000..56c9e1b --- /dev/null +++ b/lib/webpack.js @@ -0,0 +1,100 @@ +const path = require('path') +const { parse: parseURL } = require('url') +const { pathExists, outputFile, HTTP_RE, resolveURL } = require('./utils') +const createHttpCache = require('./http-cache') +const fetch = require('./fetch') +const { CACHE_DIR } = require('./constants') + +const RESOLVER_ID = 'import-http-resolver' + +class ImportHttpPlugin { + constructor(options = {}) { + this.reload = options.reload + this.cacheDir = options.cacheDir + ? path.resolve(options.cacheDir) + : CACHE_DIR + this.depsDir = path.join(this.cacheDir, 'deps') + this.httpCache = createHttpCache(path.join(this.cacheDir, 'requests.json')) + // Map + this.fileModuleCache = new Map() + } + + getFilePathFromURL(url) { + const { host, pathname, protocol } = parseURL(url) + // Where should this url be in the disk + return path.join(this.depsDir, protocol.replace(':', ''), host, pathname) + } + + apply(compiler) { + compiler.resolverFactory.hooks.resolver + .for('normal') + .tap(RESOLVER_ID, resolver => { + const resolvedHook = resolver.ensureHook(`resolve`) + + resolver + .getHook(`described-resolve`) + .tapAsync( + RESOLVER_ID, + async (requestContext, resolveContext, callback) => { + let id = requestContext.request + const { issuer } = requestContext.context + + // if the issuer is a URL (cached in deps dir) + // resolve the url + if ( + !HTTP_RE.test(id) && + issuer && + this.fileModuleCache.has(issuer) && + !this.fileModuleCache.has(id) + ) { + id = resolveURL(this.fileModuleCache.get(issuer), id) + } + + if (!HTTP_RE.test(id)) { + return callback() + } + + const canResolve = (request, url) => { + this.fileModuleCache.set(request, url) + resolver.doResolve( + resolvedHook, + Object.assign({}, requestContext, { + request + }), + null, + resolveContext, + callback + ) + } + + try { + // Let's get the actual URL + const url = await this.httpCache.get(id) + let file + + if (url && !this.reload) { + file = this.getFilePathFromURL(url) + if (await pathExists(file)) { + return canResolve(file, url) + } + } + + // We have never requested this before + const res = await fetch(id) + file = this.getFilePathFromURL(res.url) + await this.httpCache.set(id, res.url) + const content = await res + .buffer() + .then(buff => buff.toString('utf8')) + await outputFile(file, content, 'utf8') + canResolve(file, res.url) + } catch (error) { + callback(error) + } + } + ) + }) + } +} + +module.exports = ImportHttpPlugin diff --git a/package.json b/package.json index e4976e6..1462c44 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,9 @@ "description": "Import modules from http instead of local node_modules", "main": "lib/index.js", "files": [ - "lib" + "lib", + "rollup.js", + "webpack.js" ], "scripts": { "test": "npm run lint && jest", diff --git a/rollup.js b/rollup.js new file mode 100644 index 0000000..543b228 --- /dev/null +++ b/rollup.js @@ -0,0 +1 @@ +module.exports = require('./lib/rollup') diff --git a/test/index.test.js b/test/index.test.js index 5a60871..81fe39a 100644 --- a/test/index.test.js +++ b/test/index.test.js @@ -1,7 +1,7 @@ const path = require('path') const util = require('util') const webpack = require('webpack') -const HttpImport = require('..') +const { WebpackPlugin } = require('..') jest.setTimeout(30000) @@ -15,7 +15,7 @@ test('main', async () => { filename: 'main.js' }, plugins: [ - new HttpImport({ + new WebpackPlugin({ reload: true, cacheDir: path.join(__dirname, 'fixture/cache') }) diff --git a/webpack.js b/webpack.js new file mode 100644 index 0000000..ef60e97 --- /dev/null +++ b/webpack.js @@ -0,0 +1 @@ +module.exports = require('./lib/webpack')