diff --git a/CHANGELOG.md b/CHANGELOG.md index d8fb5a49..ea2ae137 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +# [1.11.2] - 2017-06-19 + +## Changed +* modify the controller to make it initialise each datasource's data provider just before it is used, then destroy it after the data is returned + # [1.11.0] - 2017-06-11 ## Added diff --git a/dadi/lib/auth/index.js b/dadi/lib/auth/index.js index ad0c43bc..fb5b28e3 100755 --- a/dadi/lib/auth/index.js +++ b/dadi/lib/auth/index.js @@ -10,7 +10,7 @@ var log = require('@dadi/logger') // This attaches middleware to the passed in app instance module.exports = function (server) { server.app.use(function (req, res, next) { - log.info({module: 'auth'}, 'Retrieving access token for "' + req.url + '"') + log.debug({module: 'auth'}, 'Retrieving access token for "' + req.url + '"') help.timer.start('auth') return help.getToken().then(function (bearerToken) { diff --git a/dadi/lib/cache/datasource.js b/dadi/lib/cache/datasource.js index 0bf83b50..c9f3116a 100644 --- a/dadi/lib/cache/datasource.js +++ b/dadi/lib/cache/datasource.js @@ -7,16 +7,19 @@ var merge = require('deepmerge') var path = require('path') var url = require('url') -var Cache = require(path.join(__dirname, '/index.js')) +// var Cache = require(path.join(__dirname, '/index.js')) +var DadiCache = require('@dadi/cache') var config = require(path.join(__dirname, '/../../../config.js')) +var log = require('@dadi/logger') /** * Creates a new DatasourceCache singleton for caching datasource results * @constructor */ var DatasourceCache = function () { - this.cache = Cache().cache + // this.cache = Cache().cache this.cacheOptions = config.get('caching') + this.cache = new DadiCache(this.cacheOptions) DatasourceCache.numInstances = (DatasourceCache.numInstances || 0) + 1 // console.log('DatasourceCache:', DatasourceCache.numInstances) @@ -27,32 +30,112 @@ var DatasourceCache = function () { this.enabled = !(directoryEnabled === false && redisEnabled === false) } +/** + * Get datasource data from the cache if it exists + * @param {object} datasource - a datasource schema object containing the datasource settings + * @param {fn} done - the method to call when finished, accepts 1 arg: + * if the cache key was found, returns {Buffer} data + * if the cache key was not found, returns false + */ +DatasourceCache.prototype.getFromCache = function (opts, done) { + debug('get (%s)', opts.name) + + if (!this.cachingEnabled(opts)) { + return done(false) + } + + if (this.stillCaching) { + return done(false) + } + + var filename = this.getFilename(opts) + var options = this.getOptions(opts) + + var buffers = [] + + // attempt to get from the cache + this.cache.get(filename, options).then((stream) => { + debug('serving %s from cache (%s)', opts.name, filename) + log.info('serving %s from cache (%s)', opts.name, filename) + + stream.on('data', (chunk) => { + if (chunk) { + buffers.push(chunk) + } + }) + + stream.on('end', () => { + return done(Buffer.concat(buffers)) + }) + }).catch(() => { + // key doesn't exist in cache + return done(false) + }) +} + +/** + * Cache the supplied data it caching is enabled for the datasource + * + * @param {Object} datasource - the datasource instance + * @param {Buffer} data - the body of the response as a Buffer + * @param {fn} done - the method to call when finished, accepts args (Boolean written) + */ +DatasourceCache.prototype.cacheResponse = function (opts, data, done) { + var enabled = this.cachingEnabled(opts) + + if (!enabled) { + return done(false) + } + + if (this.stillCaching) { + // console.log('stillCaching...') + return done(false) + } + + debug('write to cache (%s)', opts.name) + + var filename = this.getFilename(opts) + var options = this.getOptions(opts) + + // console.log('> CACHE RESPONSE') + // console.log('is Buffer?', Buffer.isBuffer(data)) + // console.log(filename, opts.endpoint) + + this.stillCaching = true + + this.cache.set(filename, data, options).then(() => { + // console.log('< CACHE RESPONSE', filename) + this.stillCaching = false + return done(true) + }) +} + /** * * @param {object} datasource - a datasource schema object containing the datasource settings */ -DatasourceCache.prototype.cachingEnabled = function (datasource) { +DatasourceCache.prototype.cachingEnabled = function (opts) { var enabled = this.enabled // check the querystring for a no cache param - if (typeof datasource.provider.endpoint !== 'undefined') { - var query = url.parse(datasource.provider.endpoint, true).query + if (typeof opts.endpoint !== 'undefined') { + var query = url.parse(opts.endpoint, true).query if (query.cache && query.cache === 'false') { enabled = false } } - if (datasource.source.type === 'static') { - enabled = false - } + // if (datasource.source.type === 'static') { + // enabled = false + // } if (config.get('debug')) { enabled = false } - var options = this.getOptions(datasource) + var options = this.getOptions(opts) - debug('options (%s): %o', datasource.name, options) + debug('options (%s): %o', opts.name, options) // enabled if the datasource caching block says it's enabled return enabled && (options.directory.enabled || options.redis.enabled) @@ -66,13 +149,13 @@ DatasourceCache.prototype.cachingEnabled = function (datasource) { * a unique cacheKey instead * @param {object} datasource - a datasource schema object containing the datasource settings */ -DatasourceCache.prototype.getFilename = function (datasource) { - var filename = crypto.createHash('sha1').update(datasource.name).digest('hex') +DatasourceCache.prototype.getFilename = function (opts) { + var filename = crypto.createHash('sha1').update(opts.name).digest('hex') - if (datasource.provider.cacheKey) { - filename += '_' + crypto.createHash('sha1').update(datasource.provider.cacheKey).digest('hex') + if (opts.cacheKey) { + filename += '_' + crypto.createHash('sha1').update(opts.cacheKey).digest('hex') } else { - filename += '_' + crypto.createHash('sha1').update(datasource.provider.endpoint).digest('hex') + filename += '_' + crypto.createHash('sha1').update(opts.endpoint).digest('hex') } return filename @@ -83,73 +166,16 @@ DatasourceCache.prototype.getFilename = function (datasource) { * @param {object} datasource - a datasource schema object containing the datasource settings * @returns {object} options for the cache */ -DatasourceCache.prototype.getOptions = function (datasource) { - var options = merge(this.cacheOptions, datasource.schema.datasource.caching || {}) +DatasourceCache.prototype.getOptions = function (opts) { + var options = merge(this.cacheOptions, opts.caching || {}) options.directory.extension = 'json' return options } -/** - * - * @param {object} datasource - a datasource schema object containing the datasource settings - */ -DatasourceCache.prototype.getFromCache = function (datasource, done) { - debug('get (%s)', datasource.name) - - if (!this.cachingEnabled(datasource)) { - return done(false) - } - - var filename = this.getFilename(datasource) - var options = this.getOptions(datasource) - - var data = '' - - // attempt to get from the cache - this.cache.get(filename, options).then((stream) => { - debug('serving %s from cache (%s)', datasource.name, filename) - - stream.on('data', (chunk) => { - if (chunk) data += chunk - }) - - stream.on('end', () => { - return done(data) - }) - }).catch(() => { - // key doesn't exist in cache - return done(false) - }) -} - -/** - * - */ -DatasourceCache.prototype.cacheResponse = function (datasource, data, done) { - var enabled = this.cachingEnabled(datasource) - - if (!enabled) { - return done(false) - } - - debug('write to cache (%s)', datasource.name) - - var filename = this.getFilename(datasource) - var options = this.getOptions(datasource) - - this.cache.set(filename, data, options).then(() => { - return done(true) - }) -} +module.exports._reset = function () {} module.exports = function () { return new DatasourceCache() } - -module.exports._reset = function () { - -} - -module.exports.DatasourceCache = DatasourceCache diff --git a/dadi/lib/controller/index.js b/dadi/lib/controller/index.js index cce25925..1d353088 100755 --- a/dadi/lib/controller/index.js +++ b/dadi/lib/controller/index.js @@ -14,6 +14,7 @@ var log = require('@dadi/logger') var Datasource = require(path.join(__dirname, '/../datasource')) var Event = require(path.join(__dirname, '/../event')) +var Providers = require(path.join(__dirname, '/../providers')) var View = require(path.join(__dirname, '/../view')) // helpers @@ -293,9 +294,9 @@ Controller.prototype.loadData = function (req, res, data, done) { _.each(self.datasources, function (ds, key) { if (ds.chained) { - chainedDatasources[key] = ds + chainedDatasources[key] = _.clone(ds) } else { - primaryDatasources[key] = ds + primaryDatasources[key] = _.clone(ds) } }) @@ -328,23 +329,39 @@ Controller.prototype.loadData = function (req, res, data, done) { }) } - processSearchParameters(ds.schema.datasource.key, ds, req) - help.timer.start('datasource: ' + ds.name) + ds.provider = new Providers[ds.source.type]() + ds.provider.initialise(ds, ds.schema) + + // var requestUrl = processSearchParameters(ds.schema.datasource.key, ds, req) + processSearchParameters(ds.schema.datasource.key, ds, req) + + /** + * Call the data provider's load method to obtain data + * for this datasource + * @returns err, {Object} result, {Object} dsResponse + */ + // ds.provider.load(requestUrl, function (err, result, dsResponse) { ds.provider.load(req.url, function (err, result, dsResponse) { help.timer.stop('datasource: ' + ds.name) - if (err) return done(err) - if (dsResponse) { - return done(null, result, dsResponse) + if (ds.provider.destroy) { + ds.provider.destroy() } + ds.provider = null + + if (err) return done(err) + + if (dsResponse) return done(null, result, dsResponse) + + // TODO: simplify this, doesn't require a try/catch if (result) { try { - data[ds.schema.datasource.key] = (typeof result === 'object' ? result : JSON.parse(result)) + data[ds.schema.datasource.key] = result } catch (e) { - console.log('Provider Load Error:', ds.name, ds.provider.endpoint) + console.log('Provider Load Error:', ds.name, req.url) console.log(e) } } @@ -460,17 +477,25 @@ Controller.prototype.processChained = function (chainedDatasources, data, req, d chainedDatasource.schema.datasource.filter = JSON.parse(filter) } - chainedDatasource.provider.buildEndpoint(chainedDatasource.schema, function () {}) + chainedDatasource.provider = new Providers[chainedDatasource.source.type]() + chainedDatasource.provider.initialise(chainedDatasource, chainedDatasource.schema) + + // var requestUrl = chainedDatasource.provider.buildEndpoint(chainedDatasource.schema.datasource) + chainedDatasource.provider.buildEndpoint(chainedDatasource.schema.datasource) + + // debug('datasource (load): %s %s', chainedDatasource.name, requestUrl) + debug('datasource (load): %s %s', chainedDatasource.name, chainedDatasource.provider.endpoint) - debug('datasource (load): %s %o', chainedDatasource.name, chainedDatasource.schema.datasource.filter) - chainedDatasource.provider.load(req.url, function (err, result) { + // chainedDatasource.provider.load(requestUrl, (err, chainedData) => { + chainedDatasource.provider.load(req.url, (err, chainedData) => { if (err) log.error({module: 'controller'}, err) help.timer.stop('datasource: ' + chainedDatasource.name + ' (chained)') - if (result) { + // TODO: simplify this, doesn't require a try/catch + if (chainedData) { try { - data[chainedKey] = (typeof result === 'object' ? result : JSON.parse(result)) + data[chainedKey] = chainedData } catch (e) { log.error({module: 'controller'}, e) } @@ -488,6 +513,7 @@ Controller.prototype.processChained = function (chainedDatasources, data, req, d function processSearchParameters (key, datasource, req) { // process each of the datasource's requestParams, testing for their existence // in the querystring's request params e.g. /car-reviews/:make/:model + // return datasource.processRequest(key, req) datasource.processRequest(key, req) } diff --git a/dadi/lib/controller/router.js b/dadi/lib/controller/router.js index c6c9b10e..3117c361 100644 --- a/dadi/lib/controller/router.js +++ b/dadi/lib/controller/router.js @@ -86,12 +86,11 @@ Router.prototype.loadRewrites = function (options, done) { return cb(null) } - if (response) { - response = JSON.parse(response) - } + // if (response) { + // response = JSON.parse(response) + // } if (response.results) { - // api.in(self.rewritesDatasource).find().then(function (response) var idx = 0 _.each(response.results, function (rule) { @@ -335,14 +334,15 @@ module.exports = function (server, options) { ds.provider.processRequest(ds.page.name, req) - ds.provider.load(req.url, function (err, result) { + ds.provider.load(req.url, function (err, data) { if (err) { console.log('Error loading data in Router Rewrite module') return next(err) } - if (result) { - var results = (typeof result === 'object') ? result : JSON.parse(result) + if (data) { + // var results = JSON.parse(data.toString()) + var results = data if (results && results.results && results.results.length > 0 && results.results[0].rule === req.url) { var rule = results.results[0] diff --git a/dadi/lib/datasource/index.js b/dadi/lib/datasource/index.js index 73bcf864..e2417824 100644 --- a/dadi/lib/datasource/index.js +++ b/dadi/lib/datasource/index.js @@ -98,15 +98,23 @@ Datasource.prototype.loadDatasource = function (done) { * @param {IncomingMessage} req - the original HTTP request */ Datasource.prototype.processRequest = function (datasource, req) { - this.schema.datasource.filter = this.originalFilter + // console.log('> DS PROCESS REQUEST') + // console.log(this.schema.datasource.filter) + // this.schema.datasource.filter = this.originalFilter + // var filter = this.originalFilter || {} + + var datasourceParams = _.clone(this.schema.datasource) + datasourceParams.filter = this.originalFilter || {} var query = JSON.parse(JSON.stringify(url.parse(req.url, true).query)) // handle the cache flag if (query.cache && query.cache === 'false') { - this.schema.datasource.cache = false + // this.schema.datasource.cache = false + datasourceParams.cache = false } else { - delete this.schema.datasource.cache + // delete this.schema.datasource.cache + delete datasourceParams.cache } // if (req.headers && req.headers.referer) { @@ -125,8 +133,15 @@ Datasource.prototype.processRequest = function (datasource, req) { }) // handle pagination param - if (this.schema.datasource.paginate) { - this.schema.datasource.page = query.page || + // if (this.schema.datasource.paginate) { + // this.schema.datasource.page = query.page || + // (requestParamsPage && req.params[requestParamsPage]) || + // req.params.page || + // 1 + // } + + if (datasourceParams.paginate) { + datasourceParams.page = query.page || (requestParamsPage && req.params[requestParamsPage]) || req.params.page || 1 @@ -142,7 +157,8 @@ Datasource.prototype.processRequest = function (datasource, req) { // URI encode each querystring value _.each(query, (value, key) => { if (key === 'filter') { - _.extend(this.schema.datasource.filter, JSON.parse(value)) + // _.extend(this.schema.datasource.filter, JSON.parse(value)) + datasourceParams.filter = _.extend(datasourceParams.filter, JSON.parse(value)) } }) } @@ -150,7 +166,15 @@ Datasource.prototype.processRequest = function (datasource, req) { // Regular expression search for {param.nameOfParam} and replace with requestParameters var paramRule = /("\{)(\bparams.\b)(.*?)(\}")/gmi - this.schema.datasource.filter = JSON.parse(JSON.stringify(this.schema.datasource.filter).replace(paramRule, function (match, p1, p2, p3, p4, offset, string) { + // this.schema.datasource.filter = JSON.parse(JSON.stringify(this.schema.datasource.filter).replace(paramRule, function (match, p1, p2, p3, p4, offset, string) { + // if (req.params[p3]) { + // return req.params[p3] + // } else { + // return match + // } + // })) + + datasourceParams.filter = JSON.parse(JSON.stringify(datasourceParams.filter).replace(paramRule, function (match, p1, p2, p3, p4, offset, string) { if (req.params[p3]) { return req.params[p3] } else { @@ -173,7 +197,8 @@ Datasource.prototype.processRequest = function (datasource, req) { paramValue = obj.type === 'Number' ? Number(paramValue) : encodeURIComponent(paramValue) if (obj.target === 'filter') { - this.schema.datasource.filter[obj.field] = paramValue + // this.schema.datasource.filter[obj.field] = paramValue + datasourceParams.filter[obj.field] = paramValue } else if (obj.target === 'endpoint') { var placeholderRegex = new RegExp('{' + obj.field + '}', 'ig') this.source.modifiedEndpoint = this.schema.datasource.source.endpoint.replace(placeholderRegex, paramValue) @@ -181,19 +206,29 @@ Datasource.prototype.processRequest = function (datasource, req) { } else { if (obj.target === 'filter') { // param not found in request, remove it from the datasource filter - if (this.schema.datasource.filter[obj.field]) { - delete this.schema.datasource.filter[obj.field] + // if (this.schema.datasource.filter[obj.field]) { + // delete this.schema.datasource.filter[obj.field] + // } + if (datasourceParams.filter[obj.field]) { + delete datasourceParams.filter[obj.field] } } } }) if (this.schema.datasource.filterEventResult) { - this.schema.datasource.filter = _.extend(this.schema.datasource.filter, this.schema.datasource.filterEventResult) + // this.schema.datasource.filter = _.extend(this.schema.datasource.filter, this.schema.datasource.filterEventResult) + datasourceParams.filter = _.extend(datasourceParams.filter, this.schema.datasource.filterEventResult) } if (typeof this.provider.processRequest === 'function') { - this.provider.processRequest(req) + if (this.provider.hasOwnProperty('processSchemaParams') && this.provider.processSchemaParams === false) { + // return this.provider.processRequest(req) + this.provider.processRequest(req) + } else { + // return this.provider.processRequest(datasourceParams) + this.provider.processRequest(datasourceParams) + } } } diff --git a/dadi/lib/datasource/preload.js b/dadi/lib/datasource/preload.js index fe7a132d..1d688a0d 100644 --- a/dadi/lib/datasource/preload.js +++ b/dadi/lib/datasource/preload.js @@ -3,6 +3,7 @@ var path = require('path') var config = require(path.resolve(path.join(__dirname, '/../../../config'))) var Datasource = require(path.join(__dirname, '/../datasource')) +var Providers = require(path.join(__dirname, '/../providers')) var Preload = function () { this.data = {} @@ -18,14 +19,29 @@ Preload.prototype.init = function (options) { return } - datasource.provider.load(null, (err, result) => { + datasource.provider = new Providers[datasource.source.type]() + datasource.provider.initialise(datasource, datasource.schema) + + // var requestUrl = datasource.processRequest('preload', req) + // datasource.processRequest('preload', null) + + // datasource.provider.load(requestUrl, (err, data) => { + datasource.provider.load(null, (err, data) => { if (err) console.log(err) - if (result) { + + if (datasource.provider.destroy) { + datasource.provider.destroy() + } + + datasource.provider = null + + // TODO: simplify this, doesn't require a try/catch + if (data) { try { - var results = (typeof result === 'object' ? result : JSON.parse(result)) + var results = data this.data[source] = results.results ? results.results : results } catch (e) { - console.log('Preload Load Error:', datasource.name, datasource.provider.endpoint) + console.log('Preload Load Error:', datasource.name) console.log(e) } } diff --git a/dadi/lib/datasource/route-validator.js b/dadi/lib/datasource/route-validator.js index 3ca8e1d9..d75863a9 100644 --- a/dadi/lib/datasource/route-validator.js +++ b/dadi/lib/datasource/route-validator.js @@ -1,5 +1,6 @@ var path = require('path') var Datasource = require(path.join(__dirname, '/../datasource')) +var Providers = require(path.join(__dirname, '/../providers')) var RouteValidator = function () { this.validationDatasources = {} @@ -19,14 +20,26 @@ RouteValidator.prototype.get = function (route, param, options, req) { }) } + datasource.provider = new Providers[datasource.source.type]() + datasource.provider.initialise(datasource, datasource.schema) + + // var requestUrl = datasource.processRequest(route.path, req) datasource.processRequest(route.path, req) - return datasource.provider.load(null, (err, result) => { + // return datasource.provider.load(requestUrl, (err, data) => { + return datasource.provider.load(req.url, (err, data) => { if (err) return reject(err) - if (result) { + if (datasource.provider && datasource.provider.destroy) { + datasource.provider.destroy() + } + + datasource.provider = null + + // TODO: simplify this, doesn't require a try/catch + if (data) { try { - var results = (typeof result === 'object' ? result : JSON.parse(result)) + var results = data // JSON.parse(data.toString()) if (results.results && results.results.length > 0) { return resolve('') @@ -34,7 +47,7 @@ RouteValidator.prototype.get = function (route, param, options, req) { return reject('') } } catch (e) { - console.log('RouteValidator Load Error:', datasource.name, datasource.provider.endpoint) + console.log('RouteValidator Load Error:', datasource.name, req.url) console.log(e) return reject('') diff --git a/dadi/lib/providers/remote.js b/dadi/lib/providers/remote.js index 8ecf3462..f63cb03e 100644 --- a/dadi/lib/providers/remote.js +++ b/dadi/lib/providers/remote.js @@ -15,12 +15,16 @@ const BearerAuthStrategy = require(path.join(__dirname, '/../auth/bearer')) const DatasourceCache = require(path.join(__dirname, '/../cache/datasource')) const RemoteProvider = function () { - this.dataCache = DatasourceCache() + this.dataCache = new DatasourceCache() RemoteProvider.numInstances = (RemoteProvider.numInstances || 0) + 1 // console.log('RemoteProvider:', RemoteProvider.numInstances) } +RemoteProvider.prototype.destroy = function () { + RemoteProvider.numInstances = (RemoteProvider.numInstances || 0) - 1 +} + /** * initialise - initialises the datasource provider * @@ -40,9 +44,13 @@ RemoteProvider.prototype.initialise = function initialise (datasource, schema) { * * @return {void} */ -RemoteProvider.prototype.buildEndpoint = function buildEndpoint () { +RemoteProvider.prototype.buildEndpoint = function buildEndpoint (datasourceParams) { + if (!datasourceParams) { + datasourceParams = this.schema.datasource + } + const apiConfig = config.get('api') - const source = this.schema.datasource.source + const source = datasourceParams.source || this.datasource.source const protocol = source.protocol || 'http' const host = source.host || apiConfig.host @@ -51,206 +59,257 @@ RemoteProvider.prototype.buildEndpoint = function buildEndpoint () { const uri = [protocol, '://', host, (port !== '' ? ':' : ''), port, '/', this.datasource.source.modifiedEndpoint || source.endpoint].join('') - this.endpoint = this.processDatasourceParameters(this.schema, uri) + // return this.processDatasourceParameters(datasourceParams, uri) + this.endpoint = this.processDatasourceParameters(datasourceParams, uri) } /** - * getHeaders + * Load data from the specified datasource * - * @param {fn} done - callback - * @return {void} + * @param {string} requestUrl - datasource endpoint to load + * @param {fn} done - callback on error or completion */ -RemoteProvider.prototype.getHeaders = function getHeaders (done) { - var headers = { - 'accept-encoding': 'gzip' +RemoteProvider.prototype.load = function (requestUrl, done) { + this.options = { + protocol: this.datasource.source.protocol || config.get('api.protocol'), + host: this.datasource.source.host || config.get('api.host'), + port: this.datasource.source.port || config.get('api.port'), + path: url.parse(this.endpoint).path, + // path: url.parse(requestUrl).path, + method: 'GET' } - if (this.datasource.requestHeaders) { - delete this.datasource.requestHeaders['host'] - delete this.datasource.requestHeaders['content-length'] - delete this.datasource.requestHeaders['accept'] + this.options.agent = this.keepAliveAgent(this.options.protocol) + this.options.protocol = this.options.protocol + ':' - if (this.datasource.requestHeaders['content-type'] !== 'application/json') { - this.datasource.requestHeaders['content-type'] = 'application/json' + var cacheOptions = { + name: this.datasource.name, + caching: this.schema.datasource.caching, + // endpoint: requestUrl + endpoint: this.endpoint + } + + this.dataCache.getFromCache(cacheOptions, (cachedData) => { + // data found in the cache, parse into JSON + // and return to whatever called load() + if (cachedData) { + try { + cachedData = JSON.parse(cachedData.toString()) + return done(null, cachedData) + } catch (err) { + log.error('Remote: cache data incomplete, making HTTP request: ' + err + '(' + cacheOptions.endpoint + ')') + } } - headers = _.extend(headers, this.datasource.requestHeaders) - } + // debug('load %s', requestUrl) + debug('load %s', this.endpoint) - // If the data-source has its own auth strategy, use it. - // Otherwise, authenticate with the main server via bearer token - if (this.authStrategy) { - // This could eventually become a switch statement that handles different auth types - if (this.authStrategy.getType() === 'bearer') { - this.authStrategy.getToken(this.authStrategy, (err, bearerToken) => { - if (err) { - return done(err) - } + this.getHeaders((err, headers) => { + err && done(err) - headers['Authorization'] = 'Bearer ' + bearerToken + this.options = _.extend(this.options, headers) - return done(null, { headers: headers }) - }) - } - } else { - try { - help.getToken(this.datasource).then((bearerToken) => { - headers['Authorization'] = 'Bearer ' + bearerToken + log.info({module: 'remote'}, 'GET datasource "' + this.datasource.schema.datasource.key + '": ' + decodeURIComponent(this.endpoint)) - help.timer.stop('auth') - return done(null, { headers: headers }) - }).catch((errorData) => { - const err = new Error() - err.name = errorData.title - err.message = errorData.detail - err.remoteIp = config.get('api.host') - err.remotePort = config.get('api.port') - err.path = config.get('auth.tokenUrl') + const agent = (this.options.protocol === 'https') ? https : http - if (errorData.stack) { - console.log(errorData.stack) - } + let request = agent.request(this.options) - help.timer.stop('auth') + request.on('response', (res) => { + // this.handleResponse(requestUrl, res, done) + this.handleResponse(this.endpoint, res, done) + }) + + request.on('error', (err) => { + // const message = err.toString() + ". Couldn't request data from " + requestUrl + const message = err.toString() + ". Couldn't request data from " + this.endpoint + + err.name = 'GetData' + err.message = message + err.remoteIp = this.options.host + err.remotePort = this.options.port return done(err) }) - } catch (err) { - console.log(err.stack) - } - } + + request.end() + }) + }) } /** - * handleResponse + * Takes the response from the server and turns it into a Buffer, + * decompressing it if required. Calls processOutput() with the Buffer. * - * @param {response} res - response + * @param {http.ServerResponse} res - the full HTTP response * @param {fn} done - callback * @return {void} */ -RemoteProvider.prototype.handleResponse = function handleResponse (res, done) { - const encoding = res.headers['content-encoding'] ? res.headers['content-encoding'] : '' - let output = '' - - if (encoding === 'gzip') { - const gunzip = zlib.createGunzip() - const buffer = [] - - gunzip.on('data', (data) => { - buffer.push(data.toString()) - }).on('end', () => { - output = buffer.join('') - this.processOutput(res, output, (err, data, res) => { - if (err) return done(err) - return done(null, data, res) +RemoteProvider.prototype.handleResponse = function (requestUrl, res, done) { + setImmediate(() => { + var encoding = res.headers['content-encoding'] ? res.headers['content-encoding'] : '' + var buffers = [] + var output + + if (encoding === 'gzip') { + const gunzip = zlib.createGunzip() + + gunzip.on('data', (data) => { + buffers.push(data) + }).on('end', () => { + output = Buffer.concat(buffers) + + this.processOutput(requestUrl, res, output, (err, data, res) => { + if (err) return done(err) + return done(null, data, res) + }) + }).on('error', (err) => { + done(err) }) - }).on('error', (err) => { - done(err) - }) - res.pipe(gunzip) - } else { - res.on('data', (chunk) => { - output += chunk - }) - - res.on('end', () => { - this.processOutput(res, output, (err, data, res) => { - if (err) return done(err) - return done(null, data, res) + res.pipe(gunzip) + } else { + res.on('data', (chunk) => { + buffers.push(chunk) }) - }) - } -} -/** - * keepAliveAgent - returns http|https module depending on config - * - * @param {string} protocol - * @return {module} http|https - */ -RemoteProvider.prototype.keepAliveAgent = function keepAliveAgent (protocol) { - return (protocol === 'https') - ? new https.Agent({ keepAlive: true }) - : new http.Agent({ keepAlive: true }) + res.on('end', () => { + output = Buffer.concat(buffers) + + this.processOutput(requestUrl, res, output, (err, data, res) => { + if (err) return done(err) + return done(null, data, res) + }) + }) + } + }) } /** - * load - loads data form the datasource + * Processes the response from the server, caching it if it's a 200 * - * @param {string} requestUrl - url of the web request (not used) - * @param {fn} done - callback on error or completion - * @return {void} + * @param {http.ServerResponse} res - the full HTTP response + * @param {Buffer} data - the body of the response as a Buffer + * @param {fn} done - the method to call when finished, accepts args (err, data, res) */ -RemoteProvider.prototype.load = function (requestUrl, done) { - this.requestUrl = requestUrl - - this.options = { - protocol: this.datasource.source.protocol || config.get('api.protocol'), - host: this.datasource.source.host || config.get('api.host'), - port: this.datasource.source.port || config.get('api.port'), - path: url.parse(this.endpoint).path, - method: 'GET' - } - - this.options.agent = this.keepAliveAgent(this.options.protocol) - this.options.protocol = this.options.protocol + ':' +RemoteProvider.prototype.processOutput = function (requestUrl, res, data, done) { + setImmediate(() => { + // Return a 202 Accepted response immediately, + // along with the datasource response + if (res.statusCode === 202) { + return done(null, JSON.parse(data.toString()), res) + } - this.dataCache.getFromCache(this.datasource, (cachedData) => { - if (cachedData) return done(null, cachedData) + // return 5xx error as the datasource response + if (res.statusCode && /^5/.exec(res.statusCode)) { + data = { + 'results': [], + 'errors': [{ + 'code': 'WEB-0005', + 'title': 'Datasource Timeout', + 'details': "The datasource '" + this.datasource.name + "' timed out: " + res.statusMessage + ' (' + res.statusCode + ')' + ': ' + this.endpoint + }] + } + } else if (res.statusCode === 404) { + data = { + 'results': [], + 'errors': [{ + 'code': 'WEB-0004', + 'title': 'Datasource Not Found', + 'details': 'Datasource "' + this.datasource.name + '" failed. ' + res.statusMessage + ' (' + res.statusCode + ')' + ': ' + this.endpoint + }] + } + } else if (res.statusCode && !/200|400/.exec(res.statusCode)) { + // if the error is anything other than Success or Bad Request, error + const err = new Error() + err.message = 'Datasource "' + this.datasource.name + '" failed. ' + res.statusMessage + ' (' + res.statusCode + ')' + ': ' + this.endpoint + if (data) err.message += '\n' + data.toString() - debug('load %s', this.endpoint) + err.remoteIp = this.options.host + err.remotePort = this.options.port - this.getHeaders((err, headers) => { - err && done(err) + log.error({module: 'remote'}, res.statusMessage + ' (' + res.statusCode + ')' + ': ' + this.endpoint) - this.options = _.extend(this.options, headers) + console.log(err) + // return done(err) + throw (err) + } - log.info({module: 'remote'}, "GET datasource '" + this.datasource.schema.datasource.key + "': " + this.options.path) + // Cache 200 responses + if (res.statusCode === 200) { + log.info( + {module: 'remote'}, + 'GOT datasource "' + + this.datasource.schema.datasource.key + + '": ' + decodeURIComponent(this.endpoint) + + // '": ' + decodeURIComponent(requestUrl) + + ' (HTTP 200, ' + + require('humanize-plus').fileSize(Buffer.byteLength(data)) + ')' + ) + + // log.info( + // {module: 'remote'}, '> CALL cacheResponse ' + requestUrl + ' ' + this.dataCache.stillCaching) + + var cacheOptions = { + name: this.datasource.name, + caching: this.schema.datasource.caching, + // endpoint: requestUrl + endpoint: this.endpoint + } - const agent = (this.options.protocol === 'https') ? https : http - let request = agent.request(this.options, (res) => { - this.handleResponse(res, done) + this.dataCache.cacheResponse(cacheOptions, data, written => { + // console.log(typeof data, Buffer.isBuffer(data)) + // console.log(written, this.dataCache.stillCaching) + return done(null, JSON.parse(data.toString())) }) + } else { + if (Buffer.isBuffer(data)) { + data = data.toString() + } - request.on('error', (err) => { - const message = err.toString() + ". Couldn't request data from " + this.datasource.endpoint - err.name = 'GetData' - err.message = message - err.remoteIp = this.options.host - err.remotePort = this.options.port - return done(err) - }) + if (typeof data === 'string') { + data = JSON.parse(data) + } - request.end() - }) + return done(null, data) + } }) } /** - * processDatasourceParameters - adds querystring parameters to the datasource endpoint using properties defined in the schema + * Called on every request, rebuilds the datasource endpoint + * + * @param {http.IncomingMessage} req - the full HTTP request object + */ +RemoteProvider.prototype.processRequest = function (datasourceParams) { + // return this.buildEndpoint(datasourceParams) + this.buildEndpoint(datasourceParams) +} + +/** + * Adds querystring parameters to the datasource endpoint using + * properties defined in the schema * * @param {Object} schema - the datasource schema * @param {type} uri - the original datasource endpoint * @returns {string} uri with query string appended */ -RemoteProvider.prototype.processDatasourceParameters = function processDatasourceParameters (schema, uri) { +RemoteProvider.prototype.processDatasourceParameters = function (datasourceParams, uri) { debug('processDatasourceParameters %s', uri) let query = (uri.indexOf('?') > 0) ? '&' : '?' const params = [ - { 'count': (schema.datasource.count || 0) }, - { 'skip': (schema.datasource.skip) }, - { 'page': (schema.datasource.page || 1) }, - { 'referer': schema.datasource.referer }, - { 'filter': schema.datasource.filter || {} }, - { 'fields': schema.datasource.fields || {} }, - { 'sort': this.processSortParameter(schema.datasource.sort) } + { 'count': (datasourceParams.count || 0) }, + { 'skip': (datasourceParams.skip) }, + { 'page': (datasourceParams.page || 1) }, + { 'filter': datasourceParams.filter || {} }, + { 'fields': datasourceParams.fields || {} }, + { 'sort': this.processSortParameter(datasourceParams.sort) } ] // pass cache flag to API endpoint - if (schema.datasource.hasOwnProperty('cache')) { - params.push({ 'cache': schema.datasource.cache }) + if (datasourceParams.hasOwnProperty('cache')) { + params.push({ 'cache': datasourceParams.cache }) } params.forEach((param) => { @@ -265,73 +324,81 @@ RemoteProvider.prototype.processDatasourceParameters = function processDatasourc } /** - * processOutput + * Requests an Authorization token and sets up the request headers + * with encoding and Authorization values * - * @param {response} res - * @param {string} data - * @param {fn} done - * @return {void} + * @param {fn} done - returns the request headers */ -RemoteProvider.prototype.processOutput = function processOutput (res, data, done) { - // Return a 202 Accepted response immediately, - // along with the datasource response - if (res.statusCode === 202) { - return done(null, JSON.parse(data), res) +RemoteProvider.prototype.getHeaders = function (done) { + var headers = { + 'accept-encoding': 'gzip' } - // return 5xx error as the datasource response - if (res.statusCode && /^5/.exec(res.statusCode)) { - data = { - 'results': [], - 'errors': [{ - 'code': 'WEB-0005', - 'title': 'Datasource Timeout', - 'details': "The datasource '" + this.datasource.name + "' timed out: " + res.statusMessage + ' (' + res.statusCode + ')' + ': ' + this.endpoint - }] - } - } else if (res.statusCode === 404) { - data = { - 'results': [], - 'errors': [{ - 'code': 'WEB-0004', - 'title': 'Datasource Not Found', - 'details': 'Datasource "' + this.datasource.name + '" failed. ' + res.statusMessage + ' (' + res.statusCode + ')' + ': ' + this.endpoint - }] + if (this.datasource.requestHeaders) { + delete this.datasource.requestHeaders['host'] + delete this.datasource.requestHeaders['content-length'] + delete this.datasource.requestHeaders['accept'] + + if (this.datasource.requestHeaders['content-type'] !== 'application/json') { + this.datasource.requestHeaders['content-type'] = 'application/json' } - } else if (res.statusCode && !/200|400/.exec(res.statusCode)) { - // if the error is anything other than Success or Bad Request, error - const err = new Error() - err.message = 'Datasource "' + this.datasource.name + '" failed. ' + res.statusMessage + ' (' + res.statusCode + ')' + ': ' + this.endpoint - if (data) err.message += '\n' + data - err.remoteIp = this.options.host - err.remotePort = this.options.port + headers = _.extend(headers, this.datasource.requestHeaders) + } - log.error({module: 'remote'}, res.statusMessage + ' (' + res.statusCode + ')' + ': ' + this.endpoint) + // If the data-source has its own auth strategy, use it. + // Otherwise, authenticate with the main server via bearer token + if (this.authStrategy) { + // This could eventually become a switch statement that handles different auth types + if (this.authStrategy.getType() === 'bearer') { + this.authStrategy.getToken(this.authStrategy, (err, bearerToken) => { + if (err) { + return done(err) + } - console.log(err) - // return done(err) - throw (err) - } + headers['Authorization'] = 'Bearer ' + bearerToken - // Cache 200 responses - if (res.statusCode === 200) { - this.dataCache.cacheResponse(this.datasource, data, written => { - return done(null, data) - }) + return done(null, { headers: headers }) + }) + } } else { - return done(null, data) + try { + help.getToken(this.datasource).then((bearerToken) => { + headers['Authorization'] = 'Bearer ' + bearerToken + + help.timer.stop('auth') + return done(null, { headers: headers }) + }).catch((errorData) => { + const err = new Error() + err.name = errorData.title + err.message = errorData.detail + err.remoteIp = config.get('api.host') + err.remotePort = config.get('api.port') + err.path = config.get('auth.tokenUrl') + + if (errorData.stack) { + console.log(errorData.stack) + } + + help.timer.stop('auth') + return done(err) + }) + } catch (err) { + console.log(err.stack) + } } } /** - * processRequest - called on every request, rebuild buildEndpoint + * Returns http|https keepAliveAgent depending on specified protocol * - * @param {obj} req - web request object - * @return {void} + * @param {string} protocol - the protocol for the current request + * @returns {module} http|https keepAliveAgent */ -RemoteProvider.prototype.processRequest = function processRequest (req) { - this.buildEndpoint() +RemoteProvider.prototype.keepAliveAgent = function (protocol) { + return (protocol === 'https') + ? new https.Agent({ keepAlive: true }) + : new http.Agent({ keepAlive: true }) } /** diff --git a/dadi/lib/providers/rss.js b/dadi/lib/providers/rss.js index 4433cebe..50f01eb8 100644 --- a/dadi/lib/providers/rss.js +++ b/dadi/lib/providers/rss.js @@ -18,6 +18,7 @@ const RSSProvider = function () {} RSSProvider.prototype.initialise = function initialise (datasource, schema) { this.datasource = datasource this.schema = schema + this.processSchemaParams = false // this.setAuthStrategy() } @@ -69,10 +70,23 @@ RSSProvider.prototype.load = function load (requestUrl, done) { // const queryParams = this.buildQueryParams() this.cacheKey = [this.endpoint, encodeURIComponent(JSON.stringify(this.schema.datasource))].join('+') - this.dataCache = DatasourceCache() - - this.dataCache.getFromCache(this.datasource, (cachedData) => { - if (cachedData) return done(null, cachedData) + this.dataCache = new DatasourceCache() + + var cacheOptions = { + name: this.datasource.name, + caching: this.schema.datasource.caching, + cacheKey: this.cacheKey + } + + this.dataCache.getFromCache(cacheOptions, (cachedData) => { + if (cachedData) { + try { + cachedData = JSON.parse(cachedData.toString()) + return done(null, cachedData) + } catch (err) { + console.error('RSS: cache data incomplete, making HTTP request: ' + err + '(' + cacheOptions.cacheKey + ')') + } + } const items = [] const feedparser = new FeedParser() @@ -97,7 +111,13 @@ RSSProvider.prototype.load = function load (requestUrl, done) { feedparser.on('error', done) feedparser.on('end', () => { - context.dataCache.cacheResponse(this.datasource, JSON.stringify(items), () => {}) + var cacheOptions = { + name: this.datasource.name, + caching: this.schema.datasource.caching, + cacheKey: this.cacheKey + } + + context.dataCache.cacheResponse(cacheOptions, JSON.stringify(items), () => {}) return done(null, items) }) diff --git a/dadi/lib/providers/twitter.js b/dadi/lib/providers/twitter.js index 0abfbdaf..50ddc96c 100644 --- a/dadi/lib/providers/twitter.js +++ b/dadi/lib/providers/twitter.js @@ -57,10 +57,23 @@ TwitterProvider.prototype.load = function load (requestUrl, done) { const queryParams = this.buildQueryParams() this.cacheKey = [endpoint, encodeURIComponent(JSON.stringify(this.schema.datasource))].join('+') - this.dataCache = DatasourceCache() + this.dataCache = new DatasourceCache() - this.dataCache.getFromCache(this.datasource, (cachedData) => { - if (cachedData) return done(null, cachedData) + var cacheOptions = { + name: this.datasource.name, + caching: this.schema.datasource.caching, + cacheKey: this.cacheKey + } + + this.dataCache.getFromCache(cacheOptions, (cachedData) => { + if (cachedData) { + try { + cachedData = JSON.parse(cachedData.toString()) + return done(null, cachedData) + } catch (err) { + console.error('Twitter: cache data incomplete, making HTTP request: ' + err + '(' + cacheOptions.cacheKey + ')') + } + } this.twitterApi.query() .select(endpoint) @@ -100,7 +113,14 @@ TwitterProvider.prototype.processOutput = function processOutput (res, data, don if (res.statusCode === 200) { data = this.processFields(data) - this.dataCache.cacheResponse(this.datasource, JSON.stringify(data), () => { + + var cacheOptions = { + name: this.datasource.name, + caching: this.schema.datasource.caching, + cacheKey: this.cacheKey + } + + this.dataCache.cacheResponse(cacheOptions, JSON.stringify(data), () => { // }) } diff --git a/dadi/lib/providers/wordpress.js b/dadi/lib/providers/wordpress.js index 520836c5..8dd84169 100644 --- a/dadi/lib/providers/wordpress.js +++ b/dadi/lib/providers/wordpress.js @@ -20,6 +20,7 @@ WordPressProvider.prototype.initialise = function initialise (datasource, schema this.datasource = datasource this.schema = schema this.setAuthStrategy() + this.processSchemaParams = false this.wordpressApi = new Purest({ provider: 'wordpress', version: 'v1.1' @@ -32,7 +33,7 @@ WordPressProvider.prototype.initialise = function initialise (datasource, schema * @param {obj} req - web request object * @return {void} */ -WordPressProvider.prototype.buildEndpoint = function buildEndpoint (req) { +WordPressProvider.prototype.buildEndpoint = function (req) { const endpointParams = this.schema.datasource.endpointParams || [] const endpoint = this.schema.datasource.source.endpoint @@ -80,15 +81,28 @@ WordPressProvider.prototype.buildQueryParams = function buildQueryParams () { * @param {fn} done - callback on error or completion * @return {void} */ -WordPressProvider.prototype.load = function load (requestUrl, done) { +WordPressProvider.prototype.load = function (requestUrl, done) { try { const queryParams = this.buildQueryParams() this.cacheKey = [this.endpoint, encodeURIComponent(JSON.stringify(this.schema.datasource))].join('+') - this.dataCache = DatasourceCache() - - this.dataCache.getFromCache(this.datasource, (cachedData) => { - if (cachedData) return done(null, cachedData) + this.dataCache = new DatasourceCache() + + var cacheOptions = { + name: this.datasource.name, + caching: this.schema.datasource.caching, + cacheKey: this.cacheKey + } + + this.dataCache.getFromCache(cacheOptions, (cachedData) => { + if (cachedData) { + try { + cachedData = JSON.parse(cachedData.toString()) + return done(null, cachedData) + } catch (err) { + log.error('Wordpress: cache data incomplete, making HTTP request: ' + err + '(' + cacheOptions.cacheKey + ')') + } + } this.wordpressApi.query() .select(this.endpoint) @@ -125,8 +139,14 @@ WordPressProvider.prototype.processOutput = function processOutput (res, data, d return done(err) } + var cacheOptions = { + name: this.datasource.name, + caching: this.schema.datasource.caching, + cacheKey: this.cacheKey + } + if (res.statusCode === 200) { - this.dataCache.cacheResponse(this.datasource, JSON.stringify(data), () => { + this.dataCache.cacheResponse(cacheOptions, JSON.stringify(data), () => { // }) } @@ -141,6 +161,7 @@ WordPressProvider.prototype.processOutput = function processOutput (res, data, d * @return {void} */ WordPressProvider.prototype.processRequest = function processRequest (req) { + // return this.buildEndpoint(req) this.buildEndpoint(req) } diff --git a/package.json b/package.json index 06773390..54bdb18c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@dadi/web", - "version": "1.11.1", + "version": "1.11.2", "description": "Web frontend and template layer for @dadi/api", "main": "main.js", "scripts": { @@ -37,6 +37,7 @@ "event-stream": "^3.3.1", "express-session": "1.12.1", "feedparser": "^1.1.4", + "humanize-plus": "^1.8.2", "js-beautify": "^1.5.10", "json5": "0.2.0", "kinesis": "^1.2.2", diff --git a/test/unit/controller.js b/test/unit/controller.js index 89c59fcc..3f2f5a50 100644 --- a/test/unit/controller.js +++ b/test/unit/controller.js @@ -279,7 +279,7 @@ describe('Controller', function (done) { var call = providerSpy.secondCall var provider = call.thisValue - var q = require('url').parse(provider.endpoint, true).query + var q = require('url').parse(provider.options.path, true).query var filter = q.filter var filterObj = JSON.parse(filter) should.exist(filterObj._id) @@ -333,15 +333,19 @@ describe('Controller', function (done) { var filterDatasource = providerSpy.thisValues[1] + var q = require('url').parse(filterDatasource.options.path, true).query + var filter = q.filter + var filterObj = JSON.parse(filter) + filterDatasource.schema.datasource.filterEventResult.should.exist filterDatasource.schema.datasource.filterEventResult.x.should.exist filterDatasource.schema.datasource.filterEventResult.x.should.eql('1') - filterDatasource.schema.datasource.filter.x.should.exist - filterDatasource.schema.datasource.filter.x.should.eql('1') + filterObj.x.should.exist + filterObj.x.should.eql('1') - filterDatasource.schema.datasource.filter.y.should.exist - filterDatasource.schema.datasource.filter.y.should.eql('2') + filterObj.y.should.exist + filterObj.y.should.eql('2') done() }) diff --git a/test/unit/data-providers.js b/test/unit/data-providers.js index dcd77e50..0eb983b7 100644 --- a/test/unit/data-providers.js +++ b/test/unit/data-providers.js @@ -102,7 +102,10 @@ describe('Data Providers', function (done) { .end((err, res) => { providerSpy.restore() providerSpy.called.should.eql(true) - providerSpy.firstCall.args[1].should.eql(text) + + var buffer = providerSpy.firstCall.args[2] + + buffer.toString().should.eql(text) done() }) @@ -602,7 +605,7 @@ describe('Data Providers', function (done) { TestHelper.startServer(pages).then(() => { var connectionString = 'http://' + config.get('server.host') + ':' + config.get('server.port') var client = request(connectionString) - + console.log(pages[0].routes[0].path) client .get(pages[0].routes[0].path + '?json=true') .end((err, res) => { diff --git a/test/unit/datasource.js b/test/unit/datasource.js index 52eeb243..06d5d160 100644 --- a/test/unit/datasource.js +++ b/test/unit/datasource.js @@ -178,6 +178,7 @@ describe('Datasource', function (done) { var dsName = 'car-makes' new Datasource(p, dsName, TestHelper.getPathOptions()).init(function (err, ds) { + ds.provider.buildEndpoint() ds.provider.endpoint.should.eql('http://127.0.0.1:3000/1.0/cars/makes?count=20&page=1&filter={}&fields={"name":1,"_id":0}&sort={"name":1}') done() }) @@ -193,7 +194,7 @@ describe('Datasource', function (done) { new Datasource(p, dsName, TestHelper.getPathOptions()).init(function (err, ds) { ds.schema.datasource.skip = 5 - ds.processRequest(dsName, req) + ds.provider.buildEndpoint() ds.provider.endpoint.should.eql('http://127.0.0.1:3000/1.0/cars/makes?count=20&skip=5&page=1&filter={}&fields={"name":1,"_id":0}&sort={"name":1}') done() }) @@ -205,6 +206,7 @@ describe('Datasource', function (done) { var dsName = 'car-makes' new Datasource(null, dsName, TestHelper.getPathOptions()).init(function (err, ds) { + ds.provider.buildEndpoint() ds.provider.endpoint.should.eql('http://127.0.0.1:3000/1.0/cars/makes?count=20&page=1&filter={}&fields={"name":1,"_id":0}&sort={"name":1}') done() }) @@ -221,7 +223,7 @@ describe('Datasource', function (done) { new Datasource(null, dsName, TestHelper.getPathOptions()).init(function (err, ds) { delete ds.schema.datasource.source.host // delete ds.schema.datasource.source.port - ds.provider.buildEndpoint(ds.schema, function () {}) + ds.provider.buildEndpoint() ds.provider.endpoint.should.eql('http://api.example.com:3000/1.0/cars/makes?count=20&page=1&filter={}&fields={"name":1,"_id":0}&sort={"name":1}') done() }) @@ -237,7 +239,7 @@ describe('Datasource', function (done) { new Datasource(null, dsName, TestHelper.getPathOptions()).init(function (err, ds) { delete ds.schema.datasource.source.port - ds.provider.buildEndpoint(ds.schema, function () {}) + ds.provider.buildEndpoint() ds.provider.endpoint.should.eql('http://127.0.0.1:80/1.0/cars/makes?count=20&page=1&filter={}&fields={"name":1,"_id":0}&sort={"name":1}') done() }) @@ -254,7 +256,7 @@ describe('Datasource', function (done) { new Datasource(null, dsName, TestHelper.getPathOptions()).init(function (err, ds) { delete ds.schema.datasource.source.host delete ds.schema.datasource.source.port - ds.provider.buildEndpoint(ds.schema, function () {}) + ds.provider.buildEndpoint() ds.provider.endpoint.should.eql('http://api.example.com:80/1.0/cars/makes?count=20&page=1&filter={}&fields={"name":1,"_id":0}&sort={"name":1}') done() }) @@ -349,9 +351,9 @@ describe('Datasource', function (done) { new Datasource(p, dsName, TestHelper.getPathOptions()).init(function (err, ds) { var params = { 'make': 'bmw' } var req = { params: params, url: '/1.0/cars/makes' } - var endpoint = ds.provider.processDatasourceParameters(dsSchema, req.url) + ds.provider.processDatasourceParameters(dsSchema, req.url) Datasource.Datasource.prototype.loadDatasource.restore() - endpoint.should.eql('/1.0/cars/makes?count=20&page=1&filter={}&fields={"name":1,"_id":0}&sort={"name":1}') + decodeURIComponent(require('url').parse(ds.provider.endpoint).path).should.eql('/1.0/cars/makes?count=20&page=1&filter={}&fields={"name":1,"_id":0}&sort={"name":1}') done() }) }) @@ -370,9 +372,9 @@ describe('Datasource', function (done) { new Datasource(p, dsName, TestHelper.getPathOptions()).init(function (err, ds) { var params = { 'make': 'bmw' } var req = { params: params, url: '/1.0/cars/makes' } - var endpoint = ds.provider.processDatasourceParameters(dsSchema, req.url) + ds.provider.processDatasourceParameters(dsSchema, req.url) Datasource.Datasource.prototype.loadDatasource.restore() - endpoint.should.eql('/1.0/cars/makes?count=20&page=1&filter={}&fields={"name":1,"_id":0}&sort={"name":1,"age":-1}') + decodeURIComponent(require('url').parse(ds.provider.endpoint).path).should.eql('/1.0/cars/makes?count=20&page=1&filter={}&fields={"name":1,"_id":0}&sort={"name":1,"age":-1}') done() }) }) @@ -391,10 +393,10 @@ describe('Datasource', function (done) { new Datasource(p, dsName, options).init(function (err, ds) { var params = { 'make': 'bmw' } var req = { params: params, url: '/1.0/cars/makes' } - var endpoint = ds.provider.processDatasourceParameters(dsSchema, req.url) + ds.provider.processDatasourceParameters(dsSchema, req.url) Datasource.Datasource.prototype.loadDatasource.restore() - endpoint.should.eql('/1.0/cars/makes?count=20&page=1&filter={}&fields={"name":1,"_id":0}&sort={"name":1}') + decodeURIComponent(require('url').parse(ds.provider.endpoint).path).should.eql('/1.0/cars/makes?count=20&page=1&filter={}&fields={"name":1,"_id":0}&sort={"name":1}') done() }) }) @@ -413,10 +415,10 @@ describe('Datasource', function (done) { new Datasource(p, dsName, options).init(function (err, ds) { var params = { 'make': 'bmw' } var req = { params: params, url: '/1.0/cars/makes' } - var endpoint = ds.provider.processDatasourceParameters(dsSchema, req.url) + ds.provider.processDatasourceParameters(dsSchema, req.url) Datasource.Datasource.prototype.loadDatasource.restore() - endpoint.should.eql('/1.0/cars/makes?count=20&page=1&filter={}&fields={"name":1,"_id":0}&sort={"name":1}') + decodeURIComponent(require('url').parse(ds.provider.endpoint).path).should.eql('/1.0/cars/makes?count=20&page=1&filter={}&fields={"name":1,"_id":0}&sort={"name":1}') done() }) }) @@ -435,9 +437,9 @@ describe('Datasource', function (done) { new Datasource(p, dsName, options).init(function (err, ds) { var params = { 'make': 'bmw' } var req = { params: params, url: '/1.0/cars/makes' } - var endpoint = ds.provider.processDatasourceParameters(dsSchema, req.url) + ds.provider.processDatasourceParameters(dsSchema, req.url) Datasource.Datasource.prototype.loadDatasource.restore() - endpoint.should.eql('/1.0/cars/makes?count=20&page=1&filter={}&fields={"name":1,"_id":0}&sort={"name":1,"age":-1}') + decodeURIComponent(require('url').parse(ds.provider.endpoint).path).should.eql('/1.0/cars/makes?count=20&page=1&filter={}&fields={"name":1,"_id":0}&sort={"name":1,"age":-1}') done() }) }) @@ -550,7 +552,7 @@ describe('Datasource', function (done) { new Datasource(p, dsName, TestHelper.getPathOptions()).init(function (err, ds) { ds.processRequest(dsName, req) ds.provider.endpoint.should.eql('http://127.0.0.1:3000/1.0/cars/makes?count=20&page=3&filter={"name":"bmw"}&fields={"name":1,"_id":0}&sort={"name":1}&cache=false') - ds.schema.datasource.cache.should.eql(false) + // ds.schema.datasource.cache.should.eql(false) done() }) }) @@ -568,12 +570,13 @@ describe('Datasource', function (done) { new Datasource(p, dsName, TestHelper.getPathOptions()).init(function (err, ds) { ds.processRequest(dsName, req) ds.provider.endpoint.should.eql('http://127.0.0.1:3000/1.0/cars/makes?count=20&page=3&filter={"name":"bmw"}&fields={"name":1,"_id":0}&sort={"name":1}&cache=false') - ds.schema.datasource.cache.should.eql(false) + // ds.schema.datasource.cache.should.eql(false) req = { params: params, url: '/1.0/cars/makes' } ds.processRequest(dsName, req) ds.provider.endpoint.should.eql('http://127.0.0.1:3000/1.0/cars/makes?count=20&page=3&filter={"name":"bmw"}&fields={"name":1,"_id":0}&sort={"name":1}') - ;(typeof ds.schema.datasource.cache === 'undefined').should.eql(true) + + // ;(typeof ds.schema.datasource.cache === 'undefined').should.eql(true) done() }) diff --git a/test/unit/ds_cache.js b/test/unit/ds_cache.js index 23fdc2e6..bf0b0152 100644 --- a/test/unit/ds_cache.js +++ b/test/unit/ds_cache.js @@ -24,7 +24,7 @@ var cachepath describe('Datasource Cache', function (done) { beforeEach(function (done) { - datasourceCache._reset() + // datasourceCache._reset() TestHelper.resetConfig().then(() => { TestHelper.disableApiConfig().then(() => { @@ -164,7 +164,11 @@ describe('Datasource Cache', function (done) { var dsCache = new datasourceCache(ds) var expectToFind = crypto.createHash('sha1').update(ds.schema.datasource.key).digest('hex') var dsCache = datasourceCache() - dsCache.getFilename(ds).indexOf(expectToFind).should.be.above(-1) + dsCache.getFilename({ + name: ds.name, + caching: cacheConfig.caching, + endpoint: '' + }).indexOf(expectToFind).should.be.above(-1) done() }) }) @@ -209,9 +213,17 @@ describe('Datasource Cache', function (done) { // process the http request so parameters are injected datasource.processRequest(datasource.schema.datasource.key, { url: '/1.0/makes/ford' , params: { 'make': 'ford' } }) - var filename1 = dsCache.getFilename(datasource) + var filename1 = dsCache.getFilename({ + name: datasource.name, + caching: cacheConfig.caching, + endpoint: datasource.provider.endpoint + }) datasource.processRequest(datasource.schema.datasource.key, { url: '/1.0/makes/mazda' , params: { 'make': 'mazda' } }) - var filename2 = dsCache.getFilename(datasource) + var filename2 = dsCache.getFilename({ + name: datasource.name, + caching: cacheConfig.caching, + endpoint: datasource.provider.endpoint + }) filename1.should.not.eql(filename2) @@ -239,7 +251,13 @@ describe('Datasource Cache', function (done) { ds.schema.datasource.caching.ttl = 1000 var c = cache(server.object) var dsCache = datasourceCache() - dsCache.getOptions(ds).ttl.should.eql(1000) + var opts = dsCache.getOptions({ + name: ds.name, + caching: ds.schema.datasource.caching, + endpoint: '' + }) + + opts.ttl.should.eql(1000) done() }) }) @@ -268,7 +286,12 @@ describe('Datasource Cache', function (done) { delete ds.schema.datasource.caching.directory.path ds.schema.datasource.caching.redis.enabled = false - var options = dsCache.getOptions(ds) + var options = dsCache.getOptions({ + name: ds.name, + caching: ds.schema.datasource.caching, + endpoint: '' + }) + options.directory.path.should.eql(cacheConfig.caching.directory.path) ds.schema.datasource.caching.directory.path = p @@ -298,7 +321,11 @@ describe('Datasource Cache', function (done) { ds.schema.datasource.caching.directory.enabled = false ds.schema.datasource.caching.redis.enabled = true - var options = dsCache.getOptions(ds) + var options = dsCache.getOptions({ + name: ds.name, + caching: ds.schema.datasource.caching, + endpoint: '' + }) options.directory.enabled.should.eql(false) options.redis.enabled.should.eql(true) @@ -329,7 +356,12 @@ describe('Datasource Cache', function (done) { ds.schema.datasource.caching.redis.enabled = true ds.schema.datasource.caching.redis.port = 13057 - var options = dsCache.getOptions(ds) + var options = dsCache.getOptions({ + name: ds.name, + caching: ds.schema.datasource.caching, + endpoint: '' + }) + options.directory.enabled.should.eql(false) options.redis.enabled.should.eql(true) options.redis.port.should.eql(13057) @@ -358,7 +390,13 @@ describe('Datasource Cache', function (done) { ds.schema.datasource.caching.directory.enabled = false var c = cache(server.object) var dsCache = datasourceCache() - dsCache.cachingEnabled(ds).should.eql(false) + + dsCache.cachingEnabled({ + name: ds.name, + caching: ds.schema.datasource.caching, + endpoint: '' + }).should.eql(false) + done() }) }) @@ -380,7 +418,11 @@ describe('Datasource Cache', function (done) { ds.provider.endpoint += '&cache=false' var c = cache(server.object) var dsCache = datasourceCache() - dsCache.cachingEnabled(ds).should.eql(false) + dsCache.cachingEnabled({ + name: ds.name, + caching: ds.schema.datasource.caching, + endpoint: ds.provider.endpoint + }).should.eql(false) done() }) }) @@ -401,7 +443,11 @@ describe('Datasource Cache', function (done) { ds.schema.datasource.caching.directory.enabled = true var c = cache(server.object) var dsCache = datasourceCache() - dsCache.cachingEnabled(ds).should.eql(true) + dsCache.cachingEnabled({ + name: ds.name, + caching: ds.schema.datasource.caching, + endpoint: '' + }).should.eql(true) done() }) }) @@ -438,9 +484,13 @@ describe('Datasource Cache', function (done) { setTimeout(function() { var dsCache = datasourceCache() - dsCache.getFromCache(ds, function (data) { + dsCache.getFromCache({ + name: ds.name, + caching: ds.schema.datasource.caching, + endpoint: ds.provider.endpoint + }, function (data) { data.should.not.eql(false) - data.should.eql(expected) + data.toString().should.eql(expected) done() }) }, 1000) @@ -474,7 +524,11 @@ describe('Datasource Cache', function (done) { if (err) console.log(err.toString()) var dsCache = datasourceCache() - dsCache.getFromCache(ds, function (data) { + dsCache.getFromCache({ + name: ds.name, + caching: ds.schema.datasource.caching, + endpoint: ds.provider.endpoint + }, function (data) { data.should.eql(false) done() }) @@ -501,7 +555,11 @@ describe('Datasource Cache', function (done) { var data = 'ds content from filesystem' var dsCache = datasourceCache() - dsCache.getFromCache(ds, function (data) { + dsCache.getFromCache({ + name: ds.name, + caching: ds.schema.datasource.caching, + endpoint: '' + }, function (data) { data.should.eql(false) done() }) @@ -538,7 +596,11 @@ describe('Datasource Cache', function (done) { setTimeout(function () { var dsCache = datasourceCache() - dsCache.getFromCache(ds, function (data) { + dsCache.getFromCache({ + name: ds.name, + caching: ds.schema.datasource.caching, + endpoint: ds.provider.endpoint + }, function (data) { data.should.eql(false) done() }) @@ -572,7 +634,11 @@ describe('Datasource Cache', function (done) { var data = 'ds content from filesystem' var dsCache = datasourceCache() - dsCache.cacheResponse(ds, data, function () { + dsCache.cacheResponse({ + name: ds.name, + caching: ds.schema.datasource.caching, + endpoint: ds.provider.endpoint + }, data, function () { fs.readFile(cachepath, function (err, content) { content.toString().should.eql(data) done() @@ -610,7 +676,11 @@ describe('Datasource Cache', function (done) { c.redisClient = redisClient var dsCache = datasourceCache() - dsCache.cacheResponse(ds, data, function () { + dsCache.cacheResponse({ + name: ds.name, + caching: ds.schema.datasource.caching, + endpoint: ds.provider.endpoint + }, data, function () { done() }) })