From 4901ecc517e43cade6e5ae621009f248b49be273 Mon Sep 17 00:00:00 2001 From: James Lambie Date: Thu, 16 Mar 2017 13:13:23 +1300 Subject: [PATCH] =?UTF-8?q?fix:=20don=E2=80=99t=20modify=20original=20sche?= =?UTF-8?q?ma=20endpoint?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit otherwise placeholders no longer exist --- README.md | 2 +- dadi/lib/datasource/index.js | 3 +- dadi/lib/index.js | 46 +---------- dadi/lib/providers/remote.js | 34 ++++---- dadi/lib/providers/rss.js | 4 +- test/rss.xml | 151 +++++++++++++++++++++++++++++++++++ test/unit/config.js | 14 +--- test/unit/controller.js | 3 +- test/unit/data-providers.js | 9 ++- test/unit/ds_cache.js | 52 ++++++++++++ 10 files changed, 235 insertions(+), 83 deletions(-) create mode 100644 test/rss.xml diff --git a/README.md b/README.md index ceb6344e..7c4e5324 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ DADI Web [![npm (scoped)](https://img.shields.io/npm/v/@dadi/web.svg?maxAge=10800&style=flat-square)](https://www.npmjs.com/package/@dadi/web) -[![coverage](https://img.shields.io/badge/coverage-67%25-yellow.svg?style=flat?style=flat-square)](https://github.com/dadi/web) +[![coverage](https://img.shields.io/badge/coverage-68%25-yellow.svg?style=flat?style=flat-square)](https://github.com/dadi/web) [![Build Status](https://travis-ci.org/dadi/web.svg?branch=master)](https://travis-ci.org/dadi/web) [![JavaScript Style Guide](https://img.shields.io/badge/code%20style-standard-brightgreen.svg?style=flat-square)](http://standardjs.com/) [![semantic-release](https://img.shields.io/badge/%20%20%F0%9F%93%A6%F0%9F%9A%80-semantic--release-e10079.svg?style=flat-square)](https://github.com/semantic-release/semantic-release) diff --git a/dadi/lib/datasource/index.js b/dadi/lib/datasource/index.js index 765915fa..eeebe4df 100644 --- a/dadi/lib/datasource/index.js +++ b/dadi/lib/datasource/index.js @@ -142,6 +142,7 @@ Datasource.prototype.processRequest = function (datasource, req) { // Regular expression search for {param.nameOfParam} and replace with requestParameters var paramRule = /("\{)(\bparams.\b)(.*?)(\}")/gmi + this.schema.datasource.filter = JSON.parse(JSON.stringify(this.schema.datasource.filter).replace(paramRule, function (match, p1, p2, p3, p4, offset, string) { if (req.params[p3]) { return req.params[p3] @@ -168,7 +169,7 @@ Datasource.prototype.processRequest = function (datasource, req) { this.schema.datasource.filter[obj.field] = paramValue } else if (obj.target === 'endpoint') { var placeholderRegex = new RegExp('{' + obj.field + '}', 'ig') - this.schema.datasource.source.endpoint = this.schema.datasource.source.endpoint.replace(placeholderRegex, paramValue) + this.source.modifiedEndpoint = this.schema.datasource.source.endpoint.replace(placeholderRegex, paramValue) } } else { if (obj.target === 'filter') { diff --git a/dadi/lib/index.js b/dadi/lib/index.js index 8394fe57..bba8cb08 100755 --- a/dadi/lib/index.js +++ b/dadi/lib/index.js @@ -718,7 +718,7 @@ function onConnection (socket) { function onListening (e) { // check that our API connection is valid - help.isApiAvailable(function (err, result) { + help.isApiAvailable((err, result) => { if (err) { console.log(err) console.log() @@ -774,47 +774,3 @@ function onError (err) { process.exit(0) } } - -// function processSortParameter (obj) { -// var sort = {} -// if (typeof obj !== 'object' || obj === null) return sort -// -// _.each(obj, function (value, key) { -// if (typeof value === 'object' && value.hasOwnProperty('field') && value.hasOwnProperty('order')) { -// sort[value.field] = (value.order === 'asc') ? 1 : -1 -// } -// }) -// -// return sort -// } - -// function parseRoutes (endpoints, req_path) { -// var params = {} -// var route_path = '' -// _.each(endpoints, function (endpoint) { -// var paths = endpoint.page.route.paths -// var req_path_items = req_path.split('/') -// _.each(paths, function (path) { -// path_items = path.split('/') -// if (path_items.length == req_path_items.length) { -// var alias = _.filter(path_items, function (item) { -// return item == '' || item.slice(0, 1) != ':' -// }) -// -// if (_.difference(alias, _.intersection(path_items, req_path_items)).length == 0) { -// _.each(path_items, function (item, index) { -// if (item != '' && item.slice(0, 1) == ':') { -// params[item.slice(1)] = req_path_items[index] -// } -// }) -// route_path = path -// } -// } -// }) -// }) -// -// return { -// route_path: route_path, -// params: params -// } -// } diff --git a/dadi/lib/providers/remote.js b/dadi/lib/providers/remote.js index b511a266..8f1b6a9c 100644 --- a/dadi/lib/providers/remote.js +++ b/dadi/lib/providers/remote.js @@ -44,7 +44,7 @@ RemoteProvider.prototype.buildEndpoint = function buildEndpoint () { const port = source.port || apiConfig.port const uri = [protocol, '://', host, (port !== '' ? ':' : ''), - port, '/', source.endpoint].join('') + port, '/', this.datasource.source.modifiedEndpoint || source.endpoint].join('') this.endpoint = this.processDatasourceParameters(this.schema, uri) } @@ -111,7 +111,6 @@ RemoteProvider.prototype.getHeaders = function getHeaders (done) { * @return {void} */ RemoteProvider.prototype.handleResponse = function handleResponse (res, done) { - const self = this const encoding = res.headers['content-encoding'] ? res.headers['content-encoding'] : '' let output = '' @@ -123,7 +122,7 @@ RemoteProvider.prototype.handleResponse = function handleResponse (res, done) { buffer.push(data.toString()) }).on('end', () => { output = buffer.join('') - self.processOutput(res, output, (err, data, res) => { + this.processOutput(res, output, (err, data, res) => { if (err) return done(err) return done(null, data, res) }) @@ -138,7 +137,7 @@ RemoteProvider.prototype.handleResponse = function handleResponse (res, done) { }) res.on('end', () => { - self.processOutput(res, output, (err, data, res) => { + this.processOutput(res, output, (err, data, res) => { if (err) return done(err) return done(null, data, res) }) @@ -166,8 +165,6 @@ RemoteProvider.prototype.keepAliveAgent = function keepAliveAgent (protocol) { * @return {void} */ RemoteProvider.prototype.load = function (requestUrl, done) { - const self = this - this.requestUrl = requestUrl this.dataCache = DatasourceCache() @@ -187,24 +184,24 @@ RemoteProvider.prototype.load = function (requestUrl, done) { debug('load %s', this.endpoint) - self.getHeaders((err, headers) => { + this.getHeaders((err, headers) => { err && done(err) - self.options = _.extend(self.options, headers) + this.options = _.extend(this.options, headers) - log.info({module: 'helper'}, "GET datasource '" + self.datasource.schema.datasource.key + "': " + self.options.path) + log.info({module: 'helper'}, "GET datasource '" + this.datasource.schema.datasource.key + "': " + this.options.path) - const agent = (self.options.protocol === 'https') ? https : http - let request = agent.request(self.options, (res) => { - self.handleResponse(res, done) + const agent = (this.options.protocol === 'https') ? https : http + let request = agent.request(this.options, (res) => { + this.handleResponse(res, done) }) request.on('error', (err) => { - const message = err.toString() + ". Couldn't request data from " + self.datasource.endpoint + const message = err.toString() + ". Couldn't request data from " + this.datasource.endpoint err.name = 'GetData' err.message = message - err.remoteIp = self.options.host - err.remotePort = self.options.port + err.remoteIp = this.options.host + err.remotePort = this.options.port return done(err) }) @@ -221,6 +218,7 @@ RemoteProvider.prototype.load = function (requestUrl, done) { * @returns {string} uri with query string appended */ RemoteProvider.prototype.processDatasourceParameters = function processDatasourceParameters (schema, uri) { + debug('processDatasourceParameters %s', uri) let query = '?' const params = [ @@ -258,8 +256,6 @@ RemoteProvider.prototype.processDatasourceParameters = function processDatasourc * @return {void} */ RemoteProvider.prototype.processOutput = function processOutput (res, data, done) { - const self = this - // Return a 202 Accepted response immediately, // along with the datasource response if (res.statusCode === 202) { @@ -291,8 +287,8 @@ RemoteProvider.prototype.processOutput = function processOutput (res, data, done err.message = 'Datasource "' + this.datasource.name + '" failed. ' + res.statusMessage + ' (' + res.statusCode + ')' + ': ' + this.endpoint if (data) err.message += '\n' + data - err.remoteIp = self.options.host - err.remotePort = self.options.port + err.remoteIp = this.options.host + err.remotePort = this.options.port log.error({module: 'helper'}, res.statusMessage + ' (' + res.statusCode + ')' + ': ' + this.endpoint) diff --git a/dadi/lib/providers/rss.js b/dadi/lib/providers/rss.js index afa451ee..4433cebe 100644 --- a/dadi/lib/providers/rss.js +++ b/dadi/lib/providers/rss.js @@ -76,6 +76,7 @@ RSSProvider.prototype.load = function load (requestUrl, done) { const items = [] const feedparser = new FeedParser() + const req = request(this.endpoint, { pool: false, headers: { @@ -86,6 +87,7 @@ RSSProvider.prototype.load = function load (requestUrl, done) { // request events req.on('error', done) + req.on('response', (res) => { if (res.statusCode !== 200) return done('Bad status code', null) res.pipe(feedparser) @@ -96,7 +98,7 @@ RSSProvider.prototype.load = function load (requestUrl, done) { feedparser.on('end', () => { context.dataCache.cacheResponse(this.datasource, JSON.stringify(items), () => {}) - done(null, items) + return done(null, items) }) feedparser.on('readable', function () { diff --git a/test/rss.xml b/test/rss.xml new file mode 100644 index 00000000..57ea10d5 --- /dev/null +++ b/test/rss.xml @@ -0,0 +1,151 @@ + + + + FeedForAll Sample Feed + RSS is a fascinating technology. The uses for RSS are expanding daily. Take a closer look at how various industries are using the benefits of RSS in their businesses. + http://www.feedforall.com/industry-solutions.htm + Computers/Software/Internet/Site Management/Content Management + Copyright 2004 NotePage, Inc. + http://blogs.law.harvard.edu/tech/rss + en-us + Tue, 19 Oct 2004 13:39:14 -0400 + marketing@feedforall.com + Tue, 19 Oct 2004 13:38:55 -0400 + webmaster@feedforall.com + FeedForAll Beta1 (0.0.1.8) + + http://www.feedforall.com/ffalogo48x48.gif + FeedForAll Sample Feed + http://www.feedforall.com/industry-solutions.htm + FeedForAll Sample Feed + 48 + 48 + + + RSS Solutions for Restaurants + <b>FeedForAll </b>helps Restaurant's communicate with customers. Let your customers know the latest specials or events.<br> +<br> +RSS feed uses include:<br> +<i><font color="#FF0000">Daily Specials <br> +Entertainment <br> +Calendar of Events </i></font> + http://www.feedforall.com/restaurant.htm + Computers/Software/Internet/Site Management/Content Management + http://www.feedforall.com/forum + Tue, 19 Oct 2004 11:09:11 -0400 + + + RSS Solutions for Schools and Colleges + FeedForAll helps Educational Institutions communicate with students about school wide activities, events, and schedules.<br> +<br> +RSS feed uses include:<br> +<i><font color="#0000FF">Homework Assignments <br> +School Cancellations <br> +Calendar of Events <br> +Sports Scores <br> +Clubs/Organization Meetings <br> +Lunches Menus </i></font> + http://www.feedforall.com/schools.htm + Computers/Software/Internet/Site Management/Content Management + http://www.feedforall.com/forum + Tue, 19 Oct 2004 11:09:09 -0400 + + + RSS Solutions for Computer Service Companies + FeedForAll helps Computer Service Companies communicate with clients about cyber security and related issues. <br> +<br> +Uses include:<br> +<i><font color="#0000FF">Cyber Security Alerts <br> +Specials<br> +Job Postings </i></font> + http://www.feedforall.com/computer-service.htm + Computers/Software/Internet/Site Management/Content Management + http://www.feedforall.com/forum + Tue, 19 Oct 2004 11:09:07 -0400 + + + RSS Solutions for Governments + FeedForAll helps Governments communicate with the general public about positions on various issues, and keep the community aware of changes in important legislative issues. <b><i><br> +</b></i><br> +RSS uses Include:<br> +<i><font color="#00FF00">Legislative Calendar<br> +Votes<br> +Bulletins</i></font> + http://www.feedforall.com/government.htm + Computers/Software/Internet/Site Management/Content Management + http://www.feedforall.com/forum + Tue, 19 Oct 2004 11:09:05 -0400 + + + RSS Solutions for Politicians + FeedForAll helps Politicians communicate with the general public about positions on various issues, and keep the community notified of their schedule. <br> +<br> +Uses Include:<br> +<i><font color="#FF0000">Blogs<br> +Speaking Engagements <br> +Statements<br> + </i></font> + http://www.feedforall.com/politics.htm + Computers/Software/Internet/Site Management/Content Management + http://www.feedforall.com/forum + Tue, 19 Oct 2004 11:09:03 -0400 + + + RSS Solutions for Meteorologists + FeedForAll helps Meteorologists communicate with the general public about storm warnings and weather alerts, in specific regions. Using RSS meteorologists are able to quickly disseminate urgent and life threatening weather warnings. <br> +<br> +Uses Include:<br> +<i><font color="#0000FF">Weather Alerts<br> +Plotting Storms<br> +School Cancellations </i></font> + http://www.feedforall.com/weather.htm + Computers/Software/Internet/Site Management/Content Management + http://www.feedforall.com/forum + Tue, 19 Oct 2004 11:09:01 -0400 + + + RSS Solutions for Realtors & Real Estate Firms + FeedForAll helps Realtors and Real Estate companies communicate with clients informing them of newly available properties, and open house announcements. RSS helps to reach a targeted audience and spread the word in an inexpensive, professional manner. <font color="#0000FF"><br> +</font><br> +Feeds can be used for:<br> +<i><font color="#FF0000">Open House Dates<br> +New Properties For Sale<br> +Mortgage Rates</i></font> + http://www.feedforall.com/real-estate.htm + Computers/Software/Internet/Site Management/Content Management + http://www.feedforall.com/forum + Tue, 19 Oct 2004 11:08:59 -0400 + + + RSS Solutions for Banks / Mortgage Companies + FeedForAll helps <b>Banks, Credit Unions and Mortgage companies</b> communicate with the general public about rate changes in a prompt and professional manner. <br> +<br> +Uses include:<br> +<i><font color="#0000FF">Mortgage Rates<br> +Foreign Exchange Rates <br> +Bank Rates<br> +Specials</i></font> + http://www.feedforall.com/banks.htm + Computers/Software/Internet/Site Management/Content Management + http://www.feedforall.com/forum + Tue, 19 Oct 2004 11:08:57 -0400 + + + RSS Solutions for Law Enforcement + <b>FeedForAll</b> helps Law Enforcement Professionals communicate with the general public and other agencies in a prompt and efficient manner. Using RSS police are able to quickly disseminate urgent and life threatening information. <br> +<br> +Uses include:<br> +<i><font color="#0000FF">Amber Alerts<br> +Sex Offender Community Notification <br> +Weather Alerts <br> +Scheduling <br> +Security Alerts <br> +Police Report <br> +Meetings</i></font> + http://www.feedforall.com/law-enforcement.htm + Computers/Software/Internet/Site Management/Content Management + http://www.feedforall.com/forum + Tue, 19 Oct 2004 11:08:56 -0400 + + + \ No newline at end of file diff --git a/test/unit/config.js b/test/unit/config.js index 3ef0202c..e208cfe0 100644 --- a/test/unit/config.js +++ b/test/unit/config.js @@ -13,7 +13,6 @@ var TestHelper = require(__dirname + '/../help')() var config = require(path.resolve(path.join(__dirname, '/../../config'))) -var connectionString = 'http://' + config.get('server.host') + ':' + config.get('server.port') var testConfigPath = './config/config.test.json' var domainConfigPath @@ -22,7 +21,6 @@ function createDomainConfig () { server: { host: '127.0.0.1', port: 5111, - redirectPort: 0, protocol: 'http' } }).then(() => { @@ -49,7 +47,7 @@ function cleanup () { } } -describe.skip('Config', function (done) { +describe('Config', function (done) { before(function(done) { delete require.cache[__dirname + '/../../dadi/lib/api'] api = require(__dirname + '/../../dadi/lib/api') @@ -69,13 +67,7 @@ describe.skip('Config', function (done) { }) it('should load a domain specific config file if available', function (done) { - var apiConfig = { - api: { - enabled: false - } - } - - TestHelper.updateConfig(apiConfig).then(() => { + TestHelper.disableApiConfig().then(() => { createDomainConfig() setTimeout(function() { @@ -84,6 +76,8 @@ describe.skip('Config', function (done) { delete require.cache[path.resolve(path.join(__dirname, '/../../config'))] + var connectionString = 'http://' + config.get('server.host') + ':' + config.get('server.port') + TestHelper.startServer(pages).then(() => { var client = request(connectionString) client diff --git a/test/unit/controller.js b/test/unit/controller.js index 610a8669..89c59fcc 100644 --- a/test/unit/controller.js +++ b/test/unit/controller.js @@ -294,7 +294,7 @@ describe('Controller', function (done) { }) }) - describe.skip('Datasource Filter Events', function (done) { + describe('Datasource Filter Events', function (done) { it('should run an attached `filterEvent` before datasource loads', function (done) { TestHelper.enableApiConfig().then(() => { var pages = TestHelper.setUpPages() @@ -321,7 +321,6 @@ describe('Controller', function (done) { var providerSpy = sinon.spy(remoteProvider.prototype, 'load') TestHelper.startServer(pages).then(() => { - var client = request(connectionString) client .get(pages[0].routes[0].path + '?json=true') diff --git a/test/unit/data-providers.js b/test/unit/data-providers.js index bc44cda0..4565f927 100644 --- a/test/unit/data-providers.js +++ b/test/unit/data-providers.js @@ -624,10 +624,11 @@ describe('Data Providers', function (done) { }) }) - it.skip('should return data when no error is encountered', function(done) { + it('should return data when no error is encountered', function(done) { var host = 'http://www.feedforall.com' var path = '/sample.xml' - var scope = nock(host).get(path).reply(200, { x: 'y' }) + + var scope = nock(host).get(path).replyWithFile(200, __dirname + '/../rss.xml') TestHelper.disableApiConfig().then(() => { TestHelper.updateConfig({'allowJsonView': true}).then(() => { @@ -641,8 +642,8 @@ describe('Data Providers', function (done) { client .get(pages[0].routes[0].path + '?json=true') .end((err, res) => { - should.exist(res.body['rss']) - res.body['rss'].should.eql({x:'y'}) + should.exist(res.body.rss) + should.exist(res.body.rss[0].title) done() }) }) diff --git a/test/unit/ds_cache.js b/test/unit/ds_cache.js index a453f54c..8d4dd8ec 100644 --- a/test/unit/ds_cache.js +++ b/test/unit/ds_cache.js @@ -1,3 +1,4 @@ +var _ = require('underscore') var exec = require('child_process').exec var fs = require('fs') var path = require('path') @@ -167,6 +168,57 @@ describe('Datasource Cache', function (done) { done() }) }) + + it('should use different cache filenames when datasource endpoints use placeholders', function (done) { + var cacheConfig = { + caching: { + directory: { + enabled: true + }, + redis: { + enabled: false + } + } + } + + var name = 'test' + var schema = TestHelper.getPageSchema() + var p = page(name, schema) + var dsName = 'car-makes' + var options = TestHelper.getPathOptions() + var dsSchema = _.clone(TestHelper.getSchemaFromFile(options.datasourcePath, dsName)) + + dsSchema.datasource.source.endpoint = '1.0/makes/{make}' + dsSchema.datasource.requestParams = [ + { + param: 'make', + field: 'make', + target: 'endpoint' + } + ] + + sinon.stub(Datasource.Datasource.prototype, 'loadDatasource').yields(null, dsSchema) + + TestHelper.updateConfig(cacheConfig).then(() => { + var c = cache(server.object) + + new Datasource(p, dsName, options).init((err, datasource) => { + + Datasource.Datasource.prototype.loadDatasource.restore() + var dsCache = datasourceCache() + + // process the http request so parameters are injected + datasource.processRequest(datasource.schema.datasource.key, { url: '/1.0/makes/ford' , params: { 'make': 'ford' } }) + var filename1 = dsCache.getFilename(datasource) + datasource.processRequest(datasource.schema.datasource.key, { url: '/1.0/makes/mazda' , params: { 'make': 'mazda' } }) + var filename2 = dsCache.getFilename(datasource) + + filename1.should.not.eql(filename2) + + done() + }) + }) + }) }) describe('cachingEnabled', function (done) {