From e42504013163741e65691a3e2544582e7e5b867d Mon Sep 17 00:00:00 2001 From: James Lambie Date: Sun, 12 Feb 2017 09:21:59 +0800 Subject: [PATCH 1/6] feat: add debug statements instead of log statements --- dadi/lib/api/index.js | 3 ++- dadi/lib/cache/index.js | 3 +-- dadi/lib/controller/router.js | 5 ++--- dadi/lib/dust/index.js | 2 ++ dadi/lib/middleware/index.js | 5 +++-- main.js | 5 +++-- package.json | 5 +++-- 7 files changed, 16 insertions(+), 12 deletions(-) diff --git a/dadi/lib/api/index.js b/dadi/lib/api/index.js index 6dd786ac..1917518e 100755 --- a/dadi/lib/api/index.js +++ b/dadi/lib/api/index.js @@ -1,4 +1,5 @@ var _ = require('underscore') +var debug = require('debug')('api') var fs = require('fs') var http = require('http') var https = require('https') @@ -108,7 +109,7 @@ Api.prototype.use = function (path, handler) { regex: regex }) - log.warn({module: 'api'}, 'Loaded ' + path) + debug('loaded %s', path) this.paths.sort((a, b) => { return b.order - a.order diff --git a/dadi/lib/cache/index.js b/dadi/lib/cache/index.js index 04a07204..5b8bf3c0 100755 --- a/dadi/lib/cache/index.js +++ b/dadi/lib/cache/index.js @@ -1,10 +1,10 @@ /** * @module Cache */ +var _ = require('underscore') var crypto = require('crypto') var path = require('path') var url = require('url') -var _ = require('underscore') var config = require(path.join(__dirname, '/../../../config.js')) var log = require('@dadi/logger') @@ -18,7 +18,6 @@ var cache = new DadiCache(config.get('caching')) * @param {Server} server - the main server instance */ var Cache = function (server) { - log.info({module: 'cache'}, 'Cache logging started.') this.server = server this.enabled = config.get('caching.directory.enabled') || config.get('caching.redis.enabled') this.encoding = 'utf8' diff --git a/dadi/lib/controller/router.js b/dadi/lib/controller/router.js index a579fc50..3ecb38d3 100644 --- a/dadi/lib/controller/router.js +++ b/dadi/lib/controller/router.js @@ -3,6 +3,7 @@ REWRITE INFO: https://github.com/tinganho/connect-modrewrite */ var _ = require('underscore') +var debug = require('debug')('router') var es = require('event-stream') var fs = require('fs') var path = require('path') @@ -22,8 +23,6 @@ var RouteValidator = require(path.join(__dirname, '../datasource/route-validator var rewriteFunction = null var Router = function (server, options) { - log.info({module: 'router'}, 'Router logging started.') - this.data = {} this.params = {} this.constraints = {} @@ -149,7 +148,7 @@ Router.prototype.constrain = function (route, constraint) { // add constraint from /{routesPath}/constraints.js if it exists if (this.handlers[constraint]) { this.constraints[route] = this.handlers[constraint] - log.info({module: 'router'}, "Added route constraint function '%s' for '%s'", constraint, route) + debug('added route constraint function "%s" for %s', constraint, route) } else { var error = "Route constraint '" + constraint + "' not found. Is it defined in '" + this.options.routesPath + "/constraints.js'?" var err = new Error(error) diff --git a/dadi/lib/dust/index.js b/dadi/lib/dust/index.js index c9c2ba0c..fa4245f5 100644 --- a/dadi/lib/dust/index.js +++ b/dadi/lib/dust/index.js @@ -1,5 +1,6 @@ 'use strict' +const debug = require('debug')('dust') const dust = require('dustjs-linkedin') const fs = require('fs') const mkdirp = require('mkdirp') @@ -63,6 +64,7 @@ Dust.prototype.load = function (source, templateName) { } Dust.prototype.loadDirectory = function (directory, prefix, recursive) { + debug('loadDirectory %o %s %s', directory, prefix, recursive) prefix = prefix || '' return new Promise((resolve, reject) => { diff --git a/dadi/lib/middleware/index.js b/dadi/lib/middleware/index.js index a4c6400d..71b8261a 100644 --- a/dadi/lib/middleware/index.js +++ b/dadi/lib/middleware/index.js @@ -1,11 +1,12 @@ /** * @module Middleware */ -var path = require('path') +var debug = require('debug')('middleware') var log = require('@dadi/logger') +var path = require('path') var Middleware = function (name, options) { - log.info({module: 'middleware'}, 'Middleware logging started (' + name + ').') + debug('loaded %s', name) this.name = name this.options = options || {} diff --git a/main.js b/main.js index 290e2ed0..8fcd7d80 100644 --- a/main.js +++ b/main.js @@ -1,6 +1,7 @@ var chokidar = require('chokidar') var cluster = require('cluster') var config = require('./config') +var debug = require('debug')('cluster') var fs = require('fs') var path = require('path') @@ -66,11 +67,11 @@ if (config.get('cluster')) { } } else { // Single thread start - log.info('Starting DADI Web in single thread mode.') + debug('starting DADI Web in single thread mode.') app = require(path.join(__dirname, '/index.js')) app.start(function () { - log.info('Process ' + process.pid + ' is listening for incoming requests') + debug('process ' + process.pid + ' is listening for incoming requests') }) } diff --git a/package.json b/package.json index ceb80553..5ba33fab 100644 --- a/package.json +++ b/package.json @@ -25,10 +25,11 @@ "colors": "latest", "common-dustjs-helpers": "^0.1.0", "compression": "^1.6.0", - "connect-mongo": "1.0.1", + "connect-mongo": "latest", "connect-redis": "3.0.2", "console-stamp": "^0.2.2", "convict": "2.0.0", + "debug": "^2.6.1", "dustjs-helpers": "^1.7.3", "dustjs-linkedin": "^2.7.2", "event-stream": "^3.3.1", @@ -40,7 +41,7 @@ "marked": "^0.3.5", "mkdirp": "^0.5.1", "moment": "^2.10.6", - "mongodb": "1.4.7", + "mongodb": "2.2.x", "node-uuid": "latest", "path-to-regexp": "^1.2.1", "perfy": "^1.1.0", From bbb226d90ff97f518d830a6e7752010e28d62e74 Mon Sep 17 00:00:00 2001 From: James Lambie Date: Sun, 12 Feb 2017 09:22:39 +0800 Subject: [PATCH 2/6] feat: allow array of partial paths --- dadi/lib/index.js | 209 +++++++++++++++++++++------------------------- 1 file changed, 97 insertions(+), 112 deletions(-) diff --git a/dadi/lib/index.js b/dadi/lib/index.js index cf1d373c..b3edb989 100755 --- a/dadi/lib/index.js +++ b/dadi/lib/index.js @@ -7,6 +7,7 @@ var bodyParser = require('body-parser') var colors = require('colors') // eslint-disable-line var compress = require('compression') var crypto = require('crypto') +var debug = require('debug')('server') var dust = require('./dust') var enableDestroy = require('server-destroy') var fs = require('fs') @@ -20,13 +21,8 @@ var toobusy = require('toobusy-js') var url = require('url') var dadiStatus = require('@dadi/status') -var MongoStore -if (nodeVersion < 1) { - MongoStore = require('connect-mongo/es5')(session) -} else { - MongoStore = require('connect-mongo')(session) - var RedisStore = require('connect-redis')(session) -} +var MongoStore = require('connect-mongo')(session) +var RedisStore = require('connect-redis')(session) // let's ensure there's at least a dev config file here var devConfigPath = path.join(__dirname, '/../../config/config.development.json') @@ -60,8 +56,6 @@ log.init(config.get('logging'), config.get('aws'), process.env.NODE_ENV) var Server = function () { this.components = {} this.monitors = {} - - log.info({module: 'server'}, 'Server logging started.') } Server.prototype.start = function (done) { @@ -69,7 +63,7 @@ Server.prototype.start = function (done) { this.readyState = 2 - var options = this.loadPaths(config.get('paths') || {}) + var options = this.loadPaths() // create app var app = this.app = api() @@ -156,18 +150,6 @@ Server.prototype.start = function (done) { app.use(session(sessionOptions)) } - app.use('/config', function (req, res, next) { - var hash = crypto.createHash('md5').update(config.get('secret') + config.get('app.name')).digest('hex') - console.log(hash) - if (url.parse(req.url, true).query.secret === hash) { - res.statusCode = 200 - // res.end(config.toString()) - res.end(config.getSchemaString()) - } else { - next() - } - }) - // set up cache var cacheLayer = cache(self) @@ -284,40 +266,40 @@ Server.prototype.stop = function (done) { }) } -Server.prototype.loadPaths = function (paths) { - var options = {} - - options.datasourcePath = path.resolve(paths.datasources || path.join(__dirname, '/../../app/datasources')) - options.eventPath = path.resolve(paths.events || path.join(__dirname, '/../../app/events')) - options.pagePath = path.resolve(paths.pages || path.join(__dirname, '/../../app/pages')) - options.partialPath = path.resolve(paths.partials || path.join(__dirname, '/../../app/partials')) - options.routesPath = path.resolve(paths.routes || path.join(__dirname, '/../../app/routes')) - options.middlewarePath = path.resolve(paths.middleware || path.join(__dirname, '/../../app/middleware')) +Server.prototype.resolvePaths = function (paths) { + if (Array.isArray(paths)) { + return _.map(paths, (p) => { + return path.resolve(p) + }) + } else { + return [path.resolve(paths)] + } +} - options.filtersPath = path.resolve(paths.filters || path.join(__dirname, '/../../app/utils/filters')) - options.helpersPath = path.resolve(paths.helpers || path.join(__dirname, '/../../app/utils/helpers')) +Server.prototype.loadPaths = function () { + var paths = config.get('paths') + var options = {} - options.tokenWalletsPath = path.resolve(paths.tokenWallets || path.join(__dirname, '/../../.wallet')) - if (paths.media) options.mediaPath = path.resolve(paths.media) + options.datasourcePath = path.resolve(paths.datasources) + options.eventPath = path.resolve(paths.events) + options.filtersPath = path.resolve(paths.filters) + options.helpersPath = path.resolve(paths.helpers) + options.mediaPath = path.resolve(paths.media) + options.middlewarePath = path.resolve(paths.middleware) + options.pagePath = path.resolve(paths.pages) + options.partialPaths = this.resolvePaths(paths.partials) + options.publicPath = path.resolve(paths.public) + options.routesPath = path.resolve(paths.routes) + options.tokenWalletsPath = path.resolve(paths.tokenWallets) - if (paths.public) options.publicPath = path.resolve(paths.public) - - _.each(options, (path, key) => { - fs.stat(path, (err, stats) => { - if (err) { - if (err.code === 'ENOENT') { - this.ensureDirectories(options, () => { - // - }) - } - } - }) - }) + this.ensureDirectories(options, () => {}) return options } Server.prototype.loadApi = function (options) { + debug('loadApi %O', options) + this.app.use('/api/flush', (req, res, next) => { if (help.validateRequestMethod(req, res, 'POST') && help.validateRequestCredentials(req, res)) { return help.clearCache(req, (err) => { @@ -363,43 +345,43 @@ Server.prototype.loadApi = function (options) { } }) - this.ensureDirectories(options, (text) => { - // load routes - this.updatePages(options.pagePath, options, false) + // load routes + this.updatePages(options.pagePath, options, false) - // Load middleware - this.initMiddleware(options.middlewarePath, options) + // Load middleware + this.initMiddleware(options.middlewarePath, options) - // compile all dust templates - this.compile(options) + // compile all dust templates + this.compile(options) - this.addMonitor(options.datasourcePath, (dsFile) => { - this.updatePages(options.pagePath, options, true) - }) + this.addMonitor(options.datasourcePath, (dsFile) => { + this.updatePages(options.pagePath, options, true) + }) - this.addMonitor(options.eventPath, (eventFile) => { - this.updatePages(options.pagePath, options, true) - }) + this.addMonitor(options.eventPath, (eventFile) => { + this.updatePages(options.pagePath, options, true) + }) - this.addMonitor(options.pagePath, (pageFile) => { - this.updatePages(options.pagePath, options, true) - this.compile(options) - }) + this.addMonitor(options.pagePath, (pageFile) => { + this.updatePages(options.pagePath, options, true) + this.compile(options) + }) - this.addMonitor(options.partialPath, (partialFile) => { + _.each(options.partialPaths, (partialPath) => { + this.addMonitor(partialPath, (partialFile) => { this.compile(options) }) + }) - this.addMonitor(options.routesPath, (file) => { - if (this.app.Router) { - this.app.Router.loadRewrites(options, () => { - this.app.Router.loadRewriteModule() - }) - } - }) - - log.info({module: 'server'}, 'Load complete.') + this.addMonitor(options.routesPath, (file) => { + if (this.app.Router) { + this.app.Router.loadRewrites(options, () => { + this.app.Router.loadRewriteModule() + }) + } }) + + debug('load complete') } Server.prototype.initMiddleware = function (directoryPath, options) { @@ -428,25 +410,25 @@ Server.prototype.loadMiddleware = function (directoryPath, options) { } Server.prototype.updatePages = function (directoryPath, options, reload) { - if (!fs.existsSync(directoryPath)) return + if (fs.existsSync(directoryPath)) { + var pages = fs.readdirSync(directoryPath) - var pages = fs.readdirSync(directoryPath) + pages.forEach((page) => { + if (path.extname(page) !== '.json') return - pages.forEach((page) => { - if (path.extname(page) !== '.json') return + // get the full path to the page file + var pageFilepath = path.join(directoryPath, page) - // get the full path to the page file - var pageFilepath = path.join(directoryPath, page) + // strip the filename minus the extension + // to use as the page name + var name = page.slice(0, page.indexOf('.')) - // strip the filename minus the extension - // to use as the page name - var name = page.slice(0, page.indexOf('.')) - - this.addRoute({ - name: name, - filepath: pageFilepath - }, options, reload) - }) + this.addRoute({ + name: name, + filepath: pageFilepath + }, options, reload) + }) + } } Server.prototype.addRoute = function (obj, options, reload) { @@ -543,6 +525,7 @@ Server.prototype.getComponent = function (key) { } Server.prototype.addMonitor = function (filepath, callback) { + debug('addMonitor %s', filepath) filepath = path.normalize(filepath) // only add one watcher per path @@ -561,17 +544,14 @@ Server.prototype.removeMonitor = function (filepath) { Server.prototype.compile = function (options) { var templatePath = options.pagePath - var partialPath = options.partialPath - - var self = this + var partialPaths = options.partialPaths - // reset the dust cache so - // templates can be reloaded + // reset the dust cache so templates can be reloaded dust.clearCache() // Get a list of templates to render based on the registered components - var componentTemplates = Object.keys(self.components).map(function (route) { - return path.join(templatePath, self.components[route].page.template) + var componentTemplates = Object.keys(this.components).map((route) => { + return path.join(templatePath, this.components[route].page.template) }) // Load component templates @@ -582,7 +562,9 @@ Server.prototype.compile = function (options) { }) .then(function () { // Load partials - return dust.loadDirectory(partialPath, 'partials', true) + return _.each(partialPaths, (partialPath) => { + return dust.loadDirectory(partialPath, path.basename(partialPath), true) + }) }) .then(function () { // Load filters @@ -593,6 +575,7 @@ Server.prototype.compile = function (options) { return dust.requireDirectory(options.helpersPath) }) .then(function () { + debug('templates loaded %O', Object.keys(dust.templates).sort()) // Write client-side files dust.writeClientsideFiles() }) @@ -648,25 +631,27 @@ Server.prototype.getSessionStore = function (sessionConfig, env) { * @api public */ Server.prototype.ensureDirectories = function (options, done) { - // create workspace directories if they don't exist - // permissions default to 0777 + // create workspace directories if they don't exist; permissions default to 0777 var idx = 0 + _.each(options, (dir) => { - mkdirp(dir, {}, (err, made) => { - if (err) { - log.error({module: 'server'}, err) - console.log(err) - } + if (Array.isArray(dir)) { + this.ensureDirectories(dir, () => {}) + } else { + mkdirp(dir, {}, (err, made) => { + if (err) { + log.error({module: 'server'}, err) + } - if (made) { - log.info({module: 'server'}, 'Created directory ' + made) - console.log('Created directory ' + made) - } + if (made) { + debug('Created directory %s', made) + } - idx++ + idx++ - if (idx === Object.keys(options).length) return done() - }) + if (idx === Object.keys(options).length) return done() + }) + } }) } From 163210d9e35f8d31e741992d22f632655a1833e8 Mon Sep 17 00:00:00 2001 From: James Lambie Date: Sun, 12 Feb 2017 09:22:47 +0800 Subject: [PATCH 3/6] chore: update coverage --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5067e212..d51e50cc 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ DADI Web [![npm (scoped)](https://img.shields.io/npm/v/@dadi/web.svg?maxAge=10800&style=flat-square)](https://www.npmjs.com/package/@dadi/web) -[![coverage](https://img.shields.io/badge/coverage-63%25-yellow.svg?style=flat-square)](https://github.com/dadi/web) +[![coverage](https://img.shields.io/badge/coverage-64%25-yellow.svg?style=flat-square)](https://github.com/dadi/web) [![Build Status](https://travis-ci.org/dadi/web.svg?branch=master)](https://travis-ci.org/dadi/web) [![JavaScript Style Guide](https://img.shields.io/badge/code%20style-standard-brightgreen.svg?style=flat-square)](http://standardjs.com/) [![semantic-release](https://img.shields.io/badge/%20%20%F0%9F%93%A6%F0%9F%9A%80-semantic--release-e10079.svg?style=flat-square)](https://github.com/semantic-release/semantic-release) From 5987dd2de06abda347a797f837b46e7ba1c4f05a Mon Sep 17 00:00:00 2001 From: James Lambie Date: Sun, 12 Feb 2017 09:50:39 +0800 Subject: [PATCH 4/6] feat: start in cluster mode by default --- config.js | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/config.js b/config.js index d4597376..765ee707 100644 --- a/config.js +++ b/config.js @@ -489,22 +489,16 @@ var conf = convict({ cluster: { doc: 'If true, Web runs in cluster mode, starting a worker for each CPU core', format: Boolean, - default: false + default: true }, debug: { doc: 'If true, debug mode is enabled and a panel containing the JSON loaded for each page is displayed alongside the normal content.', format: Boolean, default: false }, - secret: { - doc: 'A value that must be passed to requests for the /config route, which allows viewing the application config in the browser', - format: String, - default: '1dc10073-ca36-4373-a646-0d1092caf4a5', - env: 'CONFIG_SECRET' - }, data: { preload: { - doc: '', + doc: 'An array of datasources to load at startup', format: Array, default: [] } From 848c1f020aba2317cedda385039c7fcf181860d9 Mon Sep 17 00:00:00 2001 From: James Lambie Date: Sun, 12 Feb 2017 09:56:14 +0800 Subject: [PATCH 5/6] style: add prefix to debug messages --- dadi/lib/api/index.js | 2 +- dadi/lib/controller/router.js | 2 +- dadi/lib/dust/index.js | 2 +- dadi/lib/index.js | 2 +- dadi/lib/middleware/index.js | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/dadi/lib/api/index.js b/dadi/lib/api/index.js index 1917518e..fa6427d6 100755 --- a/dadi/lib/api/index.js +++ b/dadi/lib/api/index.js @@ -1,5 +1,5 @@ var _ = require('underscore') -var debug = require('debug')('api') +var debug = require('debug')('web:api') var fs = require('fs') var http = require('http') var https = require('https') diff --git a/dadi/lib/controller/router.js b/dadi/lib/controller/router.js index 3ecb38d3..17228464 100644 --- a/dadi/lib/controller/router.js +++ b/dadi/lib/controller/router.js @@ -3,7 +3,7 @@ REWRITE INFO: https://github.com/tinganho/connect-modrewrite */ var _ = require('underscore') -var debug = require('debug')('router') +var debug = require('debug')('web:router') var es = require('event-stream') var fs = require('fs') var path = require('path') diff --git a/dadi/lib/dust/index.js b/dadi/lib/dust/index.js index fa4245f5..54e87e98 100644 --- a/dadi/lib/dust/index.js +++ b/dadi/lib/dust/index.js @@ -1,6 +1,6 @@ 'use strict' -const debug = require('debug')('dust') +const debug = require('debug')('web:dust') const dust = require('dustjs-linkedin') const fs = require('fs') const mkdirp = require('mkdirp') diff --git a/dadi/lib/index.js b/dadi/lib/index.js index b3edb989..4d7e8104 100755 --- a/dadi/lib/index.js +++ b/dadi/lib/index.js @@ -7,7 +7,7 @@ var bodyParser = require('body-parser') var colors = require('colors') // eslint-disable-line var compress = require('compression') var crypto = require('crypto') -var debug = require('debug')('server') +var debug = require('debug')('web:server') var dust = require('./dust') var enableDestroy = require('server-destroy') var fs = require('fs') diff --git a/dadi/lib/middleware/index.js b/dadi/lib/middleware/index.js index 71b8261a..1b6a478c 100644 --- a/dadi/lib/middleware/index.js +++ b/dadi/lib/middleware/index.js @@ -1,7 +1,7 @@ /** * @module Middleware */ -var debug = require('debug')('middleware') +var debug = require('debug')('web:middleware') var log = require('@dadi/logger') var path = require('path') From 5b35f731ebc37358874a861a98acbfd37ae49785 Mon Sep 17 00:00:00 2001 From: James Lambie Date: Sun, 12 Feb 2017 10:01:56 +0800 Subject: [PATCH 6/6] style: remove unused variables --- dadi/lib/index.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/dadi/lib/index.js b/dadi/lib/index.js index 4d7e8104..c0632214 100755 --- a/dadi/lib/index.js +++ b/dadi/lib/index.js @@ -6,7 +6,6 @@ var _ = require('underscore') var bodyParser = require('body-parser') var colors = require('colors') // eslint-disable-line var compress = require('compression') -var crypto = require('crypto') var debug = require('debug')('web:server') var dust = require('./dust') var enableDestroy = require('server-destroy') @@ -18,7 +17,6 @@ var serveFavicon = require('serve-favicon') var serveStatic = require('serve-static') var session = require('express-session') var toobusy = require('toobusy-js') -var url = require('url') var dadiStatus = require('@dadi/status') var MongoStore = require('connect-mongo')(session)