index.js

var request = require('request')
var fs = require('fs')
var path = require('path')
var log = require('single-line-log').stdout
var progress = require('progress-stream')
var prettyBytes = require('pretty-bytes')
var throttle = require('throttleit')
var EventEmitter = require('events').EventEmitter
var debug = require('debug')('nugget')

function noop () {}

module.exports = function (urls, opts, cb) {
  if (!Array.isArray(urls)) urls = [urls]
  if (urls.length === 1) opts.singleTarget = true

  var defaultProps = {}

  if (opts.sockets) {
    var sockets = +opts.sockets
    defaultProps.pool = {maxSockets: sockets}
  }

  if (opts.proxy) {
    defaultProps.proxy = opts.proxy
  }

  if (opts.strictSSL !== null) {
    defaultProps.strictSSL = opts.strictSSL
  }

  if (Object.keys(defaultProps).length > 0) {
    request = request.defaults(defaultProps)
  }

  var downloads = []
  var errors = []
  var pending = 0
  var truncated = urls.length * 2 >= (process.stdout.rows - 15)

  urls.forEach(function (url) {
    debug('start dl', url)
    pending++
    var dl = startDownload(url, opts, function done (err) {
      debug('done dl', url, pending)
      if (err) {
        debug('error dl', url, err)
        errors.push(err)
        dl.error = err.message
      }
      if (truncated) {
        var i = downloads.indexOf(dl)
        downloads.splice(i, 1)
        downloads.push(dl)
      }
      if (--pending === 0) {
        render()
        cb(errors.length ? errors : undefined)
      }
    })

    downloads.push(dl)

    dl.on('start', function (progressStream) {
      throttledRender()
    })

    dl.on('progress', function (data) {
      debug('progress', url, data.percentage)

      dl.speed = data.speed
      if (dl.percentage === 100) render()
      else throttledRender()
    })
  })

  var _log = opts.quiet ? noop : log
  render()
  var throttledRender = throttle(render, opts.frequency || 250)

  if (opts.singleTarget) return downloads[0]
  else return downloads

  function render () {
    var height = process.stdout.rows
    var rendered = 0
    var output = ''
    var totalSpeed = 0
    downloads.forEach(function (dl) {
      if (2 * rendered >= height - 15) return
      rendered++
      if (dl.error) {
        output += 'Downloading ' + path.basename(dl.target) + '\n'
        output += 'Error: ' + dl.error + '\n'
        return
      }
      var pct = dl.percentage
      var speed = dl.speed || 0
      var total = dl.fileSize
      totalSpeed += speed
      var bar = Array(Math.floor(45 * pct / 100)).join('=') + '>'
      while (bar.length < 45) bar += ' '
      output += 'Downloading ' + path.basename(dl.target) + '\n' +
      '[' + bar + '] ' + pct.toFixed(1) + '%'
      if (total) output += ' of ' + prettyBytes(total)
      output += ' (' + prettyBytes(speed) + '/s)\n'
    })
    if (rendered < downloads.length) output += '\n... and ' + (downloads.length - rendered) + ' more\n'
    if (downloads.length > 1) output += '\nCombined Speed: ' + prettyBytes(totalSpeed) + '/s\n'
    _log(output)
  }

  function startDownload (url, opts, cb) {
    var targetName = path.basename(url).split('?')[0]
    if (opts.singleTarget && opts.target) targetName = opts.target
    var target = path.resolve(opts.dir || process.cwd(), targetName)
    var origTarget = target
    if (opts.tmpfile) {
      target = target + '.tmp'
    }
    if (opts.resume) {
      resume(url, opts, cb)
    } else {
      download(url, opts, cb)
    }

    var progressEmitter = new EventEmitter()
    progressEmitter.target = target
    progressEmitter.speed = 0
    progressEmitter.percentage = 0

    function onprogress (p) {
      var pct = p.percentage
      progressEmitter.progress = p
      progressEmitter.percentage = pct
      progressEmitter.emit('progress', p)
    }

    return progressEmitter

    function resume (url, opts, cb) {
      var onStat = function (err, stats) {
        if (err && err.code === 'ENOENT') {
          return download(url, opts, cb)
        }
        if (err) {
          return cb(err)
        }
        var offset = stats.size
        var req = request.get(url)

        req.on('error', cb)
        req.on('response', function (resp) {
          resp.destroy()

          var length = parseInt(resp.headers['content-length'], 10)

          // file is already downloaded.
          if (length === offset) {
            onprogress({percentage: 100})
            return cb()
          }

          if (!isNaN(length) && length > offset && /bytes/.test(resp.headers['accept-ranges'])) {
            opts.range = [offset, length]
          }

          download(url, opts, cb)
        })
      }
      if (opts.tmpfile) {
        fs.stat(origTarget, function (err, origStats) {
          if (err && err.code === 'ENOENT') {
            fs.stat(target, onStat)
          } else {
            // file is already downloaded
            onprogress({percentage: 100})
            cb()
          }
        })
      } else {
        fs.stat(target, onStat)
      }
      
    }

    function download (url, opts, cb) {
      var headers = opts.headers || {}
      if (opts.range) {
        headers.Range = 'bytes=' + opts.range[0] + '-' + opts.range[1]
      }
      var read = request(url, { headers: headers })

      read.on('error', cb)
      read.on('response', function (resp) {
        debug('response', url, resp.statusCode)
        if (resp.statusCode > 299 && !opts.force) return cb(new Error('GET ' + url + ' returned ' + resp.statusCode))
        var write = fs.createWriteStream(target, {flags: opts.resume ? 'a' : 'w'})
        write.on('error', cb)
        write.on('finish', async () => {
          if (opts.tmpfile) {
            fs.rename(target, origTarget, cb)
          } else {
            process.nextTick(cb)
          }
        })

        var fullLen
        var contentLen = Number(resp.headers['content-length'])
        var range = resp.headers['content-range']
        if (range) {
          fullLen = Number(range.split('/')[1])
        } else {
          fullLen = contentLen
        }

        progressEmitter.fileSize = fullLen
        if (range) {
          var downloaded = fullLen - contentLen
        }
        var progressStream = progress({ length: fullLen, transferred: downloaded }, onprogress)
        progressEmitter.emit('start', progressStream)

        resp
          .pipe(progressStream)
          .pipe(write)
      })
    }
  }
}