Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add --bundled-dependencies flag to deploy* #91

Merged
merged 5 commits into from
Oct 24, 2017
Merged

add --bundled-dependencies flag to deploy* #91

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
bd644f9
add --bundled-dependencies flag to deployhostedfunction
noahdietz Oct 13, 2017
e3ee6d1
unify deploy* handling of node_modules
noahdietz Oct 20, 2017
e4b6e17
add bundled-dependencies to deployproxy
noahdietz Oct 20, 2017
bbbba65
add bundled-dependencies options to README
noahdietz Oct 23, 2017
2cbc7df
remove debug log
noahdietz Oct 24, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
65 changes: 7 additions & 58 deletions lib/commands/deployhostedfunction.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ var unzipProxy = require('../deploycommon').unzipProxy;
var copyFile = require('../deploycommon').copyFile;
var handleUploadResult = require('../deploycommon').handleUploadResult;
var usePackedSource = require('../deploycommon').usePackedSource;
var uploadSource = require('../deploycommon').uploadSource;

const APP_YAML = 'app.yaml';

Expand Down Expand Up @@ -122,13 +123,17 @@ module.exports.run = function(opts, cb) {
opts.remoteNpm = false;

steps.push(function(done) {
usePackedSource(opts, done);
usePackedSource(opts.directory, opts, function(err, packedDirectory) {
// set the target directory to upload to the packed directory
opts.directory = packedDirectory
return done(err);
});
})
}

steps = steps.concat([
function(done) {
uploadHostedSource(opts, request, done);
uploadSource(opts.directory, 'hosted', opts, request, done);
},
function(done) {
createTarget(opts, request, done);
Expand Down Expand Up @@ -285,62 +290,6 @@ function getDeploymentInfo(opts, request, done) {
});
}

function uploadHostedSource(opts, request, done) {

// Get a list of entries, broken down by which are directories
ziputils.enumerateDirectory(opts.directory, 'hosted', opts.remoteNpm, function(err, entries) {
if (err) { return done(err); }

if (opts.debug) { console.log('Directories to upload: %j', entries); }

async.eachLimit(entries, opts.asynclimit, function(entry, entryDone) {
var uri =
util.format('%s/v1/o/%s/apis/%s/revisions/%d/resources?type=hosted&name=%s',
opts.baseuri, opts.organization, opts.api,
opts.deploymentVersion, entry.resourceName);
if (entry.directory) {
// ZIP up all directories, possibly with additional file prefixes
ziputils.zipDirectory(entry.fileName, entry.zipEntryName, function(err, zipBuf) {
if (err) {
entryDone(err);
} else {
if (opts.verbose) {
console.log('Uploading resource %s of size %d', entry.resourceName, zipBuf.length);
}
request({
uri: uri,
method: 'POST',
json: false,
headers: { 'Content-Type': 'application/octet-stream' },
body: zipBuf
}, function(err, req, body) {
handleUploadResult(err, req, entry.fileName, entryDone);
});
}
});

} else {
if (opts.verbose) {
console.log('Uploading resource %s', entry.resourceName);
}
var httpReq = request({
uri: uri,
method: 'POST',
json: false,
headers: { 'Content-Type': 'application/octet-stream' }
}, function(err, req, body) {
handleUploadResult(err, req, entry.fileName, entryDone);
});

var fileStream = fs.createReadStream(entry.fileName);
fileStream.pipe(httpReq);
}
}, function(err) {
done(err);
});
});
}

// Create a target endpoint that references the Hosted Function target
function createTarget(opts, request, done) {
var targetDoc = mustache.render(
Expand Down
66 changes: 7 additions & 59 deletions lib/commands/deploynodeapp.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ var unzipProxy = require('../deploycommon').unzipProxy;
var copyFile = require('../deploycommon').copyFile;
var handleUploadResult = require('../deploycommon').handleUploadResult;
var usePackedSource = require('../deploycommon').usePackedSource;
var uploadSource = require('../deploycommon').uploadSource;

// By default, do not run NPM remotely
var DefaultResolveModules = false;
Expand Down Expand Up @@ -138,13 +139,17 @@ module.exports.run = function(opts, cb) {
opts.remoteNpm = false;

steps.push(function(done) {
usePackedSource(opts, done);
usePackedSource(opts.directory, opts, function(err, packedDirectory) {
// set the target directory to upload to the packed directory
opts.directory = packedDirectory
return done(err);
});
})
}

steps = steps.concat([
function(done) {
uploadNodeSource(opts, request, done);
uploadSource(opts.directory, 'node', opts, request, done);
},
function(done) {
createTarget(opts, request, done);
Expand Down Expand Up @@ -331,63 +336,6 @@ function getDeploymentInfo(opts, request, done) {
});
}

function uploadNodeSource(opts, request, done) {

// Get a list of entries, broken down by which are directories,
// and with special handling for the node_modules directory.
ziputils.enumerateDirectory(opts.directory, 'node', opts.remoteNpm, function(err, entries) {
if (err) { return done(err); }

if (opts.debug) { console.log('Directories to upload: %j', entries); }

async.eachLimit(entries, opts.asynclimit, function(entry, entryDone) {
var uri =
util.format('%s/v1/o/%s/apis/%s/revisions/%d/resources?type=node&name=%s',
opts.baseuri, opts.organization, opts.api,
opts.deploymentVersion, entry.resourceName);
if (entry.directory) {
// ZIP up all directories, possibly with additional file prefixes
ziputils.zipDirectory(entry.fileName, entry.zipEntryName, function(err, zipBuf) {
if (err) {
entryDone(err);
} else {
if (opts.verbose) {
console.log('Uploading resource %s of size %d', entry.resourceName, zipBuf.length);
}
request({
uri: uri,
method: 'POST',
json: false,
headers: { 'Content-Type': 'application/octet-stream' },
body: zipBuf
}, function(err, req, body) {
handleUploadResult(err, req, entry.fileName, entryDone);
});
}
});

} else {
if (opts.verbose) {
console.log('Uploading resource %s', entry.resourceName);
}
var httpReq = request({
uri: uri,
method: 'POST',
json: false,
headers: { 'Content-Type': 'application/octet-stream' }
}, function(err, req, body) {
handleUploadResult(err, req, entry.fileName, entryDone);
});

var fileStream = fs.createReadStream(entry.fileName);
fileStream.pipe(httpReq);
}
}, function(err) {
done(err);
});
});
}

// Create a target endpoint that references the Node.js script
function createTarget(opts, request, done) {
var targetDoc = mustache.render(
Expand Down
58 changes: 54 additions & 4 deletions lib/commands/deployproxy.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ var fsutils = require('../fsutils');
var options = require('../options');
var ziputils = require('../ziputils');
var parseDeployments = require('./parsedeployments');
var usePackedSource = require('../deploycommon').usePackedSource;
var uploadSource = require('../deploycommon').uploadSource;

var ProxyBase = 'apiproxy';
var XmlExp = /(.+)\.xml$/i;
Expand All @@ -22,6 +24,8 @@ var BASE_PATH_REGEXP = /<BasePath[^>]*>(.*?)<\/BasePath>/;

// By default, do not run NPM remotely
var DefaultResolveModules = false;
// used when bundled-dependencies is active to potentially force remoteNPM
var noNodeSource = false;

var descriptor = defaults.defaultDescriptor({
api: {
Expand Down Expand Up @@ -57,6 +61,10 @@ var descriptor = defaults.defaultDescriptor({
name: 'Upload Modules',
shortOption: 'U',
toggle: true
},
'bundled-dependencies' : {
name: 'Upload dependencies from bundledDependencies',
toggle: true
}
});
module.exports.descriptor = descriptor;
Expand Down Expand Up @@ -290,7 +298,10 @@ function uploadResources(opts, request, done) {
util.format('%s/v1/o/%s/apis/%s/revisions/%d/resources?type=%s&name=%s',
opts.baseuri, opts.organization, opts.api,
opts.deploymentVersion, entry.resourceType, entry.resourceName);
if (entry.directory) {
if(opts['bundled-dependencies'] && (entry.resourceType === 'node' || entry.resourceType === 'hosted')) {
// skip node/hosted files we will handle these special
return entryDone();
} else if (entry.directory) {
// ZIP up all directories, possibly with additional file prefixes
ziputils.zipDirectory(entry.fileName, entry.zipEntryName, function(err, zipBuf) {
if (err) {
Expand All @@ -311,7 +322,6 @@ function uploadResources(opts, request, done) {
});
}
});

} else {
if (opts.verbose) {
console.log('Uploading %s resource %s', entry.resourceType, entry.resourceName);
Expand All @@ -329,7 +339,46 @@ function uploadResources(opts, request, done) {
fileStream.pipe(httpReq);
}
}, function(err) {
done(err);
if (opts['bundled-dependencies']) {
var steps = [];

// check for hosted resources
var hostedDir = path.join(resBaseDir, 'hosted');
if (fs.existsSync(hostedDir)) {
steps.push(function(stepDone) {
// package hosted resources and upload the packed directory
usePackedSource(hostedDir, opts, function(err, packedDirectory) {
if (err) {
return stepDone(err);
}

uploadSource(packedDirectory, 'hosted', opts, request, stepDone)
});
});
}

// check for node resources
var nodeDir = path.join(resBaseDir, 'node');
if (fs.existsSync(nodeDir)) {
steps.push(function(stepDone) {
// package node resources and upload the packed directory
usePackedSource(nodeDir, opts, function(err, packedDirectory) {
if (err) {
return stepDone(err);
}

uploadSource(packedDirectory, 'node', opts, request, stepDone)
});
});
} else {
// no node resources, so do not force remoteNPM
noNodeSource = true;
}

return async.series(steps, done);
}

return done(err);
});
}

Expand Down Expand Up @@ -553,7 +602,8 @@ function uploadProxies(opts, request, done) {
}

function runNpm(opts, request, done) {
if (!opts.remoteNpm) {
console.log(opts)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Remove debug logging

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ah good catch, thanks

if (!opts.remoteNpm && (!opts['bundled-dependencies'] || noNodeSource)) {
done();
} else {
if (opts.verbose) {
Expand Down
63 changes: 59 additions & 4 deletions lib/deploycommon.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -247,7 +247,62 @@ module.exports.copyFile = function(source, target, cb) {
});
}

module.exports.usePackedSource = function(opts, cb) {
module.exports.uploadSource = function(sourceDir, type, opts, request, done) {
// Get a list of entries, broken down by which are directories
ziputils.enumerateDirectory(sourceDir, type, opts.remoteNpm, function(err, entries) {
if (err) { return done(err); }

if (opts.debug) { console.log('Directories to upload: %j', entries); }

async.eachLimit(entries, opts.asynclimit, function(entry, entryDone) {
var uri =
util.format('%s/v1/o/%s/apis/%s/revisions/%d/resources?type=%s&name=%s',
opts.baseuri, opts.organization, opts.api,
opts.deploymentVersion, type, entry.resourceName);
if (entry.directory) {
// ZIP up all directories, possibly with additional file prefixes
ziputils.zipDirectory(entry.fileName, entry.zipEntryName, function(err, zipBuf) {
if (err) {
entryDone(err);
} else {
if (opts.verbose) {
console.log('Uploading resource %s of size %d', entry.resourceName, zipBuf.length);
}
request({
uri: uri,
method: 'POST',
json: false,
headers: { 'Content-Type': 'application/octet-stream' },
body: zipBuf
}, function(err, req, body) {
handleUploadResult(err, req, entry.fileName, entryDone);
});
}
});

} else {
if (opts.verbose) {
console.log('Uploading resource %s', entry.resourceName);
}
var httpReq = request({
uri: uri,
method: 'POST',
json: false,
headers: { 'Content-Type': 'application/octet-stream' }
}, function(err, req, body) {
handleUploadResult(err, req, entry.fileName, entryDone);
});

var fileStream = fs.createReadStream(entry.fileName);
fileStream.pipe(httpReq);
}
}, function(err) {
done(err);
});
});
}

module.exports.usePackedSource = function(sourceDir, opts, cb) {
if (opts.debug) {
console.log('packaging bundled dependencies for upload')
}
Expand All @@ -257,7 +312,7 @@ module.exports.usePackedSource = function(opts, cb) {
return cb(err);
}

fs.copy(opts.directory, tempDir, function(err) {
fs.copy(sourceDir, tempDir, function(err) {
if (err) {
return cb(err)
}
Expand All @@ -277,13 +332,13 @@ module.exports.usePackedSource = function(opts, cb) {
var packageArchive = path.join(tempDir, packageName);
fs.createReadStream(packageArchive).pipe(zlib.createGunzip()).pipe(tar.extract(tempDir)).on('finish', function() {
fs.removeSync(packageArchive) // remove the pack archive so it doesn't show up in the proxy
opts.directory = path.join(tempDir, 'package'); // npm pack archives all sources, so we will use this as the deploy target now

if (opts.debug) {
console.log('bundled dependencies ready for upload')
}

return cb()
// return path to packed directory
return cb(undefined, path.join(tempDir, 'package'))
}).on('error', function(err) {
return cb(err);
});
Expand Down
2 changes: 1 addition & 1 deletion lib/ziputils.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ module.exports.enumerateResourceDirectory = function(baseDir, remoteNpm) {
var stat = fs.statSync(fullName);
if (stat.isDirectory()) {
visitDirectory(fullName, fileList, '', '',
type, (type === 'node'), remoteNpm);
type, (type === 'node' || type === 'hosted'), remoteNpm);
}
});
return fileList;
Expand Down