From e1452233a01364aba7446a123c64832309f5fa89 Mon Sep 17 00:00:00 2001 From: David Lawrence Date: Tue, 22 Jan 2019 11:23:57 -0800 Subject: [PATCH 1/8] add options to build requirements as a lambda layer --- README.md | 32 ++++++++++++++++++++++++++++++++ index.js | 12 ++++++++++++ lib/inject.js | 4 ++++ lib/layer.js | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 97 insertions(+) create mode 100644 lib/layer.js diff --git a/README.md b/README.md index d8e5568a..7bb7e0e3 100644 --- a/README.md +++ b/README.md @@ -146,6 +146,38 @@ custom: ``` This will remove all folders within the installed requirements that match the names in `slimPatterns` + +### Lamba Layer +Another method for dealing with large dependencies is to put them into a +[Lambda Layer](https://docs.aws.amazon.com/lambda/latest/dg/configuration-layers.html). +Simply add the `layer` option to the configuration. +```yaml +custom: + pythonRequirements: + layer: true +``` +The requirements will be zipped up and a layer will be created automatically. +Now just add the reference to the functions that will use the layer. +```yaml +functions: + hello: + handler: handler.hello + layers: + - {Ref: PythonRequirementsLambdaLayer} +``` +If the layer requires additional or custom configuration, add them onto the `layer` option. +```yaml +custom: + pythonRequirements: + layer: + name: ${self:provider.stage}-layerName + description: Python requirements lamba layer + compatibleRuntimes: + - python3.7 + licenseInfo: GPLv3 + allowedAccounts: + - '*' +``` ## Omitting Packages You can omit a package from deployment with the `noDeploy` option. Note that dependencies of omitted packages must explicitly be omitted too. By default, diff --git a/index.js b/index.js index 470fcd67..a0164d83 100644 --- a/index.js +++ b/index.js @@ -10,6 +10,7 @@ const { packRequirements } = require('./lib/zip'); const { injectAllRequirements } = require('./lib/inject'); +const { layerRequirements } = require('./lib/layer'); const { installAllRequirements } = require('./lib/pip'); const { pipfileToRequirements } = require('./lib/pipenv'); const { cleanup, cleanupCache } = require('./lib/clean'); @@ -31,6 +32,8 @@ class ServerlessPythonRequirements { slimPatterns: false, slimPatternsAppendDefaults: true, zip: false, + inject: true, + layer: false, cleanupZipHelper: true, invalidateCaches: false, fileName: 'requirements.txt', @@ -94,6 +97,14 @@ class ServerlessPythonRequirements { }`; options.dockerImage = options.dockerImage || defaultImage; } + if (options.layer) { + // If layers are being used, dependencies should not be injected. + options.inject = false; + // If layer was set as a boolean, set it to an empty object to use the layer defaults. + if (options.layer === true) { + options.layer = {}; + } + } return options; } @@ -167,6 +178,7 @@ class ServerlessPythonRequirements { } return BbPromise.bind(this) .then(removeVendorHelper) + .then(layerRequirements) .then(() => injectAllRequirements.bind(this)( arguments[1].functionObj && diff --git a/lib/inject.js b/lib/inject.js index 973ba99b..64f7b613 100644 --- a/lib/inject.js +++ b/lib/inject.js @@ -72,6 +72,10 @@ function moveModuleUp(source, target, module) { * @return {Promise} the combined promise for requirements injection. */ function injectAllRequirements(funcArtifact) { + if (!this.options.inject) { + return BbPromise.resolve(); + } + this.serverless.cli.log('Injecting required Python packages to package...'); if (this.serverless.service.package.individually) { diff --git a/lib/layer.js b/lib/layer.js new file mode 100644 index 00000000..24ea2b72 --- /dev/null +++ b/lib/layer.js @@ -0,0 +1,49 @@ +const BbPromise = require('bluebird'); +const fse = require('fs-extra'); +const path = require('path'); +const JSZip = require('jszip'); +const { writeZip, addTree } = require('./zipTree'); + +BbPromise.promisifyAll(fse); + +/** + * Zip up requirements to be used as layer package. + * @return {Promise} the JSZip object constructed. + */ +function zipRequirements() { + return addTree(new JSZip(), path.join('.serverless', 'requirements')).then(zip => + writeZip(zip, path.join('.serverless', 'pythonRequirements.zip')) + ); +} + +/** + * Creates a layer on the serverless service for the requirements zip. + * @return {Promise} + */ +function createLayers() { + this.serverless.service.layers['pythonRequirements'] = Object.assign({ + artifact: path.join('.serverless', 'pythonRequirements.zip'), + name: `${this.serverless.service.stage}-python-requirements`, + description: 'Python requirements generated by serverless-python-requirements.', + }, this.options.layer); + + return BbPromise.resolve(); +} + +/** + * Creates a layer from the installed requirements. + * @return {Promise} the combined promise for requirements layer. + */ +function layerRequirements() { + if (!this.options.layer) { + return BbPromise.resolve() + } + + this.serverless.cli.log('Packaging Python Requirements Lambda Layer...'); + + return BbPromise.bind(this) + .then(zipRequirements) + .then(createLayers) +} + +module.exports = { layerRequirements }; From 81d68649ef62f23ff6cffc85434f39a5ff4a7321 Mon Sep 17 00:00:00 2001 From: David Lawrence Date: Tue, 22 Jan 2019 12:03:25 -0800 Subject: [PATCH 2/8] fix formatting issues from prettier --- lib/layer.js | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/lib/layer.js b/lib/layer.js index 24ea2b72..8c18d4db 100644 --- a/lib/layer.js +++ b/lib/layer.js @@ -11,9 +11,9 @@ BbPromise.promisifyAll(fse); * @return {Promise} the JSZip object constructed. */ function zipRequirements() { - return addTree(new JSZip(), path.join('.serverless', 'requirements')).then(zip => - writeZip(zip, path.join('.serverless', 'pythonRequirements.zip')) - ); + return addTree(new JSZip(), path.join('.serverless', 'requirements')).then( + zip => writeZip(zip, path.join('.serverless', 'pythonRequirements.zip')) + ); } /** @@ -21,11 +21,15 @@ function zipRequirements() { * @return {Promise} */ function createLayers() { - this.serverless.service.layers['pythonRequirements'] = Object.assign({ + this.serverless.service.layers['pythonRequirements'] = Object.assign( + { artifact: path.join('.serverless', 'pythonRequirements.zip'), name: `${this.serverless.service.stage}-python-requirements`, - description: 'Python requirements generated by serverless-python-requirements.', - }, this.options.layer); + description: + 'Python requirements generated by serverless-python-requirements.' + }, + this.options.layer + ); return BbPromise.resolve(); } @@ -36,14 +40,14 @@ function createLayers() { */ function layerRequirements() { if (!this.options.layer) { - return BbPromise.resolve() + return BbPromise.resolve(); } this.serverless.cli.log('Packaging Python Requirements Lambda Layer...'); return BbPromise.bind(this) .then(zipRequirements) - .then(createLayers) + .then(createLayers); } module.exports = { layerRequirements }; From 32dd65940111b3316707afef96e035ecec308844 Mon Sep 17 00:00:00 2001 From: David Lawrence Date: Tue, 5 Feb 2019 21:54:52 -0800 Subject: [PATCH 3/8] remove inject option. layer option can be used to infer what to do --- index.js | 3 --- lib/inject.js | 3 ++- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/index.js b/index.js index a0164d83..db8dbdc3 100644 --- a/index.js +++ b/index.js @@ -32,7 +32,6 @@ class ServerlessPythonRequirements { slimPatterns: false, slimPatternsAppendDefaults: true, zip: false, - inject: true, layer: false, cleanupZipHelper: true, invalidateCaches: false, @@ -98,8 +97,6 @@ class ServerlessPythonRequirements { options.dockerImage = options.dockerImage || defaultImage; } if (options.layer) { - // If layers are being used, dependencies should not be injected. - options.inject = false; // If layer was set as a boolean, set it to an empty object to use the layer defaults. if (options.layer === true) { options.layer = {}; diff --git a/lib/inject.js b/lib/inject.js index 64f7b613..1abbb531 100644 --- a/lib/inject.js +++ b/lib/inject.js @@ -72,7 +72,8 @@ function moveModuleUp(source, target, module) { * @return {Promise} the combined promise for requirements injection. */ function injectAllRequirements(funcArtifact) { - if (!this.options.inject) { + if (this.options.layer) { + // The requirements will be placed in a Layer, so just resolve return BbPromise.resolve(); } From 6f32335fb8b93367e7ec43c241c721868d8e61d3 Mon Sep 17 00:00:00 2001 From: David Lawrence Date: Tue, 5 Feb 2019 23:18:54 -0800 Subject: [PATCH 4/8] prepend the path in the zip folder depending on the runtime --- lib/layer.js | 56 +++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 45 insertions(+), 11 deletions(-) diff --git a/lib/layer.js b/lib/layer.js index 8c18d4db..d627fd6e 100644 --- a/lib/layer.js +++ b/lib/layer.js @@ -2,31 +2,62 @@ const BbPromise = require('bluebird'); const fse = require('fs-extra'); const path = require('path'); const JSZip = require('jszip'); -const { writeZip, addTree } = require('./zipTree'); +const { + writeZip, + addTree +} = require('./zipTree'); BbPromise.promisifyAll(fse); +/** + * Get the paths for the compatible runtimes of the layer + * @param {string[]} list of runtime paths + */ +function getRunTimeBuildPaths() { + const runtimepaths = { + 'python2.7': 'python', + 'python3.6': path.join('python', 'lib', 'python3.6', 'site-packages'), + 'python3.7': path.join('python', 'lib', 'python3.7', 'site-packages'), + }; + + let runtimes = [] + + // Defer to Layer config first + if (this.options.layer.compatibleRuntimes) { + runtimes = this.options.layer.compatibleRuntimes; + // If none provided, assume the provider runtime + } else if (this.serverless.service.provider.runtime) { + runtimes = [this.serverless.service.provider.runtime]; + // If still no runtime found, just assume latest python + } else { + runtimes = ['python3.7']; + } + + return BbPromise.resolve(runtimes.map(runtime => runtimepaths[runtime])); +} + /** * Zip up requirements to be used as layer package. + * @param {string[]} list of paths where the requirements should be put in the layer * @return {Promise} the JSZip object constructed. */ -function zipRequirements() { - return addTree(new JSZip(), path.join('.serverless', 'requirements')).then( - zip => writeZip(zip, path.join('.serverless', 'pythonRequirements.zip')) - ); +function zipRequirements(runtimepaths) { + const rootZip = new JSZip(); + const src = path.join('.serverless', 'requirements') + + return BbPromise.each(runtimepaths, (runtimepath) => addTree(rootZip.folder(runtimepath), src)) + .then(() => writeZip(rootZip, path.join('.serverless', 'pythonRequirementsLayer.zip'))) } /** * Creates a layer on the serverless service for the requirements zip. - * @return {Promise} + * @return {Promise} empty promise */ function createLayers() { - this.serverless.service.layers['pythonRequirements'] = Object.assign( - { + this.serverless.service.layers['pythonRequirements'] = Object.assign({ artifact: path.join('.serverless', 'pythonRequirements.zip'), name: `${this.serverless.service.stage}-python-requirements`, - description: - 'Python requirements generated by serverless-python-requirements.' + description: 'Python requirements generated by serverless-python-requirements.' }, this.options.layer ); @@ -46,8 +77,11 @@ function layerRequirements() { this.serverless.cli.log('Packaging Python Requirements Lambda Layer...'); return BbPromise.bind(this) + .then(getRunTimeBuildPaths) .then(zipRequirements) .then(createLayers); } -module.exports = { layerRequirements }; +module.exports = { + layerRequirements +}; From 6f4dc94928731ad83f2d16fa37649882792028e9 Mon Sep 17 00:00:00 2001 From: David Lawrence Date: Tue, 5 Feb 2019 23:30:41 -0800 Subject: [PATCH 5/8] default the compatible runtimes to the provider runtime. (If its undefined, serverless core will remove it when creating the layer.) --- lib/layer.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/layer.js b/lib/layer.js index d627fd6e..33233251 100644 --- a/lib/layer.js +++ b/lib/layer.js @@ -57,7 +57,8 @@ function createLayers() { this.serverless.service.layers['pythonRequirements'] = Object.assign({ artifact: path.join('.serverless', 'pythonRequirements.zip'), name: `${this.serverless.service.stage}-python-requirements`, - description: 'Python requirements generated by serverless-python-requirements.' + description: 'Python requirements generated by serverless-python-requirements.', + compatibleRuntimes: [this.serverless.service.provider.runtime], }, this.options.layer ); From 84edb812e8d026b1be7fc0915b18a1351ae2e7e6 Mon Sep 17 00:00:00 2001 From: David Lawrence Date: Tue, 5 Feb 2019 23:35:28 -0800 Subject: [PATCH 6/8] fix formatting with prettier --- lib/layer.js | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/lib/layer.js b/lib/layer.js index 33233251..193856fb 100644 --- a/lib/layer.js +++ b/lib/layer.js @@ -2,10 +2,7 @@ const BbPromise = require('bluebird'); const fse = require('fs-extra'); const path = require('path'); const JSZip = require('jszip'); -const { - writeZip, - addTree -} = require('./zipTree'); +const { writeZip, addTree } = require('./zipTree'); BbPromise.promisifyAll(fse); @@ -17,10 +14,10 @@ function getRunTimeBuildPaths() { const runtimepaths = { 'python2.7': 'python', 'python3.6': path.join('python', 'lib', 'python3.6', 'site-packages'), - 'python3.7': path.join('python', 'lib', 'python3.7', 'site-packages'), + 'python3.7': path.join('python', 'lib', 'python3.7', 'site-packages') }; - let runtimes = [] + let runtimes = []; // Defer to Layer config first if (this.options.layer.compatibleRuntimes) { @@ -43,10 +40,13 @@ function getRunTimeBuildPaths() { */ function zipRequirements(runtimepaths) { const rootZip = new JSZip(); - const src = path.join('.serverless', 'requirements') + const src = path.join('.serverless', 'requirements'); - return BbPromise.each(runtimepaths, (runtimepath) => addTree(rootZip.folder(runtimepath), src)) - .then(() => writeZip(rootZip, path.join('.serverless', 'pythonRequirementsLayer.zip'))) + return BbPromise.each(runtimepaths, runtimepath => + addTree(rootZip.folder(runtimepath), src) + ).then(() => + writeZip(rootZip, path.join('.serverless', 'pythonRequirementsLayer.zip')) + ); } /** @@ -54,11 +54,13 @@ function zipRequirements(runtimepaths) { * @return {Promise} empty promise */ function createLayers() { - this.serverless.service.layers['pythonRequirements'] = Object.assign({ + this.serverless.service.layers['pythonRequirements'] = Object.assign( + { artifact: path.join('.serverless', 'pythonRequirements.zip'), name: `${this.serverless.service.stage}-python-requirements`, - description: 'Python requirements generated by serverless-python-requirements.', - compatibleRuntimes: [this.serverless.service.provider.runtime], + description: + 'Python requirements generated by serverless-python-requirements.', + compatibleRuntimes: [this.serverless.service.provider.runtime] }, this.options.layer ); From 0cb238d3154b23b0adfab87f993a9ddc1b61d0ea Mon Sep 17 00:00:00 2001 From: David Lawrence Date: Tue, 5 Feb 2019 23:48:39 -0800 Subject: [PATCH 7/8] add name to contributors --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index a046eac2..f0297b66 100644 --- a/README.md +++ b/README.md @@ -455,3 +455,4 @@ zipinfo .serverless/xxx.zip * [@andrewfarley](https://github.com/andrewfarley) - Implemented download caching and static caching * [@bweigel](https://github.com/bweigel) - adding the `slimPatternsAppendDefaults` option & fixing per-function packaging when some functions don't have requirements & Porting tests from bats to js! * [@squaresurf](https://github.com/squaresurf) - adding usePoetry option + * [@david-mk-lawrence](https://github.com/david-mk-lawrence) - added Lambda Layer support From 79c078fe72d22611157ef58c774477739fb7eab9 Mon Sep 17 00:00:00 2001 From: David Lawrence Date: Wed, 6 Feb 2019 20:25:07 -0800 Subject: [PATCH 8/8] always use "python" as the path in the layer. this works for python2.7, python3.6 and python3.7 --- lib/layer.js | 38 ++++---------------------------------- 1 file changed, 4 insertions(+), 34 deletions(-) diff --git a/lib/layer.js b/lib/layer.js index 193856fb..e0a57358 100644 --- a/lib/layer.js +++ b/lib/layer.js @@ -6,46 +6,17 @@ const { writeZip, addTree } = require('./zipTree'); BbPromise.promisifyAll(fse); -/** - * Get the paths for the compatible runtimes of the layer - * @param {string[]} list of runtime paths - */ -function getRunTimeBuildPaths() { - const runtimepaths = { - 'python2.7': 'python', - 'python3.6': path.join('python', 'lib', 'python3.6', 'site-packages'), - 'python3.7': path.join('python', 'lib', 'python3.7', 'site-packages') - }; - - let runtimes = []; - - // Defer to Layer config first - if (this.options.layer.compatibleRuntimes) { - runtimes = this.options.layer.compatibleRuntimes; - // If none provided, assume the provider runtime - } else if (this.serverless.service.provider.runtime) { - runtimes = [this.serverless.service.provider.runtime]; - // If still no runtime found, just assume latest python - } else { - runtimes = ['python3.7']; - } - - return BbPromise.resolve(runtimes.map(runtime => runtimepaths[runtime])); -} - /** * Zip up requirements to be used as layer package. - * @param {string[]} list of paths where the requirements should be put in the layer * @return {Promise} the JSZip object constructed. */ -function zipRequirements(runtimepaths) { +function zipRequirements() { const rootZip = new JSZip(); const src = path.join('.serverless', 'requirements'); + const runtimepath = 'python'; - return BbPromise.each(runtimepaths, runtimepath => - addTree(rootZip.folder(runtimepath), src) - ).then(() => - writeZip(rootZip, path.join('.serverless', 'pythonRequirementsLayer.zip')) + return addTree(rootZip.folder(runtimepath), src).then(() => + writeZip(rootZip, path.join('.serverless', 'pythonRequirements.zip')) ); } @@ -80,7 +51,6 @@ function layerRequirements() { this.serverless.cli.log('Packaging Python Requirements Lambda Layer...'); return BbPromise.bind(this) - .then(getRunTimeBuildPaths) .then(zipRequirements) .then(createLayers); }