Merge branch 'master' into la-pipenv-modules

Author: leaveningagent

README.md

@@ -174,7 +174,7 @@ custom:
     strip: false
 ```
 
-### Lamba Layer
+### Lambda Layer
 Another method for dealing with large dependencies is to put them into a
 [Lambda Layer](https://docs.aws.amazon.com/lambda/latest/dg/configuration-layers.html).
 Simply add the `layer` option to the configuration.
@@ -439,6 +439,35 @@ zipinfo .serverless/xxx.zip
 ```
 (If you can't see the library, you might need to adjust your package include/exclude configuration in `serverless.yml`.)
 
+## Optimising packaging time
+
+If you wish to exclude most of the files in your project, and only include the source files of your lambdas and their dependencies you may well use an approach like this:
+
+```yaml
+package:
+  individually: false
+  include:
+    - "./src/lambda_one/**"
+    - "./src/lambda_two/**"
+  exclude:
+    - "**"
+```
+
+This will be very slow. Serverless adds a default `"**"` include. If you are using the `cacheLocation` parameter to this plugin, this will result in all of the cached files' names being loaded and then subsequently discarded because of the exclude pattern. To avoid this happening you can add a negated include pattern, as is observed in https://github.com/serverless/serverless/pull/5825.
+
+Use this approach instead:
+
+```yaml
+package:
+  individually: false
+  include:
+    - "!./**"
+    - "./src/lambda_one/**"
+    - "./src/lambda_two/**"
+  exclude:
+    - "**"
+```
+
 ## Contributors
  * [@dschep](https://github.com/dschep) - Lead developer & maintainer
  * [@azurelogic](https://github.com/azurelogic) - logging & documentation fixes

lib/layer.js

@@ -25,6 +25,9 @@ function zipRequirements() {
  * @return {Promise} empty promise
  */
 function createLayers() {
+  if (!this.serverless.service.layers) {
+    this.serverless.service.layers = {};
+  }
   this.serverless.service.layers['pythonRequirements'] = Object.assign(
     {
       artifact: path.join('.serverless', 'pythonRequirements.zip'),

lib/pip.js

@@ -613,10 +613,9 @@ function installAllRequirements() {
       !fse.existsSync(symlinkPath) &&
       reqsInstalledAt != symlinkPath
     ) {
-      // Windows can't symlink so we have to copy on Windows,
-      // it's not as fast, but at least it works
+      // Windows can't symlink so we have to use junction on Windows
       if (process.platform == 'win32') {
-        fse.copySync(reqsInstalledAt, symlinkPath);
+        fse.symlink(reqsInstalledAt, symlinkPath, 'junction');
       } else {
         fse.symlink(reqsInstalledAt, symlinkPath);
       }

lib/poetry.js

@@ -1,15 +1,14 @@
+const fs = require('fs');
 const fse = require('fs-extra');
 const path = require('path');
 const { spawnSync } = require('child_process');
+const tomlParse = require('@iarna/toml/parse-string');
 
 /**
  * poetry install
  */
 function pyprojectTomlToRequirements() {
-  if (
-    !this.options.usePoetry ||
-    !fse.existsSync(path.join(this.servicePath, 'pyproject.toml'))
-  ) {
+  if (!this.options.usePoetry || !isPoetryProject(this.servicePath)) {
     return;
   }
 
@@ -40,4 +39,29 @@ function pyprojectTomlToRequirements() {
   );
 }
 
+/**
+ * Check if pyproject.toml file exists and is a poetry project.
+ */
+function isPoetryProject(servicePath) {
+  const pyprojectPath = path.join(servicePath, 'pyproject.toml');
+
+  if (!fse.existsSync(pyprojectPath)) {
+    return false;
+  }
+
+  const pyprojectToml = fs.readFileSync(pyprojectPath);
+  const pyproject = tomlParse(pyprojectToml);
+
+  const buildSystemReqs =
+    (pyproject['build-system'] && pyproject['build-system']['requires']) || [];
+
+  for (var i = 0; i < buildSystemReqs.length; i++) {
+    if (buildSystemReqs[i].startsWith('poetry')) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
 module.exports = { pyprojectTomlToRequirements };

package.json

@@ -50,6 +50,7 @@
     "tape": "*"
   },
   "dependencies": {
+    "@iarna/toml": "^2.2.3",
     "appdirectory": "^0.1.0",
     "bluebird": "^3.0.6",
     "fs-extra": "^7.0.0",
@@ -61,8 +62,8 @@
     "lodash.uniqby": "^4.0.0",
     "lodash.values": "^4.3.0",
     "rimraf": "^2.6.2",
-    "shell-quote": "^1.6.1",
-    "sha256-file": "1.0.0"
+    "sha256-file": "1.0.0",
+    "shell-quote": "^1.6.1"
   },
   "eslintConfig": {
     "extends": "eslint:recommended",

test.js

@@ -712,6 +712,17 @@ test("pipenv py3.6 doesn't package bottle with noDeploy option", t => {
   t.end();
 });
 
+test('non build pyproject.toml uses requirements.txt', t => {
+  process.chdir('tests/non_build_pyproject');
+  const path = npm(['pack', '../..']);
+  npm(['i', path]);
+  sls(['package']);
+  const zipfiles = listZipFiles('.serverless/sls-py-req-test.zip');
+  t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged');
+  t.false(zipfiles.includes(`boto3${sep}__init__.py`), 'boto3 is NOT packaged');
+  t.end();
+});
+
 test('poetry py3.6 can package flask with default options', t => {
   process.chdir('tests/poetry');
   const path = npm(['pack', '../..']);

tests/non_build_pyproject/.gitignore

@@ -0,0 +1,22 @@
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Serverless
+.serverless
+.requirements
+unzip_requirements.py

tests/non_build_pyproject/handler.py

@@ -0,0 +1,5 @@
+import requests
+
+
+def hello(event, context):
+    return requests.get('https://httpbin.org/get').json()

tests/non_build_pyproject/package.json

@@ -0,0 +1,14 @@
+{
+  "name": "example",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "serverless-python-requirements": "file:serverless-python-requirements-4.2.5.tgz"
+  }
+}

tests/non_build_pyproject/pyproject.toml

@@ -0,0 +1,10 @@
+[tool.black]
+line-length = 79
+py36 = true
+skip-string-normalization = true
+exclude = '''
+/(
+    \.serverless
+    | node_modules
+)/
+'''

tests/non_build_pyproject/requirements.txt

@@ -0,0 +1,2 @@
+flask
+boto3

tests/non_build_pyproject/serverless.yml

@@ -0,0 +1,21 @@
+service: sls-py-req-test
+
+provider:
+  name: aws
+  runtime: python3.6
+
+plugins:
+  - serverless-python-requirements
+custom:
+  pythonRequirements:
+    usePoetry: false
+
+package:
+  exclude:
+    - '**/*'
+  include:
+    - handler.py
+
+functions:
+  hello:
+    handler: handler.hello