install celery with sqs support

you need to `pip install celery[sqs]` to get the additional
dependencies that celery needs to use SQS queues - there are two libs -
boto3 and pycurl.

pycurl is a bunch of python handles around curl, so needs to be
installed from source so it can link to your curl/ssl libs. On paas and
in docker this works fine (needed to add `libcurl4-openssl-dev` to the
docker container), but on macos it can't find openssl. We need to pass
a couple of flags in:

* set the environment variable PYCURL_SSL_LIBRARY=openssl
* pass in the global options `build_ext` and `-I{openssl_headers_path}`.

As shown here:
pycurl/pycurl#530 (comment)

Env var is no biggie, but using any install-option flags disables
wheels for the whole pip install run. (See
pypa/pip#2677 and
pypa/pip#4118 for more context on the
install-options flags). A whole bunch of our dependencies don't
install nicely from source (but do from wheel), so this commit installs
pycurl separately as an initial step, with the requisite flags, and
then installs the rest of the requirements as before.

I've updated the makefile and bootstrap.sh files to reflect this, but
if you run `pip install -r requirements.txt` from scratch you will run
into issues.

Author: Leo Hemsted

Makefile

@@ -67,6 +67,7 @@ production: ## Set environment to production
 .PHONY: dependencies
 dependencies: venv ## Install build dependencies
 	mkdir -p ${PIP_ACCEL_CACHE}
+	. venv/bin/activate && make install-pycurl
 	. venv/bin/activate && PIP_ACCEL_CACHE=${PIP_ACCEL_CACHE} pip-accel install -r requirements_for_test.txt
 
 .PHONY: generate-version-file
@@ -97,6 +98,7 @@ test: venv generate-version-file ## Run tests
 freeze-requirements:
 	rm -rf venv-freeze
 	virtualenv -p python3 venv-freeze
+	$(call install-pycurl, $$(pwd)/venv-freeze/bin/pip)
 	$$(pwd)/venv-freeze/bin/pip install -r requirements-app.txt
 	echo '# pyup: ignore file' > requirements.txt
 	echo '# This file is autogenerated. Do not edit it manually.' >> requirements.txt
@@ -105,6 +107,15 @@ freeze-requirements:
 	$$(pwd)/venv-freeze/bin/pip freeze -r <(sed '/^--/d' requirements-app.txt) | sed -n '/The following requirements were added by pip freeze/,$$p' >> requirements.txt
 	rm -rf venv-freeze
 
+define install-pycurl
+	# install pycurl separately to avoid flags disabling wheels for other packages
+	PYCURL_SSL_LIBRARY=openssl ${1} install pycurl --global-option="build_ext" --global-option="-I/usr/local/opt/openssl/include"
+endef
+
+.PHONY: install-pycurl
+install-pycurl:
+	$(call install-pycurl, pip)
+
 .PHONY: test-requirements
 test-requirements:
 	@diff requirements-app.txt requirements.txt | grep '<' \

docker/Dockerfile

@@ -22,6 +22,7 @@ RUN \
 		libffi-dev \
 		python-dev \
 		jq \
+		libcurl4-openssl-dev \
 	&& echo "Clean up" \
 	&& rm -rf /var/lib/apt/lists/* /tmp/*
 

requirements-app.txt

@@ -2,7 +2,7 @@
 # with package version changes made in requirements-app.txt
 
 cffi==1.11.5
-celery==4.2.1
+celery[sqs]==4.2.1
 docopt==0.6.2
 Flask-Bcrypt==0.7.1
 flask-marshmallow==0.9.0

requirements.txt

@@ -4,7 +4,7 @@
 # with package version changes made in requirements-app.txt
 
 cffi==1.11.5
-celery==4.2.1
+celery[sqs]==4.2.1
 docopt==0.6.2
 Flask-Bcrypt==0.7.1
 flask-marshmallow==0.9.0
@@ -37,7 +37,7 @@ amqp==2.3.2
 bcrypt==3.1.4
 billiard==3.5.0.4
 bleach==2.1.3
-boto3==1.6.16
+boto3==1.9.16
 certifi==2018.8.24
 chardet==3.0.4
 Click==7.0
@@ -60,6 +60,7 @@ orderedset==2.0.1
 phonenumbers==8.9.4
 pyasn1==0.4.4
 pycparser==2.19
+pycurl==7.43.0.2
 PyPDF2==1.26.0
 python-dateutil==2.7.3
 python-editor==1.0.3

scripts/bootstrap.sh

@@ -28,6 +28,7 @@ fi
 # we need the version file to exist otherwise the app will blow up
 make generate-version-file
 
+make install-pycurl
 # Install Python development dependencies
 pip3 install -r requirements_for_test.txt