Merge pull request #106 from scrapinghub/option-to-schedule-with-environment

Add option to schedule with custom environment

Author: vshlapakov

scrapinghub/client/jobs.py

@@ -189,7 +189,7 @@ def list(self, count=None, start=None, spider=None, state=None,
 
     def run(self, spider=None, units=None, priority=None, meta=None,
             add_tag=None, job_args=None, job_settings=None, cmd_args=None,
-            **params):
+            environment=None, **params):
         """Schedule a new job and returns its job key.
 
         :param spider: a spider name string
@@ -201,6 +201,7 @@ def run(self, spider=None, units=None, priority=None, meta=None,
         :param job_args: (optional) a dictionary with job arguments.
         :param job_settings: (optional) a dictionary with job settings.
         :param cmd_args: (optional) a string with script command args.
+        :param environment: (option) a dictionary with custom environment
         :param \*\*params: (optional) additional keyword args.
 
         :return: a job instance, representing the scheduled job.
@@ -222,12 +223,15 @@ def run(self, spider=None, units=None, priority=None, meta=None,
             cleaned_args = {k: v for k, v in job_args.items()
                             if k not in params}
             params.update(cleaned_args)
+        if environment and not isinstance(environment, dict):
+            raise ValueError("environment should be a dictionary")
 
         params['project'] = self.project_id
         params['spider'] = spider or self.spider.name
 
         update_kwargs(params, units=units, priority=priority, add_tag=add_tag,
-                      cmd_args=cmd_args, job_settings=job_settings, meta=meta)
+                      cmd_args=cmd_args, job_settings=job_settings, meta=meta,
+                      environment=environment)
 
         # FIXME improve to run multiple jobs
         try:

tests/client/cassetes/test_job/test_job_start_with_environment.gz

@@ -0,0 +1 @@
+eJydlI9S20YQxgETSCmQpOmfpDStogaqEkuyZWNsE6c1hhKCjV3sA5L22p6lwydsS1qdZIw7numj9HH6WD1ZMIW00+lU0oy00ulu97vft7/NjGFKQfMD6nPbdWD6YBot2k5AfWIG4gWHGQwJZQyzYpBPIaQ8gDtjmFPQbNu1LmH+tDc1NeX57jk1g5IRH2vUGdi+6/SpE5RWN7dXDeO4fJQWt9VM+fkkqk4ioxJHR8atb5Noc2eNe7ZF/RLjaiDWVeMQ7qJ5Rol44vDeGBYUtFxxRcZOoFap0wkYvI9hESXSqTwsEXSvbJrUC9Rdx3Qt2+nAMoZ7aKkzsr2kZNGzHgko3CdoLh4HDzB8gBLr+jo8JGgBceqr5Y6YHD7E8BF65F0GzHVUbvrEE7OxsK0bWkrLwMditMjDoRPd4BMMj9BCl1JPJT17QOExQYvXebYuPQqfYlhBT4nn9WyTRP/oQ/Xi4kI9c/2+Gvo9GiVMLfiMoKVyKFb17dFkHDzB8Dkytgm3Telt/7+eb3LwBQnRXJ+KuSyQ0Gyj3mzBU5QIfRtk9IwFgVfU9UxGi6+NdDGfyqd0Uajuh452zsXaX4bork+5J9Cg8GwMqwqa4wEJQg5rY/hKUNKnnJMOBQXN1A/gazQblQHrB3+ETGzXcwHOgPiXkMSgouVY9KRUcd2uTUEjaNm8EqkXb6aOIYVmMpuQFgJefwsiAQ0MGXT/poCTDLME3SG9nnsBGxhyaCmqMinVG639+mETNsU2cRqoZrxgHkMB/W5y/yxwu9RRLTpou8PSxXDPLfRORvvtw3qzUnl9Zti17czwZPTqfFSzWHevFVad7aNKAexa86DeaofH+UK7edxP504b1fPLLYkOPVvoVDqhVlJKZaUdakpGKl2Q0tlidqOYSUt7tdaWVCPDCK9SJp3NFnKp1JbUIAEr6VtSk/Rp0w5oqUqGUCQhmx/DVqS1H0H84nRV2O5XOZZeLspuV07K527btkRwZUM9refkMZR4GI7hpcKEb79R2PxhtA/fKkyQXmYPCROobrPHhAljVN41xg5hgrbd/0Xbd7f9s4fh1b/4Z1+UKKHE3m4LXjMZvbyikQeuL2jSBn3txj+a6fZ1US3Xb5Sq3+g6cBAyQWdVYYLKmsJiGA/ZNYZ15W+gNTB8L0Az4Eg0A5G2aInQxNBCKyfU745o2NFTWkHLSY1JCXpGy2lZQKJI8y/bH2M4uWX7U4IeDNVoOtuk6nWbfYPhLUoI2OAHgmatSOkfMWD0JMZl4xqX/C1c4CeCVohwDOdqlLzv9tQJ6aroDR3bgZ8x/IKm14G8a5U2BvMfrGLFWFH24lSa4BR1abkoyVF/lpPR/ci4ig1B0pkgiYban/fZCVs=

tests/client/test_job.py

@@ -1,5 +1,7 @@
 from collections import Iterator
 
+import pytest
+
 from scrapinghub.client.items import Items
 from scrapinghub.client.jobs import Job
 from scrapinghub.client.jobs import JobMeta
@@ -53,6 +55,14 @@ def test_job_start(spider):
     assert job.metadata.get('priority') == 2
 
 
+def test_job_start_with_environment(spider):
+    with pytest.raises(ValueError):
+        spider.jobs.run(environment='wrong-env')
+    env = {'VAR1': 'VAL1', 'VAR2': 'VAL2'}
+    job = spider.jobs.run(environment=env)
+    assert job.metadata.get('environment') == env
+
+
 def test_job_start_extras(spider):
     job = spider.jobs.run()
     extras = {