diff --git a/README.rst b/README.rst index 4a9c07b..d26157b 100644 --- a/README.rst +++ b/README.rst @@ -24,7 +24,7 @@ ____________ .. code-block:: bash $ pip install --upgrade scaleapi - + .. code-block:: bash $ conda install -c conda-forge scaleapi @@ -66,7 +66,7 @@ __ https://docs.scale.com/reference .. code-block:: python from scaleapi.tasks import TaskType - from scaleapi.exceptions import ScaleDuplicateTask + from scaleapi.exceptions import ScaleDuplicateResource payload = dict( project = "test_project", @@ -86,7 +86,7 @@ __ https://docs.scale.com/reference try: client.create_task(TaskType.ImageAnnotation, **payload) - except ScaleDuplicateTask as err: + except ScaleDuplicateResource as err: print(err.message) # If unique_id is already used for a different task @@ -149,8 +149,8 @@ Accessing ``task.params`` child objects directly at task level is **deprecated** task.params["geometries"] # task.geometries is DEPRECATED task.params["attachment"] # task.attachment is DEPRECATED -List Tasks -^^^^^^^^^^ +Retrieve List of Tasks +^^^^^^^^^^^^^^^^^^^^^^ Retrieve a list of `Task` objects, with filters for: ``project_name``, ``batch_name``, ``type``, ``status``, ``review_status``, ``unique_id``, ``completed_after``, ``completed_before``, ``updated_after``, ``updated_before``, @@ -191,6 +191,28 @@ __ https://docs.scale.com/reference#list-multiple-tasks task_list = list(tasks) print(f"{len(task_list))} tasks retrieved") +Get Tasks Count +^^^^^^^^^^^^^^^ + +``get_tasks_count()`` method returns the number of tasks with the given optional parameters for: ``project_name``, ``batch_name``, ``type``, ``status``, +``review_status``, ``unique_id``, ``completed_after``, ``completed_before``, ``updated_after``, ``updated_before``, +``created_after``, ``created_before`` and ``tags``. + +.. code-block :: python + + from scaleapi.tasks import TaskReviewStatus, TaskStatus + + task_count = client.get_tasks_count( + project_name = "My Project", + created_after = "2020-09-08", + completed_before = "2021-04-01", + status = TaskStatus.Completed, + review_status = TaskReviewStatus.Accepted + ) + + print(task_count) # 1923 + + Cancel Task ^^^^^^^^^^^ @@ -214,12 +236,16 @@ __ https://docs.scale.com/reference#batch-creation .. code-block:: python - client.create_batch( + batch = client.create_batch( project = "test_project", callback = "http://www.example.com/callback", batch_name = "batch_name_01_07_2021" ) + print(batch.name) # batch_name_01_07_2021 + +Throws ``ScaleDuplicateResource`` exception if a batch with the same name already exists. + Finalize Batch ^^^^^^^^^^^^^^^ @@ -321,6 +347,8 @@ __ https://docs.scale.com/reference#project-creation print(project.name) # Test_Project +Throws ``ScaleDuplicateResource`` exception if a project with the same name already exists. + Retrieve Project ^^^^^^^^^^^^^^^^ @@ -445,7 +473,7 @@ as a `ScaleException` parent type and child exceptions: - ``ScaleUnauthorized``: 401 - Unauthorized -- No valid API key provided. - ``ScaleNotEnabled``: 402 - Not enabled -- Please contact sales@scaleapi.com before creating this type of task. - ``ScaleResourceNotFound``: 404 - Not Found -- The requested resource doesn't exist. -- ``ScaleDuplicateTask``: 409 - Conflict -- The provided idempotency key or unique_id is already in use for a different request. +- ``ScaleDuplicateResource``: 409 - Conflict -- Object already exists with same name, idempotency key or unique_id. - ``ScaleTooManyRequests``: 429 - Too Many Requests -- Too many requests hit the API too quickly. - ``ScaleInternalError``: 500 - Internal Server Error -- We had a problem with our server. Try again later. - ``ScaleServiceUnavailable``: 503 - Server Timeout From Request Queueing -- Try again later. diff --git a/docs/migration_guide.md b/docs/migration_guide.md index 6c2fa15..5d8777a 100644 --- a/docs/migration_guide.md +++ b/docs/migration_guide.md @@ -81,7 +81,7 @@ SDK now supports auto-retry in case of a `408`, `429` or `5xx` error occurs. ### New Exceptions New error types are introduces if you want to handle specific exception cases. -`ScaleInvalidRequest`, `ScaleUnauthorized`, `ScaleNotEnabled`, `ScaleResourceNotFound`, `ScaleDuplicateTask`, `ScaleTooManyRequests`, `ScaleInternalError` and `ScaleTimeoutError`. +`ScaleInvalidRequest`, `ScaleUnauthorized`, `ScaleNotEnabled`, `ScaleResourceNotFound`, `ScaleDuplicateResource`, `ScaleTooManyRequests`, `ScaleInternalError` and `ScaleTimeoutError`. All new error types are child of the existing `ScaleException` which can be used to handle all cases. diff --git a/docs/pypi_update_guide.md b/docs/pypi_update_guide.md index a89d4dc..e9a1241 100644 --- a/docs/pypi_update_guide.md +++ b/docs/pypi_update_guide.md @@ -6,15 +6,29 @@ _Creating and deploying a new package version is easy!_ 1. Ensure you're on the latest `master` branch -2. Ensure you have access to a PyPI account that is a maintainer of [scaleapi](https://pypi.org/project/scaleapi/) on PyPI +2. Ensure the master already has an incremented version -### Deployment Steps: +3. *(Required if you are manually publishing to PyPI)* Ensure you have access to a PyPI account that is a maintainer of [scaleapi](https://pypi.org/project/scaleapi/) on PyPI -**Step 0: Critical - Bump Project Version** +**How to Bump Project Version** Ensure `_version.py` has an updated project version. If not, please increment the project version, commit and push the changes. We use [semantic versioning](https://packaging.python.org/guides/distributing-packages-using-setuptools/#semantic-versioning-preferred). If you are adding a meaningful feature, bump the minor version. If you are fixing a bug, bump the incremental version. +### Deployment: + + +#### Automated Deployment and Publish with CircleCI: + +Our repo already has a publish worklow built into the CircleCI. It's trigerred when there's a new release on GitHub, with a specific version tag. + +In order to deploy and publish a new version: +- Create a new [Release](https://github.com/scaleapi/scaleapi-python-client/releases) on GitHub +- Create and assign a new tag in the release page with the following template: `vX.Y.Z` Please make sure `X.Y.Z` is matching the version in the `_version.py`. + - *i.e.* If the version in `_version.py` is **2.3.1** then the tag should be **v2.3.1** +- Provide release notes by following the [Release Notes Template](release_notes_template.md) and filling relevant sections to your changes. +#### *(Unpreferred)* Manual Deployment and Publish: + **Step 1: Run Publish Script** @@ -36,4 +50,4 @@ SCALE_TEST_API_KEY="{apikey}|{userid}|test" ./publish.sh runtest **Step 3: Create a New Release** -Create a [new release](https://github.com/scaleapi/scaleapi-python-client/releases/new) on GitHub with a matching version tag _(i.e. v2.0.1)_. Please provide a summary about new features and fixed bugs in the Release Notes. +Create a [new release](https://github.com/scaleapi/scaleapi-python-client/releases/new) on GitHub with a matching version tag _(i.e. v2.0.1)_. Provide release notes by following the [Release Notes Template](release_notes_template.md) and filling relevant sections to your changes. diff --git a/docs/release_notes_template.md b/docs/release_notes_template.md new file mode 100644 index 0000000..642e0db --- /dev/null +++ b/docs/release_notes_template.md @@ -0,0 +1,20 @@ +## Upgrade Steps +- [IF ANY IMMEDIATE ACTION IS REQUIRED] + +## Breaking Changes +- [BIG NEW FEATURES] + +## New Features +- [ANY NEW FEATURES] + +## Bug Fixes +- [IF AN ISSUE IS RESOLVED, PLEASE LINK TO ISSUE#] + +## Improvements +- [IMPROVEMENTS ON EXISTING FEATURES] + +## Known Issues +- [KEEP SECTION UPDATED WITH KNOWN ISSUES] + +## Other Changes and Notes +- [MISC] diff --git a/scaleapi/__init__.py b/scaleapi/__init__.py index 37492b3..70ae627 100644 --- a/scaleapi/__init__.py +++ b/scaleapi/__init__.py @@ -262,32 +262,24 @@ def get_tasks( next_token = None has_more = True - while has_more: - tasks_args = { - "next_token": next_token, - "start_time": created_after, - "end_time": created_before, - "project": project_name, - "batch": batch_name, - "completed_before": completed_before, - "completed_after": completed_after, - "tags": tags, - "updated_before": updated_before, - "updated_after": updated_after, - "unique_id": unique_id, - } - - if status: - tasks_args["status"] = status.value - if task_type: - tasks_args["type"] = task_type.value - if review_status: - if isinstance(review_status, List): - value = ",".join(map(lambda x: x.value, review_status)) - else: - value = review_status.value + tasks_args = self._process_tasks_endpoint_args( + project_name, + batch_name, + task_type, + status, + review_status, + unique_id, + completed_after, + completed_before, + updated_after, + updated_before, + created_after, + created_before, + tags, + ) - tasks_args["customer_review_status"] = value + while has_more: + tasks_args["next_token"] = next_token tasks = self.tasks(**tasks_args) for task in tasks.docs: @@ -296,6 +288,145 @@ def get_tasks( next_token = tasks.next_token has_more = tasks.has_more + def get_tasks_count( + self, + project_name: str, + batch_name: str = None, + task_type: TaskType = None, + status: TaskStatus = None, + review_status: Union[List[TaskReviewStatus], TaskReviewStatus] = None, + unique_id: Union[List[str], str] = None, + completed_after: str = None, + completed_before: str = None, + updated_after: str = None, + updated_before: str = None, + created_after: str = None, + created_before: str = None, + tags: Union[List[str], str] = None, + ) -> int: + """Returns number of tasks with given filters. + + Args: + project_name (str): + Project Name + + batch_name (str, optional): + Batch Name + + task_type (TaskType, optional): + Task type to filter i.e. `TaskType.TextCollection` + + status (TaskStatus, optional): + Task status i.e. `TaskStatus.Completed` + + review_status (List[TaskReviewStatus] | TaskReviewStatus): + The status of the audit result of the task. + Input can be a single element or a list of + TaskReviewStatus. i.e. `TaskReviewStatus.Accepted` to + filter the tasks that you accepted after audit. + + unique_id (List[str] | str, optional): + The unique_id of a task. Multiple unique IDs can be + specified at the same time as a list. + + completed_after (str, optional): + The minimum value of `completed_at` in UTC timezone + ISO format: 'YYYY-MM-DD HH:MM:SS.mmmmmm' + + completed_before (str, optional): + The maximum value of `completed_at` in UTC timezone + ISO format: 'YYYY-MM-DD HH:MM:SS.mmmmmm' + + updated_after (str, optional): + The minimum value of `updated_at` in UTC timezone + ISO format: 'YYYY-MM-DD HH:MM:SS.mmmmmm' + + updated_before (str, optional): + The maximum value of `updated_at` in UTC timezone + ISO format: 'YYYY-MM-DD HH:MM:SS.mmmmmm' + + created_after (str, optional): + The minimum value of `created_at` in UTC timezone + ISO format: 'YYYY-MM-DD HH:MM:SS.mmmmmm' + + created_before (str, optional): + The maximum value of `created_at` in UTC timezone + ISO format: 'YYYY-MM-DD HH:MM:SS.mmmmmm' + + tags (List[str] | str, optional): + The tags of a task; multiple tags can be + specified as a list. + + Returns: + int: + Returns number of tasks + """ + + tasks_args = self._process_tasks_endpoint_args( + project_name, + batch_name, + task_type, + status, + review_status, + unique_id, + completed_after, + completed_before, + updated_after, + updated_before, + created_after, + created_before, + tags, + ) + + tasks_args["limit"] = 1 + + tasks = self.tasks(**tasks_args) + return tasks.total + + @staticmethod + def _process_tasks_endpoint_args( + project_name: str, + batch_name: str = None, + task_type: TaskType = None, + status: TaskStatus = None, + review_status: Union[List[TaskReviewStatus], TaskReviewStatus] = None, + unique_id: Union[List[str], str] = None, + completed_after: str = None, + completed_before: str = None, + updated_after: str = None, + updated_before: str = None, + created_after: str = None, + created_before: str = None, + tags: Union[List[str], str] = None, + ): + """Generates args for /tasks endpoint.""" + tasks_args = { + "start_time": created_after, + "end_time": created_before, + "project": project_name, + "batch": batch_name, + "completed_before": completed_before, + "completed_after": completed_after, + "tags": tags, + "updated_before": updated_before, + "updated_after": updated_after, + "unique_id": unique_id, + } + + if status: + tasks_args["status"] = status.value + if task_type: + tasks_args["type"] = task_type.value + if review_status: + if isinstance(review_status, List): + value = ",".join(map(lambda x: x.value, review_status)) + else: + value = review_status.value + + tasks_args["customer_review_status"] = value + + return tasks_args + def create_task(self, task_type: TaskType, **kwargs) -> Task: """This method can be used for any Scale supported task type. Parameters may differ based on the given task_type. diff --git a/scaleapi/_version.py b/scaleapi/_version.py index d4ec4d4..7fdc4d4 100644 --- a/scaleapi/_version.py +++ b/scaleapi/_version.py @@ -1,2 +1,2 @@ -__version__ = "2.1.0" +__version__ = "2.2.0" __package_name__ = "scaleapi" diff --git a/scaleapi/exceptions.py b/scaleapi/exceptions.py index d429a9e..f1e58db 100644 --- a/scaleapi/exceptions.py +++ b/scaleapi/exceptions.py @@ -48,9 +48,9 @@ class ScaleResourceNotFound(ScaleException): code = 404 -class ScaleDuplicateTask(ScaleException): - """409 - Conflict -- The provided idempotency key or unique_id is - already in use for a different request. +class ScaleDuplicateResource(ScaleException): + """409 - Conflict -- Object already exists with same name, + idempotency key or unique_id. """ code = 409 @@ -89,7 +89,7 @@ class ScaleTimeoutError(ScaleException): ScaleUnauthorized.code: ScaleUnauthorized, ScaleNotEnabled.code: ScaleNotEnabled, ScaleResourceNotFound.code: ScaleResourceNotFound, - ScaleDuplicateTask.code: ScaleDuplicateTask, + ScaleDuplicateResource.code: ScaleDuplicateResource, ScaleTooManyRequests.code: ScaleTooManyRequests, ScaleInternalError.code: ScaleInternalError, ScaleTimeoutError.code: ScaleTimeoutError, diff --git a/tests/test_client.py b/tests/test_client.py index fc2c6f0..02ad4df 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -10,7 +10,7 @@ import scaleapi from scaleapi.batches import BatchStatus from scaleapi.exceptions import ( - ScaleDuplicateTask, + ScaleDuplicateResource, ScaleInvalidRequest, ScaleResourceNotFound, ScaleUnauthorized, @@ -72,7 +72,7 @@ def make_a_task(unique_id: str = None, batch: str = None): def test_uniquekey_fail(): unique_key = str(uuid.uuid4()) make_a_task(unique_key) - with pytest.raises(ScaleDuplicateTask): + with pytest.raises(ScaleDuplicateResource): make_a_task(unique_key) @@ -336,6 +336,12 @@ def test_get_tasks(): assert task.id in task_ids +def test_get_tasks_count(): + tasks_count = client.tasks(project=TEST_PROJECT_NAME).total + get_tasks_count = client.get_tasks_count(project_name=TEST_PROJECT_NAME) + assert tasks_count == get_tasks_count + + def test_finalize_batch(): batch = create_a_batch() batch = client.finalize_batch(batch.name)