From 01cc30fb64b5ef95cddda339e92b10728392eaa6 Mon Sep 17 00:00:00 2001 From: Alexander Piskun Date: Mon, 26 Feb 2024 00:34:30 +0300 Subject: [PATCH 1/8] NextcloudApp: polishing API Signed-off-by: Alexander Piskun --- CHANGELOG.md | 10 ++++++++++ nc_py_api/_session.py | 11 ++++------- nc_py_api/_version.py | 2 +- nc_py_api/ex_app/__init__.py | 7 ++++++- nc_py_api/ex_app/integration_fastapi.py | 20 +++++++++++++------- nc_py_api/ex_app/misc.py | 5 +++++ 6 files changed, 39 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8068bde6..47fdfd34 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,16 @@ All notable changes to this project will be documented in this file. +## [0.12.0 - 2024-02-28] + +Update with new features only for `NextcloudApp` class. #233 + +### Added + +- `ex_app.get_computation_device` function for retrieving GPU type(only with AppAPI `2.3.0`). +- `ex_app.integration_fastapi.fetch_models_task` are now public function, added `progress_init_start_value` param. +- Global authentication when used now sets `request.scope["username"]` for easy use. + ## [0.11.0 - 2024-02-17] ### Added diff --git a/nc_py_api/_session.py b/nc_py_api/_session.py index 44c8caf1..e5614832 100644 --- a/nc_py_api/_session.py +++ b/nc_py_api/_session.py @@ -123,7 +123,7 @@ def __init__(self, **kwargs): super().__init__(**kwargs) self.aa_version = self._get_config_value("aa_version", raise_not_found=False, **kwargs) if not self.aa_version: - self.aa_version = "1.0.0" + self.aa_version = "2.2.0" self.app_name = self._get_config_value("app_id", **kwargs) self.app_version = self._get_config_value("app_version", **kwargs) self.app_secret = self._get_config_value("app_secret", **kwargs) @@ -459,7 +459,7 @@ def __init__(self, **kwargs): self.cfg = AppConfig(**kwargs) super().__init__(**kwargs) - def sign_check(self, request: HTTPConnection) -> None: + def sign_check(self, request: HTTPConnection) -> str: headers = { "AA-VERSION": request.headers.get("AA-VERSION", ""), "EX-APP-ID": request.headers.get("EX-APP-ID", ""), @@ -474,13 +474,10 @@ def sign_check(self, request: HTTPConnection) -> None: if headers["EX-APP-ID"] != self.cfg.app_name: raise ValueError(f"Invalid EX-APP-ID:{headers['EX-APP-ID']} != {self.cfg.app_name}") - our_version = self.adapter.headers.get("EX-APP-VERSION", "") - if headers["EX-APP-VERSION"] != our_version: - raise ValueError(f"Invalid EX-APP-VERSION:{headers['EX-APP-VERSION']} <=> {our_version}") - - app_secret = get_username_secret_from_headers(headers)[1] + username, app_secret = get_username_secret_from_headers(headers) if app_secret != self.cfg.app_secret: raise ValueError(f"Invalid App secret:{app_secret} != {self.cfg.app_secret}") + return username class NcSessionApp(NcSessionAppBasic, NcSessionBasic): diff --git a/nc_py_api/_version.py b/nc_py_api/_version.py index 043f7841..9b683c0c 100644 --- a/nc_py_api/_version.py +++ b/nc_py_api/_version.py @@ -1,3 +1,3 @@ """Version of nc_py_api.""" -__version__ = "0.11.0" +__version__ = "0.12.0.dev0" diff --git a/nc_py_api/ex_app/__init__.py b/nc_py_api/ex_app/__init__.py index f8cfe0c8..8575b572 100644 --- a/nc_py_api/ex_app/__init__.py +++ b/nc_py_api/ex_app/__init__.py @@ -9,7 +9,12 @@ set_handlers, talk_bot_msg, ) -from .misc import get_model_path, persistent_storage, verify_version +from .misc import ( + get_computation_device, + get_model_path, + persistent_storage, + verify_version, +) from .ui.files_actions import UiActionFileInfo from .ui.settings import SettingsField, SettingsFieldType, SettingsForm from .uvicorn_fastapi import run_app diff --git a/nc_py_api/ex_app/integration_fastapi.py b/nc_py_api/ex_app/integration_fastapi.py index feb207f0..6ec11e29 100644 --- a/nc_py_api/ex_app/integration_fastapi.py +++ b/nc_py_api/ex_app/integration_fastapi.py @@ -60,6 +60,7 @@ def set_handlers( default_init: bool = True, models_to_fetch: dict[str, dict] | None = None, map_app_static: bool = True, + progress_init_start_value: int = 0, ): """Defines handlers for the application. @@ -77,6 +78,8 @@ def set_handlers( :param map_app_static: Should be folders ``js``, ``css``, ``l10n``, ``img`` automatically mounted in FastAPI or not. .. note:: First, presence of these directories in the current working dir is checked, then one directory higher. + + :param progress_init_start_value: The "init" progress value from which the download of models starts. """ if models_to_fetch is not None and default_init is False: raise ValueError("`models_to_fetch` can be defined only with `default_init`=True.") @@ -103,7 +106,9 @@ async def heartbeat_callback(): @fast_api_app.post("/init") async def init_callback(b_tasks: BackgroundTasks, nc: typing.Annotated[NextcloudApp, Depends(nc_app)]): - b_tasks.add_task(__fetch_models_task, nc, models_to_fetch if models_to_fetch else {}) + b_tasks.add_task( + fetch_models_task, nc, models_to_fetch if models_to_fetch else {}, progress_init_start_value + ) return JSONResponse(content={}) if map_app_static: @@ -120,10 +125,11 @@ def __map_app_static_folders(fast_api_app: FastAPI): fast_api_app.mount(f"/{mnt_dir}", staticfiles.StaticFiles(directory=mnt_dir_path), name=mnt_dir) -def __fetch_models_task(nc: NextcloudApp, models: dict[str, dict]) -> None: +def fetch_models_task(nc: NextcloudApp, models: dict[str, dict], progress_init_start_value: int = 0) -> None: + """Use for cases when you want to define custom `/init` but still need to easy download models.""" if models: - current_progress = 0 - percent_for_each = min(int(100 / len(models)), 99) + current_progress = progress_init_start_value + percent_for_each = min(int((100 - progress_init_start_value) / len(models)), 99) for model in models: if model.startswith(("http://", "https://")): __fetch_model_as_file(current_progress, percent_for_each, nc, model, models[model]) @@ -206,9 +212,9 @@ def __request_sign_check_if_needed(request: HTTPConnection, nextcloud_app: Nextc _request_sign_check(request, nextcloud_app) -def _request_sign_check(request: HTTPConnection, nextcloud_app: NextcloudApp | AsyncNextcloudApp) -> None: +def _request_sign_check(request: HTTPConnection, nextcloud_app: NextcloudApp | AsyncNextcloudApp) -> str: try: - nextcloud_app._session.sign_check(request) # noqa pylint: disable=protected-access + return nextcloud_app._session.sign_check(request) # noqa pylint: disable=protected-access except ValueError as e: print(e) raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED) from None @@ -238,7 +244,7 @@ async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: url_path = conn.url.path.lstrip("/") if not fnmatch.filter(self._disable_for, url_path): try: - _request_sign_check(conn, AsyncNextcloudApp()) + scope["username"] = _request_sign_check(conn, AsyncNextcloudApp()) except HTTPException as exc: response = self._on_error(exc.status_code, exc.detail) await response(scope, receive, send) diff --git a/nc_py_api/ex_app/misc.py b/nc_py_api/ex_app/misc.py index 2f4610e7..27f38ea9 100644 --- a/nc_py_api/ex_app/misc.py +++ b/nc_py_api/ex_app/misc.py @@ -50,3 +50,8 @@ def get_model_path(model_name: str) -> str: from huggingface_hub import snapshot_download # noqa isort:skip pylint: disable=C0415 disable=E0401 return snapshot_download(model_name, local_files_only=True, cache_dir=persistent_storage()) + + +def get_computation_device() -> str: + """Returns computation device(`ROCM` or `CUDA`) if it is defined in the environment variable.""" + return os.environ.get("COMPUTE_DEVICE", "") From a06b048411bcfe55f07423cfb808a6332c9b3ac9 Mon Sep 17 00:00:00 2001 From: Alexander Piskun Date: Mon, 26 Feb 2024 00:52:06 +0300 Subject: [PATCH 2/8] removed ExApp version check test Signed-off-by: Alexander Piskun --- tests/_app_security_checks.py | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/tests/_app_security_checks.py b/tests/_app_security_checks.py index ce497669..a08fc578 100644 --- a/tests/_app_security_checks.py +++ b/tests/_app_security_checks.py @@ -32,7 +32,7 @@ def sign_request(req_headers: dict, secret=None, user: str = ""): result = httpx.put(request_url, headers=headers) assert result.status_code == 200 # Invalid EX-APP-ID - old_app_name = headers["EX-APP-ID"] + old_app_name = headers.get("EX-APP-ID") headers["EX-APP-ID"] = "unknown_app" sign_request(headers) result = httpx.put(request_url, headers=headers) @@ -41,16 +41,3 @@ def sign_request(req_headers: dict, secret=None, user: str = ""): sign_request(headers) result = httpx.put(request_url, headers=headers) assert result.status_code == 200 - # Invalid EX-APP-VERSION - sign_request(headers) - result = httpx.put(request_url, headers=headers) - assert result.status_code == 200 - old_version = headers["EX-APP-VERSION"] - headers["EX-APP-VERSION"] = "999.0.0" - sign_request(headers) - result = httpx.put(request_url, headers=headers) - assert result.status_code == 401 - headers["EX-APP-VERSION"] = old_version - sign_request(headers) - result = httpx.put(request_url, headers=headers) - assert result.status_code == 200 From 08f9534357842ad83e7acbe146f54bf3f283da16 Mon Sep 17 00:00:00 2001 From: Alexander Piskun <13381981+bigcat88@users.noreply.github.com> Date: Sun, 17 Mar 2024 12:42:06 +0300 Subject: [PATCH 3/8] reflect last changes in AppAPI Signed-off-by: Alexander Piskun --- CHANGELOG.md | 2 +- docs/conf.py | 2 +- examples/as_app/skeleton/Makefile | 34 ++++------------------ examples/as_app/skeleton/requirements.txt | 2 +- examples/as_app/talk_bot/HOW_TO_INSTALL.md | 9 ++---- examples/as_app/talk_bot/Makefile | 23 +++------------ examples/as_app/talk_bot_ai/Makefile | 23 +++------------ examples/as_app/to_gif/Makefile | 34 ++++------------------ nc_py_api/ex_app/integration_fastapi.py | 9 ++---- scripts/dev_register.sh | 2 +- 10 files changed, 28 insertions(+), 112 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 47fdfd34..f65aa5fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ Update with new features only for `NextcloudApp` class. #233 ### Added -- `ex_app.get_computation_device` function for retrieving GPU type(only with AppAPI `2.3.0`). +- `ex_app.get_computation_device` function for retrieving GPU type(only with AppAPI `2.4.0`+). - `ex_app.integration_fastapi.fetch_models_task` are now public function, added `progress_init_start_value` param. - Global authentication when used now sets `request.scope["username"]` for easy use. diff --git a/docs/conf.py b/docs/conf.py index 59c9153a..546a1c18 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -34,7 +34,7 @@ # General information about the project. project = "NcPyApi" -copyright = str(now.year) + " Nextcloud GmbH" # noqa +copyright = str(now.year) + f" {project} Authors" # noqa # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the diff --git a/examples/as_app/skeleton/Makefile b/examples/as_app/skeleton/Makefile index 75c72188..081a3a77 100644 --- a/examples/as_app/skeleton/Makefile +++ b/examples/as_app/skeleton/Makefile @@ -9,10 +9,6 @@ help: @echo " " @echo " build-push build image and upload to ghcr.io" @echo " " - @echo " deploy27 deploy Skeleton to registered 'docker_dev' for Nextcloud 27" - @echo " deploy28 deploy Skeleton to registered 'docker_dev' for Nextcloud 28" - @echo " deploy deploy Skeleton to registered 'docker_dev' for Nextcloud Last" - @echo " " @echo " run27 install Skeleton for Nextcloud 27" @echo " run28 install Skeleton for Nextcloud 28" @echo " run install Skeleton for Nextcloud Last" @@ -29,59 +25,41 @@ build-push: docker login ghcr.io docker buildx build --push --platform linux/arm64/v8,linux/amd64 --tag ghcr.io/cloud-py-api/skeleton:latest . -.PHONY: deploy27 -deploy27: - docker exec master-stable27-1 sudo -u www-data php occ app_api:app:unregister skeleton --silent --force || true - docker exec master-stable27-1 sudo -u www-data php occ app_api:app:deploy skeleton docker_dev \ - --info-xml https://raw.githubusercontent.com/cloud-py-api/nc_py_api/main/examples/as_app/skeleton/appinfo/info.xml - -.PHONY: deploy28 -deploy28: - docker exec master-stable28-1 sudo -u www-data php occ app_api:app:unregister skeleton --silent --force || true - docker exec master-stable28-1 sudo -u www-data php occ app_api:app:deploy skeleton docker_dev \ - --info-xml https://raw.githubusercontent.com/cloud-py-api/nc_py_api/main/examples/as_app/skeleton/appinfo/info.xml - -.PHONY: deploy -deploy: - docker exec master-nextcloud-1 sudo -u www-data php occ app_api:app:unregister skeleton --silent --force || true - docker exec master-nextcloud-1 sudo -u www-data php occ app_api:app:deploy skeleton docker_dev \ - --info-xml https://raw.githubusercontent.com/cloud-py-api/nc_py_api/main/examples/as_app/skeleton/appinfo/info.xml - .PHONY: run27 run27: docker exec master-stable27-1 sudo -u www-data php occ app_api:app:unregister skeleton --silent --force || true - docker exec master-stable27-1 sudo -u www-data php occ app_api:app:register skeleton docker_dev --force-scopes \ + docker exec master-stable27-1 sudo -u www-data php occ app_api:app:register skeleton --force-scopes \ --info-xml https://raw.githubusercontent.com/cloud-py-api/nc_py_api/main/examples/as_app/skeleton/appinfo/info.xml .PHONY: run28 run28: docker exec master-stable28-1 sudo -u www-data php occ app_api:app:unregister skeleton --silent --force || true - docker exec master-stable28-1 sudo -u www-data php occ app_api:app:register skeleton docker_dev --force-scopes \ + docker exec master-stable28-1 sudo -u www-data php occ app_api:app:register skeleton --force-scopes \ --info-xml https://raw.githubusercontent.com/cloud-py-api/nc_py_api/main/examples/as_app/skeleton/appinfo/info.xml .PHONY: run run: docker exec master-nextcloud-1 sudo -u www-data php occ app_api:app:unregister skeleton --silent --force || true - docker exec master-nextcloud-1 sudo -u www-data php occ app_api:app:register skeleton docker_dev --force-scopes \ + docker exec master-nextcloud-1 sudo -u www-data php occ app_api:app:register skeleton --force-scopes \ --info-xml https://raw.githubusercontent.com/cloud-py-api/nc_py_api/main/examples/as_app/skeleton/appinfo/info.xml .PHONY: register27 register27: docker exec master-stable27-1 sudo -u www-data php occ app_api:app:unregister skeleton --silent --force || true docker exec master-stable27-1 sudo -u www-data php occ app_api:app:register skeleton manual_install --json-info \ - "{\"appid\":\"skeleton\",\"name\":\"App Skeleton\",\"daemon_config_name\":\"manual_install\",\"version\":\"1.0.0\",\"secret\":\"12345\",\"port\":9030,\"scopes\":[],\"system_app\":0}" \ + "{\"id\":\"skeleton\",\"name\":\"App Skeleton\",\"daemon_config_name\":\"manual_install\",\"version\":\"1.0.0\",\"secret\":\"12345\",\"port\":9030,\"scopes\":[],\"system\":0}" \ --force-scopes --wait-finish .PHONY: register28 register28: docker exec master-stable28-1 sudo -u www-data php occ app_api:app:unregister skeleton --silent --force || true docker exec master-stable28-1 sudo -u www-data php occ app_api:app:register skeleton manual_install --json-info \ - "{\"appid\":\"skeleton\",\"name\":\"App Skeleton\",\"daemon_config_name\":\"manual_install\",\"version\":\"1.0.0\",\"secret\":\"12345\",\"port\":9030,\"scopes\":[],\"system_app\":0}" \ + "{\"id\":\"skeleton\",\"name\":\"App Skeleton\",\"daemon_config_name\":\"manual_install\",\"version\":\"1.0.0\",\"secret\":\"12345\",\"port\":9030,\"scopes\":[],\"system\":0}" \ --force-scopes --wait-finish .PHONY: register register: docker exec master-nextcloud-1 sudo -u www-data php occ app_api:app:unregister skeleton --silent --force || true docker exec master-nextcloud-1 sudo -u www-data php occ app_api:app:register skeleton manual_install --json-info \ - "{\"appid\":\"skeleton\",\"name\":\"App Skeleton\",\"daemon_config_name\":\"manual_install\",\"version\":\"1.0.0\",\"secret\":\"12345\",\"port\":9030,\"scopes\":[],\"system_app\":0}" \ + "{\"id\":\"skeleton\",\"name\":\"App Skeleton\",\"daemon_config_name\":\"manual_install\",\"version\":\"1.0.0\",\"secret\":\"12345\",\"port\":9030,\"scopes\":[],\"system\":0}" \ --force-scopes --wait-finish diff --git a/examples/as_app/skeleton/requirements.txt b/examples/as_app/skeleton/requirements.txt index e4455715..b3cf46e2 100644 --- a/examples/as_app/skeleton/requirements.txt +++ b/examples/as_app/skeleton/requirements.txt @@ -1 +1 @@ -nc_py_api[app]>=0.5.0 +nc_py_api[app]>=0.10.0 diff --git a/examples/as_app/talk_bot/HOW_TO_INSTALL.md b/examples/as_app/talk_bot/HOW_TO_INSTALL.md index 79789217..00b31b5e 100644 --- a/examples/as_app/talk_bot/HOW_TO_INSTALL.md +++ b/examples/as_app/talk_bot/HOW_TO_INSTALL.md @@ -3,12 +3,7 @@ How To Install 1. [Install AppAPI](https://apps.nextcloud.com/apps/app_api) 2. Create a deployment daemon according to the [instructions](https://cloud-py-api.github.io/app_api/CreationOfDeployDaemon.html#create-deploy-daemon) of the AppPI -3. php occ app_api:app:deploy talk_bot "daemon_deploy_name" \ +3. php occ app_api:app:register talk_bot --force-scopes \ --info-xml https://raw.githubusercontent.com/cloud-py-api/nc_py_api/main/examples/as_app/talk_bot/appinfo/info.xml - to deploy a docker image with Bot to docker. - -4. php occ app_api:app:register talk_bot "daemon_deploy_name" --force-scopes \ ---info-xml https://raw.githubusercontent.com/cloud-py-api/nc_py_api/main/examples/as_app/talk_bot/appinfo/info.xml - - to call its **enable** handler and accept all required API scopes by default. + to deploy and install this ExApp. diff --git a/examples/as_app/talk_bot/Makefile b/examples/as_app/talk_bot/Makefile index 25d32dcb..885d91c2 100644 --- a/examples/as_app/talk_bot/Makefile +++ b/examples/as_app/talk_bot/Makefile @@ -9,9 +9,6 @@ help: @echo " " @echo " build-push build image and upload to ghcr.io" @echo " " - @echo " deploy deploy example to registered 'docker_dev' for Nextcloud Last" - @echo " deploy27 deploy example to registered 'docker_dev' for Nextcloud 27" - @echo " " @echo " run install TalkBot for Nextcloud Last" @echo " run27 install TalkBot for Nextcloud 27" @echo " " @@ -26,40 +23,28 @@ build-push: docker login ghcr.io docker buildx build --push --platform linux/arm64/v8,linux/amd64 --tag ghcr.io/cloud-py-api/talk_bot:latest . -.PHONY: deploy -deploy: - docker exec master-nextcloud-1 sudo -u www-data php occ app_api:app:unregister talk_bot --silent --force || true - docker exec master-nextcloud-1 sudo -u www-data php occ app_api:app:deploy talk_bot docker_dev \ - --info-xml https://raw.githubusercontent.com/cloud-py-api/nc_py_api/main/examples/as_app/talk_bot/appinfo/info.xml - -.PHONY: deploy27 -deploy27: - docker exec master-stable27-1 sudo -u www-data php occ app_api:app:unregister talk_bot --silent --force || true - docker exec master-stable27-1 sudo -u www-data php occ app_api:app:deploy talk_bot docker_dev \ - --info-xml https://raw.githubusercontent.com/cloud-py-api/nc_py_api/main/examples/as_app/talk_bot/appinfo/info.xml - .PHONY: run run: docker exec master-nextcloud-1 sudo -u www-data php occ app_api:app:unregister talk_bot --silent --force || true - docker exec master-nextcloud-1 sudo -u www-data php occ app_api:app:register talk_bot docker_dev --force-scopes \ + docker exec master-nextcloud-1 sudo -u www-data php occ app_api:app:register talk_bot --force-scopes \ --info-xml https://raw.githubusercontent.com/cloud-py-api/nc_py_api/main/examples/as_app/talk_bot/appinfo/info.xml .PHONY: run27 run27: docker exec master-stable27-1 sudo -u www-data php occ app_api:app:unregister talk_bot --silent --force || true - docker exec master-stable27-1 sudo -u www-data php occ app_api:app:register talk_bot docker_dev --force-scopes \ + docker exec master-stable27-1 sudo -u www-data php occ app_api:app:register talk_bot --force-scopes \ --info-xml https://raw.githubusercontent.com/cloud-py-api/nc_py_api/main/examples/as_app/talk_bot/appinfo/info.xml .PHONY: register register: docker exec master-nextcloud-1 sudo -u www-data php occ app_api:app:unregister talk_bot --silent --force || true docker exec master-nextcloud-1 sudo -u www-data php occ app_api:app:register talk_bot manual_install --json-info \ - "{\"appid\":\"talk_bot\",\"name\":\"TalkBot\",\"daemon_config_name\":\"manual_install\",\"version\":\"1.0.0\",\"secret\":\"12345\",\"port\":9032,\"scopes\":[\"TALK\", \"TALK_BOT\"],\"system_app\":0}" \ + "{\"id\":\"talk_bot\",\"name\":\"TalkBot\",\"daemon_config_name\":\"manual_install\",\"version\":\"1.0.0\",\"secret\":\"12345\",\"port\":9032,\"scopes\":[\"TALK\", \"TALK_BOT\"],\"system\":0}" \ --force-scopes --wait-finish .PHONY: register27 register27: docker exec master-stable27-1 sudo -u www-data php occ app_api:app:unregister talk_bot --force || true docker exec master-stable27-1 sudo -u www-data php occ app_api:app:register talk_bot manual_install --json-info \ - "{\"appid\":\"talk_bot\",\"name\":\"TalkBot\",\"daemon_config_name\":\"manual_install\",\"version\":\"1.0.0\",\"secret\":\"12345\",\"port\":9032,\"scopes\":[\"TALK\", \"TALK_BOT\"],\"system_app\":0}" \ + "{\"id\":\"talk_bot\",\"name\":\"TalkBot\",\"daemon_config_name\":\"manual_install\",\"version\":\"1.0.0\",\"secret\":\"12345\",\"port\":9032,\"scopes\":[\"TALK\", \"TALK_BOT\"],\"system\":0}" \ --force-scopes --wait-finish diff --git a/examples/as_app/talk_bot_ai/Makefile b/examples/as_app/talk_bot_ai/Makefile index ca412f75..19217a24 100644 --- a/examples/as_app/talk_bot_ai/Makefile +++ b/examples/as_app/talk_bot_ai/Makefile @@ -9,9 +9,6 @@ help: @echo " " @echo " build-push build image and upload to ghcr.io" @echo " " - @echo " deploy deploy example to registered 'docker_dev' for Nextcloud Last" - @echo " deploy27 deploy example to registered 'docker_dev' for Nextcloud 27" - @echo " " @echo " run install TalkBotAI for Nextcloud Last" @echo " run27 install TalkBotAI for Nextcloud 27" @echo " " @@ -26,40 +23,28 @@ build-push: docker login ghcr.io docker buildx build --push --platform linux/arm64/v8,linux/amd64 --tag ghcr.io/cloud-py-api/talk_bot_ai:latest . -.PHONY: deploy -deploy: - docker exec master-nextcloud-1 sudo -u www-data php occ app_api:app:unregister talk_bot_ai --silent --force || true - docker exec master-nextcloud-1 sudo -u www-data php occ app_api:app:deploy talk_bot_ai docker_dev \ - --info-xml https://raw.githubusercontent.com/cloud-py-api/nc_py_api/main/examples/as_app/talk_bot_ai/appinfo/info.xml - -.PHONY: deploy27 -deploy27: - docker exec master-stable27-1 sudo -u www-data php occ app_api:app:unregister talk_bot_ai --silent --force || true - docker exec master-stable27-1 sudo -u www-data php occ app_api:app:deploy talk_bot_ai docker_dev \ - --info-xml https://raw.githubusercontent.com/cloud-py-api/nc_py_api/main/examples/as_app/talk_bot_ai/appinfo/info.xml - .PHONY: run run: docker exec master-nextcloud-1 sudo -u www-data php occ app_api:app:unregister talk_bot_ai --silent --force || true - docker exec master-nextcloud-1 sudo -u www-data php occ app_api:app:register talk_bot_ai docker_dev --force-scopes \ + docker exec master-nextcloud-1 sudo -u www-data php occ app_api:app:register talk_bot_ai --force-scopes \ --info-xml https://raw.githubusercontent.com/cloud-py-api/nc_py_api/main/examples/as_app/talk_bot_ai/appinfo/info.xml .PHONY: run27 run27: docker exec master-stable27-1 sudo -u www-data php occ app_api:app:unregister talk_bot_ai --silent --force || true - docker exec master-stable27-1 sudo -u www-data php occ app_api:app:register talk_bot_ai docker_dev --force-scopes \ + docker exec master-stable27-1 sudo -u www-data php occ app_api:app:register talk_bot_ai --force-scopes \ --info-xml https://raw.githubusercontent.com/cloud-py-api/nc_py_api/main/examples/as_app/talk_bot_ai/appinfo/info.xml .PHONY: register register: docker exec master-nextcloud-1 sudo -u www-data php occ app_api:app:unregister talk_bot_ai --silent --force || true docker exec master-nextcloud-1 sudo -u www-data php occ app_api:app:register talk_bot_ai manual_install --json-info \ - "{\"appid\":\"talk_bot_ai\",\"name\":\"TalkBotAI\",\"daemon_config_name\":\"manual_install\",\"version\":\"1.0.0\",\"secret\":\"12345\",\"port\":9034,\"scopes\":[\"TALK\", \"TALK_BOT\"],\"system_app\":0}" \ + "{\"id\":\"talk_bot_ai\",\"name\":\"TalkBotAI\",\"daemon_config_name\":\"manual_install\",\"version\":\"1.0.0\",\"secret\":\"12345\",\"port\":9034,\"scopes\":[\"TALK\", \"TALK_BOT\"],\"system\":0}" \ --force-scopes --wait-finish .PHONY: register27 register27: docker exec master-stable27-1 sudo -u www-data php occ app_api:app:unregister talk_bot_ai --silent --force || true docker exec master-stable27-1 sudo -u www-data php occ app_api:app:register talk_bot_ai manual_install --json-info \ - "{\"appid\":\"talk_bot_ai\",\"name\":\"TalkBotAI\",\"daemon_config_name\":\"manual_install\",\"version\":\"1.0.0\",\"secret\":\"12345\",\"port\":9034,\"scopes\":[\"TALK\", \"TALK_BOT\"],\"system_app\":0}" \ + "{\"id\":\"talk_bot_ai\",\"name\":\"TalkBotAI\",\"daemon_config_name\":\"manual_install\",\"version\":\"1.0.0\",\"secret\":\"12345\",\"port\":9034,\"scopes\":[\"TALK\", \"TALK_BOT\"],\"system\":0}" \ --force-scopes --wait-finish diff --git a/examples/as_app/to_gif/Makefile b/examples/as_app/to_gif/Makefile index 8ed9a46d..c0f64515 100644 --- a/examples/as_app/to_gif/Makefile +++ b/examples/as_app/to_gif/Makefile @@ -9,10 +9,6 @@ help: @echo " " @echo " build-push build image and upload to ghcr.io" @echo " " - @echo " deploy27 deploy example to registered 'docker_dev' for Nextcloud 27" - @echo " deploy28 deploy example to registered 'docker_dev' for Nextcloud 28" - @echo " deploy deploy example to registered 'docker_dev' for Nextcloud Last" - @echo " " @echo " run27 install ToGif for Nextcloud 27" @echo " run28 install ToGif for Nextcloud 28" @echo " run install ToGif for Nextcloud Last" @@ -29,59 +25,41 @@ build-push: docker login ghcr.io docker buildx build --push --platform linux/arm64/v8,linux/amd64 --tag ghcr.io/cloud-py-api/to_gif:latest . -.PHONY: deploy27 -deploy27: - docker exec master-stable27-1 sudo -u www-data php occ app_api:app:unregister to_gif --silent --force || true - docker exec master-stable27-1 sudo -u www-data php occ app_api:app:deploy to_gif docker_dev \ - --info-xml https://raw.githubusercontent.com/cloud-py-api/nc_py_api/main/examples/as_app/to_gif/appinfo/info.xml - -.PHONY: deploy28 -deploy28: - docker exec master-stable28-1 sudo -u www-data php occ app_api:app:unregister to_gif --silent --force || true - docker exec master-stable28-1 sudo -u www-data php occ app_api:app:deploy to_gif docker_dev \ - --info-xml https://raw.githubusercontent.com/cloud-py-api/nc_py_api/main/examples/as_app/to_gif/appinfo/info.xml - -.PHONY: deploy -deploy: - docker exec master-nextcloud-1 sudo -u www-data php occ app_api:app:unregister to_gif --silent --force || true - docker exec master-nextcloud-1 sudo -u www-data php occ app_api:app:deploy to_gif docker_dev \ - --info-xml https://raw.githubusercontent.com/cloud-py-api/nc_py_api/main/examples/as_app/to_gif/appinfo/info.xml - .PHONY: run27 run27: docker exec master-stable27-1 sudo -u www-data php occ app_api:app:unregister to_gif --silent --force || true - docker exec master-stable27-1 sudo -u www-data php occ app_api:app:register to_gif docker_dev --force-scopes \ + docker exec master-stable27-1 sudo -u www-data php occ app_api:app:register to_gif --force-scopes \ --info-xml https://raw.githubusercontent.com/cloud-py-api/nc_py_api/main/examples/as_app/to_gif/appinfo/info.xml .PHONY: run28 run28: docker exec master-stable28-1 sudo -u www-data php occ app_api:app:unregister to_gif --silent --force || true - docker exec master-stable28-1 sudo -u www-data php occ app_api:app:register to_gif docker_dev --force-scopes \ + docker exec master-stable28-1 sudo -u www-data php occ app_api:app:register to_gif --force-scopes \ --info-xml https://raw.githubusercontent.com/cloud-py-api/nc_py_api/main/examples/as_app/to_gif/appinfo/info.xml .PHONY: run run: docker exec master-nextcloud-1 sudo -u www-data php occ app_api:app:unregister to_gif --silent --force || true - docker exec master-nextcloud-1 sudo -u www-data php occ app_api:app:register to_gif docker_dev --force-scopes \ + docker exec master-nextcloud-1 sudo -u www-data php occ app_api:app:register to_gif --force-scopes \ --info-xml https://raw.githubusercontent.com/cloud-py-api/nc_py_api/main/examples/as_app/to_gif/appinfo/info.xml .PHONY: register27 register27: docker exec master-stable27-1 sudo -u www-data php occ app_api:app:unregister to_gif --silent --force || true docker exec master-stable27-1 sudo -u www-data php occ app_api:app:register to_gif manual_install --json-info \ - "{\"appid\":\"to_gif\",\"name\":\"to_gif\",\"daemon_config_name\":\"manual_install\",\"version\":\"1.0.0\",\"secret\":\"12345\",\"port\":9031,\"scopes\":[\"FILES\", \"NOTIFICATIONS\"],\"system_app\":0}" \ + "{\"id\":\"to_gif\",\"name\":\"to_gif\",\"daemon_config_name\":\"manual_install\",\"version\":\"1.0.0\",\"secret\":\"12345\",\"port\":9031,\"scopes\":[\"FILES\", \"NOTIFICATIONS\"],\"system\":0}" \ --force-scopes --wait-finish .PHONY: register28 register28: docker exec master-stable28-1 sudo -u www-data php occ app_api:app:unregister to_gif --silent --force || true docker exec master-stable28-1 sudo -u www-data php occ app_api:app:register to_gif manual_install --json-info \ - "{\"appid\":\"to_gif\",\"name\":\"to_gif\",\"daemon_config_name\":\"manual_install\",\"version\":\"1.0.0\",\"secret\":\"12345\",\"port\":9031,\"scopes\":[\"FILES\", \"NOTIFICATIONS\"],\"system_app\":0}" \ + "{\"id\":\"to_gif\",\"name\":\"to_gif\",\"daemon_config_name\":\"manual_install\",\"version\":\"1.0.0\",\"secret\":\"12345\",\"port\":9031,\"scopes\":[\"FILES\", \"NOTIFICATIONS\"],\"system\":0}" \ --force-scopes --wait-finish .PHONY: register register: docker exec master-nextcloud-1 sudo -u www-data php occ app_api:app:unregister to_gif --silent --force || true docker exec master-nextcloud-1 sudo -u www-data php occ app_api:app:register to_gif manual_install --json-info \ - "{\"appid\":\"to_gif\",\"name\":\"to_gif\",\"daemon_config_name\":\"manual_install\",\"version\":\"1.0.0\",\"secret\":\"12345\",\"port\":9031,\"scopes\":[\"FILES\", \"NOTIFICATIONS\"],\"system_app\":0}" \ + "{\"id\":\"to_gif\",\"name\":\"to_gif\",\"daemon_config_name\":\"manual_install\",\"version\":\"1.0.0\",\"secret\":\"12345\",\"port\":9031,\"scopes\":[\"FILES\", \"NOTIFICATIONS\"],\"system\":0}" \ --force-scopes --wait-finish diff --git a/nc_py_api/ex_app/integration_fastapi.py b/nc_py_api/ex_app/integration_fastapi.py index 6ec11e29..7e78b8f7 100644 --- a/nc_py_api/ex_app/integration_fastapi.py +++ b/nc_py_api/ex_app/integration_fastapi.py @@ -60,7 +60,6 @@ def set_handlers( default_init: bool = True, models_to_fetch: dict[str, dict] | None = None, map_app_static: bool = True, - progress_init_start_value: int = 0, ): """Defines handlers for the application. @@ -78,8 +77,6 @@ def set_handlers( :param map_app_static: Should be folders ``js``, ``css``, ``l10n``, ``img`` automatically mounted in FastAPI or not. .. note:: First, presence of these directories in the current working dir is checked, then one directory higher. - - :param progress_init_start_value: The "init" progress value from which the download of models starts. """ if models_to_fetch is not None and default_init is False: raise ValueError("`models_to_fetch` can be defined only with `default_init`=True.") @@ -106,9 +103,7 @@ async def heartbeat_callback(): @fast_api_app.post("/init") async def init_callback(b_tasks: BackgroundTasks, nc: typing.Annotated[NextcloudApp, Depends(nc_app)]): - b_tasks.add_task( - fetch_models_task, nc, models_to_fetch if models_to_fetch else {}, progress_init_start_value - ) + b_tasks.add_task(fetch_models_task, nc, models_to_fetch if models_to_fetch else {}, 0) return JSONResponse(content={}) if map_app_static: @@ -125,7 +120,7 @@ def __map_app_static_folders(fast_api_app: FastAPI): fast_api_app.mount(f"/{mnt_dir}", staticfiles.StaticFiles(directory=mnt_dir_path), name=mnt_dir) -def fetch_models_task(nc: NextcloudApp, models: dict[str, dict], progress_init_start_value: int = 0) -> None: +def fetch_models_task(nc: NextcloudApp, models: dict[str, dict], progress_init_start_value: int) -> None: """Use for cases when you want to define custom `/init` but still need to easy download models.""" if models: current_progress = progress_init_start_value diff --git a/scripts/dev_register.sh b/scripts/dev_register.sh index cdf2be1a..5126724f 100644 --- a/scripts/dev_register.sh +++ b/scripts/dev_register.sh @@ -7,6 +7,6 @@ echo "unregistering nc_py_api as an app for $1 container" docker exec "$1" sudo -u www-data php occ app_api:app:unregister nc_py_api --silent --force || true echo "registering nc_py_api as an app for $1 container" docker exec "$1" sudo -u www-data php occ app_api:app:register nc_py_api manual_install --json-info \ - "{\"appid\":\"nc_py_api\",\"name\":\"nc_py_api\",\"daemon_config_name\":\"manual_install\",\"version\":\"1.0.0\",\"secret\":\"12345\",\"scopes\":[\"ALL\"],\"port\":9009,\"system_app\":1}" \ + "{\"id\":\"nc_py_api\",\"name\":\"nc_py_api\",\"daemon_config_name\":\"manual_install\",\"version\":\"1.0.0\",\"secret\":\"12345\",\"scopes\":[\"ALL\"],\"port\":9009,\"system\":1}" \ --force-scopes --wait-finish echo "nc_py_api for $1 is ready to use" From 63aba92916f3a887e0c67bae3d73c1596dcafa04 Mon Sep 17 00:00:00 2001 From: Alexander Piskun <13381981+bigcat88@users.noreply.github.com> Date: Sun, 17 Mar 2024 13:54:51 +0300 Subject: [PATCH 4/8] removed `ApiScope` as it was useful only with `optional` api scopes feature Signed-off-by: Alexander Piskun --- docs/reference/ExApp.rst | 3 --- nc_py_api/ex_app/__init__.py | 2 +- nc_py_api/ex_app/defs.py | 31 ------------------------------- 3 files changed, 1 insertion(+), 35 deletions(-) diff --git a/docs/reference/ExApp.rst b/docs/reference/ExApp.rst index fe508e1b..89757127 100644 --- a/docs/reference/ExApp.rst +++ b/docs/reference/ExApp.rst @@ -6,9 +6,6 @@ External Application Constants --------- -.. autoclass:: ApiScope - :members: - .. autoclass:: LogLvl :members: diff --git a/nc_py_api/ex_app/__init__.py b/nc_py_api/ex_app/__init__.py index 8575b572..657a0884 100644 --- a/nc_py_api/ex_app/__init__.py +++ b/nc_py_api/ex_app/__init__.py @@ -1,6 +1,6 @@ """All possible ExApp stuff for NextcloudApp that can be used.""" -from .defs import ApiScope, LogLvl +from .defs import LogLvl from .integration_fastapi import ( AppAPIAuthMiddleware, anc_app, diff --git a/nc_py_api/ex_app/defs.py b/nc_py_api/ex_app/defs.py index be2e2cab..5296e164 100644 --- a/nc_py_api/ex_app/defs.py +++ b/nc_py_api/ex_app/defs.py @@ -16,34 +16,3 @@ class LogLvl(enum.IntEnum): """Error log level""" FATAL = 4 """Fatal log level""" - - -class ApiScope(enum.IntEnum): - """Defined API scopes.""" - - SYSTEM = 2 - """Allows access to the System APIs.""" - FILES = 10 - """Allows access to the Nextcloud file base.""" - FILES_SHARING = 11 - """Allows access to APIs that provide File Sharing.""" - USER_INFO = 30 - """Allows access to APIs that work with users.""" - USER_STATUS = 31 - """Allows access to APIs that work with users statuses.""" - NOTIFICATIONS = 32 - """Allows access to APIs that provide Notifications.""" - WEATHER_STATUS = 33 - """Allows access to APIs that provide Weather status.""" - TALK = 50 - """Allows access to Talk API endpoints.""" - TALK_BOT = 60 - """Allows to register Talk Bots.""" - AI_PROVIDERS = 61 - """Allows to register AI providers.""" - ACTIVITIES = 110 - """Activity App endpoints.""" - NOTES = 120 - """Notes App endpoints.""" - ALL = 9999 - """All endpoints allowed.""" From e6886e1b90a70ac57a138303ada661276a72d029 Mon Sep 17 00:00:00 2001 From: Alexander Piskun Date: Sun, 17 Mar 2024 14:01:08 +0300 Subject: [PATCH 5/8] try to fix tests for `restore_version` --- nc_py_api/files/files.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nc_py_api/files/files.py b/nc_py_api/files/files.py index f0180c0c..8891a62e 100644 --- a/nc_py_api/files/files.py +++ b/nc_py_api/files/files.py @@ -330,10 +330,10 @@ def restore_version(self, file_object: FsNode) -> None: """ require_capabilities("files.versioning", self._session.capabilities) dest = self._session.cfg.dav_endpoint + f"/versions/{self._session.user}/restore/{file_object.name}" - headers = Headers({"Destination": dest}, encoding="utf-8") + headers = Headers({"Destination": quote(dest)}, encoding="utf-8") response = self._session.adapter_dav.request( "MOVE", - f"/versions/{self._session.user}/{file_object.user_path}", + quote(f"/versions/{self._session.user}/{file_object.user_path}"), headers=headers, ) check_error(response, f"restore_version: user={self._session.user}, src={file_object.user_path}") @@ -810,10 +810,10 @@ async def restore_version(self, file_object: FsNode) -> None: """ require_capabilities("files.versioning", await self._session.capabilities) dest = self._session.cfg.dav_endpoint + f"/versions/{await self._session.user}/restore/{file_object.name}" - headers = Headers({"Destination": dest}, encoding="utf-8") + headers = Headers({"Destination": quote(dest)}, encoding="utf-8") response = await self._session.adapter_dav.request( "MOVE", - f"/versions/{await self._session.user}/{file_object.user_path}", + quote(f"/versions/{await self._session.user}/{file_object.user_path}"), headers=headers, ) check_error(response, f"restore_version: user={await self._session.user}, src={file_object.user_path}") From ba4b712500c7357318aa660c4d48b1936cb0c423 Mon Sep 17 00:00:00 2001 From: Alexander Piskun Date: Sun, 17 Mar 2024 14:13:50 +0300 Subject: [PATCH 6/8] try to fix tests for `restore_version` --- nc_py_api/files/files.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nc_py_api/files/files.py b/nc_py_api/files/files.py index 8891a62e..255f0b1f 100644 --- a/nc_py_api/files/files.py +++ b/nc_py_api/files/files.py @@ -330,7 +330,7 @@ def restore_version(self, file_object: FsNode) -> None: """ require_capabilities("files.versioning", self._session.capabilities) dest = self._session.cfg.dav_endpoint + f"/versions/{self._session.user}/restore/{file_object.name}" - headers = Headers({"Destination": quote(dest)}, encoding="utf-8") + headers = Headers({"Destination": dest}, encoding="utf-8") response = self._session.adapter_dav.request( "MOVE", quote(f"/versions/{self._session.user}/{file_object.user_path}"), @@ -810,7 +810,7 @@ async def restore_version(self, file_object: FsNode) -> None: """ require_capabilities("files.versioning", await self._session.capabilities) dest = self._session.cfg.dav_endpoint + f"/versions/{await self._session.user}/restore/{file_object.name}" - headers = Headers({"Destination": quote(dest)}, encoding="utf-8") + headers = Headers({"Destination": dest}, encoding="utf-8") response = await self._session.adapter_dav.request( "MOVE", quote(f"/versions/{await self._session.user}/{file_object.user_path}"), From 430cbe8998292885d8b53b4ddce37a299fd1c7e7 Mon Sep 17 00:00:00 2001 From: Alexander Piskun Date: Sun, 17 Mar 2024 14:26:29 +0300 Subject: [PATCH 7/8] added "sleep" to test --- tests/actual_tests/files_test.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/actual_tests/files_test.py b/tests/actual_tests/files_test.py index 4f44788d..7720a87b 100644 --- a/tests/actual_tests/files_test.py +++ b/tests/actual_tests/files_test.py @@ -1,6 +1,7 @@ import contextlib import math import os +import time import zipfile from datetime import datetime from io import BytesIO @@ -1073,6 +1074,7 @@ def test_file_versions(nc_any, dest_path): assert version_str.find("File version") != -1 assert version_str.find("bytes size") != -1 nc_any.files.restore_version(versions[0]) + time.sleep(0.5) assert nc_any.files.download(new_file) == b"22" @@ -1096,6 +1098,7 @@ async def test_file_versions_async(anc_any, dest_path): assert version_str.find("File version") != -1 assert version_str.find("bytes size") != -1 await anc_any.files.restore_version(versions[0]) + time.sleep(0.5) assert await anc_any.files.download(new_file) == b"22" From d311d316272e5574a6dd8e509f6cfb0033343cce Mon Sep 17 00:00:00 2001 From: Alexander Piskun Date: Sun, 17 Mar 2024 14:44:49 +0300 Subject: [PATCH 8/8] updated Security.md [ci skip] --- .github/SECURITY.md | 30 ++++++------------------------ 1 file changed, 6 insertions(+), 24 deletions(-) diff --git a/.github/SECURITY.md b/.github/SECURITY.md index 41754673..10b90970 100644 --- a/.github/SECURITY.md +++ b/.github/SECURITY.md @@ -6,31 +6,13 @@ Only the latest non beta release version of `nc_py_api` are currently being supp ## Reporting a Vulnerability about nc_py_api -Officially, Nextcloud is not responsible for this project; the project is developed in the community’s free time. +Officially, Nextcloud is not responsible for this project; the project is developing by the community. -Please report security vulnerabilities to bigcat88@icloud.com with `nc-py-api` in the subject line. -If there is no response within 24 hours, then create an Issue, -without technical details, to report on the previously sent mail. +Please use GitHub's Private Vulnerability Reporting feature. -## Reporting a Vulnerability about Nextcloud +If there is no response within 48 hours, then create an Issue, +without technical details, to report on the previously reported vulnerability. -Transparency and Security are vital. -If you have discovered a security issue with Nextcloud, -please read our responsible disclosure guidelines and contact us at [hackerone.com/nextcloud](https://hackerone.com/nextcloud). -Your report should include: +## Conclusion -- Product version -- A vulnerability description -- Reproduction steps - -If in the scope of the project, a member of the security team confirms the vulnerability, determines its impact, and develops a fix. -Otherwise, the team will contact the maintainer and make sure the issue gets fixed. -The fix will be applied to the main branch, tested, and packaged in the next security release. -The vulnerability will be publicly announced after the release. - -Finally, your name will be added to the [hall of fame](https://hackerone.com/nextcloud/thanks) -as a thank-you from the entire Nextcloud community. - -Note our [threat model](https://nextcloud.com/security/threat-model) to know what is expected behavior. - -Please visit https://nextcloud.com/security/ for further information about security. +We welcome and appreciate contributions to the security and advancement of open-source projects.