From 2484b7131eae0fffdac41e9ca764b22c334bf883 Mon Sep 17 00:00:00 2001 From: Bastien Abadie Date: Thu, 21 Nov 2019 13:57:38 +0100 Subject: [PATCH 1/4] tools: Remove TaskclusterConfig --- tools/code_coverage_tools/taskcluster.py | 117 ----------------------- tools/requirements.txt | 1 - 2 files changed, 118 deletions(-) delete mode 100644 tools/code_coverage_tools/taskcluster.py diff --git a/tools/code_coverage_tools/taskcluster.py b/tools/code_coverage_tools/taskcluster.py deleted file mode 100644 index 30d37d3d5..000000000 --- a/tools/code_coverage_tools/taskcluster.py +++ /dev/null @@ -1,117 +0,0 @@ -# -*- coding: utf-8 -*- -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -import copy -import os - -import structlog -import taskcluster - -try: - import toml -except ImportError: - import pytoml as toml - -logger = structlog.get_logger(__name__) - -TASKCLUSTER_DATE_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ" - - -class TaskclusterConfig(object): - """ - Local configuration used to access Taskcluster service and objects - """ - - def __init__(self): - self.options = None - self.secrets = None - - def auth(self, client_id=None, access_token=None): - """ - Build Taskcluster credentials options - Supports, by order of preference: - * directly provided credentials - * credentials from local configuration - * credentials from environment variables - * taskclusterProxy - """ - self.options = {"maxRetries": 12} - - if client_id is None and access_token is None: - # Credentials preference: Use local config from release-services - xdg = os.path.expanduser(os.environ.get("XDG_CONFIG_HOME", "~/.config")) - config = os.path.join(xdg, "please", "config.toml") - try: - assert os.path.exists(config), "No user config available" - data = toml.load(open(config)) - client_id = data["common"]["taskcluster_client_id"] - access_token = data["common"]["taskcluster_access_token"] - assert ( - client_id is not None and access_token is not None - ), "Missing values in user folder" - logger.info("Using taskcluster credentials from local configuration") - except Exception: - # Credentials preference: Use env. variables - client_id = os.environ.get("TASKCLUSTER_CLIENT_ID") - access_token = os.environ.get("TASKCLUSTER_ACCESS_TOKEN") - logger.info("Using taskcluster credentials from environment") - else: - logger.info("Using taskcluster credentials from cli") - - if client_id is not None and access_token is not None: - # Use provided credentials - self.options["credentials"] = { - "clientId": client_id, - "accessToken": access_token, - } - self.options["rootUrl"] = "https://firefox-ci-tc.services.mozilla.com" - - else: - # with taskclusterProxy - root_url = "http://taskcluster" - logger.info("Taskcluster Proxy enabled", url=root_url) - self.options["rootUrl"] = root_url - - def get_service(self, service_name): - """ - Build a Taskcluster service instance using current authentication - """ - assert self.options is not None, "Not authenticated" - service = getattr(taskcluster, service_name.capitalize(), None) - assert service is not None, "Invalid Taskcluster service {}".format( - service_name - ) - return service(self.options) - - def load_secrets(self, name, project_name, required=[], existing=dict()): - """ - Fetch a specific set of secrets by name and verify that the required - secrets exist. - - Merge secrets in the following order (the latter overrides the former): - - `existing` argument - - common secrets, specified under the `common` key in the secrets - object - - project specific secrets, specified under the `project_name` key in - the secrets object - """ - assert name is not None, "Missing Taskcluster secret name" - self.secrets = dict() - if existing: - self.secrets = copy.deepcopy(existing) - - secrets_service = self.get_service("secrets") - all_secrets = secrets_service.get(name).get("secret", dict()) - logger.info("Loaded Taskcluster secret", name=name) - - secrets_common = all_secrets.get("common", dict()) - self.secrets.update(secrets_common) - - secrets_app = all_secrets.get(project_name, dict()) - self.secrets.update(secrets_app) - - for required_secret in required: - if required_secret not in self.secrets: - raise Exception(f"Missing value {required_secret} in secrets.") diff --git a/tools/requirements.txt b/tools/requirements.txt index f4421312d..44b91a91b 100644 --- a/tools/requirements.txt +++ b/tools/requirements.txt @@ -1,6 +1,5 @@ logbook structlog -taskcluster raven # limit async-timeout version to avoid using the alpha 4.0.0a0 From e5467e81d399f200cdef0fd9f09fd21769e0b91a Mon Sep 17 00:00:00 2001 From: Bastien Abadie Date: Thu, 21 Nov 2019 14:02:23 +0100 Subject: [PATCH 2/4] backend: Use official TaskclusterConfig --- backend/code_coverage_backend/__init__.py | 4 ++-- backend/code_coverage_backend/backend/__init__.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/code_coverage_backend/__init__.py b/backend/code_coverage_backend/__init__.py index 5a25c398e..12820f1d4 100644 --- a/backend/code_coverage_backend/__init__.py +++ b/backend/code_coverage_backend/__init__.py @@ -2,6 +2,6 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from code_coverage_tools.taskcluster import TaskclusterConfig +from taskcluster.helper import TaskclusterConfig -taskcluster = TaskclusterConfig() +taskcluster = TaskclusterConfig("https://firefox-ci-tc.services.mozilla.com") diff --git a/backend/code_coverage_backend/backend/__init__.py b/backend/code_coverage_backend/backend/__init__.py index 59cc14045..2874a5af5 100644 --- a/backend/code_coverage_backend/backend/__init__.py +++ b/backend/code_coverage_backend/backend/__init__.py @@ -20,7 +20,7 @@ def create_app(): taskcluster.auth() taskcluster.load_secrets( os.environ.get("TASKCLUSTER_SECRET"), - code_coverage_backend.config.PROJECT_NAME, + prefixes=["common", "backend", "code-coverage-backend"], required=["GOOGLE_CLOUD_STORAGE", "APP_CHANNEL"], existing={"REDIS_URL": os.environ.get("REDIS_URL", "redis://localhost:6379")}, ) From 1dd51fbc1fc53b9cd398d7e43083d115159b8a26 Mon Sep 17 00:00:00 2001 From: Bastien Abadie Date: Thu, 21 Nov 2019 14:04:57 +0100 Subject: [PATCH 3/4] bot: Use official TaskclusterConfig --- bot/code_coverage_bot/config.py | 1 - bot/code_coverage_bot/secrets.py | 3 +-- bot/code_coverage_bot/taskcluster.py | 4 ++-- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/bot/code_coverage_bot/config.py b/bot/code_coverage_bot/config.py index 05a67bab9..be4d1d28c 100644 --- a/bot/code_coverage_bot/config.py +++ b/bot/code_coverage_bot/config.py @@ -3,7 +3,6 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -PROJECT_NAME = "code-coverage-bot" HG_BASE = "https://hg.mozilla.org/" MOZILLA_CENTRAL_REPOSITORY = "{}mozilla-central".format(HG_BASE) TRY_REPOSITORY = "{}try".format(HG_BASE) diff --git a/bot/code_coverage_bot/secrets.py b/bot/code_coverage_bot/secrets.py index 5540661ca..ac07235e5 100644 --- a/bot/code_coverage_bot/secrets.py +++ b/bot/code_coverage_bot/secrets.py @@ -3,7 +3,6 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from code_coverage_bot import config from code_coverage_bot.taskcluster import taskcluster_config @@ -19,7 +18,7 @@ class Secrets(dict): def load(self, taskcluster_secret): taskcluster_config.load_secrets( taskcluster_secret, - config.PROJECT_NAME, + prefixes=["common", "code-coverage-bot"], required=[ Secrets.APP_CHANNEL, Secrets.BACKEND_HOST, diff --git a/bot/code_coverage_bot/taskcluster.py b/bot/code_coverage_bot/taskcluster.py index 267023ae6..c6277d627 100644 --- a/bot/code_coverage_bot/taskcluster.py +++ b/bot/code_coverage_bot/taskcluster.py @@ -7,12 +7,12 @@ import requests import structlog import taskcluster +from taskcluster.helper import TaskclusterConfig from code_coverage_bot.utils import retry -from code_coverage_tools.taskcluster import TaskclusterConfig logger = structlog.getLogger(__name__) -taskcluster_config = TaskclusterConfig() +taskcluster_config = TaskclusterConfig("https://firefox-ci-tc.services.mozilla.com") def get_task(branch, revision, platform): From d5d7120d73b4daae3cb1ab2fe809f9786ea621d5 Mon Sep 17 00:00:00 2001 From: Bastien Abadie Date: Tue, 3 Dec 2019 10:21:51 +0100 Subject: [PATCH 4/4] tools: Use taskcluster 24.0.0 --- tools/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/requirements.txt b/tools/requirements.txt index 44b91a91b..9e58dcf24 100644 --- a/tools/requirements.txt +++ b/tools/requirements.txt @@ -1,5 +1,6 @@ logbook structlog +taskcluster==24.0.0 raven # limit async-timeout version to avoid using the alpha 4.0.0a0