diff --git a/.isort.cfg b/.isort.cfg index 310bc0c1d..2673af0b1 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -1,6 +1,6 @@ [settings] known_first_party = code_coverage_backend,code_coverage_bot,code_coverage_events,code_coverage_tools,conftest,firefox_code_coverage -known_third_party = connexion,datadog,dateutil,fakeredis,flask,flask_cors,flask_talisman,google,hglib,jsone,jsonschema,libmozdata,libmozevent,logbook,magic,pytest,pytz,raven,redis,requests,responses,setuptools,structlog,taskcluster,tenacity,werkzeug,zstandard +known_third_party = connexion,datadog,dateutil,fakeredis,flask,flask_cors,flask_talisman,google,hglib,jsone,jsonschema,libmozdata,libmozevent,logbook,magic,pytest,pytz,raven,redis,requests,responses,setuptools,structlog,taskcluster,tenacity,werkzeug,yaml,zstandard force_single_line = True default_section=FIRSTPARTY line_length=159 diff --git a/README.md b/README.md index 3425cc86d..418f9cc2c 100644 --- a/README.md +++ b/README.md @@ -6,3 +6,7 @@ This project has 4 parts: * `backend` is a Python API built with Flask, that serves the aggregated code coverage data, in an efficient way, * `frontend` is a vanilla Javascript SPA displaying code coverage data in your browser, * `addon` is a Web Extension for Firefox, extending several Mozilla websites with code coverage data. + +## Help + +You can reach us on our Matrix instance: [#codecoverage:mozilla.org](https://chat.mozilla.org/#/room/#codecoverage:mozilla.org) diff --git a/backend/README.md b/backend/README.md index 1a09f9186..16870df7d 100644 --- a/backend/README.md +++ b/backend/README.md @@ -37,14 +37,27 @@ docker run -v /tmp/ccov-redis:/data -p 6379:6379 redis The development webserver will run on **http://localhost:8000** -Using default secret `project-relman/code-coverage/dev`: +You need to setup a local YAML configuration file, with the following content: + +```yaml +--- +common: + APP_CHANNEL: dev + GOOGLE_CLOUD_STORAGE: null +``` + +Using your local configuration: ```shell -./run.sh +LOCAL_CONFIGURATION=/path/to/code-coverage-conf.yml ./run.sh ``` -You can specify any other secret as: +You can specify a firefox-ci Taskcluster secret (e.g. project-relman/code-coverage/dev) instead using: ```shell TASKCLUSTER_SECRET=path/to/secret ./run.sh ``` + +## Help + +You can reach us on our Matrix instance: [#codecoverage:mozilla.org](https://chat.mozilla.org/#/room/#codecoverage:mozilla.org) diff --git a/backend/code_coverage_backend/backend/__init__.py b/backend/code_coverage_backend/backend/__init__.py index 2874a5af5..bc8255d56 100644 --- a/backend/code_coverage_backend/backend/__init__.py +++ b/backend/code_coverage_backend/backend/__init__.py @@ -6,6 +6,7 @@ import os.path import structlog +import yaml import code_coverage_backend.datadog import code_coverage_backend.gcp @@ -17,12 +18,20 @@ def create_app(): # Load secrets from Taskcluster + local_secrets_path = os.environ.get("LOCAL_CONFIGURATION") + if local_secrets_path is not None: + assert os.path.exists( + local_secrets_path + ), f"Invalid local secrets path {local_secrets_path}" taskcluster.auth() taskcluster.load_secrets( os.environ.get("TASKCLUSTER_SECRET"), prefixes=["common", "backend", "code-coverage-backend"], required=["GOOGLE_CLOUD_STORAGE", "APP_CHANNEL"], existing={"REDIS_URL": os.environ.get("REDIS_URL", "redis://localhost:6379")}, + local_secrets=yaml.safe_load(open(local_secrets_path)) + if local_secrets_path + else None, ) # Configure logger diff --git a/backend/code_coverage_backend/datadog.py b/backend/code_coverage_backend/datadog.py index efdada95b..f63bdfee3 100644 --- a/backend/code_coverage_backend/datadog.py +++ b/backend/code_coverage_backend/datadog.py @@ -25,7 +25,7 @@ def get_stats(): app_channel = taskcluster.secrets["APP_CHANNEL"] - if taskcluster.secrets["DATADOG_API_KEY"]: + if taskcluster.secrets.get("DATADOG_API_KEY"): datadog.initialize( api_key=taskcluster.secrets["DATADOG_API_KEY"], host_name=f"coverage.{app_channel}.moz.tools", diff --git a/bot/README.md b/bot/README.md new file mode 100644 index 000000000..eea3d56df --- /dev/null +++ b/bot/README.md @@ -0,0 +1,66 @@ +# Code Coverage Bot + +This project runs as [Taskcluster hooks](https://firefox-ci-tc.services.mozilla.com/hooks) on the firefox-ci instance, to extract and store the code coverage information from mozilla-central and try builds. + +It's built using Python 3.8 and few dependencies. + +## Developer setup + +Requirements: + +- Python 3.8 +- Mercurial 5.3 +- (optional) [virtualenv](https://virtualenv.pypa.io/en/stable/) +- (optional) [virtualenvwrapper](https://virtualenvwrapper.readthedocs.io/en/latest/) + +Setup on your computer: + +```console +# If you have virtualenvwrapper: +mkvirtualenv -p /usr/bin/python3.8 code-coverage-bot + +# Mandatory steps +pip install -r requirements.txt -r requirements-dev.txt +pip install -e . +pre-commit install +``` + +Check linting (it should be automatically done before any commit): + +```console +pre-commint run -a +``` + +Check unit tests: + +```console +pytest -v +``` + +Write your local configuration as YAML: + +```yaml +--- +common: + APP_CHANNEL: dev +bot: + BACKEND_HOST: 'http://localhost:8000' + EMAIL_ADDRESSES: [] + PHABRICATOR_TOKEN: api-xxx + PHABRICATOR_ENABLED: false + PHABRICATOR_URL: 'https://phabricator-dev.allizom.org/api/' + GOOGLE_CLOUD_STORAGE: null +``` + +Run the bot (in cron mode): + +```console +mkdir -p build/{cache,work} # or elsewhere on your system +code-coverage-cron --cache-root=build/cache --working-dir=build/work --local-configuration=path/to/code-coverage.yml +``` + +The repo mode (with `code-coverage-repo`) is harder to use, as it requires a Google Cloud Storage and a Phabricator account. + +## Help + +You can reach us on our Matrix instance: [#codecoverage:mozilla.org](https://chat.mozilla.org/#/room/#codecoverage:mozilla.org) diff --git a/bot/code_coverage_bot/cli.py b/bot/code_coverage_bot/cli.py index 5786f15ce..605a7b68a 100644 --- a/bot/code_coverage_bot/cli.py +++ b/bot/code_coverage_bot/cli.py @@ -6,6 +6,8 @@ import argparse import os +import yaml + from code_coverage_bot.secrets import secrets from code_coverage_bot.taskcluster import taskcluster_config from code_coverage_tools.log import init_logger @@ -38,6 +40,11 @@ def setup_cli(ask_repository=True, ask_revision=True): help="Taskcluster Secret path", default=os.environ.get("TASKCLUSTER_SECRET"), ) + parser.add_argument( + "--local-configuration", + help="Path to a local YAML configuration file", + type=open, + ) parser.add_argument("--taskcluster-client-id", help="Taskcluster Client ID") parser.add_argument("--taskcluster-access-token", help="Taskcluster Access token") args = parser.parse_args() @@ -46,7 +53,12 @@ def setup_cli(ask_repository=True, ask_revision=True): taskcluster_config.auth(args.taskcluster_client_id, args.taskcluster_access_token) # Then load secrets - secrets.load(args.taskcluster_secret) + secrets.load( + args.taskcluster_secret, + local_secrets=yaml.safe_load(args.local_configuration) + if args.local_configuration + else None, + ) init_logger( "bot", diff --git a/bot/code_coverage_bot/secrets.py b/bot/code_coverage_bot/secrets.py index ac07235e5..9f7154d5b 100644 --- a/bot/code_coverage_bot/secrets.py +++ b/bot/code_coverage_bot/secrets.py @@ -15,10 +15,10 @@ class Secrets(dict): PHABRICATOR_TOKEN = "PHABRICATOR_TOKEN" GOOGLE_CLOUD_STORAGE = "GOOGLE_CLOUD_STORAGE" - def load(self, taskcluster_secret): + def load(self, taskcluster_secret=None, local_secrets=None): taskcluster_config.load_secrets( taskcluster_secret, - prefixes=["common", "code-coverage-bot"], + prefixes=["common", "bot"], required=[ Secrets.APP_CHANNEL, Secrets.BACKEND_HOST, @@ -27,6 +27,7 @@ def load(self, taskcluster_secret): Secrets.PHABRICATOR_URL, Secrets.PHABRICATOR_TOKEN, ], + local_secrets=local_secrets, ) self.update(taskcluster_config.secrets) diff --git a/frontend/README.md b/frontend/README.md index 223be688a..4d1148ae2 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -1,7 +1,7 @@ # Mozilla Code Coverage frontend This is a simple JavaScript application displaying information from the Mozilla code coverage backend (hosted as https://coverage.moz.tools) -You can use it right now from [mozilla.github.io/code-coverage-reports](https://mozilla.github.io/code-coverage-reports/) +You can use it right now from [coverage.moz.tools](https://coverage.moz.tools/) ## File viewer @@ -11,4 +11,25 @@ You can browse the aggregated code coverage data for mozilla-central, per direct ## Zero coverage reports -We've also [developed a page](https://mozilla.github.io/code-coverage-reports/zero_coverage_report.html) to easily browse the directories & files with no coverage: these files are interesting because they can sometimes be removed altogether from mozilla-central... +We've also [developed a page](https://coverage.moz.tools/#view=zero) to easily browse the directories & files with no coverage: these files are interesting because they can sometimes be removed altogether from mozilla-central... + +## Developer setup + +It's a pretty simple Single Page application setup using webpack: + +```console +npm install +npm run start +``` + +The frontend should now be available on http://localhost:9000/ and will use a backend running on http://localhost:8000/ + +You can specify another remote backend like so: + +``` +BACKEND_URL=https://api.coverage.moz.tools npm run start +``` + +## Help + +You can reach us on our Matrix instance: [#codecoverage:mozilla.org](https://chat.mozilla.org/#/room/#codecoverage:mozilla.org) diff --git a/frontend/screenshot.png b/frontend/screenshot.png index 38fee7ddd..f649acbf3 100644 Binary files a/frontend/screenshot.png and b/frontend/screenshot.png differ