From 34628fe9c04d774c84f63251fbd49358200cdd63 Mon Sep 17 00:00:00 2001 From: Bastien Abadie Date: Wed, 7 Aug 2019 17:55:09 +0200 Subject: [PATCH 1/5] Fix sentry usage & report grcov errors. --- .../code_coverage_backend/backend/__init__.py | 2 +- bot/ci/bootstrap.sh | 2 +- bot/code_coverage_bot/cli.py | 3 +- bot/code_coverage_bot/utils.py | 10 ++++++- tools/code_coverage_tools/log.py | 30 +++++++++++-------- 5 files changed, 30 insertions(+), 17 deletions(-) diff --git a/backend/code_coverage_backend/backend/__init__.py b/backend/code_coverage_backend/backend/__init__.py index 63f52f132..9c40f3639 100644 --- a/backend/code_coverage_backend/backend/__init__.py +++ b/backend/code_coverage_backend/backend/__init__.py @@ -30,7 +30,7 @@ def create_app(): code_coverage_backend.config.PROJECT_NAME, PAPERTRAIL_HOST=taskcluster.secrets.get("PAPERTRAIL_HOST"), PAPERTRAIL_PORT=taskcluster.secrets.get("PAPERTRAIL_PORT"), - SENTRY_DSN=taskcluster.secrets.get("SENTRY_DSN"), + sentry_dsn=taskcluster.secrets.get("SENTRY_DSN"), ) logger = structlog.get_logger(__name__) diff --git a/bot/ci/bootstrap.sh b/bot/ci/bootstrap.sh index 0611f5def..0d782eba0 100755 --- a/bot/ci/bootstrap.sh +++ b/bot/ci/bootstrap.sh @@ -1,5 +1,5 @@ #!/bin/bash -ex -GRCOV_VERSION="v0.5.1" +GRCOV_VERSION="v0.5.2" MERCURIAL_VERSION="4.8" apt-get update diff --git a/bot/code_coverage_bot/cli.py b/bot/code_coverage_bot/cli.py index ffbc83e73..01b46967d 100644 --- a/bot/code_coverage_bot/cli.py +++ b/bot/code_coverage_bot/cli.py @@ -49,9 +49,10 @@ def main(): init_logger( config.PROJECT_NAME, + channel=secrets.get('APP_CHANNEL', 'dev'), PAPERTRAIL_HOST=secrets.get("PAPERTRAIL_HOST"), PAPERTRAIL_PORT=secrets.get("PAPERTRAIL_PORT"), - SENTRY_DSN=secrets.get("SENTRY_DSN"), + sentry_dsn=secrets.get("SENTRY_DSN"), ) c = CodeCov(args.repository, args.revision, args.task_name_filter, args.cache_root) diff --git a/bot/code_coverage_bot/utils.py b/bot/code_coverage_bot/utils.py index 570075bb0..07dc23746 100644 --- a/bot/code_coverage_bot/utils.py +++ b/bot/code_coverage_bot/utils.py @@ -69,12 +69,20 @@ def run_check(command, **kwargs): output, error = proc.communicate() if proc.returncode != 0: - log.info( + if isinstance(output, bytes): + output = output.decode('utf-8') + if isinstance(error, bytes): + error = error.decode('utf-8') + + # Use error to send log to sentry + log.error( f"Command failed with code: {proc.returncode}", + exit=proc.returncode, command=" ".join(command), output=output, error=error, ) + raise RunException(f"`{command[0]}` failed with code: {proc.returncode}.") return output diff --git a/tools/code_coverage_tools/log.py b/tools/code_coverage_tools/log.py index 8195b88f7..bec83374f 100644 --- a/tools/code_coverage_tools/log.py +++ b/tools/code_coverage_tools/log.py @@ -7,9 +7,12 @@ import logbook import logbook.more +import raven +import raven.handlers.logbook import structlog + class UnstructuredRenderer(structlog.processors.KeyValueRenderer): def __call__(self, logger, method_name, event_dict): event = None @@ -42,23 +45,24 @@ def setup_papertrail(project_name, channel, PAPERTRAIL_HOST, PAPERTRAIL_PORT): papertrail.push_application() -def setup_sentry(project_name, channel, SENTRY_DSN): +def setup_sentry(name, channel, dsn): """ Setup sentry account using taskcluster secrets """ - import raven - import raven.handlers.logbook + # Detect environment + if 'TASK_ID' in os.environ: + site = 'taskcluster' + elif 'DYNO' in os.environ: + site = 'heroku' + else: + site = 'unknown' sentry_client = raven.Client( - dsn=SENTRY_DSN, - site=project_name, - name="mozilla/release-services", + dsn=dsn, + site=site, + name=name, environment=channel, - # TODO: - # release=read(VERSION) we need to promote that as well via secrets - # tags=... - # repos=... ) sentry_handler = raven.handlers.logbook.SentryHandler( @@ -73,7 +77,7 @@ def init_logger( level=logbook.INFO, PAPERTRAIL_HOST=None, PAPERTRAIL_PORT=None, - SENTRY_DSN=None, + sentry_dsn=None, ): if not channel: @@ -89,8 +93,8 @@ def init_logger( setup_papertrail(project_name, channel, PAPERTRAIL_HOST, PAPERTRAIL_PORT) # Log to senty - if channel and SENTRY_DSN: - setup_sentry(project_name, channel, SENTRY_DSN) + if channel and sentry_dsn: + setup_sentry(project_name, channel, sentry_dsn) def logbook_factory(*args, **kwargs): # Logger given to structlog From 2abc5ae4e8bd61329c1701b82113212fc401554a Mon Sep 17 00:00:00 2001 From: Bastien Abadie Date: Wed, 7 Aug 2019 18:00:33 +0200 Subject: [PATCH 2/5] Add raven as requirement --- tools/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/requirements.txt b/tools/requirements.txt index 5e64758fb..5befef9a8 100644 --- a/tools/requirements.txt +++ b/tools/requirements.txt @@ -1,3 +1,4 @@ logbook structlog taskcluster +raven From 8dd6512d0ecaa9c5b2c105774288ae5b025bf6c2 Mon Sep 17 00:00:00 2001 From: Bastien Abadie Date: Wed, 7 Aug 2019 18:06:36 +0200 Subject: [PATCH 3/5] Fix styling. --- .isort.cfg | 2 +- bot/code_coverage_bot/cli.py | 2 +- bot/code_coverage_bot/utils.py | 4 ++-- tools/code_coverage_tools/log.py | 18 ++++++------------ 4 files changed, 10 insertions(+), 16 deletions(-) diff --git a/.isort.cfg b/.isort.cfg index f3baeb563..028d5a09d 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -1,6 +1,6 @@ [settings] known_first_party = code_coverage_backend,code_coverage_bot,code_coverage_tools,conftest -known_third_party = connexion,datadog,dateutil,fakeredis,flask,flask_cors,flask_talisman,google,hglib,jsone,jsonschema,libmozdata,logbook,pytest,pytz,redis,requests,responses,setuptools,structlog,taskcluster,werkzeug,zstandard +known_third_party = connexion,datadog,dateutil,fakeredis,flask,flask_cors,flask_talisman,google,hglib,jsone,jsonschema,libmozdata,logbook,pytest,pytz,raven,redis,requests,responses,setuptools,structlog,taskcluster,werkzeug,zstandard force_single_line = True default_section=FIRSTPARTY line_length=159 diff --git a/bot/code_coverage_bot/cli.py b/bot/code_coverage_bot/cli.py index 01b46967d..b6601233b 100644 --- a/bot/code_coverage_bot/cli.py +++ b/bot/code_coverage_bot/cli.py @@ -49,7 +49,7 @@ def main(): init_logger( config.PROJECT_NAME, - channel=secrets.get('APP_CHANNEL', 'dev'), + channel=secrets.get("APP_CHANNEL", "dev"), PAPERTRAIL_HOST=secrets.get("PAPERTRAIL_HOST"), PAPERTRAIL_PORT=secrets.get("PAPERTRAIL_PORT"), sentry_dsn=secrets.get("SENTRY_DSN"), diff --git a/bot/code_coverage_bot/utils.py b/bot/code_coverage_bot/utils.py index 07dc23746..5871833ba 100644 --- a/bot/code_coverage_bot/utils.py +++ b/bot/code_coverage_bot/utils.py @@ -70,9 +70,9 @@ def run_check(command, **kwargs): if proc.returncode != 0: if isinstance(output, bytes): - output = output.decode('utf-8') + output = output.decode("utf-8") if isinstance(error, bytes): - error = error.decode('utf-8') + error = error.decode("utf-8") # Use error to send log to sentry log.error( diff --git a/tools/code_coverage_tools/log.py b/tools/code_coverage_tools/log.py index bec83374f..942832922 100644 --- a/tools/code_coverage_tools/log.py +++ b/tools/code_coverage_tools/log.py @@ -12,7 +12,6 @@ import structlog - class UnstructuredRenderer(structlog.processors.KeyValueRenderer): def __call__(self, logger, method_name, event_dict): event = None @@ -51,19 +50,14 @@ def setup_sentry(name, channel, dsn): """ # Detect environment - if 'TASK_ID' in os.environ: - site = 'taskcluster' - elif 'DYNO' in os.environ: - site = 'heroku' + if "TASK_ID" in os.environ: + site = "taskcluster" + elif "DYNO" in os.environ: + site = "heroku" else: - site = 'unknown' + site = "unknown" - sentry_client = raven.Client( - dsn=dsn, - site=site, - name=name, - environment=channel, - ) + sentry_client = raven.Client(dsn=dsn, site=site, name=name, environment=channel) sentry_handler = raven.handlers.logbook.SentryHandler( sentry_client, level=logbook.WARNING, bubble=True From 01c247180820dd3da24db04ee2c3e591551c210e Mon Sep 17 00:00:00 2001 From: Bastien Abadie Date: Fri, 9 Aug 2019 10:21:29 +0200 Subject: [PATCH 4/5] Use latest grcov release --- bot/ci/bootstrap.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/ci/bootstrap.sh b/bot/ci/bootstrap.sh index 0d782eba0..3599eb48e 100755 --- a/bot/ci/bootstrap.sh +++ b/bot/ci/bootstrap.sh @@ -1,5 +1,5 @@ #!/bin/bash -ex -GRCOV_VERSION="v0.5.2" +GRCOV_VERSION="v0.5.3" MERCURIAL_VERSION="4.8" apt-get update From e5c6bb1e7c69a610326337981bb6889aba137d5e Mon Sep 17 00:00:00 2001 From: Bastien Abadie Date: Fri, 9 Aug 2019 10:35:40 +0200 Subject: [PATCH 5/5] Always decode output/error --- bot/code_coverage_bot/utils.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/bot/code_coverage_bot/utils.py b/bot/code_coverage_bot/utils.py index 5871833ba..38c56fcef 100644 --- a/bot/code_coverage_bot/utils.py +++ b/bot/code_coverage_bot/utils.py @@ -69,10 +69,8 @@ def run_check(command, **kwargs): output, error = proc.communicate() if proc.returncode != 0: - if isinstance(output, bytes): - output = output.decode("utf-8") - if isinstance(error, bytes): - error = error.decode("utf-8") + output = output and output.decode("utf-8") or "" + error = error and error.decode("utf-8") or "" # Use error to send log to sentry log.error(