From e7092d49b8561e207174db9e8af93588aa651168 Mon Sep 17 00:00:00 2001 From: Bill Prin Date: Thu, 27 Apr 2017 11:08:01 -0700 Subject: [PATCH 1/8] Add error reporting system test --- error_reporting/nox.py | 19 +++++++ error_reporting/tests/system.py | 96 +++++++++++++++++++++++++++++++++ 2 files changed, 115 insertions(+) create mode 100644 error_reporting/tests/system.py diff --git a/error_reporting/nox.py b/error_reporting/nox.py index 746417ccd3e1..2f90922121d1 100644 --- a/error_reporting/nox.py +++ b/error_reporting/nox.py @@ -63,6 +63,25 @@ def lint_setup_py(session): session.run( 'python', 'setup.py', 'check', '--restructuredtext', '--strict') +@nox.session +@nox.parametrize('python_version', ['2.7', '3.6']) +def system_tests(session, python_version): + """Run the system test suite.""" + + # Sanity check: Only run system tests if the environment variable is set. + if not os.environ.get('GOOGLE_APPLICATION_CREDENTIALS', ''): + return + + # Run the system tests against latest Python 2 and Python 3 only. + session.interpreter = 'python{}'.format(python_version) + + # Install all test dependencies, then install this package into the + # virtualenv's dist-packages. + session.install('.') + + # Run py.test against the system tests. + session.run('py.test', '-vvv', 'tests/system.py') + @nox.session def cover(session): diff --git a/error_reporting/tests/system.py b/error_reporting/tests/system.py new file mode 100644 index 000000000000..2312cd8ee01b --- /dev/null +++ b/error_reporting/tests/system.py @@ -0,0 +1,96 @@ +# Copyright 2017 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import functools +import unittest + +from google.cloud import error_reporting +from google.cloud.gapic.errorreporting.v1beta1 import ( + error_stats_service_client) +from google.cloud.proto.devtools.clouderrorreporting.v1beta1 import ( + error_stats_service_pb2) +from google.protobuf.duration_pb2 import Duration + +from test_utils.retry import RetryResult + + +class _ErrorStatsGaxApi(object): + """Helper mapping Error Reporting-related APIs + + This class provides a small wrapper around making calls to the GAX + API. It's used by the system tests to find the appropriate error group + to verify the error was successfully reported. + + :type project: str + :param project: Google Cloud Project ID + """ + def __init__(self, project): + self._project = project + self._gax_api = error_stats_service_client.ErrorStatsServiceClient() + + def list_groups(self): + """Helper to list the groups that have had errors in the last hour.""" + project_name = self._gax_api.project_path(self._project) + time_range = error_stats_service_pb2.QueryTimeRange() + time_range.period = ( + error_stats_service_pb2.QueryTimeRange.PERIOD_1_HOUR + ) + + duration = Duration() + duration.seconds = 60 * 60 + + return self._gax_api.list_group_stats( + project_name, time_range, timed_count_duration=duration) + + +def _is_incremented(initial, new): + """Helper to retry until new error is counted.""" + return new == initial + 1 + + +class TestErrorReporting(unittest.TestCase): + + def setUp(self): + self._client = error_reporting.Client() + self._error_name = 'Stackdriver Error Reporting System Test' + + def _simulate_exception(self): + """Simulates an exception to verify it was reported.""" + try: + raise RuntimeError(self._error_name) + except RuntimeError: + self._client.report_exception() + + def _get_error_count(self): + """Counts the number of errors in the group of the test exception.""" + error_stats_api = _ErrorStatsGaxApi(self._client.project) + groups = error_stats_api.list_groups() + for group in groups: + if self._error_name in group.representative.message: + return group.count + + def test_report_exception(self): + """Verifies the exception reported increases the group count by one.""" + # If test has never run, group won't exist until we report first + # exception, so first simulate it just to create the group + self._simulate_exception() + + initial_count = self._get_error_count() + self._simulate_exception() + + is_incremented = functools.partial(_is_incremented, initial_count) + retry_get_count = RetryResult(is_incremented)(self._get_error_count) + new_count = retry_get_count() + + self.assertEqual(new_count, initial_count + 1) From 8d069c22845517b3c5c2240020284e92f732e60b Mon Sep 17 00:00:00 2001 From: Bill Prin Date: Tue, 2 May 2017 13:08:44 -0700 Subject: [PATCH 2/8] DHermes review --- error_reporting/tests/system.py | 70 ++++++++++++++++----------------- 1 file changed, 34 insertions(+), 36 deletions(-) diff --git a/error_reporting/tests/system.py b/error_reporting/tests/system.py index 2312cd8ee01b..bbe167ed10e6 100644 --- a/error_reporting/tests/system.py +++ b/error_reporting/tests/system.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import functools +import time import unittest from google.cloud import error_reporting @@ -22,75 +22,73 @@ error_stats_service_pb2) from google.protobuf.duration_pb2 import Duration -from test_utils.retry import RetryResult +def setUpModule(): + Config.CLIENT = error_reporting.Client() -class _ErrorStatsGaxApi(object): - """Helper mapping Error Reporting-related APIs - This class provides a small wrapper around making calls to the GAX +class Config(object): + """Run-time configuration to be modified at set-up. + + This is a mutable stand-in to allow test set-up to modify + global state. + """ + CLIENT = None + + +def _list_groups(project): + """List Error Groups from the last 60 seconds. + + This class provides a wrapper around making calls to the GAX API. It's used by the system tests to find the appropriate error group to verify the error was successfully reported. :type project: str :param project: Google Cloud Project ID """ - def __init__(self, project): - self._project = project - self._gax_api = error_stats_service_client.ErrorStatsServiceClient() - - def list_groups(self): - """Helper to list the groups that have had errors in the last hour.""" - project_name = self._gax_api.project_path(self._project) - time_range = error_stats_service_pb2.QueryTimeRange() - time_range.period = ( - error_stats_service_pb2.QueryTimeRange.PERIOD_1_HOUR - ) + gax_api = error_stats_service_client.ErrorStatsServiceClient() + project_name = gax_api.project_path(project) - duration = Duration() - duration.seconds = 60 * 60 + time_range = error_stats_service_pb2.QueryTimeRange() + time_range.period = ( + error_stats_service_pb2.QueryTimeRange.PERIOD_1_HOUR + ) - return self._gax_api.list_group_stats( - project_name, time_range, timed_count_duration=duration) + duration = Duration() + duration.seconds = 60 * 60 + return gax_api.list_group_stats( + project_name, time_range, timed_count_duration=duration) -def _is_incremented(initial, new): - """Helper to retry until new error is counted.""" - return new == initial + 1 +ERROR_NAME = 'Stackdriver Error Reporting System Test' class TestErrorReporting(unittest.TestCase): - def setUp(self): - self._client = error_reporting.Client() - self._error_name = 'Stackdriver Error Reporting System Test' - def _simulate_exception(self): """Simulates an exception to verify it was reported.""" try: - raise RuntimeError(self._error_name) + raise RuntimeError(ERROR_NAME) except RuntimeError: - self._client.report_exception() + Config.CLIENT.report_exception() def _get_error_count(self): """Counts the number of errors in the group of the test exception.""" - error_stats_api = _ErrorStatsGaxApi(self._client.project) - groups = error_stats_api.list_groups() + groups = _list_groups(Config.CLIENT.project) for group in groups: - if self._error_name in group.representative.message: + if ERROR_NAME in group.representative.message: return group.count def test_report_exception(self): - """Verifies the exception reported increases the group count by one.""" # If test has never run, group won't exist until we report first # exception, so first simulate it just to create the group self._simulate_exception() + time.sleep(2) initial_count = self._get_error_count() self._simulate_exception() - is_incremented = functools.partial(_is_incremented, initial_count) - retry_get_count = RetryResult(is_incremented)(self._get_error_count) - new_count = retry_get_count() + time.sleep(2) + new_count = self._get_error_count() self.assertEqual(new_count, initial_count + 1) From 9d9eee064dff414518787ff70a910dde2bfdcfbf Mon Sep 17 00:00:00 2001 From: Bill Prin Date: Tue, 2 May 2017 14:19:01 -0700 Subject: [PATCH 3/8] add local deps --- error_reporting/nox.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/error_reporting/nox.py b/error_reporting/nox.py index 2f90922121d1..24e8849892b3 100644 --- a/error_reporting/nox.py +++ b/error_reporting/nox.py @@ -63,6 +63,7 @@ def lint_setup_py(session): session.run( 'python', 'setup.py', 'check', '--restructuredtext', '--strict') + @nox.session @nox.parametrize('python_version', ['2.7', '3.6']) def system_tests(session, python_version): @@ -77,6 +78,7 @@ def system_tests(session, python_version): # Install all test dependencies, then install this package into the # virtualenv's dist-packages. + session.install('flake8', *LOCAL_DEPS) session.install('.') # Run py.test against the system tests. From 2037fb3d0d98b651a6248354269e18f95b06dd2c Mon Sep 17 00:00:00 2001 From: Bill Prin Date: Wed, 3 May 2017 11:29:34 -0700 Subject: [PATCH 4/8] right test deps --- error_reporting/nox.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/error_reporting/nox.py b/error_reporting/nox.py index 24e8849892b3..4623b99c456a 100644 --- a/error_reporting/nox.py +++ b/error_reporting/nox.py @@ -78,7 +78,9 @@ def system_tests(session, python_version): # Install all test dependencies, then install this package into the # virtualenv's dist-packages. - session.install('flake8', *LOCAL_DEPS) + session.install('mock', 'pytest', *LOCAL_DEPS) + session.install('../test_utils/', '../bigquery/', '../pubsub/', + '../storage/') session.install('.') # Run py.test against the system tests. From 35fa46393c71269fd58f3c40ca0f08c5cb429f5a Mon Sep 17 00:00:00 2001 From: Bill Prin Date: Wed, 3 May 2017 11:32:15 -0700 Subject: [PATCH 5/8] fix test deps again --- error_reporting/nox.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/error_reporting/nox.py b/error_reporting/nox.py index 4623b99c456a..1deed376b6e7 100644 --- a/error_reporting/nox.py +++ b/error_reporting/nox.py @@ -79,8 +79,7 @@ def system_tests(session, python_version): # Install all test dependencies, then install this package into the # virtualenv's dist-packages. session.install('mock', 'pytest', *LOCAL_DEPS) - session.install('../test_utils/', '../bigquery/', '../pubsub/', - '../storage/') + session.install('../test_utils/') session.install('.') # Run py.test against the system tests. From ca2c440c5cba76f506a00b839caf934f4d101cff Mon Sep 17 00:00:00 2001 From: Bill Prin Date: Wed, 3 May 2017 16:14:20 -0700 Subject: [PATCH 6/8] dhermies review --- error_reporting/tests/system.py | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/error_reporting/tests/system.py b/error_reporting/tests/system.py index bbe167ed10e6..8d768659f2fb 100644 --- a/error_reporting/tests/system.py +++ b/error_reporting/tests/system.py @@ -23,6 +23,8 @@ from google.protobuf.duration_pb2 import Duration +ERROR_NAME = 'Stackdriver Error Reporting System Test' + def setUpModule(): Config.CLIENT = error_reporting.Client() @@ -46,31 +48,28 @@ def _list_groups(project): :type project: str :param project: Google Cloud Project ID """ - gax_api = error_stats_service_client.ErrorStatsServiceClient() + gax_api = error_stats_service_client.ErrorStatsServiceClient( + credentials=Config.CLIENT._credentials) project_name = gax_api.project_path(project) time_range = error_stats_service_pb2.QueryTimeRange() - time_range.period = ( - error_stats_service_pb2.QueryTimeRange.PERIOD_1_HOUR - ) + time_range.period = error_stats_service_pb2.QueryTimeRange.PERIOD_1_HOUR - duration = Duration() - duration.seconds = 60 * 60 + duration = Duration(seconds=60*60) return gax_api.list_group_stats( project_name, time_range, timed_count_duration=duration) -ERROR_NAME = 'Stackdriver Error Reporting System Test' +def _simulate_exception(): + """Simulates an exception to verify it was reported.""" + try: + raise RuntimeError(ERROR_NAME) + except RuntimeError: + Config.CLIENT.report_exception() -class TestErrorReporting(unittest.TestCase): - def _simulate_exception(self): - """Simulates an exception to verify it was reported.""" - try: - raise RuntimeError(ERROR_NAME) - except RuntimeError: - Config.CLIENT.report_exception() +class TestErrorReporting(unittest.TestCase): def _get_error_count(self): """Counts the number of errors in the group of the test exception.""" From a9c66c6a0be5144aa10ae62a05d2550918fbf3f4 Mon Sep 17 00:00:00 2001 From: Bill Prin Date: Thu, 4 May 2017 10:52:44 -0700 Subject: [PATCH 7/8] fix test --- error_reporting/tests/system.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/error_reporting/tests/system.py b/error_reporting/tests/system.py index 8d768659f2fb..1adadf869465 100644 --- a/error_reporting/tests/system.py +++ b/error_reporting/tests/system.py @@ -81,11 +81,11 @@ def _get_error_count(self): def test_report_exception(self): # If test has never run, group won't exist until we report first # exception, so first simulate it just to create the group - self._simulate_exception() + _simulate_exception() time.sleep(2) initial_count = self._get_error_count() - self._simulate_exception() + _simulate_exception() time.sleep(2) new_count = self._get_error_count() From 5cee9a8fd8c6d167b78027dc7b189be6b1de55ae Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Thu, 4 May 2017 15:30:54 -0700 Subject: [PATCH 8/8] Using a unique class name for each stacktrace. Also adding a retry until the count comes back as expected. --- error_reporting/tests/system.py | 86 ++++++++++++++++++++++----------- 1 file changed, 58 insertions(+), 28 deletions(-) diff --git a/error_reporting/tests/system.py b/error_reporting/tests/system.py index 1adadf869465..3dfafbb6cb07 100644 --- a/error_reporting/tests/system.py +++ b/error_reporting/tests/system.py @@ -12,7 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import time +import functools +import operator import unittest from google.cloud import error_reporting @@ -22,8 +23,12 @@ error_stats_service_pb2) from google.protobuf.duration_pb2 import Duration +from test_utils.retry import RetryResult +from test_utils.system import unique_resource_id + + +ERROR_MSG = 'Stackdriver Error Reporting System Test' -ERROR_NAME = 'Stackdriver Error Reporting System Test' def setUpModule(): Config.CLIENT = error_reporting.Client() @@ -38,56 +43,81 @@ class Config(object): CLIENT = None -def _list_groups(project): +def _list_groups(client): """List Error Groups from the last 60 seconds. This class provides a wrapper around making calls to the GAX API. It's used by the system tests to find the appropriate error group to verify the error was successfully reported. - :type project: str - :param project: Google Cloud Project ID + :type client: :class:`~google.cloud.error_reporting.client.Client` + :param client: The client containing a project and credentials. + + :rtype: :class:`~google.gax.ResourceIterator` + :returns: Iterable of :class:`~.error_stats_service_pb2.ErrorGroupStats`. """ gax_api = error_stats_service_client.ErrorStatsServiceClient( - credentials=Config.CLIENT._credentials) - project_name = gax_api.project_path(project) + credentials=client._credentials) + project_name = gax_api.project_path(client.project) time_range = error_stats_service_pb2.QueryTimeRange() time_range.period = error_stats_service_pb2.QueryTimeRange.PERIOD_1_HOUR - duration = Duration(seconds=60*60) + duration = Duration(seconds=60 * 60) return gax_api.list_group_stats( project_name, time_range, timed_count_duration=duration) -def _simulate_exception(): - """Simulates an exception to verify it was reported.""" +def _simulate_exception(class_name, client): + """Simulates an exception to verify it was reported. + + :type class_name: str + :param class_name: The name of a custom error class to + create (and raise). + + :type client: :class:`~google.cloud.error_reporting.client.Client` + :param client: The client that will report the exception. + """ + custom_exc = type(class_name, (RuntimeError,), {}) try: - raise RuntimeError(ERROR_NAME) + raise custom_exc(ERROR_MSG) except RuntimeError: - Config.CLIENT.report_exception() + client.report_exception() -class TestErrorReporting(unittest.TestCase): +def _get_error_count(class_name, client): + """Counts the number of errors in the group of the test exception. - def _get_error_count(self): - """Counts the number of errors in the group of the test exception.""" - groups = _list_groups(Config.CLIENT.project) - for group in groups: - if ERROR_NAME in group.representative.message: - return group.count + :type class_name: str + :param class_name: The name of a custom error class used. + + :type client: :class:`~google.cloud.error_reporting.client.Client` + :param client: The client containing a project and credentials. + + :rtype: int + :returns: Group count for errors that match ``class_name``. If no + match is found, returns :data:`None`. + """ + groups = _list_groups(client) + for group in groups: + if class_name in group.representative.message: + return group.count + + +class TestErrorReporting(unittest.TestCase): def test_report_exception(self): - # If test has never run, group won't exist until we report first - # exception, so first simulate it just to create the group - _simulate_exception() - time.sleep(2) + # Get a class name unique to this test case. + class_name = 'RuntimeError' + unique_resource_id('_') - initial_count = self._get_error_count() - _simulate_exception() + # Simulate an error: group won't exist until we report + # first exception. + _simulate_exception(class_name, Config.CLIENT) - time.sleep(2) - new_count = self._get_error_count() + is_one = functools.partial(operator.eq, 1) + is_one.__name__ = 'is_one' # partial() has no name. + wrapped_get_count = RetryResult(is_one)(_get_error_count) - self.assertEqual(new_count, initial_count + 1) + error_count = wrapped_get_count(class_name, Config.CLIENT) + self.assertEqual(error_count, 1)