make: add target to make a code coverage report

Patch introduces integration with coverage.py module [1] that allows to
gather code coverage and build a simple report about covered statements
and branches. We have a test-run.py module itself and also we have Python
helpers for Tarantool tests written in Python. These Python tests runs a
separate processes using subprocess module. To make it possible to gather code
coverage for these helpers there are special changes added to
lib/tarantool_server.py file.

Additionally patch introduces integration with Coveralls
service that allows to evaluate incremental code coverage changes like
we do it for Tarantool [2]. Code coverage for test-run is available on
page by URL [3].

Name                              Stmts   Miss Branch BrPart    Cover
---------------------------------------------------------------------
dispatcher.py                       281     57     94     13    76.0%
lib/__init__.py                      45      0      6      1    98.0%
lib/admin_connection.py              71     50     22      0    22.6%
lib/app_server.py                   162    104     38      2    33.0%
lib/box_connection.py                54     33     12      0    31.8%
lib/colorer.py                      111     31     42      7    64.7%
lib/connpool.py                      85     65     18      0    19.4%
lib/inspector.py                     81     49     26      1    34.6%
lib/options.py                       81     15     18      6    78.8%
lib/preprocessor.py                 329    290    162      0     7.9%
lib/pytap13.py                      128    110     58      1    10.2%
lib/server.py                       111     62     40      2    35.1%
lib/server_mixins.py                126     83     28      0    27.9%
lib/tarantool_connection.py         126     89     22      0    25.0%
lib/tarantool_server.py             729    547    268      9    20.8%
lib/test.py                         249    181     86      0    20.3%
lib/test_suite.py                   191     76     68     12    57.5%
lib/unittest_server.py               63     25     10      2    57.5%
lib/utils.py                        216    113     86     12    42.7%
lib/worker.py                       251    179     70      3    25.2%
listeners.py                        205     78     70      6    60.0%
test-run.py                         149     76     44      9    42.5%
test/unittest/test_lib_utils.py      25      1      2      1    92.6%
---------------------------------------------------------------------
TOTAL                              3869   2314   1290     87    35.2%

1. https://coverage.readthedocs.io/en/latest/config.html
2. https://coveralls.io/github/tarantool/tarantool
3. https://coveralls.io/github/tarantool/test-run

Author: ligurio

.coveragerc

@@ -0,0 +1,13 @@
+[run]
+branch = True
+concurrency =
+    gevent
+    multiprocessing
+
+[report]
+precision = 1
+include =
+    ./*
+omit =
+    lib/tarantool-python/*
+    lib/msgpack-python/*

.github/workflows/test.yml

@@ -58,3 +58,13 @@ jobs:
     - name: run regression tests
       run: |
        make test
+    - name: code coverage
+      if: ${{ matrix.python-version == '3.8' && matrix.tarantool-version == '2.7' }}
+      run: |
+       make coverage
+    - name: upload coverage data to coveralls.io
+      if: ${{ matrix.python-version == '3.8' && matrix.tarantool-version == '2.7' }}
+      run: coveralls
+      env:
+        COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_TOKEN }}
+        COVERALLS_PARALLEL: true

Makefile

@@ -1,3 +1,5 @@
+MAKEFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST)))
+PROJECT_DIR := $(patsubst %/,%,$(dir $(MAKEFILE_PATH)))
 TEST_RUN_EXTRA_PARAMS?=
 PYTHON?=python
 
@@ -13,11 +15,19 @@ luacheck:
 	luacheck --config .luacheckrc .
 
 test_integration:
-	$(PYTHON) test/test-run.py --force $(TEST_RUN_EXTRA_PARAMS)
+	PYTHONPATH=$(PROJECT_DIR) $(PYTHON) test/test-run.py --force $(TEST_RUN_EXTRA_PARAMS)
 
 test_unittest:
 	$(PYTHON) -m unittest discover test/unittest/
 
 test: test_unittest test_integration
 
+coverage:
+	PYTHON="coverage run" make -f $(MAKEFILE_PATH) test
+	coverage combine
+	coverage report
+
+clean:
+	coverage erase
+
 .PHONY: lint flake8 luacheck test test_integration test_unittest

README.md

@@ -1,5 +1,7 @@
 # Tarantool Functional testing framework
 
+[![Coverage Status](https://coveralls.io/repos/github/tarantool/test-run/badge.svg)](https://coveralls.io/github/tarantool/test-run)
+
 ### Test Suite
 
 Bunch of tests, that lay down in the subfolder (recursively) with `suite.ini`

requirements-test.txt

@@ -1,2 +1,4 @@
+coverage==5.*
+coveralls==1.*
 flake8==3.7.9
 hypothesis==4.*