Drop GRPC message size limitation (#632)

Author: Hazhzeng

azure-pipelines.yml

@@ -5,12 +5,12 @@ trigger:
 - master
 
 variables:
-    DOTNET_VERSION: '2.2.x'
+    DOTNET_VERSION: '2.2.207'
 
 jobs:
 - job: Tests
   pool:
-    vmImage: 'ubuntu-16.04'
+    vmImage: 'ubuntu-18.04'
   strategy:
     matrix:
       Python36:
@@ -52,7 +52,7 @@ jobs:
 - job: Build_WINDOWS_X64
   dependsOn: 'Tests'
   pool:
-    vmImage: 'vs2017-win2016'
+    vmImage: 'windows-2019'
   strategy:
     matrix:
       Python36V2:
@@ -80,7 +80,7 @@ jobs:
 - job: Build_WINDOWS_X86
   dependsOn: 'Tests'
   pool:
-    vmImage: 'vs2017-win2016'
+    vmImage: 'windows-2019'
   strategy:
     matrix:
       Python37V2:
@@ -102,7 +102,7 @@ jobs:
 - job: Build_LINUX_X64
   dependsOn: 'Tests'
   pool:
-    vmImage: 'ubuntu-16.04'
+    vmImage: 'ubuntu-18.04'
   strategy:
     matrix:
       Python36V2:
@@ -129,7 +129,7 @@ jobs:
 - job: Build_OSX_X64
   dependsOn: 'Tests'
   pool:
-    vmImage: 'macOS-10.13'
+    vmImage: 'macOS-10.15'
   strategy:
     matrix:
       Python36V2:
@@ -161,7 +161,7 @@ jobs:
               'Build_OSX_X64'
              ]
   pool:
-      vmImage: 'vs2017-win2016'
+      vmImage: 'windows-2019'
   strategy:
     matrix:
       V2PythonWorker:

azure_functions_worker/constants.py

@@ -4,5 +4,8 @@
 RPC_HTTP_BODY_ONLY = "RpcHttpBodyOnly"
 RPC_HTTP_TRIGGER_METADATA_REMOVED = "RpcHttpTriggerMetadataRemoved"
 
+# Debug Flags
+PYAZURE_WEBHOST_DEBUG = "PYAZURE_WEBHOST_DEBUG"
+
 # Feature Flags (app settings)
 PYTHON_ROLLBACK_CWD_PATH = "PYTHON_ROLLBACK_CWD_PATH"

azure_functions_worker/dispatcher.py

@@ -44,7 +44,7 @@ class Dispatcher(metaclass=DispatcherMeta):
     _GRPC_STOP_RESPONSE = object()
 
     def __init__(self, loop, host, port, worker_id, request_id,
-                 grpc_connect_timeout, grpc_max_msg_len):
+                 grpc_connect_timeout, grpc_max_msg_len=-1):
         self._loop = loop
         self._host = host
         self._port = port
@@ -65,6 +65,7 @@ def __init__(self, loop, host, port, worker_id, request_id,
             max_workers=1)
 
         self._grpc_connect_timeout = grpc_connect_timeout
+        # This is set to -1 by default to remove the limitation on msg size
         self._grpc_max_msg_len = grpc_max_msg_len
         self._grpc_resp_queue: queue.Queue = queue.Queue()
         self._grpc_connected_fut = loop.create_future()

azure_functions_worker/main.py

@@ -36,17 +36,16 @@ def main():
 
     try:
         return aio_compat.run(start_async(
-            args.host, args.port, args.worker_id, args.request_id,
-            args.grpc_max_msg_len))
+            args.host, args.port, args.worker_id, args.request_id))
     except Exception:
         error_logger.exception('unhandled error in functions worker')
         raise
 
 
-async def start_async(host, port, worker_id, request_id, grpc_max_msg_len):
+async def start_async(host, port, worker_id, request_id):
     disp = await dispatcher.Dispatcher.connect(
         host, port, worker_id, request_id,
-        connect_timeout=5.0, max_msg_len=grpc_max_msg_len)
+        connect_timeout=5.0)
 
     disp.load_bindings()
 

azure_functions_worker/testutils.py

@@ -33,6 +33,8 @@
 from . import aio_compat
 from . import dispatcher
 from . import protos
+from .utils.common import is_envvar_true
+from .constants import PYAZURE_WEBHOST_DEBUG
 
 
 PROJECT_ROOT = pathlib.Path(__file__).parent.parent
@@ -137,7 +139,10 @@ def __new__(mcls, name, bases, dct):
                 def wrapper(self, *args, __meth__=test_case,
                             __check_log__=check_log_case, **kwargs):
 
-                    if __check_log__ is not None and callable(__check_log__):
+                    if (__check_log__ is not None
+                            and callable(__check_log__)
+                            and not is_envvar_true(PYAZURE_WEBHOST_DEBUG)):
+
                         # Check logging output for unit test scenarios
                         result = self._run_test(__meth__, *args, **kwargs)
 
@@ -177,7 +182,7 @@ def get_script_dir(cls):
     @classmethod
     def setUpClass(cls):
         script_dir = pathlib.Path(cls.get_script_dir())
-        if os.environ.get('PYAZURE_WEBHOST_DEBUG'):
+        if is_envvar_true(PYAZURE_WEBHOST_DEBUG):
             cls.host_stdout = None
         else:
             cls.host_stdout = tempfile.NamedTemporaryFile('w+t')
@@ -660,7 +665,7 @@ def start_webhost(*, script_dir=None, stdout=None):
         script_root = FUNCS_PATH
 
     if stdout is None:
-        if os.environ.get('PYAZURE_WEBHOST_DEBUG'):
+        if is_envvar_true(PYAZURE_WEBHOST_DEBUG):
             stdout = sys.stdout
         else:
             stdout = subprocess.DEVNULL

pack/scripts/win_deps.ps1

@@ -1,5 +1,7 @@
 python -m venv .env
-.env\scripts\activate
+.env\Scripts\Activate.ps1
+python -m pip install --upgrade pip
+
 python -m pip install .
 
 $depsPath = Join-Path -Path $env:BUILD_SOURCESDIRECTORY -ChildPath "deps"

setup.py

@@ -17,9 +17,9 @@
 
 
 # TODO: change this to something more stable when available.
-WEBHOST_URL = ('https://ci.appveyor.com/api/buildjobs/1fqp92o5h2gks7xe'
+WEBHOST_URL = ('https://ci.appveyor.com/api/buildjobs/19gqd7drpxhkedea'
                '/artifacts'
-               '/Functions.Binaries.2.0.12888.no-runtime.zip')
+               '/Functions.Binaries.2.0.13036.no-runtime.zip')
 
 # Extensions necessary for non-core bindings.
 AZURE_EXTENSIONS = """\
@@ -274,8 +274,8 @@ def run(self):
               'azure_functions_worker.bindings',
               'azure_functions_worker.utils'],
     install_requires=[
-        'grpcio==1.26.0',
-        'grpcio-tools==1.26.0',
+        'grpcio~=1.26.0',
+        'grpcio-tools~=1.26.0',
     ],
     extras_require={
         'dev': [