From b415088d0639588a0f56a19b65ffdc7f9ab738fa Mon Sep 17 00:00:00 2001 From: Marvin Buss Date: Wed, 27 Dec 2023 12:15:00 +0100 Subject: [PATCH 01/36] Add Opentelemetry Instrumentation for AIOHTTP --- .../fastapp/api/v1/endpoints/sample.py | 9 +++++ code/function/fastapp/main.py | 4 +- code/function/fastapp/utils.py | 38 ++++++++++++++++--- code/function/requirements.txt | 3 +- 4 files changed, 45 insertions(+), 9 deletions(-) diff --git a/code/function/fastapp/api/v1/endpoints/sample.py b/code/function/fastapp/api/v1/endpoints/sample.py index 47439a8..87fa60a 100644 --- a/code/function/fastapp/api/v1/endpoints/sample.py +++ b/code/function/fastapp/api/v1/endpoints/sample.py @@ -1,5 +1,6 @@ from typing import Any +import aiohttp from fastapi import APIRouter from fastapp.models.sample import SampleRequest, SampleResponse from fastapp.utils import setup_logging @@ -14,4 +15,12 @@ async def post_predict( data: SampleRequest, ) -> SampleResponse: logger.info(f"Received request: {data}") + + # Sample request + async with aiohttp.ClientSession() as client: + async with client.get(url="https://www.bing.com/") as response: + resp_status_code = response.status + resp_text = await response.text() + logger.info(f"Received response status code: {resp_status_code}") + return SampleResponse(output=f"Hello {data.input}") diff --git a/code/function/fastapp/main.py b/code/function/fastapp/main.py index 8834cfa..d7997b9 100644 --- a/code/function/fastapp/main.py +++ b/code/function/fastapp/main.py @@ -1,7 +1,7 @@ from fastapi import FastAPI from fastapp.api.v1.api_v1 import api_v1_router from fastapp.core.config import settings -from fastapp.utils import setup_tracer +from fastapp.utils import setup_opentelemetry def get_app() -> FastAPI: @@ -25,7 +25,7 @@ def get_app() -> FastAPI: @app.on_event("startup") async def startup_event(): """Gracefully start the application before the server reports readiness.""" - setup_tracer(app=app) + setup_opentelemetry(app=app) @app.on_event("shutdown") diff --git a/code/function/fastapp/utils.py b/code/function/fastapp/utils.py index dba9aa4..1410919 100644 --- a/code/function/fastapp/utils.py +++ b/code/function/fastapp/utils.py @@ -2,10 +2,16 @@ from logging import Logger # from azure.identity import ManagedIdentityCredential -from azure.monitor.opentelemetry.exporter import AzureMonitorTraceExporter +from azure.monitor.opentelemetry.exporter import ( + AzureMonitorMetricExporter, + AzureMonitorTraceExporter, +) from fastapi import FastAPI from fastapp.core.config import settings +from opentelemetry.instrumentation.aiohttp_client import AioHttpClientInstrumentor from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader from opentelemetry.sdk.resources import SERVICE_NAME, Resource from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import BatchSpanProcessor @@ -29,7 +35,7 @@ def setup_logging(module) -> Logger: return logger -def setup_tracer(app: FastAPI): +def setup_opentelemetry(app: FastAPI): """Setup tracer for Open Telemetry. app (FastAPI): The app to be instrumented by Open Telemetry. @@ -37,10 +43,30 @@ def setup_tracer(app: FastAPI): """ if settings.APPLICATIONINSIGHTS_CONNECTION_STRING: # credential = ManagedIdentityCredential() - exporter = AzureMonitorTraceExporter.from_connection_string( + + # Create tracer provider + tracer_exporter = AzureMonitorTraceExporter.from_connection_string( + settings.APPLICATIONINSIGHTS_CONNECTION_STRING, + # credential=credential + ) + tracer_provider = TracerProvider(resource=Resource({SERVICE_NAME: "api"})) + tracer_provider.add_span_processor(BatchSpanProcessor(tracer_exporter)) + + # Create meter provider + metrics_exporter = AzureMonitorMetricExporter.from_connection_string( settings.APPLICATIONINSIGHTS_CONNECTION_STRING, # credential=credential ) - tracer = TracerProvider(resource=Resource({SERVICE_NAME: "api"})) - tracer.add_span_processor(BatchSpanProcessor(exporter)) - FastAPIInstrumentor.instrument_app(app, tracer_provider=tracer) + reader = PeriodicExportingMetricReader( + metrics_exporter, export_interval_millis=5000 + ) + meter_provider = MeterProvider(metric_readers=[reader]) + + # Create instrumenter + FastAPIInstrumentor.instrument_app( + app, + excluded_urls=f"{settings.API_V1_STR}/health/heartbeat", + tracer_provider=tracer_provider, + meter_provider=meter_provider, + ) + AioHttpClientInstrumentor().instrument(tracer_provider=tracer_provider) diff --git a/code/function/requirements.txt b/code/function/requirements.txt index f4a40cd..77188f4 100644 --- a/code/function/requirements.txt +++ b/code/function/requirements.txt @@ -2,10 +2,11 @@ # The Python Worker is managed by Azure Functions platform # Manually managing azure-functions-worker may cause unexpected issues -# azure-identity~=1.13.0 +# azure-identity~=1.15.0 azure-functions~=1.17.0 fastapi~=0.106.0 pydantic-settings~=2.1.0 aiohttp~=3.9.1 opentelemetry-instrumentation-fastapi==0.43b0 +opentelemetry-instrumentation-aiohttp-client~=0.43b0 azure-monitor-opentelemetry-exporter==1.0.0b19 From b580baf65e6fd7a04869f7e55384268eb9cfaa82 Mon Sep 17 00:00:00 2001 From: Marvin Buss Date: Wed, 27 Dec 2023 12:20:44 +0100 Subject: [PATCH 02/36] Build sample image --- .github/workflows/_containerTemplate.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/_containerTemplate.yml b/.github/workflows/_containerTemplate.yml index b9f4bda..aa38934 100644 --- a/.github/workflows/_containerTemplate.yml +++ b/.github/workflows/_containerTemplate.yml @@ -56,7 +56,7 @@ jobs: - name: Install cosign uses: sigstore/cosign-installer@v3.3.0 id: install_cosign - if: github.event_name != 'pull_request' + if: github.event_name == 'release' with: cosign-release: 'v2.2.0' @@ -74,7 +74,7 @@ jobs: - name: Login Container Registry uses: docker/login-action@v3.0.0 id: registry_login - if: github.event_name != 'pull_request' + # if: github.event_name != 'pull_request' with: registry: ${{ inputs.registry_uri }} username: ${{ secrets.USER_NAME }} @@ -101,7 +101,7 @@ jobs: with: context: ${{ inputs.working_directory }} file: ${{ inputs.working_directory }}/Dockerfile - push: ${{ github.event_name != 'pull_request' }} + push: true # ${{ github.event_name != 'pull_request' }} tags: ${{ steps.metadata.outputs.tags }} labels: ${{ steps.metadata.outputs.labels }} cache-from: type=gha From 32a93b9ed82e21591294d18d71a7db65621691a7 Mon Sep 17 00:00:00 2001 From: Marvin Buss Date: Thu, 28 Dec 2023 10:55:09 +0100 Subject: [PATCH 03/36] Update function logging --- .../fastapp/api/v1/endpoints/sample.py | 24 ++++++-- code/function/fastapp/utils.py | 55 +++++++++++++++++-- code/function/requirements.txt | 1 + code/function/wrapper/__init__.py | 19 ++++++- 4 files changed, 88 insertions(+), 11 deletions(-) diff --git a/code/function/fastapp/api/v1/endpoints/sample.py b/code/function/fastapp/api/v1/endpoints/sample.py index 87fa60a..bec356d 100644 --- a/code/function/fastapp/api/v1/endpoints/sample.py +++ b/code/function/fastapp/api/v1/endpoints/sample.py @@ -1,19 +1,19 @@ from typing import Any import aiohttp -from fastapi import APIRouter +from fastapi import APIRouter, Request from fastapp.models.sample import SampleRequest, SampleResponse -from fastapp.utils import setup_logging +from fastapp.utils import setup_logging, setup_tracer +from opentelemetry import trace logger = setup_logging(__name__) +tracer = setup_tracer(__name__) router = APIRouter() @router.post("/sample", response_model=SampleResponse, name="sample") -async def post_predict( - data: SampleRequest, -) -> SampleResponse: +async def post_predict(data: SampleRequest, request: Request) -> SampleResponse: logger.info(f"Received request: {data}") # Sample request @@ -22,5 +22,19 @@ async def post_predict( resp_status_code = response.status resp_text = await response.text() logger.info(f"Received response status code: {resp_status_code}") + + # tracer_attributes = {"http.client_ip": request.client.host} + # with tracer.start_as_current_span( + # "dependency_span", attributes=tracer_attributes + # ) as span: + # try: + # async with aiohttp.ClientSession() as client: + # async with client.get(url="https://www.bing.com/") as response: + # resp_status_code = response.status + # resp_text = await response.text() + # logger.info(f"Received response status code: {resp_status_code}") + # except Exception as ex: + # span.set_attribute("status", "exception") + # span.record_exception(ex) return SampleResponse(output=f"Hello {data.input}") diff --git a/code/function/fastapp/utils.py b/code/function/fastapp/utils.py index 1410919..6eeb036 100644 --- a/code/function/fastapp/utils.py +++ b/code/function/fastapp/utils.py @@ -5,9 +5,11 @@ from azure.monitor.opentelemetry.exporter import ( AzureMonitorMetricExporter, AzureMonitorTraceExporter, + AzureMonitorLogExporter, ) from fastapi import FastAPI from fastapp.core.config import settings +from opentelemetry import trace from opentelemetry.instrumentation.aiohttp_client import AioHttpClientInstrumentor from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor from opentelemetry.sdk.metrics import MeterProvider @@ -15,7 +17,16 @@ from opentelemetry.sdk.resources import SERVICE_NAME, Resource from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import BatchSpanProcessor - +from opentelemetry.trace import Tracer +from opentelemetry.sdk._logs import ( + LoggerProvider, + LoggingHandler, + set_logger_provider, +) +from opentelemetry.metrics import set_meter_provider +from opentelemetry.trace import get_tracer_provider, set_tracer_provider +from opentelemetry.sdk._logs.export import BatchLogRecordProcessor +from opentelemetry.instrumentation.system_metrics import SystemMetricsInstrumentor def setup_logging(module) -> Logger: """Setup logging and event handler. @@ -26,15 +37,28 @@ def setup_logging(module) -> Logger: logger.setLevel(settings.LOGGING_LEVEL) logger.propagate = False + # Create logging handler + logging_handler = LoggingHandler() + logger.addHandler(logging_handler) + # Create stream handler - logger_stream_handler = logging.StreamHandler() - logger_stream_handler.setFormatter( + stream_handler = logging.StreamHandler() + stream_handler.setFormatter( logging.Formatter("[%(asctime)s] [%(levelname)s] [%(module)-8.8s] %(message)s") ) - logger.addHandler(logger_stream_handler) + logger.addHandler(stream_handler) return logger +def setup_tracer(module) -> Tracer: + """Setup tracer and event handler. + + RETURNS (Tracer): The tracer object to create spans. + """ + tracer = trace.get_tracer(module) + return tracer + + def setup_opentelemetry(app: FastAPI): """Setup tracer for Open Telemetry. @@ -44,13 +68,23 @@ def setup_opentelemetry(app: FastAPI): if settings.APPLICATIONINSIGHTS_CONNECTION_STRING: # credential = ManagedIdentityCredential() + # Create logger provider + logger_exporter = AzureMonitorLogExporter.from_connection_string( + settings.APPLICATIONINSIGHTS_CONNECTION_STRING, + # credential=credential + ) + logger_provider = LoggerProvider() + logger_provider.add_log_record_processor(BatchLogRecordProcessor(logger_exporter)) + set_logger_provider(logger_provider) + # Create tracer provider tracer_exporter = AzureMonitorTraceExporter.from_connection_string( settings.APPLICATIONINSIGHTS_CONNECTION_STRING, # credential=credential ) tracer_provider = TracerProvider(resource=Resource({SERVICE_NAME: "api"})) - tracer_provider.add_span_processor(BatchSpanProcessor(tracer_exporter)) + set_tracer_provider(tracer_provider) + get_tracer_provider().add_span_processor(BatchSpanProcessor(tracer_exporter)) # Create meter provider metrics_exporter = AzureMonitorMetricExporter.from_connection_string( @@ -61,6 +95,16 @@ def setup_opentelemetry(app: FastAPI): metrics_exporter, export_interval_millis=5000 ) meter_provider = MeterProvider(metric_readers=[reader]) + set_meter_provider(meter_provider) + + # Configure custom metrics + system_metrics_config = { + "system.memory.usage": ["used", "free", "cached"], + "system.cpu.time": ["idle", "user", "system", "irq"], + "system.network.io": ["transmit", "receive"], + "process.runtime.memory": ["rss", "vms"], + "process.runtime.cpu.time": ["user", "system"], + } # Create instrumenter FastAPIInstrumentor.instrument_app( @@ -70,3 +114,4 @@ def setup_opentelemetry(app: FastAPI): meter_provider=meter_provider, ) AioHttpClientInstrumentor().instrument(tracer_provider=tracer_provider) + SystemMetricsInstrumentor(config=system_metrics_config).instrument() diff --git a/code/function/requirements.txt b/code/function/requirements.txt index 77188f4..56fda21 100644 --- a/code/function/requirements.txt +++ b/code/function/requirements.txt @@ -9,4 +9,5 @@ pydantic-settings~=2.1.0 aiohttp~=3.9.1 opentelemetry-instrumentation-fastapi==0.43b0 opentelemetry-instrumentation-aiohttp-client~=0.43b0 +opentelemetry-instrumentation-system-metrics~=0.43b0 azure-monitor-opentelemetry-exporter==1.0.0b19 diff --git a/code/function/wrapper/__init__.py b/code/function/wrapper/__init__.py index b30f269..f7ecf04 100644 --- a/code/function/wrapper/__init__.py +++ b/code/function/wrapper/__init__.py @@ -1,6 +1,23 @@ import azure.functions as func from fastapp.main import app +from opentelemetry.context import attach, detach +from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator async def main(req: func.HttpRequest, context: func.Context) -> func.HttpResponse: - return await func.AsgiMiddleware(app).handle_async(req, context) + # Start distributed tracing + functions_current_context = { + "traceparent": context.trace_context.Traceparent, + "tracestate": context.trace_context.Tracestate, + } + parent_context = TraceContextTextMapPropagator().extract( + carrier=functions_current_context + ) + token = attach(parent_context) + + # Function logic + response = await func.AsgiMiddleware(app).handle_async(req, context) + + # End distributed tracing + detach(token) + return response From 576790796153343748adc1a9faf38cdd245fbd4d Mon Sep 17 00:00:00 2001 From: Marvin Buss Date: Thu, 28 Dec 2023 11:00:30 +0100 Subject: [PATCH 04/36] Fix import bug and lint --- .../fastapp/api/v1/endpoints/sample.py | 2 +- code/function/fastapp/utils.py | 25 +++++++++---------- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/code/function/fastapp/api/v1/endpoints/sample.py b/code/function/fastapp/api/v1/endpoints/sample.py index bec356d..68bdbe6 100644 --- a/code/function/fastapp/api/v1/endpoints/sample.py +++ b/code/function/fastapp/api/v1/endpoints/sample.py @@ -22,7 +22,7 @@ async def post_predict(data: SampleRequest, request: Request) -> SampleResponse: resp_status_code = response.status resp_text = await response.text() logger.info(f"Received response status code: {resp_status_code}") - + # tracer_attributes = {"http.client_ip": request.client.host} # with tracer.start_as_current_span( # "dependency_span", attributes=tracer_attributes diff --git a/code/function/fastapp/utils.py b/code/function/fastapp/utils.py index 6eeb036..f4a206f 100644 --- a/code/function/fastapp/utils.py +++ b/code/function/fastapp/utils.py @@ -3,30 +3,27 @@ # from azure.identity import ManagedIdentityCredential from azure.monitor.opentelemetry.exporter import ( + AzureMonitorLogExporter, AzureMonitorMetricExporter, AzureMonitorTraceExporter, - AzureMonitorLogExporter, ) from fastapi import FastAPI from fastapp.core.config import settings from opentelemetry import trace +from opentelemetry._logs import set_logger_provider from opentelemetry.instrumentation.aiohttp_client import AioHttpClientInstrumentor from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor +from opentelemetry.instrumentation.system_metrics import SystemMetricsInstrumentor +from opentelemetry.metrics import set_meter_provider +from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler +from opentelemetry.sdk._logs.export import BatchLogRecordProcessor from opentelemetry.sdk.metrics import MeterProvider from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader from opentelemetry.sdk.resources import SERVICE_NAME, Resource from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import BatchSpanProcessor -from opentelemetry.trace import Tracer -from opentelemetry.sdk._logs import ( - LoggerProvider, - LoggingHandler, - set_logger_provider, -) -from opentelemetry.metrics import set_meter_provider -from opentelemetry.trace import get_tracer_provider, set_tracer_provider -from opentelemetry.sdk._logs.export import BatchLogRecordProcessor -from opentelemetry.instrumentation.system_metrics import SystemMetricsInstrumentor +from opentelemetry.trace import Tracer, get_tracer_provider, set_tracer_provider + def setup_logging(module) -> Logger: """Setup logging and event handler. @@ -74,7 +71,9 @@ def setup_opentelemetry(app: FastAPI): # credential=credential ) logger_provider = LoggerProvider() - logger_provider.add_log_record_processor(BatchLogRecordProcessor(logger_exporter)) + logger_provider.add_log_record_processor( + BatchLogRecordProcessor(logger_exporter) + ) set_logger_provider(logger_provider) # Create tracer provider @@ -96,7 +95,7 @@ def setup_opentelemetry(app: FastAPI): ) meter_provider = MeterProvider(metric_readers=[reader]) set_meter_provider(meter_provider) - + # Configure custom metrics system_metrics_config = { "system.memory.usage": ["used", "free", "cached"], From 34118451fd45fade9ef95122e30828ca00a0a4e1 Mon Sep 17 00:00:00 2001 From: Marvin Buss Date: Thu, 28 Dec 2023 11:19:45 +0100 Subject: [PATCH 05/36] Remove tracer from instrumentation --- code/function/fastapp/utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/code/function/fastapp/utils.py b/code/function/fastapp/utils.py index f4a206f..d8f4d3b 100644 --- a/code/function/fastapp/utils.py +++ b/code/function/fastapp/utils.py @@ -109,8 +109,8 @@ def setup_opentelemetry(app: FastAPI): FastAPIInstrumentor.instrument_app( app, excluded_urls=f"{settings.API_V1_STR}/health/heartbeat", - tracer_provider=tracer_provider, - meter_provider=meter_provider, + # tracer_provider=tracer_provider, + # meter_provider=meter_provider, ) - AioHttpClientInstrumentor().instrument(tracer_provider=tracer_provider) + AioHttpClientInstrumentor().instrument() # tracer_provider=tracer_provider) SystemMetricsInstrumentor(config=system_metrics_config).instrument() From 4551d73b66fe7068abea2392e0ee686438f194ad Mon Sep 17 00:00:00 2001 From: Marvin Buss Date: Thu, 28 Dec 2023 11:29:37 +0100 Subject: [PATCH 06/36] Add tracer provider back --- code/function/fastapp/utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/code/function/fastapp/utils.py b/code/function/fastapp/utils.py index d8f4d3b..f4a206f 100644 --- a/code/function/fastapp/utils.py +++ b/code/function/fastapp/utils.py @@ -109,8 +109,8 @@ def setup_opentelemetry(app: FastAPI): FastAPIInstrumentor.instrument_app( app, excluded_urls=f"{settings.API_V1_STR}/health/heartbeat", - # tracer_provider=tracer_provider, - # meter_provider=meter_provider, + tracer_provider=tracer_provider, + meter_provider=meter_provider, ) - AioHttpClientInstrumentor().instrument() # tracer_provider=tracer_provider) + AioHttpClientInstrumentor().instrument(tracer_provider=tracer_provider) SystemMetricsInstrumentor(config=system_metrics_config).instrument() From 81bed13d3ac124973396b0d3bc62c5454493ad05 Mon Sep 17 00:00:00 2001 From: Marvin Buss Date: Thu, 28 Dec 2023 11:29:54 +0100 Subject: [PATCH 07/36] Add span for AIO http call --- .../fastapp/api/v1/endpoints/sample.py | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/code/function/fastapp/api/v1/endpoints/sample.py b/code/function/fastapp/api/v1/endpoints/sample.py index 68bdbe6..ac1b794 100644 --- a/code/function/fastapp/api/v1/endpoints/sample.py +++ b/code/function/fastapp/api/v1/endpoints/sample.py @@ -17,24 +17,24 @@ async def post_predict(data: SampleRequest, request: Request) -> SampleResponse: logger.info(f"Received request: {data}") # Sample request - async with aiohttp.ClientSession() as client: - async with client.get(url="https://www.bing.com/") as response: - resp_status_code = response.status - resp_text = await response.text() - logger.info(f"Received response status code: {resp_status_code}") - - # tracer_attributes = {"http.client_ip": request.client.host} - # with tracer.start_as_current_span( - # "dependency_span", attributes=tracer_attributes - # ) as span: - # try: - # async with aiohttp.ClientSession() as client: - # async with client.get(url="https://www.bing.com/") as response: - # resp_status_code = response.status - # resp_text = await response.text() - # logger.info(f"Received response status code: {resp_status_code}") - # except Exception as ex: - # span.set_attribute("status", "exception") - # span.record_exception(ex) + # async with aiohttp.ClientSession() as client: + # async with client.get(url="https://www.bing.com/") as response: + # resp_status_code = response.status + # resp_text = await response.text() + # logger.info(f"Received response status code: {resp_status_code}") + + tracer_attributes = {"http.client_ip": request.client.host} + with tracer.start_as_current_span( + "dependency_span", attributes=tracer_attributes + ) as span: + try: + async with aiohttp.ClientSession() as client: + async with client.get(url="https://www.bing.com/") as response: + resp_status_code = response.status + resp_text = await response.text() + logger.info(f"Received response status code: {resp_status_code}") + except Exception as ex: + span.set_attribute("status", "exception") + span.record_exception(ex) return SampleResponse(output=f"Hello {data.input}") From 5c2684571f7ae4f78e208dc1de2fc7e26d0105f7 Mon Sep 17 00:00:00 2001 From: Marvin Buss Date: Thu, 28 Dec 2023 11:58:10 +0100 Subject: [PATCH 08/36] Updae header --- code/function/fastapp/api/v1/endpoints/sample.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/code/function/fastapp/api/v1/endpoints/sample.py b/code/function/fastapp/api/v1/endpoints/sample.py index ac1b794..142d340 100644 --- a/code/function/fastapp/api/v1/endpoints/sample.py +++ b/code/function/fastapp/api/v1/endpoints/sample.py @@ -1,7 +1,7 @@ -from typing import Any +from typing import Annotated import aiohttp -from fastapi import APIRouter, Request +from fastapi import APIRouter, Header from fastapp.models.sample import SampleRequest, SampleResponse from fastapp.utils import setup_logging, setup_tracer from opentelemetry import trace @@ -13,7 +13,7 @@ @router.post("/sample", response_model=SampleResponse, name="sample") -async def post_predict(data: SampleRequest, request: Request) -> SampleResponse: +async def post_predict(data: SampleRequest, x_forwarded_for: Annotated[str, Header()] = "") -> SampleResponse: logger.info(f"Received request: {data}") # Sample request @@ -23,7 +23,7 @@ async def post_predict(data: SampleRequest, request: Request) -> SampleResponse: # resp_text = await response.text() # logger.info(f"Received response status code: {resp_status_code}") - tracer_attributes = {"http.client_ip": request.client.host} + tracer_attributes = {"http.client_ip": x_forwarded_for} with tracer.start_as_current_span( "dependency_span", attributes=tracer_attributes ) as span: From 8800b7b6b62d4e1f1c4df1619f03e36084a0c91f Mon Sep 17 00:00:00 2001 From: Marvin Buss Date: Thu, 28 Dec 2023 12:23:56 +0100 Subject: [PATCH 09/36] Switch to Azure Monitor Library --- .../fastapp/api/v1/endpoints/sample.py | 11 +--- code/function/fastapp/utils.py | 66 +++---------------- code/function/requirements.txt | 3 +- 3 files changed, 13 insertions(+), 67 deletions(-) diff --git a/code/function/fastapp/api/v1/endpoints/sample.py b/code/function/fastapp/api/v1/endpoints/sample.py index 142d340..1c4a6a9 100644 --- a/code/function/fastapp/api/v1/endpoints/sample.py +++ b/code/function/fastapp/api/v1/endpoints/sample.py @@ -4,7 +4,6 @@ from fastapi import APIRouter, Header from fastapp.models.sample import SampleRequest, SampleResponse from fastapp.utils import setup_logging, setup_tracer -from opentelemetry import trace logger = setup_logging(__name__) tracer = setup_tracer(__name__) @@ -13,16 +12,12 @@ @router.post("/sample", response_model=SampleResponse, name="sample") -async def post_predict(data: SampleRequest, x_forwarded_for: Annotated[str, Header()] = "") -> SampleResponse: +async def post_predict( + data: SampleRequest, x_forwarded_for: Annotated[str, Header()] = "" +) -> SampleResponse: logger.info(f"Received request: {data}") # Sample request - # async with aiohttp.ClientSession() as client: - # async with client.get(url="https://www.bing.com/") as response: - # resp_status_code = response.status - # resp_text = await response.text() - # logger.info(f"Received response status code: {resp_status_code}") - tracer_attributes = {"http.client_ip": x_forwarded_for} with tracer.start_as_current_span( "dependency_span", attributes=tracer_attributes diff --git a/code/function/fastapp/utils.py b/code/function/fastapp/utils.py index f4a206f..83d03dc 100644 --- a/code/function/fastapp/utils.py +++ b/code/function/fastapp/utils.py @@ -2,27 +2,13 @@ from logging import Logger # from azure.identity import ManagedIdentityCredential -from azure.monitor.opentelemetry.exporter import ( - AzureMonitorLogExporter, - AzureMonitorMetricExporter, - AzureMonitorTraceExporter, -) from fastapi import FastAPI from fastapp.core.config import settings from opentelemetry import trace -from opentelemetry._logs import set_logger_provider from opentelemetry.instrumentation.aiohttp_client import AioHttpClientInstrumentor -from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor from opentelemetry.instrumentation.system_metrics import SystemMetricsInstrumentor -from opentelemetry.metrics import set_meter_provider -from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler -from opentelemetry.sdk._logs.export import BatchLogRecordProcessor -from opentelemetry.sdk.metrics import MeterProvider -from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader -from opentelemetry.sdk.resources import SERVICE_NAME, Resource -from opentelemetry.sdk.trace import TracerProvider -from opentelemetry.sdk.trace.export import BatchSpanProcessor -from opentelemetry.trace import Tracer, get_tracer_provider, set_tracer_provider +from opentelemetry.trace import Tracer +from azure.monitor.opentelemetry import configure_azure_monitor def setup_logging(module) -> Logger: @@ -34,10 +20,6 @@ def setup_logging(module) -> Logger: logger.setLevel(settings.LOGGING_LEVEL) logger.propagate = False - # Create logging handler - logging_handler = LoggingHandler() - logger.addHandler(logging_handler) - # Create stream handler stream_handler = logging.StreamHandler() stream_handler.setFormatter( @@ -65,37 +47,13 @@ def setup_opentelemetry(app: FastAPI): if settings.APPLICATIONINSIGHTS_CONNECTION_STRING: # credential = ManagedIdentityCredential() - # Create logger provider - logger_exporter = AzureMonitorLogExporter.from_connection_string( - settings.APPLICATIONINSIGHTS_CONNECTION_STRING, - # credential=credential - ) - logger_provider = LoggerProvider() - logger_provider.add_log_record_processor( - BatchLogRecordProcessor(logger_exporter) + # Configure azure monitor exporter + configure_azure_monitor( + connection_string=settings.APPLICATIONINSIGHTS_CONNECTION_STRING, + disable_offline_storage=False, + # credential=credential, ) - set_logger_provider(logger_provider) - - # Create tracer provider - tracer_exporter = AzureMonitorTraceExporter.from_connection_string( - settings.APPLICATIONINSIGHTS_CONNECTION_STRING, - # credential=credential - ) - tracer_provider = TracerProvider(resource=Resource({SERVICE_NAME: "api"})) - set_tracer_provider(tracer_provider) - get_tracer_provider().add_span_processor(BatchSpanProcessor(tracer_exporter)) - - # Create meter provider - metrics_exporter = AzureMonitorMetricExporter.from_connection_string( - settings.APPLICATIONINSIGHTS_CONNECTION_STRING, - # credential=credential - ) - reader = PeriodicExportingMetricReader( - metrics_exporter, export_interval_millis=5000 - ) - meter_provider = MeterProvider(metric_readers=[reader]) - set_meter_provider(meter_provider) - + # Configure custom metrics system_metrics_config = { "system.memory.usage": ["used", "free", "cached"], @@ -106,11 +64,5 @@ def setup_opentelemetry(app: FastAPI): } # Create instrumenter - FastAPIInstrumentor.instrument_app( - app, - excluded_urls=f"{settings.API_V1_STR}/health/heartbeat", - tracer_provider=tracer_provider, - meter_provider=meter_provider, - ) - AioHttpClientInstrumentor().instrument(tracer_provider=tracer_provider) + AioHttpClientInstrumentor().instrument() SystemMetricsInstrumentor(config=system_metrics_config).instrument() diff --git a/code/function/requirements.txt b/code/function/requirements.txt index 56fda21..40f7540 100644 --- a/code/function/requirements.txt +++ b/code/function/requirements.txt @@ -7,7 +7,6 @@ azure-functions~=1.17.0 fastapi~=0.106.0 pydantic-settings~=2.1.0 aiohttp~=3.9.1 -opentelemetry-instrumentation-fastapi==0.43b0 +azure-monitor-opentelemetry~=1.1.1 opentelemetry-instrumentation-aiohttp-client~=0.43b0 opentelemetry-instrumentation-system-metrics~=0.43b0 -azure-monitor-opentelemetry-exporter==1.0.0b19 From 256d36cc2b7116a7d60d77448b4a16267b39bdae Mon Sep 17 00:00:00 2001 From: Marvin Buss Date: Thu, 28 Dec 2023 12:29:05 +0100 Subject: [PATCH 10/36] lint --- code/function/fastapp/utils.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/code/function/fastapp/utils.py b/code/function/fastapp/utils.py index 83d03dc..48969ad 100644 --- a/code/function/fastapp/utils.py +++ b/code/function/fastapp/utils.py @@ -1,6 +1,8 @@ import logging from logging import Logger +from azure.monitor.opentelemetry import configure_azure_monitor + # from azure.identity import ManagedIdentityCredential from fastapi import FastAPI from fastapp.core.config import settings @@ -8,7 +10,6 @@ from opentelemetry.instrumentation.aiohttp_client import AioHttpClientInstrumentor from opentelemetry.instrumentation.system_metrics import SystemMetricsInstrumentor from opentelemetry.trace import Tracer -from azure.monitor.opentelemetry import configure_azure_monitor def setup_logging(module) -> Logger: @@ -53,7 +54,7 @@ def setup_opentelemetry(app: FastAPI): disable_offline_storage=False, # credential=credential, ) - + # Configure custom metrics system_metrics_config = { "system.memory.usage": ["used", "free", "cached"], From 9aaf20355a32c05d4df7c5f7e533647f22ac0ad9 Mon Sep 17 00:00:00 2001 From: Marvin Buss Date: Thu, 28 Dec 2023 12:45:09 +0100 Subject: [PATCH 11/36] Test with httpx --- .../fastapp/api/v1/endpoints/sample.py | 30 ++++++++++--------- code/function/fastapp/utils.py | 4 +-- code/function/requirements.txt | 4 +-- 3 files changed, 20 insertions(+), 18 deletions(-) diff --git a/code/function/fastapp/api/v1/endpoints/sample.py b/code/function/fastapp/api/v1/endpoints/sample.py index 1c4a6a9..544f6be 100644 --- a/code/function/fastapp/api/v1/endpoints/sample.py +++ b/code/function/fastapp/api/v1/endpoints/sample.py @@ -1,6 +1,6 @@ from typing import Annotated -import aiohttp +import httpx from fastapi import APIRouter, Header from fastapp.models.sample import SampleRequest, SampleResponse from fastapp.utils import setup_logging, setup_tracer @@ -18,18 +18,20 @@ async def post_predict( logger.info(f"Received request: {data}") # Sample request - tracer_attributes = {"http.client_ip": x_forwarded_for} - with tracer.start_as_current_span( - "dependency_span", attributes=tracer_attributes - ) as span: - try: - async with aiohttp.ClientSession() as client: - async with client.get(url="https://www.bing.com/") as response: - resp_status_code = response.status - resp_text = await response.text() - logger.info(f"Received response status code: {resp_status_code}") - except Exception as ex: - span.set_attribute("status", "exception") - span.record_exception(ex) + async with httpx.AsyncClient() as client: + response = await client.get("https://www.bing.com/") + # tracer_attributes = {"http.client_ip": x_forwarded_for} + # with tracer.start_as_current_span( + # "dependency_span", attributes=tracer_attributes + # ) as span: + # try: + # async with aiohttp.ClientSession() as client: + # async with client.get(url="https://www.bing.com/") as response: + # resp_status_code = response.status + # resp_text = await response.text() + # logger.info(f"Received response status code: {resp_status_code}") + # except Exception as ex: + # span.set_attribute("status", "exception") + # span.record_exception(ex) return SampleResponse(output=f"Hello {data.input}") diff --git a/code/function/fastapp/utils.py b/code/function/fastapp/utils.py index 48969ad..3ab230c 100644 --- a/code/function/fastapp/utils.py +++ b/code/function/fastapp/utils.py @@ -7,7 +7,7 @@ from fastapi import FastAPI from fastapp.core.config import settings from opentelemetry import trace -from opentelemetry.instrumentation.aiohttp_client import AioHttpClientInstrumentor +from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor from opentelemetry.instrumentation.system_metrics import SystemMetricsInstrumentor from opentelemetry.trace import Tracer @@ -65,5 +65,5 @@ def setup_opentelemetry(app: FastAPI): } # Create instrumenter - AioHttpClientInstrumentor().instrument() + HTTPXClientInstrumentor().instrument() SystemMetricsInstrumentor(config=system_metrics_config).instrument() diff --git a/code/function/requirements.txt b/code/function/requirements.txt index 40f7540..5d01136 100644 --- a/code/function/requirements.txt +++ b/code/function/requirements.txt @@ -6,7 +6,7 @@ azure-functions~=1.17.0 fastapi~=0.106.0 pydantic-settings~=2.1.0 -aiohttp~=3.9.1 +httpx~=0.26.0 azure-monitor-opentelemetry~=1.1.1 -opentelemetry-instrumentation-aiohttp-client~=0.43b0 +opentelemetry-instrumentation-httpx~=0.43b0 opentelemetry-instrumentation-system-metrics~=0.43b0 From a143ed010bc3f624675aad37841f106b8f0385d7 Mon Sep 17 00:00:00 2001 From: Marvin Buss Date: Thu, 28 Dec 2023 17:08:10 +0100 Subject: [PATCH 12/36] Update spankind --- code/function/fastapp/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/code/function/fastapp/utils.py b/code/function/fastapp/utils.py index 3ab230c..a747426 100644 --- a/code/function/fastapp/utils.py +++ b/code/function/fastapp/utils.py @@ -9,7 +9,7 @@ from opentelemetry import trace from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor from opentelemetry.instrumentation.system_metrics import SystemMetricsInstrumentor -from opentelemetry.trace import Tracer +from opentelemetry.trace import Tracer, SpanKind def setup_logging(module) -> Logger: @@ -35,7 +35,7 @@ def setup_tracer(module) -> Tracer: RETURNS (Tracer): The tracer object to create spans. """ - tracer = trace.get_tracer(module) + tracer = trace.get_tracer(module, kind=SpanKind.CLIENT) return tracer From d796d78ee01c25f14db644c3df1abca3428d5c5f Mon Sep 17 00:00:00 2001 From: Marvin Buss Date: Thu, 28 Dec 2023 17:25:00 +0100 Subject: [PATCH 13/36] Use lower httpx version --- code/function/fastapp/api/v1/endpoints/sample.py | 16 ++-------------- code/function/fastapp/utils.py | 4 ++-- code/function/requirements.txt | 2 +- 3 files changed, 5 insertions(+), 17 deletions(-) diff --git a/code/function/fastapp/api/v1/endpoints/sample.py b/code/function/fastapp/api/v1/endpoints/sample.py index 544f6be..c39c1e0 100644 --- a/code/function/fastapp/api/v1/endpoints/sample.py +++ b/code/function/fastapp/api/v1/endpoints/sample.py @@ -15,23 +15,11 @@ async def post_predict( data: SampleRequest, x_forwarded_for: Annotated[str, Header()] = "" ) -> SampleResponse: - logger.info(f"Received request: {data}") + logger.info(f"Received request: '{data}' from ip '{x_forwarded_for}'") # Sample request async with httpx.AsyncClient() as client: response = await client.get("https://www.bing.com/") - # tracer_attributes = {"http.client_ip": x_forwarded_for} - # with tracer.start_as_current_span( - # "dependency_span", attributes=tracer_attributes - # ) as span: - # try: - # async with aiohttp.ClientSession() as client: - # async with client.get(url="https://www.bing.com/") as response: - # resp_status_code = response.status - # resp_text = await response.text() - # logger.info(f"Received response status code: {resp_status_code}") - # except Exception as ex: - # span.set_attribute("status", "exception") - # span.record_exception(ex) + logger.info(f"Received response status code: {response.status_code}") return SampleResponse(output=f"Hello {data.input}") diff --git a/code/function/fastapp/utils.py b/code/function/fastapp/utils.py index a747426..3ab230c 100644 --- a/code/function/fastapp/utils.py +++ b/code/function/fastapp/utils.py @@ -9,7 +9,7 @@ from opentelemetry import trace from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor from opentelemetry.instrumentation.system_metrics import SystemMetricsInstrumentor -from opentelemetry.trace import Tracer, SpanKind +from opentelemetry.trace import Tracer def setup_logging(module) -> Logger: @@ -35,7 +35,7 @@ def setup_tracer(module) -> Tracer: RETURNS (Tracer): The tracer object to create spans. """ - tracer = trace.get_tracer(module, kind=SpanKind.CLIENT) + tracer = trace.get_tracer(module) return tracer diff --git a/code/function/requirements.txt b/code/function/requirements.txt index 5d01136..c6272f9 100644 --- a/code/function/requirements.txt +++ b/code/function/requirements.txt @@ -6,7 +6,7 @@ azure-functions~=1.17.0 fastapi~=0.106.0 pydantic-settings~=2.1.0 -httpx~=0.26.0 +httpx~=0.25.2 azure-monitor-opentelemetry~=1.1.1 opentelemetry-instrumentation-httpx~=0.43b0 opentelemetry-instrumentation-system-metrics~=0.43b0 From 4f7de8ff9a5177f0c3209bac3f9a04da9a8d349f Mon Sep 17 00:00:00 2001 From: Marvin Buss Date: Thu, 28 Dec 2023 17:26:30 +0100 Subject: [PATCH 14/36] Update uri --- code/function/fastapp/api/v1/endpoints/sample.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/function/fastapp/api/v1/endpoints/sample.py b/code/function/fastapp/api/v1/endpoints/sample.py index c39c1e0..de9b253 100644 --- a/code/function/fastapp/api/v1/endpoints/sample.py +++ b/code/function/fastapp/api/v1/endpoints/sample.py @@ -19,7 +19,7 @@ async def post_predict( # Sample request async with httpx.AsyncClient() as client: - response = await client.get("https://www.bing.com/") + response = await client.get("https://www.bing.com") logger.info(f"Received response status code: {response.status_code}") return SampleResponse(output=f"Hello {data.input}") From efbbd80e63aa0a83cf5d36615b5855278c341144 Mon Sep 17 00:00:00 2001 From: Marvin Buss Date: Thu, 28 Dec 2023 18:08:40 +0100 Subject: [PATCH 15/36] Test with span --- .../function/fastapp/api/v1/endpoints/sample.py | 17 +++++++++++++---- code/function/requirements.txt | 2 +- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/code/function/fastapp/api/v1/endpoints/sample.py b/code/function/fastapp/api/v1/endpoints/sample.py index de9b253..f3f915b 100644 --- a/code/function/fastapp/api/v1/endpoints/sample.py +++ b/code/function/fastapp/api/v1/endpoints/sample.py @@ -4,6 +4,7 @@ from fastapi import APIRouter, Header from fastapp.models.sample import SampleRequest, SampleResponse from fastapp.utils import setup_logging, setup_tracer +from opentelemetry.trace import SpanKind logger = setup_logging(__name__) tracer = setup_tracer(__name__) @@ -15,11 +16,19 @@ async def post_predict( data: SampleRequest, x_forwarded_for: Annotated[str, Header()] = "" ) -> SampleResponse: - logger.info(f"Received request: '{data}' from ip '{x_forwarded_for}'") + logger.info(f"Received request: {data}") # Sample request - async with httpx.AsyncClient() as client: - response = await client.get("https://www.bing.com") - logger.info(f"Received response status code: {response.status_code}") + tracer_attributes = {"http.client_ip": x_forwarded_for} + with tracer.start_as_current_span( + "dependency_span", attributes=tracer_attributes, kind=SpanKind.CLIENT + ) as span: + try: + async with httpx.AsyncClient() as client: + response = await client.get("https://www.bing.com") + logger.info(f"Received response status code: {response.status_code}") + except Exception as ex: + span.set_attribute("status", "exception") + span.record_exception(ex) return SampleResponse(output=f"Hello {data.input}") diff --git a/code/function/requirements.txt b/code/function/requirements.txt index c6272f9..5d01136 100644 --- a/code/function/requirements.txt +++ b/code/function/requirements.txt @@ -6,7 +6,7 @@ azure-functions~=1.17.0 fastapi~=0.106.0 pydantic-settings~=2.1.0 -httpx~=0.25.2 +httpx~=0.26.0 azure-monitor-opentelemetry~=1.1.1 opentelemetry-instrumentation-httpx~=0.43b0 opentelemetry-instrumentation-system-metrics~=0.43b0 From cef8c8379fd5d99a9415574fad43d0d67e35217d Mon Sep 17 00:00:00 2001 From: Marvin Buss Date: Mon, 1 Jan 2024 13:00:38 +0100 Subject: [PATCH 16/36] Update config with website name and instance --- code/function/fastapp/core/config.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/code/function/fastapp/core/config.py b/code/function/fastapp/core/config.py index dc87cf7..d4e8d23 100644 --- a/code/function/fastapp/core/config.py +++ b/code/function/fastapp/core/config.py @@ -14,6 +14,8 @@ class Settings(BaseSettings): APPLICATIONINSIGHTS_CONNECTION_STRING: str = Field( default="", env="APPLICATIONINSIGHTS_CONNECTION_STRING" ) + WEBSITE_NAME: str = Field(default="", env="WEBSITE_SITE_NAME") + WEBSITE_INSTANCE_ID: str = Field(default="", env="WEBSITE_INSTANCE_ID") MY_SECRET_CONFIG: str = Field(default="", env="MY_SECRET_CONFIG") From 89bd5b41998088d55843796f81ed56c9206e6a82 Mon Sep 17 00:00:00 2001 From: Marvin Buss Date: Mon, 1 Jan 2024 13:00:49 +0100 Subject: [PATCH 17/36] Add open telemetry tracing --- code/function/wrapper/__init__.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/code/function/wrapper/__init__.py b/code/function/wrapper/__init__.py index f7ecf04..f9f663e 100644 --- a/code/function/wrapper/__init__.py +++ b/code/function/wrapper/__init__.py @@ -1,8 +1,11 @@ import azure.functions as func from fastapp.main import app +from fastapp.utils import setup_tracer from opentelemetry.context import attach, detach from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator +tracer = setup_tracer(__name__) + async def main(req: func.HttpRequest, context: func.Context) -> func.HttpResponse: # Start distributed tracing @@ -16,8 +19,11 @@ async def main(req: func.HttpRequest, context: func.Context) -> func.HttpRespons token = attach(parent_context) # Function logic - response = await func.AsgiMiddleware(app).handle_async(req, context) + try: + with tracer.start_as_current_span('wrapper') as span: + response = await func.AsgiMiddleware(app).handle_async(req, context) + finally: + # End distributed tracing + detach(token) - # End distributed tracing - detach(token) return response From e660cbdee8891b9ebf079b51e6aa469ba9e55d32 Mon Sep 17 00:00:00 2001 From: Marvin Buss Date: Mon, 1 Jan 2024 13:09:27 +0100 Subject: [PATCH 18/36] Lint --- code/function/wrapper/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/function/wrapper/__init__.py b/code/function/wrapper/__init__.py index f9f663e..2f953ee 100644 --- a/code/function/wrapper/__init__.py +++ b/code/function/wrapper/__init__.py @@ -20,7 +20,7 @@ async def main(req: func.HttpRequest, context: func.Context) -> func.HttpRespons # Function logic try: - with tracer.start_as_current_span('wrapper') as span: + with tracer.start_as_current_span("wrapper") as span: response = await func.AsgiMiddleware(app).handle_async(req, context) finally: # End distributed tracing From 610e93b32c2a4e42bc42bb5f3e07eefb66b7cbe6 Mon Sep 17 00:00:00 2001 From: Marvin Buss Date: Mon, 1 Jan 2024 13:16:03 +0100 Subject: [PATCH 19/36] Remove tracer --- .../fastapp/api/v1/endpoints/sample.py | 26 +++++++++++-------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/code/function/fastapp/api/v1/endpoints/sample.py b/code/function/fastapp/api/v1/endpoints/sample.py index f3f915b..ddb7b12 100644 --- a/code/function/fastapp/api/v1/endpoints/sample.py +++ b/code/function/fastapp/api/v1/endpoints/sample.py @@ -19,16 +19,20 @@ async def post_predict( logger.info(f"Received request: {data}") # Sample request - tracer_attributes = {"http.client_ip": x_forwarded_for} - with tracer.start_as_current_span( - "dependency_span", attributes=tracer_attributes, kind=SpanKind.CLIENT - ) as span: - try: - async with httpx.AsyncClient() as client: - response = await client.get("https://www.bing.com") - logger.info(f"Received response status code: {response.status_code}") - except Exception as ex: - span.set_attribute("status", "exception") - span.record_exception(ex) + async with httpx.AsyncClient() as client: + response = await client.get("https://www.bing.com") + logger.info(f"Received response status code: {response.status_code}") + + # tracer_attributes = {"http.client_ip": x_forwarded_for} + # with tracer.start_as_current_span( + # "dependency_span", attributes=tracer_attributes, kind=SpanKind.CLIENT + # ) as span: + # try: + # async with httpx.AsyncClient() as client: + # response = await client.get("https://www.bing.com") + # logger.info(f"Received response status code: {response.status_code}") + # except Exception as ex: + # span.set_attribute("status", "exception") + # span.record_exception(ex) return SampleResponse(output=f"Hello {data.input}") From a4ba297ce8dcbd92d4debb6f788e97b89caeba7e Mon Sep 17 00:00:00 2001 From: Marvin Buss Date: Mon, 1 Jan 2024 14:22:28 +0100 Subject: [PATCH 20/36] Switch to manual Exporter --- code/function/fastapp/core/config.py | 4 +- code/function/fastapp/utils.py | 71 +++++++++++++++++++++++++--- 2 files changed, 66 insertions(+), 9 deletions(-) diff --git a/code/function/fastapp/core/config.py b/code/function/fastapp/core/config.py index d4e8d23..c95718e 100644 --- a/code/function/fastapp/core/config.py +++ b/code/function/fastapp/core/config.py @@ -14,8 +14,8 @@ class Settings(BaseSettings): APPLICATIONINSIGHTS_CONNECTION_STRING: str = Field( default="", env="APPLICATIONINSIGHTS_CONNECTION_STRING" ) - WEBSITE_NAME: str = Field(default="", env="WEBSITE_SITE_NAME") - WEBSITE_INSTANCE_ID: str = Field(default="", env="WEBSITE_INSTANCE_ID") + WEBSITE_NAME: str = Field(default="test", env="WEBSITE_SITE_NAME") + WEBSITE_INSTANCE_ID: str = Field(default="0", env="WEBSITE_INSTANCE_ID") MY_SECRET_CONFIG: str = Field(default="", env="MY_SECRET_CONFIG") diff --git a/code/function/fastapp/utils.py b/code/function/fastapp/utils.py index 3ab230c..b546847 100644 --- a/code/function/fastapp/utils.py +++ b/code/function/fastapp/utils.py @@ -4,12 +4,28 @@ from azure.monitor.opentelemetry import configure_azure_monitor # from azure.identity import ManagedIdentityCredential +from azure.monitor.opentelemetry.exporter import ( + ApplicationInsightsSampler, + AzureMonitorLogExporter, + AzureMonitorMetricExporter, + AzureMonitorTraceExporter, +) from fastapi import FastAPI from fastapp.core.config import settings from opentelemetry import trace -from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor +from opentelemetry._logs import set_logger_provider +from opentelemetry.instrumentation.aiohttp_client import AioHttpClientInstrumentor +from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor from opentelemetry.instrumentation.system_metrics import SystemMetricsInstrumentor -from opentelemetry.trace import Tracer +from opentelemetry.metrics import set_meter_provider +from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler +from opentelemetry.sdk._logs.export import BatchLogRecordProcessor +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader +from opentelemetry.sdk.resources import SERVICE_NAME, Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry.trace import Tracer, get_tracer_provider, set_tracer_provider def setup_logging(module) -> Logger: @@ -47,13 +63,48 @@ def setup_opentelemetry(app: FastAPI): """ if settings.APPLICATIONINSIGHTS_CONNECTION_STRING: # credential = ManagedIdentityCredential() + resource = Resource.create( + { + "service.name": settings.WEBSITE_NAME, + "service.instance.id": settings.WEBSITE_INSTANCE_ID, + } + ) + + # Create logger provider + logger_exporter = AzureMonitorLogExporter.from_connection_string( + settings.APPLICATIONINSIGHTS_CONNECTION_STRING, + # credential=credential + ) + logger_provider = LoggerProvider(resource=resource) + logger_provider.add_log_record_processor( + BatchLogRecordProcessor(logger_exporter) + ) + set_logger_provider(logger_provider) + handler = LoggingHandler( + level=settings.LOGGING_LEVEL, logger_provider=logger_provider + ) + logging.getLogger().addHandler(handler) - # Configure azure monitor exporter - configure_azure_monitor( - connection_string=settings.APPLICATIONINSIGHTS_CONNECTION_STRING, - disable_offline_storage=False, - # credential=credential, + # Create tracer provider + tracer_exporter = AzureMonitorTraceExporter.from_connection_string( + settings.APPLICATIONINSIGHTS_CONNECTION_STRING, + # credential=credential ) + sampler = ApplicationInsightsSampler(1.0) + tracer_provider = TracerProvider(resource=resource, sampler=sampler) + tracer_provider.add_span_processor(BatchSpanProcessor(tracer_exporter)) + set_tracer_provider(tracer_provider) + + # Create meter provider + metrics_exporter = AzureMonitorMetricExporter.from_connection_string( + settings.APPLICATIONINSIGHTS_CONNECTION_STRING, + # credential=credential + ) + reader = PeriodicExportingMetricReader( + metrics_exporter, export_interval_millis=5000 + ) + meter_provider = MeterProvider(metric_readers=[reader]) + set_meter_provider(meter_provider) # Configure custom metrics system_metrics_config = { @@ -65,5 +116,11 @@ def setup_opentelemetry(app: FastAPI): } # Create instrumenter + FastAPIInstrumentor.instrument_app( + app, + excluded_urls=f"{settings.API_V1_STR}/health/heartbeat", + tracer_provider=tracer_provider, + meter_provider=meter_provider, + ) HTTPXClientInstrumentor().instrument() SystemMetricsInstrumentor(config=system_metrics_config).instrument() From abbfb4f8c20e1d3bb96a9dbed182d2f1a2002d6a Mon Sep 17 00:00:00 2001 From: Marvin Buss Date: Mon, 1 Jan 2024 16:56:08 +0100 Subject: [PATCH 21/36] Fix imports --- code/function/fastapp/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/function/fastapp/utils.py b/code/function/fastapp/utils.py index b546847..f5c6514 100644 --- a/code/function/fastapp/utils.py +++ b/code/function/fastapp/utils.py @@ -14,8 +14,8 @@ from fastapp.core.config import settings from opentelemetry import trace from opentelemetry._logs import set_logger_provider -from opentelemetry.instrumentation.aiohttp_client import AioHttpClientInstrumentor from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor +from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor from opentelemetry.instrumentation.system_metrics import SystemMetricsInstrumentor from opentelemetry.metrics import set_meter_provider from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler From afe6346244cf5ec117f79a15cfad5a0526d814f3 Mon Sep 17 00:00:00 2001 From: Marvin Buss Date: Mon, 1 Jan 2024 17:26:01 +0100 Subject: [PATCH 22/36] Update context --- code/function/wrapper/__init__.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/code/function/wrapper/__init__.py b/code/function/wrapper/__init__.py index 2f953ee..78341b3 100644 --- a/code/function/wrapper/__init__.py +++ b/code/function/wrapper/__init__.py @@ -16,14 +16,17 @@ async def main(req: func.HttpRequest, context: func.Context) -> func.HttpRespons parent_context = TraceContextTextMapPropagator().extract( carrier=functions_current_context ) - token = attach(parent_context) # Function logic - try: - with tracer.start_as_current_span("wrapper") as span: - response = await func.AsgiMiddleware(app).handle_async(req, context) - finally: - # End distributed tracing - detach(token) + with tracer.start_as_current_span("wrapper", context=parent_context) as span: + response = await func.AsgiMiddleware(app).handle_async(req, parent_context) + + # token = attach(parent_context) + # try: + # with tracer.start_as_current_span("wrapper") as span: + # response = await func.AsgiMiddleware(app).handle_async(req, parent_context) + # finally: + # # End distributed tracing + # detach(token) return response From b9c43acd7396e9eeca3b183a76b6085caf7a53ae Mon Sep 17 00:00:00 2001 From: Marvin Buss Date: Mon, 1 Jan 2024 17:48:56 +0100 Subject: [PATCH 23/36] Add more requests --- code/function/fastapp/api/v1/endpoints/sample.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/code/function/fastapp/api/v1/endpoints/sample.py b/code/function/fastapp/api/v1/endpoints/sample.py index ddb7b12..183f1dd 100644 --- a/code/function/fastapp/api/v1/endpoints/sample.py +++ b/code/function/fastapp/api/v1/endpoints/sample.py @@ -23,6 +23,10 @@ async def post_predict( response = await client.get("https://www.bing.com") logger.info(f"Received response status code: {response.status_code}") + with httpx.Client() as client: + response = client.get("https://www.google.com") + response = httpx.get("https://www.google.de") + # tracer_attributes = {"http.client_ip": x_forwarded_for} # with tracer.start_as_current_span( # "dependency_span", attributes=tracer_attributes, kind=SpanKind.CLIENT From c6c00fb351e860fe6c4b3bf6f6defe2fc8a28411 Mon Sep 17 00:00:00 2001 From: Marvin Buss Date: Mon, 1 Jan 2024 18:22:44 +0100 Subject: [PATCH 24/36] Test update --- code/function/fastapp/utils.py | 14 +++++++------- code/function/wrapper/__init__.py | 4 +++- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/code/function/fastapp/utils.py b/code/function/fastapp/utils.py index f5c6514..78b17fc 100644 --- a/code/function/fastapp/utils.py +++ b/code/function/fastapp/utils.py @@ -103,7 +103,7 @@ def setup_opentelemetry(app: FastAPI): reader = PeriodicExportingMetricReader( metrics_exporter, export_interval_millis=5000 ) - meter_provider = MeterProvider(metric_readers=[reader]) + meter_provider = MeterProvider(metric_readers=[reader], resource=resource) set_meter_provider(meter_provider) # Configure custom metrics @@ -116,11 +116,11 @@ def setup_opentelemetry(app: FastAPI): } # Create instrumenter - FastAPIInstrumentor.instrument_app( - app, - excluded_urls=f"{settings.API_V1_STR}/health/heartbeat", - tracer_provider=tracer_provider, - meter_provider=meter_provider, - ) + # FastAPIInstrumentor.instrument_app( + # app, + # excluded_urls=f"{settings.API_V1_STR}/health/heartbeat", + # tracer_provider=tracer_provider, + # meter_provider=meter_provider, + # ) HTTPXClientInstrumentor().instrument() SystemMetricsInstrumentor(config=system_metrics_config).instrument() diff --git a/code/function/wrapper/__init__.py b/code/function/wrapper/__init__.py index 78341b3..d909f8b 100644 --- a/code/function/wrapper/__init__.py +++ b/code/function/wrapper/__init__.py @@ -19,7 +19,9 @@ async def main(req: func.HttpRequest, context: func.Context) -> func.HttpRespons # Function logic with tracer.start_as_current_span("wrapper", context=parent_context) as span: - response = await func.AsgiMiddleware(app).handle_async(req, parent_context) + response = await func.AsgiMiddleware(app).handle_async( + req=req, # context=parent_context + ) # token = attach(parent_context) # try: From 672d2df627a1dad280eca0f9076d42d917077002 Mon Sep 17 00:00:00 2001 From: Marvin Buss Date: Mon, 1 Jan 2024 18:35:19 +0100 Subject: [PATCH 25/36] Test calling setup opentelemetry manually --- code/function/wrapper/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/code/function/wrapper/__init__.py b/code/function/wrapper/__init__.py index d909f8b..cfbe119 100644 --- a/code/function/wrapper/__init__.py +++ b/code/function/wrapper/__init__.py @@ -1,9 +1,10 @@ import azure.functions as func from fastapp.main import app -from fastapp.utils import setup_tracer +from fastapp.utils import setup_opentelemetry, setup_tracer from opentelemetry.context import attach, detach from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator +setup_opentelemetry(app=app) tracer = setup_tracer(__name__) From 20d925ff2a5c46760e6741021f11f194ecd7a93e Mon Sep 17 00:00:00 2001 From: Marvin Buss Date: Mon, 1 Jan 2024 18:48:23 +0100 Subject: [PATCH 26/36] Update startup activities --- .../fastapp/api/v1/endpoints/sample.py | 6 +++--- code/function/fastapp/main.py | 20 +++++++++---------- code/function/wrapper/__init__.py | 2 +- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/code/function/fastapp/api/v1/endpoints/sample.py b/code/function/fastapp/api/v1/endpoints/sample.py index 183f1dd..9277b26 100644 --- a/code/function/fastapp/api/v1/endpoints/sample.py +++ b/code/function/fastapp/api/v1/endpoints/sample.py @@ -23,9 +23,9 @@ async def post_predict( response = await client.get("https://www.bing.com") logger.info(f"Received response status code: {response.status_code}") - with httpx.Client() as client: - response = client.get("https://www.google.com") - response = httpx.get("https://www.google.de") + # with httpx.Client() as client: + # response = client.get("https://www.google.com") + # response = httpx.get("https://www.google.de") # tracer_attributes = {"http.client_ip": x_forwarded_for} # with tracer.start_as_current_span( diff --git a/code/function/fastapp/main.py b/code/function/fastapp/main.py index d7997b9..1415655 100644 --- a/code/function/fastapp/main.py +++ b/code/function/fastapp/main.py @@ -1,34 +1,34 @@ +from contextlib import asynccontextmanager + from fastapi import FastAPI from fastapp.api.v1.api_v1 import api_v1_router from fastapp.core.config import settings from fastapp.utils import setup_opentelemetry -def get_app() -> FastAPI: +def get_app(lifespan) -> FastAPI: """Setup the Fast API server. RETURNS (FastAPI): The FastAPI object to start the server. """ app = FastAPI( title=settings.PROJECT_NAME, + description="", version=settings.APP_VERSION, openapi_url="/openapi.json", debug=settings.DEBUG, + lifespan=lifespan, ) app.include_router(api_v1_router, prefix=settings.API_V1_STR) return app -app = get_app() - - -@app.on_event("startup") -async def startup_event(): +@asynccontextmanager +async def lifespan(app: FastAPI) -> None: """Gracefully start the application before the server reports readiness.""" setup_opentelemetry(app=app) + yield + pass -@app.on_event("shutdown") -async def shutdown_event(): - """Gracefully close connections before shutdown of the server.""" - pass +app = get_app(lifespan=lifespan) diff --git a/code/function/wrapper/__init__.py b/code/function/wrapper/__init__.py index cfbe119..007ab9e 100644 --- a/code/function/wrapper/__init__.py +++ b/code/function/wrapper/__init__.py @@ -4,7 +4,7 @@ from opentelemetry.context import attach, detach from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator -setup_opentelemetry(app=app) +# setup_opentelemetry(app=app) tracer = setup_tracer(__name__) From dbb07bfdf8b76c2a59d49dd67beffe1a6711d574 Mon Sep 17 00:00:00 2001 From: Marvin Buss Date: Mon, 1 Jan 2024 19:03:18 +0100 Subject: [PATCH 27/36] Update lifespan setup --- code/function/fastapp/main.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/code/function/fastapp/main.py b/code/function/fastapp/main.py index 1415655..5a207a1 100644 --- a/code/function/fastapp/main.py +++ b/code/function/fastapp/main.py @@ -6,7 +6,15 @@ from fastapp.utils import setup_opentelemetry -def get_app(lifespan) -> FastAPI: +@asynccontextmanager +async def lifespan(app: FastAPI) -> None: + """Gracefully start the application before the server reports readiness.""" + setup_opentelemetry(app=app) + yield + pass + + +def get_app() -> FastAPI: """Setup the Fast API server. RETURNS (FastAPI): The FastAPI object to start the server. @@ -23,12 +31,4 @@ def get_app(lifespan) -> FastAPI: return app -@asynccontextmanager -async def lifespan(app: FastAPI) -> None: - """Gracefully start the application before the server reports readiness.""" - setup_opentelemetry(app=app) - yield - pass - - -app = get_app(lifespan=lifespan) +app = get_app() From d26f0c47f5c278fe46855d9743819a1f8a57b232 Mon Sep 17 00:00:00 2001 From: Marvin Buss Date: Mon, 1 Jan 2024 19:03:36 +0100 Subject: [PATCH 28/36] Enable fastapi instrumentation --- code/function/fastapp/utils.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/code/function/fastapp/utils.py b/code/function/fastapp/utils.py index 78b17fc..8143235 100644 --- a/code/function/fastapp/utils.py +++ b/code/function/fastapp/utils.py @@ -116,11 +116,11 @@ def setup_opentelemetry(app: FastAPI): } # Create instrumenter - # FastAPIInstrumentor.instrument_app( - # app, - # excluded_urls=f"{settings.API_V1_STR}/health/heartbeat", - # tracer_provider=tracer_provider, - # meter_provider=meter_provider, - # ) + FastAPIInstrumentor.instrument_app( + app, + excluded_urls=f"{settings.API_V1_STR}/health/heartbeat", + tracer_provider=tracer_provider, + meter_provider=meter_provider, + ) HTTPXClientInstrumentor().instrument() SystemMetricsInstrumentor(config=system_metrics_config).instrument() From 6d4661680012547f658af2512363c7b32ca286e2 Mon Sep 17 00:00:00 2001 From: Marvin Buss Date: Mon, 1 Jan 2024 19:03:51 +0100 Subject: [PATCH 29/36] setup opentelemetry in Function app sync --- code/function/wrapper/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/function/wrapper/__init__.py b/code/function/wrapper/__init__.py index 007ab9e..cfbe119 100644 --- a/code/function/wrapper/__init__.py +++ b/code/function/wrapper/__init__.py @@ -4,7 +4,7 @@ from opentelemetry.context import attach, detach from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator -# setup_opentelemetry(app=app) +setup_opentelemetry(app=app) tracer = setup_tracer(__name__) From 88defcda3fc1ca8ca1794c6d5d1fd1dad471b734 Mon Sep 17 00:00:00 2001 From: Marvin Buss Date: Mon, 1 Jan 2024 19:12:17 +0100 Subject: [PATCH 30/36] Test again without manual setup --- code/function/wrapper/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/function/wrapper/__init__.py b/code/function/wrapper/__init__.py index cfbe119..007ab9e 100644 --- a/code/function/wrapper/__init__.py +++ b/code/function/wrapper/__init__.py @@ -4,7 +4,7 @@ from opentelemetry.context import attach, detach from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator -setup_opentelemetry(app=app) +# setup_opentelemetry(app=app) tracer = setup_tracer(__name__) From b1ee566f56c7cc4378b912ae524b5e8b849d1d25 Mon Sep 17 00:00:00 2001 From: Marvin Buss Date: Mon, 1 Jan 2024 19:36:03 +0100 Subject: [PATCH 31/36] Cleanup and add lifespan sync --- .../fastapp/api/v1/endpoints/sample.py | 21 ++----------------- code/function/fastapp/main.py | 5 +++++ code/function/wrapper/__init__.py | 15 +++---------- 3 files changed, 10 insertions(+), 31 deletions(-) diff --git a/code/function/fastapp/api/v1/endpoints/sample.py b/code/function/fastapp/api/v1/endpoints/sample.py index 9277b26..4d2d06b 100644 --- a/code/function/fastapp/api/v1/endpoints/sample.py +++ b/code/function/fastapp/api/v1/endpoints/sample.py @@ -3,11 +3,9 @@ import httpx from fastapi import APIRouter, Header from fastapp.models.sample import SampleRequest, SampleResponse -from fastapp.utils import setup_logging, setup_tracer -from opentelemetry.trace import SpanKind +from fastapp.utils import setup_logging logger = setup_logging(__name__) -tracer = setup_tracer(__name__) router = APIRouter() @@ -17,26 +15,11 @@ async def post_predict( data: SampleRequest, x_forwarded_for: Annotated[str, Header()] = "" ) -> SampleResponse: logger.info(f"Received request: {data}") + logger.info(f"IP of sender: {x_forwarded_for}") # Sample request async with httpx.AsyncClient() as client: response = await client.get("https://www.bing.com") logger.info(f"Received response status code: {response.status_code}") - # with httpx.Client() as client: - # response = client.get("https://www.google.com") - # response = httpx.get("https://www.google.de") - - # tracer_attributes = {"http.client_ip": x_forwarded_for} - # with tracer.start_as_current_span( - # "dependency_span", attributes=tracer_attributes, kind=SpanKind.CLIENT - # ) as span: - # try: - # async with httpx.AsyncClient() as client: - # response = await client.get("https://www.bing.com") - # logger.info(f"Received response status code: {response.status_code}") - # except Exception as ex: - # span.set_attribute("status", "exception") - # span.record_exception(ex) - return SampleResponse(output=f"Hello {data.input}") diff --git a/code/function/fastapp/main.py b/code/function/fastapp/main.py index 5a207a1..7ee394f 100644 --- a/code/function/fastapp/main.py +++ b/code/function/fastapp/main.py @@ -14,6 +14,11 @@ async def lifespan(app: FastAPI) -> None: pass +def lifespan_sync(app: FastAPI) -> None: + """Gracefully start the application before the server reports readiness.""" + setup_opentelemetry(app=app) + + def get_app() -> FastAPI: """Setup the Fast API server. diff --git a/code/function/wrapper/__init__.py b/code/function/wrapper/__init__.py index 007ab9e..05b4d38 100644 --- a/code/function/wrapper/__init__.py +++ b/code/function/wrapper/__init__.py @@ -1,10 +1,9 @@ import azure.functions as func -from fastapp.main import app -from fastapp.utils import setup_opentelemetry, setup_tracer -from opentelemetry.context import attach, detach +from fastapp.main import app, lifespan_sync +from fastapp.utils import setup_tracer from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator -# setup_opentelemetry(app=app) +lifespan_sync(app=app) tracer = setup_tracer(__name__) @@ -24,12 +23,4 @@ async def main(req: func.HttpRequest, context: func.Context) -> func.HttpRespons req=req, # context=parent_context ) - # token = attach(parent_context) - # try: - # with tracer.start_as_current_span("wrapper") as span: - # response = await func.AsgiMiddleware(app).handle_async(req, parent_context) - # finally: - # # End distributed tracing - # detach(token) - return response From cd65de30ef11a69ab837ad1c5eedb2715322a9b7 Mon Sep 17 00:00:00 2001 From: Marvin Buss Date: Mon, 1 Jan 2024 19:48:27 +0100 Subject: [PATCH 32/36] Update app settings --- code/function/fastapp/core/config.py | 2 ++ code/function/fastapp/utils.py | 20 ++++++++++++++++---- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/code/function/fastapp/core/config.py b/code/function/fastapp/core/config.py index c95718e..444393b 100644 --- a/code/function/fastapp/core/config.py +++ b/code/function/fastapp/core/config.py @@ -10,6 +10,8 @@ class Settings(BaseSettings): APP_VERSION: str = "v0.0.1" API_V1_STR: str = "/v1" LOGGING_LEVEL: int = logging.INFO + LOGGING_SAMPLING_RATIO: float = 1.0 + LOGGING_SCHEDULE_DELAY: int = 5000 DEBUG: bool = False APPLICATIONINSIGHTS_CONNECTION_STRING: str = Field( default="", env="APPLICATIONINSIGHTS_CONNECTION_STRING" diff --git a/code/function/fastapp/utils.py b/code/function/fastapp/utils.py index 8143235..9a3d10c 100644 --- a/code/function/fastapp/utils.py +++ b/code/function/fastapp/utils.py @@ -66,6 +66,7 @@ def setup_opentelemetry(app: FastAPI): resource = Resource.create( { "service.name": settings.WEBSITE_NAME, + "service.namespace": settings.WEBSITE_NAME, "service.instance.id": settings.WEBSITE_INSTANCE_ID, } ) @@ -77,7 +78,10 @@ def setup_opentelemetry(app: FastAPI): ) logger_provider = LoggerProvider(resource=resource) logger_provider.add_log_record_processor( - BatchLogRecordProcessor(logger_exporter) + BatchLogRecordProcessor( + exporter=logger_exporter, + schedule_delay_millis=settings.LOGGING_SCHEDULE_DELAY, + ) ) set_logger_provider(logger_provider) handler = LoggingHandler( @@ -90,9 +94,16 @@ def setup_opentelemetry(app: FastAPI): settings.APPLICATIONINSIGHTS_CONNECTION_STRING, # credential=credential ) - sampler = ApplicationInsightsSampler(1.0) + sampler = ApplicationInsightsSampler( + sampling_ratio=settings.LOGGING_SAMPLING_RATIO + ) tracer_provider = TracerProvider(resource=resource, sampler=sampler) - tracer_provider.add_span_processor(BatchSpanProcessor(tracer_exporter)) + tracer_provider.add_span_processor( + BatchSpanProcessor( + span_exporter=tracer_exporter, + schedule_delay_millis=settings.LOGGING_SCHEDULE_DELAY, + ) + ) set_tracer_provider(tracer_provider) # Create meter provider @@ -101,7 +112,8 @@ def setup_opentelemetry(app: FastAPI): # credential=credential ) reader = PeriodicExportingMetricReader( - metrics_exporter, export_interval_millis=5000 + exporter=metrics_exporter, + export_interval_millis=settings.LOGGING_SCHEDULE_DELAY, ) meter_provider = MeterProvider(metric_readers=[reader], resource=resource) set_meter_provider(meter_provider) From f7608ef7c31a42173b6ee51fc2c7e7f528c6b32e Mon Sep 17 00:00:00 2001 From: Marvin Buss Date: Mon, 1 Jan 2024 19:52:15 +0100 Subject: [PATCH 33/36] Add context back into fastapi call --- code/function/wrapper/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/function/wrapper/__init__.py b/code/function/wrapper/__init__.py index 05b4d38..4748e4c 100644 --- a/code/function/wrapper/__init__.py +++ b/code/function/wrapper/__init__.py @@ -20,7 +20,7 @@ async def main(req: func.HttpRequest, context: func.Context) -> func.HttpRespons # Function logic with tracer.start_as_current_span("wrapper", context=parent_context) as span: response = await func.AsgiMiddleware(app).handle_async( - req=req, # context=parent_context + req=req, context=parent_context ) return response From 51efd74f12a9f85c58b348c841aac4f3afb00411 Mon Sep 17 00:00:00 2001 From: Marvin Buss Date: Mon, 1 Jan 2024 19:59:21 +0100 Subject: [PATCH 34/36] Update function tf code --- code/infra/function.tf | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/code/infra/function.tf b/code/infra/function.tf index 30216b9..a3d6473 100644 --- a/code/infra/function.tf +++ b/code/infra/function.tf @@ -93,6 +93,14 @@ resource "azapi_resource" "function" { name = "APPLICATIONINSIGHTS_CONNECTION_STRING" value = azurerm_application_insights.application_insights.connection_string }, + { + name = "AZURE_SDK_TRACING_IMPLEMENTATION" + value = "opentelemetry" + }, + { + name = "AZURE_TRACING_ENABLED" + value = "true" + }, { name = "AZURE_FUNCTIONS_ENVIRONMENT" value = "Production" From 3365dd9eea8f45ad062d1cfe6b07af319ee6b03b Mon Sep 17 00:00:00 2001 From: Marvin Buss Date: Mon, 1 Jan 2024 19:59:34 +0100 Subject: [PATCH 35/36] Revert container app changes --- .github/workflows/_containerTemplate.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/_containerTemplate.yml b/.github/workflows/_containerTemplate.yml index aa38934..566ef78 100644 --- a/.github/workflows/_containerTemplate.yml +++ b/.github/workflows/_containerTemplate.yml @@ -74,7 +74,7 @@ jobs: - name: Login Container Registry uses: docker/login-action@v3.0.0 id: registry_login - # if: github.event_name != 'pull_request' + if: github.event_name != 'pull_request' with: registry: ${{ inputs.registry_uri }} username: ${{ secrets.USER_NAME }} @@ -101,7 +101,7 @@ jobs: with: context: ${{ inputs.working_directory }} file: ${{ inputs.working_directory }}/Dockerfile - push: true # ${{ github.event_name != 'pull_request' }} + push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.metadata.outputs.tags }} labels: ${{ steps.metadata.outputs.labels }} cache-from: type=gha From 4167aca852473cc212b33445c457518fb89e054a Mon Sep 17 00:00:00 2001 From: Marvin Buss Date: Mon, 1 Jan 2024 20:05:44 +0100 Subject: [PATCH 36/36] Fix depcrecation warning --- code/function/fastapp/core/config.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/code/function/fastapp/core/config.py b/code/function/fastapp/core/config.py index 444393b..62846bd 100644 --- a/code/function/fastapp/core/config.py +++ b/code/function/fastapp/core/config.py @@ -16,9 +16,9 @@ class Settings(BaseSettings): APPLICATIONINSIGHTS_CONNECTION_STRING: str = Field( default="", env="APPLICATIONINSIGHTS_CONNECTION_STRING" ) - WEBSITE_NAME: str = Field(default="test", env="WEBSITE_SITE_NAME") - WEBSITE_INSTANCE_ID: str = Field(default="0", env="WEBSITE_INSTANCE_ID") - MY_SECRET_CONFIG: str = Field(default="", env="MY_SECRET_CONFIG") + WEBSITE_NAME: str = Field(default="test", alias="WEBSITE_SITE_NAME") + WEBSITE_INSTANCE_ID: str = Field(default="0", alias="WEBSITE_INSTANCE_ID") + MY_SECRET_CONFIG: str = Field(default="", alias="MY_SECRET_CONFIG") settings = Settings()