feat: add inference-gateway deployment

Signed-off-by: Shane Utt <[email protected]>

Author: shaneutt

deploy/components/inference-gateway/configmaps.yaml

@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: endpoint-picker-config
+data:
+  config.yaml: |
+    pod_selector:
+      ai-aware-router-pod: true
+    routing_filters:
+    routing_scorers:
+    - name: session-affinity
+      weight: 60
+    - name: route-by-active-lora
+      weight: 50
+    routing_header: x-ai-aware-router-routing
+    session_id_header: x-ai-aware-router-session-id
+    listening_port: 9080
+    inference_port: 8000

deploy/components/inference-gateway/deployments.yaml

@@ -0,0 +1,32 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: endpoint-picker
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: endpoint-picker
+  template:
+    metadata:
+      labels:
+        app: endpoint-picker
+    spec:
+      serviceAccountName: endpoint-picker
+      containers:
+        - name: endpoint-picker
+          image: inference-router/router-ext-proc:latest
+          args:
+            - "--config-file"
+            - "/etc/endpoint-picker/config.yaml"
+          ports:
+            - name: grpc
+              containerPort: 9080
+              protocol: TCP
+          volumeMounts:
+            - name: endpoint-picker-config
+              mountPath: /etc/endpoint-picker
+      volumes:
+        - name: endpoint-picker-config
+          configMap:
+            name: endpoint-picker-config

deploy/components/inference-gateway/envoy-filters.yaml

@@ -0,0 +1,31 @@
+apiVersion: networking.istio.io/v1alpha3
+kind: EnvoyFilter
+metadata:
+  name: endpoint-picker
+spec:
+  configPatches:
+  - applyTo: HTTP_FILTER
+    match:
+      listener:
+        filterChain:
+          filter:
+            name: "envoy.filters.network.http_connection_manager"
+    patch:
+      operation: INSERT_FIRST
+      value:
+        name: envoy.filters.http.ext_proc
+        typed_config:
+          "@type": type.googleapis.com/envoy.extensions.filters.http.ext_proc.v3.ExternalProcessor
+          failure_mode_allow: false
+          allow_mode_override: true
+          processing_mode:
+            request_header_mode: "SEND"
+            response_header_mode: "SEND"
+            request_body_mode: "BUFFERED"
+            response_body_mode: "BUFFERED"
+            request_trailer_mode: "SEND"
+            response_trailer_mode: "SKIP"
+          grpc_service:
+            envoy_grpc:
+              cluster_name: outbound|9080||endpoint-picker.REPLACE_NAMESPACE.svc.cluster.local
+            timeout: 5s

deploy/components/inference-gateway/gateways.yaml

@@ -0,0 +1,14 @@
+apiVersion: gateway.networking.k8s.io/v1
+kind: Gateway
+metadata:
+  name: inference-gateway
+  labels:
+    istio.io/rev: istio-control-plane
+  annotations:
+    networking.istio.io/service-type: ClusterIP
+spec:
+  gatewayClassName: istio
+  listeners:
+  - name: default
+    port: 80
+    protocol: HTTP

deploy/components/inference-gateway/kustomization.yaml

@@ -0,0 +1,30 @@
+# ------------------------------------------------------------------------------
+# Inference Gateway
+#
+# This deploys a Gateway and the Endpoint Picker (EPP), and attaches the EPP to
+# the Gateway with an EnvoyFilter.
+#
+# Add an HTTPRoute to route traffic to VLLM, or a VLLM simulator.
+#
+# **WARNING**: The EnvoyFilter contains a variable that needs to be replaced
+# with the namespace to match the EPP's Service. For now use sed to replace it,
+# e.g.:
+#
+#   $ kubectl kustomize deploy/components/inference-gateway \
+#     | sed 's/REPLACE_NAMESPACE/mynamespace/gI' \
+#     | kubectl -n mynamespace apply -f -
+# ------------------------------------------------------------------------------
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+- configmaps.yaml
+- deployments.yaml
+- services.yaml
+- rbac.yaml
+- gateways.yaml
+- envoy-filters.yaml
+
+images:
+- name: inference-router/router-ext-proc
+  newTag: 0.0.1

deploy/components/inference-gateway/rbac.yaml

@@ -0,0 +1,31 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: endpoint-picker
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: endpoint-picker
+rules:
+  - apiGroups:
+    - ""
+    resources:
+    - "pods"
+    verbs:
+    - "get"
+    - "list"
+    - "watch"
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: endpoint-picker-binding
+subjects:
+  - kind: ServiceAccount
+    name: endpoint-picker
+roleRef:
+  kind: Role
+  name: endpoint-picker
+  apiGroup: rbac.authorization.k8s.io
+

deploy/components/inference-gateway/services.yaml

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: endpoint-picker
+spec:
+  type: ClusterIP
+  selector:
+    app: endpoint-picker
+  ports:
+    - name: grpc
+      protocol: TCP
+      port: 9080
+      targetPort: 9080