Basic NJS module to extract model name (#3877)

Problem: To support the full Gateway API Inference Extension, we need to be able to extract the model name from the client request body in certain situations.

Solution: Add a basic NJS module to extract the model name. This module will be enhanced (I've added notes) to be included in the full solution. On its own, it is not yet used.

Author: sjberman

.nvmrc

@@ -1 +1 @@
-22
+24

Makefile

@@ -33,7 +33,7 @@ GEN_CRD_API_REFERENCE_DOCS_VERSION = v0.3.0
 # renovate: datasource=go depName=sigs.k8s.io/controller-tools
 CONTROLLER_TOOLS_VERSION = v0.19.0
 # renovate: datasource=docker depName=node
-NODE_VERSION = 22
+NODE_VERSION = 24
 # renovate: datasource=docker depName=quay.io/helmpack/chart-testing
 CHART_TESTING_VERSION = v3.13.0
 # renovate: datasource=github-tags depName=dadav/helm-schema

build/Dockerfile.nginx

@@ -23,7 +23,7 @@ RUN apk add --no-cache bash \
     && ln -sf /dev/stderr /var/log/nginx/error.log
 
 COPY build/entrypoint.sh /agent/entrypoint.sh
-COPY ${NJS_DIR}/httpmatches.js /usr/lib/nginx/modules/njs/httpmatches.js
+COPY ${NJS_DIR}/ /usr/lib/nginx/modules/njs/
 COPY ${NGINX_CONF_DIR}/nginx.conf /etc/nginx/nginx.conf
 COPY ${NGINX_CONF_DIR}/grpc-error-locations.conf /etc/nginx/grpc-error-locations.conf
 COPY ${NGINX_CONF_DIR}/grpc-error-pages.conf /etc/nginx/grpc-error-pages.conf

build/Dockerfile.nginxplus

@@ -29,7 +29,7 @@ RUN apk add --no-cache bash \
     && ln -sf /dev/stderr /var/log/nginx/error.log
 
 COPY build/entrypoint.sh /agent/entrypoint.sh
-COPY ${NJS_DIR}/httpmatches.js /usr/lib/nginx/modules/njs/httpmatches.js
+COPY ${NJS_DIR}/ /usr/lib/nginx/modules/njs/
 COPY ${NGINX_CONF_DIR}/nginx-plus.conf /etc/nginx/nginx.conf
 COPY ${NGINX_CONF_DIR}/grpc-error-locations.conf /etc/nginx/grpc-error-locations.conf
 COPY ${NGINX_CONF_DIR}/grpc-error-pages.conf /etc/nginx/grpc-error-pages.conf

internal/controller/nginx/conf/nginx-plus.conf

@@ -13,6 +13,7 @@ http {
   include /etc/nginx/conf.d/*.conf;
   include /etc/nginx/mime.types;
   js_import /usr/lib/nginx/modules/njs/httpmatches.js;
+  js_import /usr/lib/nginx/modules/njs/epp.js;
 
   default_type application/octet-stream;
 

internal/controller/nginx/conf/nginx.conf

@@ -13,6 +13,7 @@ http {
   include /etc/nginx/conf.d/*.conf;
   include /etc/nginx/mime.types;
   js_import /usr/lib/nginx/modules/njs/httpmatches.js;
+  js_import /usr/lib/nginx/modules/njs/epp.js;
 
   default_type application/octet-stream;
 

internal/controller/nginx/modules/README.md

@@ -22,6 +22,7 @@ dependencies.
 
 - [httpmatches](./src/httpmatches.js): a location handler for HTTP requests. It redirects requests to an internal
   location block based on the request's headers, arguments, and method.
+- [epp](./src/epp.js): handles communication with the EndpointPicker (EPP) component. This is for acquiring a specific AI endpoint to route client traffic to when using the Gateway API Inference Extension.
 
 ### Helpful Resources for Module Development
 

internal/controller/nginx/modules/src/epp.js

@@ -0,0 +1,29 @@
+// This file contains the methods to get an AI workload endpoint from the EndpointPicker (EPP).
+
+// TODO (sberman): this module will need to be enhanced to include the following:
+// - function that sends the subrequest to the Go middleware application (to get the endpoint from EPP)
+// - if a user has specified an Exact matching condition for a model name, extract the model name from
+// the request body, and if it matches that condition, set the proper value in the X-Gateway-Model-Name header
+// (based on if we do a redirect or traffic split (see design doc)) in the subrequest. If the client request
+// already has this header set, then I don't think we need to extract the model from the body, just pass
+// through the existing header.
+// I believe we have to use js_content to call the NJS functionality. Because this takes over
+// the request, we will likely have to finish the NJS functionality with an internalRedirect to an internal
+// location that proxy_passes to the chosen endpoint.
+
+// extractModel extracts the model name from the request body.
+function extractModel(r) {
+	try {
+		var body = JSON.parse(r.requestText);
+		if (body && body.model !== undefined) {
+			return String(body.model);
+		}
+	} catch (e) {
+		r.error(`error parsing request body for model name: ${e.message}`);
+		return '';
+	}
+	r.error('request body does not contain model parameter');
+	return '';
+}
+
+export default { extractModel };

internal/controller/nginx/modules/test/epp.test.js

@@ -0,0 +1,52 @@
+import { default as epp } from '../src/epp.js';
+import { expect, describe, it } from 'vitest';
+
+function makeRequest(body) {
+	let r = {
+		// Test mocks
+		error(msg) {
+			r.variables.error = msg;
+		},
+		requestText: body,
+		variables: {},
+	};
+
+	return r;
+}
+
+describe('extractModel', () => {
+	const tests = [
+		{
+			name: 'returns the model value',
+			body: '{"model":"gpt-4"}',
+			model: 'gpt-4',
+			error: undefined,
+		},
+		{
+			name: 'returns empty string if model is missing',
+			body: '{"foo":1}',
+			model: '',
+			error: 'request body does not contain model parameter',
+		},
+		{
+			name: 'returns empty string for invalid JSON',
+			body: 'not-json',
+			model: '',
+			error: `error parsing request body for model name: Unexpected token 'o', "not-json" is not valid JSON`,
+		},
+		{
+			name: 'empty request body',
+			body: '',
+			model: '',
+			error: 'error parsing request body for model name: Unexpected end of JSON input',
+		},
+	];
+
+	tests.forEach((test) => {
+		it(test.name, () => {
+			let r = makeRequest(test.body);
+			expect(epp.extractModel(r)).to.equal(test.model);
+			expect(r.variables.error).to.equal(test.error);
+		});
+	});
+});