Get python emulator going.

Author: taeold

src/deploy/functions/runtimes/index.ts

@@ -10,7 +10,7 @@ const RUNTIMES: string[] = ["nodejs10", "nodejs12", "nodejs14", "nodejs16", "nod
 // Experimental runtimes are part of the Runtime type, but are in a
 // different list to help guard against some day accidentally iterating over
 // and printing a hidden runtime to the user.
-const EXPERIMENTAL_RUNTIMES: string[] = [];
+const EXPERIMENTAL_RUNTIMES: string[] = ["python310", "python311"];
 export type Runtime = typeof RUNTIMES[number] | typeof EXPERIMENTAL_RUNTIMES[number];
 
 /** Runtimes that can be found in existing backends but not used for new functions. */

src/deploy/functions/runtimes/python/index.ts

@@ -1,5 +1,6 @@
 import * as fs from "fs";
 import * as path from "path";
+import fetch from "node-fetch";
 import { promisify } from "util";
 
 import * as portfinder from "portfinder";
@@ -78,7 +79,7 @@ class Delegate implements runtimes.RuntimeDelegate {
         child.on("exit", resolve);
         child.on("error", reject);
       });
-      this._modulesDir = out;
+      this._modulesDir = out.trim();
     }
     return this._modulesDir;
   }

src/emulator/functionsEmulator.ts

@@ -124,15 +124,43 @@ export interface FunctionsEmulatorArgs {
   projectAlias?: string;
 }
 
-// FunctionsRuntimeInstance is the handler for a running function invocation
+/**
+ * IPC connection info of a Function Runtime.
+ */
+export class IPCConn {
+  constructor(readonly socketPath: string) {}
+
+  httpReqOpts(): http.RequestOptions {
+    return {
+      path: `/`,
+      socketPath: this.socketPath,
+    };
+  }
+}
+
+/**
+ * TCP/IP connection info of a Function Runtime.
+ */
+export class TCPConn {
+  constructor(readonly host: string, readonly port: number) {}
+
+  httpReqOpts(): http.RequestOptions {
+    return {
+      path: `/`,
+      host: this.host,
+      port: this.port,
+    };
+  }
+}
+
 export interface FunctionsRuntimeInstance {
   process: ChildProcess;
   // An emitter which sends our EmulatorLog events from the runtime.
   events: EventEmitter;
   // A cwd of the process
   cwd: string;
-  // Path to socket file used for HTTP-over-IPC comms.
-  socketPath: string;
+  // Communication info for the runtime
+  conn: IPCConn | TCPConn;
 }
 
 export interface InvokeRuntimeOpts {
@@ -355,8 +383,8 @@ export class FunctionsEmulator implements EmulatorInstance {
     return new Promise((resolve, reject) => {
       const req = http.request(
         {
+          ...worker.runtime.conn.httpReqOpts(),
           path: `/`,
-          socketPath: worker.runtime.socketPath,
           headers: headers,
         },
         resolve
@@ -1269,14 +1297,14 @@ export class FunctionsEmulator implements EmulatorInstance {
       process: childProcess,
       events: new EventEmitter(),
       cwd: backend.functionsDir,
-      socketPath,
+      conn: new IPCConn(socketPath),
     });
   }
 
   async startPython(
     backend: EmulatableBackend,
     envs: Record<string, string>
-  ): Promise<ChildProcess> {
+  ): Promise<FunctionsRuntimeInstance> {
     const args = [path.join(__dirname, "functionsEmulatorRuntime")];
 
     if (this.args.debugPort) {
@@ -1294,18 +1322,23 @@ export class FunctionsEmulator implements EmulatorInstance {
       );
     }
 
-    // Unfortunately, there isn't platform-neutral support for Unix Domain Socket or Nameed Pipe in the python
-    // ecosystem. Use regular IP+PORT combination instead.
+    // Unfortunately, there isn't platform-neutral support for Unix Domain Socket or Named Pipe in the python
+    // ecosystem. Use TCP/IP stack instead.
     const port = await portfinder.getPortPromise({
       port: 8081,
     });
-    return Promise.resolve(
-      runWithVirtualEnv([bin, ...args], backend.functionsDir, {
-        ...process.env,
-        ...envs,
-        PORT: port.toString(),
-      })
-    );
+    const childProcess = runWithVirtualEnv([bin, ...args], backend.functionsDir, {
+      ...process.env,
+      ...envs,
+      PORT: port.toString(),
+    });
+
+    return Promise.resolve({
+      process: childProcess,
+      events: new EventEmitter(),
+      cwd: backend.functionsDir,
+      conn: new TCPConn("localhost", port),
+    });
   }
 
   async startRuntime(
@@ -1315,7 +1348,12 @@ export class FunctionsEmulator implements EmulatorInstance {
     const runtimeEnv = this.getRuntimeEnvs(backend, trigger);
     const secretEnvs = await this.resolveSecretEnvs(backend, trigger);
 
-    const runtime = await this.startNode(backend, { ...runtimeEnv, ...secretEnvs });
+    let runtime;
+    if (backend.runtime!.startsWith("python")) {
+      runtime = await this.startPython(backend, { ...runtimeEnv, ...secretEnvs });
+    } else {
+      runtime = await this.startNode(backend, { ...runtimeEnv, ...secretEnvs });
+    }
     const extensionLogInfo = {
       instanceId: backend.extensionInstanceId,
       ref: backend.extensionVersion?.ref,

src/emulator/functionsRuntimeWorker.ts

@@ -8,6 +8,7 @@ import { EventEmitter } from "events";
 import { EmulatorLogger, ExtensionLogInfo } from "./emulatorLogger";
 import { FirebaseError } from "../error";
 import { Serializable } from "child_process";
+import { IncomingMessage } from "http";
 
 type LogListener = (el: EmulatorLog) => any;
 
@@ -118,12 +119,12 @@ export class RuntimeWorker {
     return new Promise((resolve) => {
       const proxy = http.request(
         {
+          ...this.runtime.conn.httpReqOpts(),
           method: req.method,
           path: req.path,
           headers: req.headers,
-          socketPath: this.runtime.socketPath,
         },
-        (_resp) => {
+        (_resp: IncomingMessage) => {
           resp.writeHead(_resp.statusCode || 200, _resp.headers);
           const piped = _resp.pipe(resp);
           piped.on("finish", () => {
@@ -178,20 +179,19 @@ export class RuntimeWorker {
 
   isSocketReady(): Promise<void> {
     return new Promise((resolve, reject) => {
-      const req = http
-        .request(
-          {
-            method: "GET",
-            path: "/__/health",
-            socketPath: this.runtime.socketPath,
-          },
-          () => {
-            // Set the worker state to IDLE for new work
-            this.readyForWork();
-            resolve();
-          }
-        )
-        .end();
+      const req = http.request(
+        {
+          ...this.runtime.conn.httpReqOpts(),
+          method: "GET",
+          path: "/__/health",
+        },
+        () => {
+          // Set the worker state to IDLE for new work
+          this.readyForWork();
+          resolve();
+        }
+      );
+      req.end();
       req.on("error", (error) => {
         reject(error);
       });

src/functions/python.ts

@@ -1,6 +1,7 @@
 import * as path from "path";
 import * as spawn from "cross-spawn";
 import * as cp from "child_process";
+import { logger } from "../logger";
 
 const DEFAULT_VENV_DIR = "venv";
 
@@ -18,6 +19,7 @@ export function runWithVirtualEnv(
   const venvActivate = path.join(cwd, venvDir, ...activateScriptPath);
   const command = process.platform === "win32" ? venvActivate : "source";
   const args = [process.platform === "win32" ? "" : venvActivate, "&&", ...commandAndArgs];
+  logger.debug(`Running command with virtualenv: command=${command}, args=${JSON.stringify(args)}`);
 
   return spawn(command, args, {
     shell: true,

src/test/emulators/functionsRuntimeWorker.spec.ts

@@ -1,7 +1,7 @@
 import * as httpMocks from "node-mocks-http";
 import * as nock from "nock";
 import { expect } from "chai";
-import { FunctionsRuntimeInstance } from "../../emulator/functionsEmulator";
+import { FunctionsRuntimeInstance, IPCConn } from "../../emulator/functionsEmulator";
 import { EventEmitter } from "events";
 import {
   RuntimeWorker,
@@ -21,7 +21,7 @@ class MockRuntimeInstance implements FunctionsRuntimeInstance {
   events: EventEmitter = new EventEmitter();
   exit: Promise<number>;
   cwd = "/home/users/dir";
-  socketPath = "/path/to/socket/foo.sock";
+  conn = new IPCConn("/path/to/socket/foo.sock");
 
   constructor() {
     this.exit = new Promise((resolve) => {