Skip to content

Commit c1cf0c5

Browse files
authored
feat: Pull upstream changes 2023/04 (#87)
1 parent a08886c commit c1cf0c5

File tree

122 files changed

+6845
-2726
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

122 files changed

+6845
-2726
lines changed

cmd/aws-lambda-rie/handlers.go

+25-6
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,10 @@ import (
1414
"strings"
1515
"time"
1616

17+
"go.amzn.com/lambda/core/statejson"
1718
"go.amzn.com/lambda/interop"
1819
"go.amzn.com/lambda/rapidcore"
20+
"go.amzn.com/lambda/rapidcore/env"
1921

2022
"github.com/google/uuid"
2123

@@ -27,6 +29,19 @@ type Sandbox interface {
2729
Invoke(responseWriter http.ResponseWriter, invoke *interop.Invoke) error
2830
}
2931

32+
type InteropServer interface {
33+
Init(i *interop.Init, invokeTimeoutMs int64) error
34+
AwaitInitialized() error
35+
FastInvoke(w http.ResponseWriter, i *interop.Invoke, direct bool) error
36+
Reserve(id string, traceID, lambdaSegmentID string) (*rapidcore.ReserveResponse, error)
37+
Reset(reason string, timeoutMs int64) (*statejson.ResetDescription, error)
38+
AwaitRelease() (*statejson.InternalStateDescription, error)
39+
Shutdown(shutdown *interop.Shutdown) *statejson.InternalStateDescription
40+
InternalState() (*statejson.InternalStateDescription, error)
41+
CurrentToken() *interop.Token
42+
Restore(restore *interop.Restore) error
43+
}
44+
3045
var initDone bool
3146

3247
func GetenvWithDefault(key string, defaultValue string) string {
@@ -57,7 +72,7 @@ func printEndReports(invokeId string, initDuration string, memorySize string, in
5772
invokeId, invokeDuration, math.Ceil(invokeDuration), memorySize, memorySize)
5873
}
5974

60-
func InvokeHandler(w http.ResponseWriter, r *http.Request, sandbox Sandbox) {
75+
func InvokeHandler(w http.ResponseWriter, r *http.Request, sandbox Sandbox, bs interop.Bootstrap) {
6176
log.Debugf("invoke: -> %s %s %v", r.Method, r.URL, r.Header)
6277
bodyBytes, err := ioutil.ReadAll(r.Body)
6378
if err != nil {
@@ -80,7 +95,7 @@ func InvokeHandler(w http.ResponseWriter, r *http.Request, sandbox Sandbox) {
8095

8196
if !initDone {
8297

83-
initStart, initEnd := InitHandler(sandbox, functionVersion, timeout)
98+
initStart, initEnd := InitHandler(sandbox, functionVersion, timeout, bs)
8499

85100
// Calculate InitDuration
86101
initTimeMS := math.Min(float64(initEnd.Sub(initStart).Nanoseconds()),
@@ -99,7 +114,6 @@ func InvokeHandler(w http.ResponseWriter, r *http.Request, sandbox Sandbox) {
99114
TraceID: r.Header.Get("X-Amzn-Trace-Id"),
100115
LambdaSegmentID: r.Header.Get("X-Amzn-Segment-Id"),
101116
Payload: bytes.NewReader(bodyBytes),
102-
CorrelationID: "invokeCorrelationID",
103117
}
104118
fmt.Println("START RequestId: " + invokePayload.ID + " Version: " + functionVersion)
105119

@@ -166,7 +180,7 @@ func InvokeHandler(w http.ResponseWriter, r *http.Request, sandbox Sandbox) {
166180
w.Write(invokeResp.Body)
167181
}
168182

169-
func InitHandler(sandbox Sandbox, functionVersion string, timeout int64) (time.Time, time.Time) {
183+
func InitHandler(sandbox Sandbox, functionVersion string, timeout int64, bs interop.Bootstrap) (time.Time, time.Time) {
170184
additionalFunctionEnvironmentVariables := map[string]string{}
171185

172186
// Add default Env Vars if they were not defined. This is a required otherwise 1p Python2.7, Python3.6, and
@@ -189,15 +203,20 @@ func InitHandler(sandbox Sandbox, functionVersion string, timeout int64) (time.T
189203
// pass to rapid
190204
sandbox.Init(&interop.Init{
191205
Handler: GetenvWithDefault("AWS_LAMBDA_FUNCTION_HANDLER", os.Getenv("_HANDLER")),
192-
CorrelationID: "initCorrelationID",
193206
AwsKey: os.Getenv("AWS_ACCESS_KEY_ID"),
194207
AwsSecret: os.Getenv("AWS_SECRET_ACCESS_KEY"),
195208
AwsSession: os.Getenv("AWS_SESSION_TOKEN"),
196209
XRayDaemonAddress: "0.0.0.0:0", // TODO
197210
FunctionName: GetenvWithDefault("AWS_LAMBDA_FUNCTION_NAME", "test_function"),
198211
FunctionVersion: functionVersion,
199-
212+
RuntimeInfo: interop.RuntimeInfo{
213+
ImageJSON: "{}",
214+
Arn: "",
215+
Version: ""},
200216
CustomerEnvironmentVariables: additionalFunctionEnvironmentVariables,
217+
SandboxType: interop.SandboxClassic,
218+
Bootstrap: bs,
219+
EnvironmentVariables: env.NewEnvironment(),
201220
}, timeout*1000)
202221
initEnd := time.Now()
203222
return initStart, initEnd

cmd/aws-lambda-rie/http.go

+4-2
Original file line numberDiff line numberDiff line change
@@ -7,16 +7,18 @@ import (
77
"net/http"
88

99
log "github.com/sirupsen/logrus"
10+
"go.amzn.com/lambda/interop"
11+
"go.amzn.com/lambda/rapidcore"
1012
)
1113

12-
func startHTTPServer(ipport string, sandbox Sandbox) {
14+
func startHTTPServer(ipport string, sandbox *rapidcore.SandboxBuilder, bs interop.Bootstrap) {
1315
srv := &http.Server{
1416
Addr: ipport,
1517
}
1618

1719
// Pass a channel
1820
http.HandleFunc("/2015-03-31/functions/function/invocations", func(w http.ResponseWriter, r *http.Request) {
19-
InvokeHandler(w, r, sandbox)
21+
InvokeHandler(w, r, sandbox.LambdaInvokeAPI(), bs)
2022
})
2123

2224
// go routine (main thread waits)

cmd/aws-lambda-rie/main.go

+44-7
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ package main
66
import (
77
"context"
88
"fmt"
9+
"net"
910
"os"
1011
"runtime/debug"
1112

@@ -21,20 +22,49 @@ const (
2122
)
2223

2324
type options struct {
24-
LogLevel string `long:"log-level" default:"info" description:"log level"`
25+
LogLevel string `long:"log-level" description:"The level of AWS Lambda Runtime Interface Emulator logs to display. Can also be set by the environment variable 'LOG_LEVEL'. Defaults to the value 'info'."`
2526
InitCachingEnabled bool `long:"enable-init-caching" description:"Enable support for Init Caching"`
27+
// Do not have a default value so we do not need to keep it in sync with the default value in lambda/rapidcore/sandbox_builder.go
28+
RuntimeAPIAddress string `long:"runtime-api-address" description:"The address of the AWS Lambda Runtime API to communicate with the Lambda execution environment."`
29+
RuntimeInterfaceEmulatorAddress string `long:"runtime-interface-emulator-address" default:"0.0.0.0:8080" description:"The address for the AWS Lambda Runtime Interface Emulator to accept HTTP request upon."`
2630
}
2731

2832
func main() {
2933
// More frequent GC reduces the tail latencies, equivalent to export GOGC=33
3034
debug.SetGCPercent(33)
3135

3236
opts, args := getCLIArgs()
33-
rapidcore.SetLogLevel(opts.LogLevel)
37+
38+
logLevel := "info"
39+
40+
// If you specify an option by using a parameter on the CLI command line, it overrides any value from either the corresponding environment variable.
41+
//
42+
// https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html
43+
if opts.LogLevel != "" {
44+
logLevel = opts.LogLevel
45+
} else if envLogLevel, envLogLevelSet := os.LookupEnv("LOG_LEVEL"); envLogLevelSet {
46+
logLevel = envLogLevel
47+
}
48+
49+
rapidcore.SetLogLevel(logLevel)
50+
51+
if opts.RuntimeAPIAddress != "" {
52+
_, _, err := net.SplitHostPort(opts.RuntimeAPIAddress)
53+
54+
if err != nil {
55+
log.WithError(err).Fatalf("The command line value for \"--runtime-api-address\" is not a valid network address %q.", opts.RuntimeAPIAddress)
56+
}
57+
}
58+
59+
_, _, err := net.SplitHostPort(opts.RuntimeInterfaceEmulatorAddress)
60+
61+
if err != nil {
62+
log.WithError(err).Fatalf("The command line value for \"--runtime-interface-emulator-address\" is not a valid network address %q.", opts.RuntimeInterfaceEmulatorAddress)
63+
}
3464

3565
bootstrap, handler := getBootstrap(args, opts)
3666
sandbox := rapidcore.
37-
NewSandboxBuilder(bootstrap).
67+
NewSandboxBuilder().
3868
AddShutdownFunc(context.CancelFunc(func() { os.Exit(0) })).
3969
SetExtensionsFlag(true).
4070
SetInitCachingFlag(opts.InitCachingEnabled)
@@ -43,10 +73,17 @@ func main() {
4373
sandbox.SetHandler(handler)
4474
}
4575

46-
go sandbox.Create()
76+
if opts.RuntimeAPIAddress != "" {
77+
sandbox.SetRuntimeAPIAddress(opts.RuntimeAPIAddress)
78+
}
79+
80+
sandboxContext, internalStateFn := sandbox.Create()
81+
// Since we have not specified a custom interop server for standalone, we can
82+
// directly reference the default interop server, which is a concrete type
83+
sandbox.DefaultInteropServer().SetSandboxContext(sandboxContext)
84+
sandbox.DefaultInteropServer().SetInternalStateGetter(internalStateFn)
4785

48-
testAPIipport := "0.0.0.0:8080"
49-
startHTTPServer(testAPIipport, sandbox)
86+
startHTTPServer(opts.RuntimeInterfaceEmulatorAddress, sandbox, bootstrap)
5087
}
5188

5289
func getCLIArgs() (options, []string) {
@@ -112,5 +149,5 @@ func getBootstrap(args []string, opts options) (*rapidcore.Bootstrap, string) {
112149
log.Panic("insufficient arguments: bootstrap not provided")
113150
}
114151

115-
return rapidcore.NewBootstrapSingleCmd(bootstrapLookupCmd, currentWorkingDir), handler
152+
return rapidcore.NewBootstrapSingleCmd(bootstrapLookupCmd, currentWorkingDir, ""), handler
116153
}

lambda/agents/agent.go

+18-57
Original file line numberDiff line numberDiff line change
@@ -4,77 +4,38 @@
44
package agents
55

66
import (
7-
"fmt"
8-
"io"
9-
"io/ioutil"
10-
"os/exec"
7+
"os"
118
"path"
12-
"syscall"
9+
"path/filepath"
1310

1411
log "github.com/sirupsen/logrus"
1512
)
1613

17-
// AgentProcess is the common interface exposed by both internal and external agent processes
18-
type AgentProcess interface {
19-
Name() string
20-
}
21-
22-
// ExternalAgentProcess represents an external agent process
23-
type ExternalAgentProcess struct {
24-
cmd *exec.Cmd
25-
}
26-
27-
// NewExternalAgentProcess returns a new external agent process
28-
func NewExternalAgentProcess(path string, env []string, stdoutWriter io.Writer, stderrWriter io.Writer) ExternalAgentProcess {
29-
command := exec.Command(path)
30-
command.Env = env
31-
32-
command.Stdout = NewNewlineSplitWriter(stdoutWriter)
33-
command.Stderr = NewNewlineSplitWriter(stderrWriter)
34-
command.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
35-
36-
return ExternalAgentProcess{
37-
cmd: command,
38-
}
39-
}
40-
41-
// Name returns the name of the agent
42-
// For external agents is the executable name
43-
func (a *ExternalAgentProcess) Name() string {
44-
return path.Base(a.cmd.Path)
45-
}
46-
47-
func (a *ExternalAgentProcess) Pid() int {
48-
return a.cmd.Process.Pid
49-
}
50-
51-
// Start starts an external agent process
52-
func (a *ExternalAgentProcess) Start() error {
53-
return a.cmd.Start()
54-
}
55-
56-
// Wait waits for the external agent process to exit
57-
func (a *ExternalAgentProcess) Wait() error {
58-
return a.cmd.Wait()
59-
}
60-
61-
// String is used to print values passed as an operand to any format that accepts a string or to an unformatted printer such as Print.
62-
func (a *ExternalAgentProcess) String() string {
63-
return fmt.Sprintf("%s (%s)", a.Name(), a.cmd.Path)
64-
}
65-
6614
// ListExternalAgentPaths return a list of external agents found in a given directory
67-
func ListExternalAgentPaths(root string) []string {
15+
func ListExternalAgentPaths(dir string, root string) []string {
6816
var agentPaths []string
69-
files, err := ioutil.ReadDir(root)
17+
if !isCanonical(dir) || !isCanonical(root) {
18+
log.Warningf("Agents base paths are not absolute and in canonical form: %s, %s", dir, root)
19+
return agentPaths
20+
}
21+
fullDir := path.Join(root, dir)
22+
files, err := os.ReadDir(fullDir)
7023
if err != nil {
7124
log.WithError(err).Warning("Cannot list external agents")
7225
return agentPaths
7326
}
7427
for _, file := range files {
7528
if !file.IsDir() {
76-
agentPaths = append(agentPaths, path.Join(root, file.Name()))
29+
// The returned path is absolute wrt to `root`. This allows
30+
// to exec the agents in their own mount namespace
31+
p := path.Join("/", dir, file.Name())
32+
agentPaths = append(agentPaths, p)
7733
}
7834
}
7935
return agentPaths
8036
}
37+
38+
func isCanonical(path string) bool {
39+
absPath, err := filepath.Abs(path)
40+
return err == nil && absPath == path
41+
}

0 commit comments

Comments
 (0)