Skip to content

Add invocation timeout handling #6

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Nov 15, 2022
Merged

Add invocation timeout handling #6

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
ecab55f
add timeout to invoke
dominikschubert Nov 11, 2022
34969de
set server timeout instead of invoke deadline
dominikschubert Nov 11, 2022
2ba328a
enable json/trace logging
dominikschubert Nov 11, 2022
b0da13f
add timeout handling
dfangl Nov 14, 2022
9d755c4
Revert "enable json/trace logging"
dfangl Nov 14, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
# LOCALSTACK CHANGES 2022-03-10: remove linker flags and add gc flags for delve debugger
# LOCALSTACK CHANGES 2022-03-28: change compile src folder
# LOCALSTACK CHANGES 2022-11-14: add --rm flag to compile-with-docker

# RELEASE_BUILD_LINKER_FLAGS disables DWARF and symbol table generation to reduce binary size
#RELEASE_BUILD_LINKER_FLAGS=-s -w
Expand All @@ -21,7 +22,7 @@ compile-lambda-linux-all:
make ARCH=arm64 compile-lambda-linux

compile-with-docker:
docker run --env GOPROXY=direct -v $(shell pwd):/LambdaRuntimeLocal -w /LambdaRuntimeLocal golang:1.18 make ARCH=${ARCH} compile-lambda-linux
docker run --rm --env GOPROXY=direct -v $(shell pwd):/LambdaRuntimeLocal -w /LambdaRuntimeLocal golang:1.18 make ARCH=${ARCH} compile-lambda-linux

compile-lambda-linux:
CGO_ENABLED=0 GOOS=linux GOARCH=${GO_ARCH_${ARCH}} go build -ldflags "${RELEASE_BUILD_LINKER_FLAGS}" -gcflags="all=-N -l" -o ${DESTINATION_${ARCH}} ./cmd/localstack
Expand Down
39 changes: 31 additions & 8 deletions cmd/localstack/custom_interop.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,9 +52,9 @@ type InvokeRequest struct {

type ErrorResponse struct {
ErrorMessage string `json:"errorMessage"`
ErrorType string `json:"errorType"`
RequestId string `json:"requestId"`
StackTrace []string `json:"stackTrace"`
ErrorType string `json:"errorType,omitempty"`
RequestId string `json:"requestId,omitempty"`
StackTrace []string `json:"stackTrace,omitempty"`
}

func NewCustomInteropServer(lsOpts *LsOpts, delegate rapidcore.InteropServer, logCollector *LogCollector) (server *CustomInteropServer) {
Expand Down Expand Up @@ -99,12 +99,36 @@ func NewCustomInteropServer(lsOpts *LsOpts, delegate rapidcore.InteropServer, lo
CorrelationID: "invokeCorrelationID",
NeedDebugLogs: true,
InvokedFunctionArn: invokeR.InvokedFunctionArn,
//DeadlineNs:
})
timeout := int(server.delegate.GetInvokeTimeout().Seconds())
isErr := false
if err != nil {
log.Fatalln(err)
switch err {
case rapidcore.ErrInvokeTimeout:
log.Debugf("Got invoke timeout")
isErr = true
errorResponse := ErrorResponse{
ErrorMessage: fmt.Sprintf(
"%s %s Task timed out after %d.00 seconds",
time.Now().Format("2006-01-02T15:04:05Z"),
invokeR.InvokeId,
timeout,
),
}
jsonErrorResponse, err := json.Marshal(errorResponse)
if err != nil {
log.Fatalln("unable to marshall json timeout response")
}
_, err = invokeResp.Write(jsonErrorResponse)
if err != nil {
log.Fatalln("unable to write to response")
}
default:
log.Fatalln(err)
}
}
inv := GetEnvOrDie("AWS_LAMBDA_FUNCTION_TIMEOUT")
timeoutDuration, _ := time.ParseDuration(inv + "s")
timeoutDuration := time.Duration(timeout) * time.Second
memorySize := GetEnvOrDie("AWS_LAMBDA_FUNCTION_MEMORY_SIZE")
PrintEndReports(invokeR.InvokeId, "", memorySize, invokeStart, timeoutDuration, logCollector)

Expand All @@ -117,8 +141,7 @@ func NewCustomInteropServer(lsOpts *LsOpts, delegate rapidcore.InteropServer, lo
var errR map[string]any
marshalErr := json.Unmarshal(invokeResp.Body, &errR)

isErr := false
if marshalErr == nil {
if !isErr && marshalErr == nil {
_, isErr = errR["errorType"]
}

Expand Down
11 changes: 9 additions & 2 deletions cmd/localstack/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
_ "net/http/pprof"
"os"
"runtime/debug"
"strconv"
)

type LsOpts struct {
Expand Down Expand Up @@ -72,12 +73,18 @@ func main() {
// initialize all flows and start runtime API
go sandbox.Create()

// get timeout
invokeTimeoutEnv := GetEnvOrDie("AWS_LAMBDA_FUNCTION_TIMEOUT")
invokeTimeoutSeconds, err := strconv.Atoi(invokeTimeoutEnv)
if err != nil {
log.Fatalln(err)
}
// start runtime init
go InitHandler(sandbox, GetEnvOrDie("AWS_LAMBDA_FUNCTION_VERSION"), 30) // TODO: replace this with a custom init
go InitHandler(sandbox, GetEnvOrDie("AWS_LAMBDA_FUNCTION_VERSION"), int64(invokeTimeoutSeconds)) // TODO: replace this with a custom init

// TODO: make the tracing server optional
// start blocking with the tracing server
err := http.ListenAndServe("0.0.0.0:"+lsOpts.InitTracingPort, http.DefaultServeMux)
err = http.ListenAndServe("0.0.0.0:"+lsOpts.InitTracingPort, http.DefaultServeMux)
if err != nil {
log.Fatal("Failed to start debug server")
}
Expand Down