From 207ef667a33e9e19050897c59e9eeeecd97ceeba Mon Sep 17 00:00:00 2001 From: Rowan Bohde Date: Tue, 11 Jun 2024 14:28:06 -0500 Subject: [PATCH] perf: reduce memory usage for chunked artifact uploads to MinIO When specifying a filesize of -1, the MinIO client uses significantly more memory during upload. Since we know the file size ahead of time, use that to avoid the extra allocations. --- routers/api/actions/artifacts_chunks.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/routers/api/actions/artifacts_chunks.go b/routers/api/actions/artifacts_chunks.go index bba8ec5f94af2..3d1a3891d9e2f 100644 --- a/routers/api/actions/artifacts_chunks.go +++ b/routers/api/actions/artifacts_chunks.go @@ -39,7 +39,7 @@ func saveUploadChunkBase(st storage.ObjectStorage, ctx *ArtifactContext, r = io.TeeReader(r, hasher) } // save chunk to storage - writtenSize, err := st.Save(storagePath, r, -1) + writtenSize, err := st.Save(storagePath, r, contentSize) if err != nil { return -1, fmt.Errorf("save chunk to storage error: %v", err) } @@ -208,7 +208,7 @@ func mergeChunksForArtifact(ctx *ArtifactContext, chunks []*chunkFileItem, st st // save merged file storagePath := fmt.Sprintf("%d/%d/%d.%s", artifact.RunID%255, artifact.ID%255, time.Now().UnixNano(), extension) - written, err := st.Save(storagePath, mergedReader, -1) + written, err := st.Save(storagePath, mergedReader, artifact.FileCompressedSize) if err != nil { return fmt.Errorf("save merged file error: %v", err) }