fix ADDON-18851: partial log lines: When sending lines longer than 16k, the first 16k gets send and then "the rest" up to the SPLUNK_LOGGING_DRIVER_TEMP_MESSAGES_BUFFER_SIZE

Author: rxie

config.json

@@ -49,7 +49,7 @@
 		{
 			"name": "SPLUNK_LOGGING_DRIVER_TEMP_MESSAGES_HOLD_DURATION",
 			"description": "Used when logs that are chunked by docker with 16kb limit. Set how long the system can wait for the next message to come.",
-			"value": "100ms",
+			"value": "5s",
 			"settable": ["value"]
 		},
 		{

message_processor.go

@@ -80,6 +80,10 @@ func (mg messageProcessor) consumeLog(lf *logPair) {
 		curRetryNumber = 0
 
 		if mg.shouldSendMessage(buf.Line) {
+			if tmpBuf.tBuf.Len() == 0 {
+				logrus.Debug("First messaging, reseting timer")
+				tmpBuf.bufferTimer = time.Now()
+			}
 			// Append to temp buffer
 			if err := tmpBuf.append(&buf); err == nil {
 				// Send message to splunk and json logger

partial_message_buffer.go

@@ -52,15 +52,19 @@ func (b *partialMsgBuffer) reset() {
 	if b.bufferReset {
 		b.tBuf.Reset()
 		b.bufferTimer = time.Now()
+		logrus.WithField("resetBufferTimer", b.bufferTimer).Debug("resetting buffer Timer")
 	}
 }
 
 func (b *partialMsgBuffer) hasHoldDurationExpired(t time.Time) bool {
 	diff := t.Sub(b.bufferTimer)
+	logrus.WithField("currentTime", t).WithField("bufferTime", b.bufferTimer).WithField("diff", diff).Debug("Timeout settings")
+	logrus.WithField("partialMsgBufferHoldDuration", partialMsgBufferHoldDuration).WithField("hasHoldDurationExpired", diff > partialMsgBufferHoldDuration).Debug("check timeout")
 	return diff > partialMsgBufferHoldDuration
 }
 
 func (b *partialMsgBuffer) hasLengthExceeded() bool {
+	logrus.WithField("buffer size limit exceeded", partialMsgBufferMaximum < b.tBuf.Len()).Debug("check size")
 	return partialMsgBufferMaximum < b.tBuf.Len()
 }
 

splunk_logger.go

@@ -69,7 +69,7 @@ const (
 	// Number of messages allowed to be queued in the channel
 	defaultStreamChannelSize = 4 * defaultPostMessagesBatchSize
 	// Partial log hold duration (if we are not reaching max buffer size)
-	defaultPartialMsgBufferHoldDuration = 100 * time.Millisecond
+	defaultPartialMsgBufferHoldDuration = 5 * time.Second
 	// Maximum buffer size for partial logging
 	defaultPartialMsgBufferMaximum = 1024 * 1024
 	// Number of retry if error happens while reading logs from docker provided fifo