diff --git a/replication/binlogsyncer.go b/replication/binlogsyncer.go index 14ee49839..6bdd88faa 100644 --- a/replication/binlogsyncer.go +++ b/replication/binlogsyncer.go @@ -76,6 +76,9 @@ type BinlogSyncerConfig struct { // Use decimal.Decimal structure for decimals. UseDecimal bool + // FloatWithTrailingZero structure for floats. + UseFloatWithTrailingZero bool + // RecvBufferSize sets the size in bytes of the operating system's receive buffer associated with the connection. RecvBufferSize int @@ -197,6 +200,7 @@ func NewBinlogSyncer(cfg BinlogSyncerConfig) *BinlogSyncer { b.parser.SetParseTime(b.cfg.ParseTime) b.parser.SetTimestampStringLocation(b.cfg.TimestampStringLocation) b.parser.SetUseDecimal(b.cfg.UseDecimal) + b.parser.SetUseFloatWithTrailingZero(b.cfg.UseFloatWithTrailingZero) b.parser.SetVerifyChecksum(b.cfg.VerifyChecksum) b.parser.SetRowsEventDecodeFunc(b.cfg.RowsEventDecodeFunc) b.parser.SetTableMapOptionalMetaDecodeFunc(b.cfg.TableMapOptionalMetaDecodeFunc) diff --git a/replication/json_binary.go b/replication/json_binary.go index e4fb7e4e3..4568fc678 100644 --- a/replication/json_binary.go +++ b/replication/json_binary.go @@ -3,6 +3,7 @@ package replication import ( "fmt" "math" + "strconv" "github.com/go-mysql-org/go-mysql/mysql" "github.com/go-mysql-org/go-mysql/utils" @@ -52,6 +53,8 @@ type ( JsonDiffOperation byte ) +type FloatWithTrailingZero float64 + const ( // The JSON value in the given path is replaced with a new value. // @@ -96,6 +99,14 @@ func (jd *JsonDiff) String() string { return fmt.Sprintf("json_diff(op:%s path:%s value:%s)", jd.Op, jd.Path, jd.Value) } +func (f FloatWithTrailingZero) MarshalJSON() ([]byte, error) { + if float64(f) == float64(int(f)) { + return []byte(strconv.FormatFloat(float64(f), 'f', 1, 64)), nil + } + + return []byte(strconv.FormatFloat(float64(f), 'f', -1, 64)), nil +} + func jsonbGetOffsetSize(isSmall bool) int { if isSmall { return jsonbSmallOffsetSize @@ -124,8 +135,9 @@ func jsonbGetValueEntrySize(isSmall bool) int { // the common JSON encoding data. func (e *RowsEvent) decodeJsonBinary(data []byte) ([]byte, error) { d := jsonBinaryDecoder{ - useDecimal: e.useDecimal, - ignoreDecodeErr: e.ignoreJSONDecodeErr, + useDecimal: e.useDecimal, + useFloatWithTrailingZero: e.useFloatWithTrailingZero, + ignoreDecodeErr: e.ignoreJSONDecodeErr, } if d.isDataShort(data, 1) { @@ -141,9 +153,10 @@ func (e *RowsEvent) decodeJsonBinary(data []byte) ([]byte, error) { } type jsonBinaryDecoder struct { - useDecimal bool - ignoreDecodeErr bool - err error + useDecimal bool + useFloatWithTrailingZero bool + ignoreDecodeErr bool + err error } func (d *jsonBinaryDecoder) decodeValue(tp byte, data []byte) interface{} { @@ -175,6 +188,9 @@ func (d *jsonBinaryDecoder) decodeValue(tp byte, data []byte) interface{} { case JSONB_UINT64: return d.decodeUint64(data) case JSONB_DOUBLE: + if d.useFloatWithTrailingZero { + return d.decodeDoubleWithTrailingZero(data) + } return d.decodeDouble(data) case JSONB_STRING: return d.decodeString(data) @@ -395,6 +411,11 @@ func (d *jsonBinaryDecoder) decodeDouble(data []byte) float64 { return v } +func (d *jsonBinaryDecoder) decodeDoubleWithTrailingZero(data []byte) FloatWithTrailingZero { + v := d.decodeDouble(data) + return FloatWithTrailingZero(v) +} + func (d *jsonBinaryDecoder) decodeString(data []byte) string { if d.err != nil { return "" diff --git a/replication/json_binary_test.go b/replication/json_binary_test.go new file mode 100644 index 000000000..01bb12352 --- /dev/null +++ b/replication/json_binary_test.go @@ -0,0 +1,403 @@ +package replication + +import ( + "testing" + + "github.com/goccy/go-json" + "github.com/stretchr/testify/require" +) + +func TestFloatWithTrailingZero_MarshalJSON(t *testing.T) { + tests := []struct { + name string + input FloatWithTrailingZero + expected string + }{ + { + name: "whole number should have .0", + input: FloatWithTrailingZero(5.0), + expected: "5.0", + }, + { + name: "negative whole number should have .0", + input: FloatWithTrailingZero(-3.0), + expected: "-3.0", + }, + { + name: "decimal number should preserve original format", + input: FloatWithTrailingZero(3.14), + expected: "3.14", + }, + { + name: "negative decimal should preserve original format", + input: FloatWithTrailingZero(-2.5), + expected: "-2.5", + }, + { + name: "zero should have .0", + input: FloatWithTrailingZero(0.0), + expected: "0.0", + }, + { + name: "very small decimal", + input: FloatWithTrailingZero(0.001), + expected: "0.001", + }, + { + name: "large whole number", + input: FloatWithTrailingZero(1000000.0), + expected: "1000000.0", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, err := tt.input.MarshalJSON() + require.NoError(t, err) + require.Equal(t, tt.expected, string(result)) + }) + } +} + +func TestRegularFloat64_MarshalJSON_TruncatesTrailingZero(t *testing.T) { + // Test that regular float64 truncates trailing zeros (the default behavior) + // This demonstrates the difference when UseFloatWithTrailingZero is NOT set + tests := []struct { + name string + input float64 + expected string + }{ + { + name: "whole number truncates .0", + input: 5.0, + expected: "5", + }, + { + name: "negative whole number truncates .0", + input: -3.0, + expected: "-3", + }, + { + name: "decimal number preserves decimals", + input: 3.14, + expected: "3.14", + }, + { + name: "negative decimal preserves decimals", + input: -2.5, + expected: "-2.5", + }, + { + name: "zero truncates .0", + input: 0.0, + expected: "0", + }, + { + name: "large whole number truncates .0", + input: 1000000.0, + expected: "1000000", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, err := json.Marshal(tt.input) + require.NoError(t, err) + require.Equal(t, tt.expected, string(result)) + }) + } +} + +func TestFloatWithTrailingZero_vs_RegularFloat64_Comparison(t *testing.T) { + // Direct comparison test showing the key difference + testCases := []struct { + name string + value float64 + withTrailing string // Expected output with FloatWithTrailingZero + withoutTrailing string // Expected output with regular float64 + }{ + { + name: "whole number 5.0", + value: 5.0, + withTrailing: "5.0", + withoutTrailing: "5", + }, + { + name: "zero", + value: 0.0, + withTrailing: "0.0", + withoutTrailing: "0", + }, + { + name: "negative whole number", + value: -42.0, + withTrailing: "-42.0", + withoutTrailing: "-42", + }, + { + name: "decimal number (no difference)", + value: 3.14, + withTrailing: "3.14", + withoutTrailing: "3.14", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // Test FloatWithTrailingZero + trailingResult, err := json.Marshal(FloatWithTrailingZero(tc.value)) + require.NoError(t, err) + require.Equal(t, tc.withTrailing, string(trailingResult)) + + // Test regular float64 + regularResult, err := json.Marshal(tc.value) + require.NoError(t, err) + require.Equal(t, tc.withoutTrailing, string(regularResult)) + + // Verify they're different for whole numbers + if tc.value == float64(int(tc.value)) { + require.NotEqual(t, string(trailingResult), string(regularResult), + "FloatWithTrailingZero and regular float64 should produce different output for whole numbers") + } + }) + } +} + +func TestJsonBinaryDecoder_decodeDoubleWithTrailingZero(t *testing.T) { + // Test the decodeDoubleWithTrailingZero method directly + decoder := &jsonBinaryDecoder{ + useFloatWithTrailingZero: true, + } + + // Test data representing float64 in binary format (little endian) + // 5.0 as IEEE 754 double precision: 0x4014000000000000 + testData := []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x14, 0x40} + + result := decoder.decodeDoubleWithTrailingZero(testData) + require.NoError(t, decoder.err) + + // Verify the result is FloatWithTrailingZero type + require.IsType(t, FloatWithTrailingZero(0), result) + require.Equal(t, FloatWithTrailingZero(5.0), result) + + // Test JSON marshaling + jsonBytes, err := json.Marshal(result) + require.NoError(t, err) + require.Equal(t, "5.0", string(jsonBytes)) +} + +func TestJsonBinaryDecoder_decodeValue_JSONB_DOUBLE(t *testing.T) { + tests := []struct { + name string + useFloatWithTrailingZero bool + expectedType interface{} + expectedJSONString string + }{ + { + name: "with trailing zero enabled", + useFloatWithTrailingZero: true, + expectedType: FloatWithTrailingZero(0), + expectedJSONString: "5.0", + }, + { + name: "with trailing zero disabled", + useFloatWithTrailingZero: false, + expectedType: float64(0), + expectedJSONString: "5", + }, + } + + // 5.0 as IEEE 754 double precision in little endian + testData := []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x14, 0x40} + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + decoder := &jsonBinaryDecoder{ + useFloatWithTrailingZero: tt.useFloatWithTrailingZero, + } + + result := decoder.decodeValue(JSONB_DOUBLE, testData) + require.NoError(t, decoder.err) + require.IsType(t, tt.expectedType, result) + + // Test JSON marshaling + jsonBytes, err := json.Marshal(result) + require.NoError(t, err) + require.Equal(t, tt.expectedJSONString, string(jsonBytes)) + }) + } +} + +func TestRowsEvent_decodeJsonBinary_WithFloatTrailingZero(t *testing.T) { + // Create a sample JSON binary data representing {"value": 5.0} + // This is a simplified test - in practice the binary format would be more complex + rowsEvent := &RowsEvent{ + useFloatWithTrailingZero: true, + } + + // Mock a simple JSON binary that would contain a double value + // In a real scenario, this would come from actual MySQL binlog data + // For this test, we'll create a minimal valid structure + + // This test would need actual MySQL JSON binary data to be fully functional + // For now, we'll test the decoding path exists and the option is respected + decoder := &jsonBinaryDecoder{ + useFloatWithTrailingZero: rowsEvent.useFloatWithTrailingZero, + } + + require.True(t, decoder.useFloatWithTrailingZero) +} + +func TestBinlogParser_SetUseFloatWithTrailingZero(t *testing.T) { + parser := NewBinlogParser() + + // Test default value + require.False(t, parser.useFloatWithTrailingZero) + + // Test setting to true + parser.SetUseFloatWithTrailingZero(true) + require.True(t, parser.useFloatWithTrailingZero) + + // Test setting to false + parser.SetUseFloatWithTrailingZero(false) + require.False(t, parser.useFloatWithTrailingZero) +} + +func TestBinlogSyncerConfig_UseFloatWithTrailingZero(t *testing.T) { + cfg := BinlogSyncerConfig{ + UseFloatWithTrailingZero: true, + } + + require.True(t, cfg.UseFloatWithTrailingZero) +} + +func TestFloatWithTrailingZero_EdgeCases(t *testing.T) { + tests := []struct { + name string + input FloatWithTrailingZero + expected string + }{ + { + name: "very large whole number", + input: FloatWithTrailingZero(1e15), + expected: "1000000000000000.0", + }, + { + name: "very small positive number", + input: FloatWithTrailingZero(1e-10), + expected: "0.0000000001", + }, + { + name: "scientific notation threshold", + input: FloatWithTrailingZero(1e6), + expected: "1000000.0", + }, + { + name: "negative zero", + input: FloatWithTrailingZero(-0.0), + expected: "0.0", + }, + { + name: "number that looks whole but has tiny fractional part", + input: FloatWithTrailingZero(5.0000000000000001), // This might be rounded to 5.0 due to float64 precision + expected: "5.0", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, err := tt.input.MarshalJSON() + require.NoError(t, err) + require.Equal(t, tt.expected, string(result)) + }) + } +} + +func TestFloatWithTrailingZero_Integration(t *testing.T) { + // Test that demonstrates the full flow with a sample JSON structure + type JSONData struct { + Price FloatWithTrailingZero `json:"price"` + Quantity FloatWithTrailingZero `json:"quantity"` + Total FloatWithTrailingZero `json:"total"` + } + + data := JSONData{ + Price: FloatWithTrailingZero(10.0), // Should become "10.0" + Quantity: FloatWithTrailingZero(2.5), // Should stay "2.5" + Total: FloatWithTrailingZero(25.0), // Should become "25.0" + } + + jsonBytes, err := json.Marshal(data) + require.NoError(t, err) + + expectedJSON := `{"price":10.0,"quantity":2.5,"total":25.0}` + require.Equal(t, expectedJSON, string(jsonBytes)) + + // Verify that regular float64 would behave differently + type RegularJSONData struct { + Price float64 `json:"price"` + Quantity float64 `json:"quantity"` + Total float64 `json:"total"` + } + + regularData := RegularJSONData{ + Price: 10.0, + Quantity: 2.5, + Total: 25.0, + } + + regularJSONBytes, err := json.Marshal(regularData) + require.NoError(t, err) + + regularExpectedJSON := `{"price":10,"quantity":2.5,"total":25}` + require.Equal(t, regularExpectedJSON, string(regularJSONBytes)) + + // Demonstrate the difference + require.NotEqual(t, string(jsonBytes), string(regularJSONBytes)) +} + +func TestRowsEvent_UseFloatWithTrailingZero_Integration(t *testing.T) { + // Test that RowsEvent properly propagates the useFloatWithTrailingZero setting + + // Create table map event (similar to existing tests in replication_test.go) + tableMapEventData := []byte("m\x00\x00\x00\x00\x00\x01\x00\x04test\x00\x03t10\x00\x02\xf5\xf6\x03\x04\n\x00\x03") + + tableMapEvent := new(TableMapEvent) + tableMapEvent.tableIDSize = 6 + err := tableMapEvent.Decode(tableMapEventData) + require.NoError(t, err) + + // Test with useFloatWithTrailingZero enabled + rowsWithTrailingZero := &RowsEvent{ + tableIDSize: 6, + tables: make(map[uint64]*TableMapEvent), + Version: 2, + useFloatWithTrailingZero: true, + } + rowsWithTrailingZero.tables[tableMapEvent.TableID] = tableMapEvent + + // Test with useFloatWithTrailingZero disabled + rowsWithoutTrailingZero := &RowsEvent{ + tableIDSize: 6, + tables: make(map[uint64]*TableMapEvent), + Version: 2, + useFloatWithTrailingZero: false, + } + rowsWithoutTrailingZero.tables[tableMapEvent.TableID] = tableMapEvent + + // Verify that the setting is properly stored + require.True(t, rowsWithTrailingZero.useFloatWithTrailingZero) + require.False(t, rowsWithoutTrailingZero.useFloatWithTrailingZero) + + // Test the decoder creation with the setting + decoderWithTrailing := &jsonBinaryDecoder{ + useFloatWithTrailingZero: rowsWithTrailingZero.useFloatWithTrailingZero, + } + + decoderWithoutTrailing := &jsonBinaryDecoder{ + useFloatWithTrailingZero: rowsWithoutTrailingZero.useFloatWithTrailingZero, + } + + require.True(t, decoderWithTrailing.useFloatWithTrailingZero) + require.False(t, decoderWithoutTrailing.useFloatWithTrailingZero) +} diff --git a/replication/parser.go b/replication/parser.go index f66345588..8d235d9ae 100644 --- a/replication/parser.go +++ b/replication/parser.go @@ -35,9 +35,10 @@ type BinlogParser struct { // used to start/stop processing stopProcessing uint32 - useDecimal bool - ignoreJSONDecodeErr bool - verifyChecksum bool + useDecimal bool + useFloatWithTrailingZero bool + ignoreJSONDecodeErr bool + verifyChecksum bool rowsEventDecodeFunc func(*RowsEvent, []byte) error @@ -198,6 +199,10 @@ func (p *BinlogParser) SetUseDecimal(useDecimal bool) { p.useDecimal = useDecimal } +func (p *BinlogParser) SetUseFloatWithTrailingZero(useFloatWithTrailingZero bool) { + p.useFloatWithTrailingZero = useFloatWithTrailingZero +} + func (p *BinlogParser) SetIgnoreJSONDecodeError(ignoreJSONDecodeErr bool) { p.ignoreJSONDecodeErr = ignoreJSONDecodeErr } @@ -406,6 +411,7 @@ func (p *BinlogParser) newRowsEvent(h *EventHeader) *RowsEvent { e.parseTime = p.parseTime e.timestampStringLocation = p.timestampStringLocation e.useDecimal = p.useDecimal + e.useFloatWithTrailingZero = p.useFloatWithTrailingZero e.ignoreJSONDecodeErr = p.ignoreJSONDecodeErr switch h.EventType { diff --git a/replication/row_event.go b/replication/row_event.go index 6fc943dd8..a83a73ca5 100644 --- a/replication/row_event.go +++ b/replication/row_event.go @@ -945,10 +945,11 @@ type RowsEvent struct { Rows [][]interface{} SkippedColumns [][]int - parseTime bool - timestampStringLocation *time.Location - useDecimal bool - ignoreJSONDecodeErr bool + parseTime bool + timestampStringLocation *time.Location + useDecimal bool + useFloatWithTrailingZero bool + ignoreJSONDecodeErr bool } // EnumRowsEventType is an abridged type describing the operation which triggered the given RowsEvent.