From a89612d289cda056b4f98a352f9255c0817fbc01 Mon Sep 17 00:00:00 2001 From: mikereiche Date: Mon, 8 Apr 2024 16:21:54 -0700 Subject: [PATCH] Store BigDecimal and BigInteger as Numbers like the Java SDK does. Also be able to read BigDecimal and BigInteger that were written as String. Also does not lose precision by converting BigDecimal to double in the transcoder. Closes #1611. --- .../core/convert/OtherConverters.java | 45 ++++------------ .../JacksonTranslationService.java | 2 +- .../core/mapping/CouchbaseSimpleTypes.java | 2 +- .../MappingCouchbaseConverterTests.java | 16 +++--- .../data/couchbase/domain/BigAirline.java | 51 +++++++++++++++++++ .../domain/BigAirlineRepository.java | 38 ++++++++++++++ ...aseRepositoryKeyValueIntegrationTests.java | 19 ++++--- ...sitoryQueryCollectionIntegrationTests.java | 20 +++++++- ...aseRepositoryQuerydslIntegrationTests.java | 7 --- 9 files changed, 142 insertions(+), 58 deletions(-) create mode 100644 src/test/java/org/springframework/data/couchbase/domain/BigAirline.java create mode 100644 src/test/java/org/springframework/data/couchbase/domain/BigAirlineRepository.java diff --git a/src/main/java/org/springframework/data/couchbase/core/convert/OtherConverters.java b/src/main/java/org/springframework/data/couchbase/core/convert/OtherConverters.java index 41eee7189..f0538eefa 100644 --- a/src/main/java/org/springframework/data/couchbase/core/convert/OtherConverters.java +++ b/src/main/java/org/springframework/data/couchbase/core/convert/OtherConverters.java @@ -24,30 +24,27 @@ import java.nio.charset.StandardCharsets; import java.time.YearMonth; import java.util.ArrayList; +import java.util.Base64; import java.util.Collection; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.UUID; -import com.couchbase.client.java.json.JsonArray; -import com.couchbase.client.java.json.JsonObject; -import com.couchbase.client.java.json.JsonValueModule; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.JsonNodeFactory; -import com.fasterxml.jackson.databind.node.ObjectNode; import org.springframework.core.convert.converter.Converter; import org.springframework.data.convert.ReadingConverter; import org.springframework.data.convert.WritingConverter; import org.springframework.data.couchbase.core.mapping.CouchbaseDocument; import org.springframework.data.couchbase.core.mapping.CouchbaseList; -import org.springframework.util.Base64Utils; import com.couchbase.client.core.encryption.CryptoManager; +import com.couchbase.client.java.json.JsonArray; +import com.couchbase.client.java.json.JsonObject; +import com.couchbase.client.java.json.JsonValueModule; import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; /** @@ -65,13 +62,11 @@ private OtherConverters() {} * @return the list of converters to register. */ public static Collection> getConvertersToRegister() { - List> converters = new ArrayList>(); + List> converters = new ArrayList<>(); converters.add(UuidToString.INSTANCE); converters.add(StringToUuid.INSTANCE); - converters.add(BigIntegerToString.INSTANCE); converters.add(StringToBigInteger.INSTANCE); - converters.add(BigDecimalToString.INSTANCE); converters.add(StringToBigDecimal.INSTANCE); converters.add(ByteArrayToString.INSTANCE); converters.add(StringToByteArray.INSTANCE); @@ -114,16 +109,7 @@ public UUID convert(String source) { } } - @WritingConverter - public enum BigIntegerToString implements Converter { - INSTANCE; - - @Override - public String convert(BigInteger source) { - return source == null ? null : source.toString(); - } - } - + // to support reading BigIntegers that were written as Strings (now discontinued) @ReadingConverter public enum StringToBigInteger implements Converter { INSTANCE; @@ -134,16 +120,7 @@ public BigInteger convert(String source) { } } - @WritingConverter - public enum BigDecimalToString implements Converter { - INSTANCE; - - @Override - public String convert(BigDecimal source) { - return source == null ? null : source.toString(); - } - } - + // to support reading BigDecimals that were written as Strings (now discontinued) @ReadingConverter public enum StringToBigDecimal implements Converter { INSTANCE; @@ -160,7 +137,7 @@ public enum ByteArrayToString implements Converter { @Override public String convert(byte[] source) { - return source == null ? null : Base64Utils.encodeToString(source); + return source == null ? null : Base64.getEncoder().encodeToString(source); } } @@ -170,7 +147,7 @@ public enum StringToByteArray implements Converter { @Override public byte[] convert(String source) { - return source == null ? null : Base64Utils.decode(source.getBytes(StandardCharsets.UTF_8)); + return source == null ? null : Base64.getDecoder().decode(source.getBytes(StandardCharsets.UTF_8)); } } diff --git a/src/main/java/org/springframework/data/couchbase/core/convert/translation/JacksonTranslationService.java b/src/main/java/org/springframework/data/couchbase/core/convert/translation/JacksonTranslationService.java index cbde49c0e..3c2cbbbc6 100644 --- a/src/main/java/org/springframework/data/couchbase/core/convert/translation/JacksonTranslationService.java +++ b/src/main/java/org/springframework/data/couchbase/core/convert/translation/JacksonTranslationService.java @@ -220,7 +220,7 @@ private Object decodePrimitive(final JsonToken token, final JsonParser parser) t case VALUE_NUMBER_INT: return parser.getNumberValue(); case VALUE_NUMBER_FLOAT: - return parser.getDoubleValue(); + return parser.getDecimalValue(); case VALUE_NULL: return null; default: diff --git a/src/main/java/org/springframework/data/couchbase/core/mapping/CouchbaseSimpleTypes.java b/src/main/java/org/springframework/data/couchbase/core/mapping/CouchbaseSimpleTypes.java index c6bbc4cc2..eeecda742 100644 --- a/src/main/java/org/springframework/data/couchbase/core/mapping/CouchbaseSimpleTypes.java +++ b/src/main/java/org/springframework/data/couchbase/core/mapping/CouchbaseSimpleTypes.java @@ -38,7 +38,7 @@ public abstract class CouchbaseSimpleTypes { Stream.of(JsonObject.class, JsonArray.class, Number.class).collect(toSet()), true); public static final SimpleTypeHolder DOCUMENT_TYPES = new SimpleTypeHolder( - Stream.of(CouchbaseDocument.class, CouchbaseList.class).collect(toSet()), true); + Stream.of(CouchbaseDocument.class, CouchbaseList.class, Number.class).collect(toSet()), true); private CouchbaseSimpleTypes() {} diff --git a/src/test/java/org/springframework/data/couchbase/core/mapping/MappingCouchbaseConverterTests.java b/src/test/java/org/springframework/data/couchbase/core/mapping/MappingCouchbaseConverterTests.java index 684416e74..ae096e5b1 100644 --- a/src/test/java/org/springframework/data/couchbase/core/mapping/MappingCouchbaseConverterTests.java +++ b/src/test/java/org/springframework/data/couchbase/core/mapping/MappingCouchbaseConverterTests.java @@ -200,12 +200,12 @@ void readsNoTypeAlias() { @Test void writesBigInteger() { CouchbaseDocument converted = new CouchbaseDocument(); - BigIntegerEntity entity = new BigIntegerEntity(new BigInteger("12345")); + BigIntegerEntity entity = new BigIntegerEntity(new BigInteger("12345678901234567890123")); converter.write(entity, converted); Map result = converted.export(); assertThat(result.get("_class")).isEqualTo(entity.getClass().getName()); - assertThat(result.get("attr0")).isEqualTo(entity.attr0.toString()); + assertThat(result.get("attr0")).isEqualTo(entity.attr0); assertThat(converted.getId()).isEqualTo(BaseEntity.ID); } @@ -213,21 +213,21 @@ void writesBigInteger() { void readsBigInteger() { CouchbaseDocument source = new CouchbaseDocument(); source.put("_class", BigIntegerEntity.class.getName()); - source.put("attr0", "12345"); + source.put("attr0", new BigInteger("12345678901234567890123")); BigIntegerEntity converted = converter.read(BigIntegerEntity.class, source); - assertThat(converted.attr0).isEqualTo(new BigInteger((String) source.get("attr0"))); + assertThat(converted.attr0).isEqualTo(source.get("attr0")); } @Test void writesBigDecimal() { CouchbaseDocument converted = new CouchbaseDocument(); - BigDecimalEntity entity = new BigDecimalEntity(new BigDecimal("123.45")); + BigDecimalEntity entity = new BigDecimalEntity(new BigDecimal("12345678901234567890123.45")); converter.write(entity, converted); Map result = converted.export(); assertThat(result.get("_class")).isEqualTo(entity.getClass().getName()); - assertThat(result.get("attr0")).isEqualTo(entity.attr0.toString()); + assertThat(result.get("attr0")).isEqualTo(entity.attr0); assertThat(converted.getId()).isEqualTo(BaseEntity.ID); } @@ -235,10 +235,10 @@ void writesBigDecimal() { void readsBigDecimal() { CouchbaseDocument source = new CouchbaseDocument(); source.put("_class", BigDecimalEntity.class.getName()); - source.put("attr0", "123.45"); + source.put("attr0", new BigDecimal("12345678901234567890123.45")); BigDecimalEntity converted = converter.read(BigDecimalEntity.class, source); - assertThat(converted.attr0).isEqualTo(new BigDecimal((String) source.get("attr0"))); + assertThat(converted.attr0).isEqualTo(source.get("attr0")); } @Test diff --git a/src/test/java/org/springframework/data/couchbase/domain/BigAirline.java b/src/test/java/org/springframework/data/couchbase/domain/BigAirline.java new file mode 100644 index 000000000..8272b765e --- /dev/null +++ b/src/test/java/org/springframework/data/couchbase/domain/BigAirline.java @@ -0,0 +1,51 @@ +/* + * Copyright 2012-2024 the original author or authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.data.couchbase.domain; + +import java.math.BigDecimal; +import java.math.BigInteger; + +import org.springframework.data.annotation.PersistenceConstructor; +import org.springframework.data.couchbase.core.mapping.Document; + +@Document +/** + * @author Michael Reiche + */ +public class BigAirline extends Airline { + BigInteger airlineNumber = new BigInteger("88881234567890123456"); // less than 63 bits, otherwise query truncates + BigDecimal airlineDecimal = new BigDecimal("888812345678901.23"); // less than 53 bits in mantissa + + @PersistenceConstructor + public BigAirline(String id, String name, String hqCountry, Number airlineNumber, Number airlineDecimal) { + super(id, name, hqCountry); + this.airlineNumber = airlineNumber != null && !airlineNumber.equals("") + ? new BigInteger(airlineNumber.toString()) + : this.airlineNumber; + this.airlineDecimal = airlineDecimal != null && !airlineDecimal.equals("") + ? new BigDecimal(airlineDecimal.toString()) + : this.airlineDecimal; + } + + public BigInteger getAirlineNumber() { + return airlineNumber; + } + + public BigDecimal getAirlineDecimal() { + return airlineDecimal; + } + +} diff --git a/src/test/java/org/springframework/data/couchbase/domain/BigAirlineRepository.java b/src/test/java/org/springframework/data/couchbase/domain/BigAirlineRepository.java new file mode 100644 index 000000000..e4167fdc1 --- /dev/null +++ b/src/test/java/org/springframework/data/couchbase/domain/BigAirlineRepository.java @@ -0,0 +1,38 @@ +/* + * Copyright 2017-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.data.couchbase.domain; + +import java.util.List; + +import org.springframework.data.couchbase.repository.CouchbaseRepository; +import org.springframework.data.couchbase.repository.DynamicProxyable; +import org.springframework.data.couchbase.repository.Query; +import org.springframework.data.querydsl.QuerydslPredicateExecutor; +import org.springframework.data.repository.query.Param; +import org.springframework.stereotype.Repository; + +/** + * @author Michael Reiche + */ +@Repository +public interface BigAirlineRepository extends CouchbaseRepository, + QuerydslPredicateExecutor, DynamicProxyable { + + @Query("#{#n1ql.selectEntity} where #{#n1ql.filter} and (name = $1)") + List getByName(@Param("airline_name") String airlineName); + +} diff --git a/src/test/java/org/springframework/data/couchbase/repository/CouchbaseRepositoryKeyValueIntegrationTests.java b/src/test/java/org/springframework/data/couchbase/repository/CouchbaseRepositoryKeyValueIntegrationTests.java index 29e427db5..ff3fafc0f 100644 --- a/src/test/java/org/springframework/data/couchbase/repository/CouchbaseRepositoryKeyValueIntegrationTests.java +++ b/src/test/java/org/springframework/data/couchbase/repository/CouchbaseRepositoryKeyValueIntegrationTests.java @@ -33,15 +33,14 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Configuration; import org.springframework.dao.DuplicateKeyException; import org.springframework.dao.OptimisticLockingFailureException; -import org.springframework.data.couchbase.config.AbstractCouchbaseConfiguration; import org.springframework.data.couchbase.core.CouchbaseTemplate; import org.springframework.data.couchbase.domain.Airline; import org.springframework.data.couchbase.domain.AirlineRepository; -import org.springframework.data.couchbase.domain.Course; +import org.springframework.data.couchbase.domain.BigAirline; import org.springframework.data.couchbase.domain.Config; +import org.springframework.data.couchbase.domain.Course; import org.springframework.data.couchbase.domain.Library; import org.springframework.data.couchbase.domain.LibraryRepository; import org.springframework.data.couchbase.domain.PersonValue; @@ -59,9 +58,6 @@ import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.junit.jupiter.SpringJUnitConfig; -import com.couchbase.client.core.deps.io.netty.handler.ssl.util.InsecureTrustManagerFactory; -import com.couchbase.client.core.env.SecurityConfig; -import com.couchbase.client.java.env.ClusterEnvironment; import com.couchbase.client.java.kv.GetResult; /** @@ -125,6 +121,17 @@ void saveReplaceUpsertInsert() { airlineRepository.delete(airline); } + @Test + @IgnoreWhen(clusterTypes = ClusterType.MOCKED) + void saveBig() { + BigAirline airline = new BigAirline(UUID.randomUUID().toString(), "MyAirline", null, null, null); + airline = airlineRepository.save(airline); + Optional foundMaybe = airlineRepository.findById(airline.getId()); + BigAirline found = (BigAirline) foundMaybe.get(); + assertEquals(found, airline); + airlineRepository.delete(airline); + } + @Test @IgnoreWhen(clusterTypes = ClusterType.MOCKED) void saveAndFindById() { diff --git a/src/test/java/org/springframework/data/couchbase/repository/query/CouchbaseRepositoryQueryCollectionIntegrationTests.java b/src/test/java/org/springframework/data/couchbase/repository/query/CouchbaseRepositoryQueryCollectionIntegrationTests.java index 56ba60dda..a04523b1c 100644 --- a/src/test/java/org/springframework/data/couchbase/repository/query/CouchbaseRepositoryQueryCollectionIntegrationTests.java +++ b/src/test/java/org/springframework/data/couchbase/repository/query/CouchbaseRepositoryQueryCollectionIntegrationTests.java @@ -28,18 +28,21 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Configuration; import org.springframework.dao.DataRetrievalFailureException; import org.springframework.data.couchbase.core.CouchbaseTemplate; import org.springframework.data.couchbase.core.ReactiveCouchbaseTemplate; import org.springframework.data.couchbase.core.RemoveResult; import org.springframework.data.couchbase.domain.Address; import org.springframework.data.couchbase.domain.AddressAnnotated; +import org.springframework.data.couchbase.domain.Airline; import org.springframework.data.couchbase.domain.Airport; import org.springframework.data.couchbase.domain.AirportRepository; import org.springframework.data.couchbase.domain.AirportRepositoryAnnotated; +import org.springframework.data.couchbase.domain.BigAirline; +import org.springframework.data.couchbase.domain.BigAirlineRepository; import org.springframework.data.couchbase.domain.ConfigScoped; import org.springframework.data.couchbase.domain.User; import org.springframework.data.couchbase.domain.UserCol; @@ -72,6 +75,7 @@ public class CouchbaseRepositoryQueryCollectionIntegrationTests extends Collecti @Autowired AirportRepositoryAnnotated airportRepositoryAnnotated; @Autowired AirportRepository airportRepository; + @Autowired BigAirlineRepository bigAirlineRepository; @Autowired UserColRepository userColRepository; @Autowired UserSubmissionAnnotatedRepository userSubmissionAnnotatedRepository; @Autowired UserSubmissionUnannotatedRepository userSubmissionUnannotatedRepository; @@ -103,6 +107,7 @@ public void beforeEach() { couchbaseTemplate.removeByQuery(User.class).inCollection(collectionName).all(); couchbaseTemplate.removeByQuery(UserCol.class).inScope(otherScope).inCollection(otherCollection).all(); couchbaseTemplate.removeByQuery(Airport.class).inCollection(collectionName).all(); + couchbaseTemplate.removeByQuery(BigAirline.class).inCollection(collectionName).all(); couchbaseTemplate.removeByQuery(Airport.class).inCollection(collectionName2).all(); couchbaseTemplate.findByQuery(Airport.class).withConsistency(REQUEST_PLUS).inCollection(collectionName).all(); } @@ -126,6 +131,19 @@ void findByKey() { userColRepository.delete(found); } + @Test + @Disabled // BigInteger and BigDecimal lose precision through Query + @IgnoreWhen(clusterTypes = ClusterType.MOCKED) + void saveBig() { + BigAirline airline = new BigAirline(UUID.randomUUID().toString(), "MyAirline", null, null, null); + airline = bigAirlineRepository.withCollection(collectionName).save(airline); + List foundMaybe = bigAirlineRepository.withCollection(collectionName) + .withOptions(QueryOptions.queryOptions().scanConsistency(REQUEST_PLUS)).getByName("MyAirline"); + BigAirline found = (BigAirline) foundMaybe.get(0); + assertEquals(found, airline); + bigAirlineRepository.withCollection(collectionName).delete(airline); + } + @Test public void myTest() { diff --git a/src/test/java/org/springframework/data/couchbase/repository/query/CouchbaseRepositoryQuerydslIntegrationTests.java b/src/test/java/org/springframework/data/couchbase/repository/query/CouchbaseRepositoryQuerydslIntegrationTests.java index 99286dd0e..95d0cd912 100644 --- a/src/test/java/org/springframework/data/couchbase/repository/query/CouchbaseRepositoryQuerydslIntegrationTests.java +++ b/src/test/java/org/springframework/data/couchbase/repository/query/CouchbaseRepositoryQuerydslIntegrationTests.java @@ -39,16 +39,12 @@ import org.springframework.context.annotation.AnnotationConfigApplicationContext; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.data.auditing.DateTimeProvider; -import org.springframework.data.couchbase.config.AbstractCouchbaseConfiguration; import org.springframework.data.couchbase.core.CouchbaseTemplate; import org.springframework.data.couchbase.core.mapping.event.ValidatingCouchbaseEventListener; import org.springframework.data.couchbase.core.query.QueryCriteriaDefinition; import org.springframework.data.couchbase.domain.Airline; import org.springframework.data.couchbase.domain.AirlineRepository; -import org.springframework.data.couchbase.domain.NaiveAuditorAware; import org.springframework.data.couchbase.domain.QAirline; -import org.springframework.data.couchbase.domain.time.AuditingDateTimeProvider; import org.springframework.data.couchbase.repository.auditing.EnableCouchbaseAuditing; import org.springframework.data.couchbase.repository.auditing.EnableReactiveCouchbaseAuditing; import org.springframework.data.couchbase.repository.config.EnableCouchbaseRepositories; @@ -63,9 +59,6 @@ import org.springframework.test.context.junit.jupiter.SpringJUnitConfig; import org.springframework.validation.beanvalidation.LocalValidatorFactoryBean; -import com.couchbase.client.core.deps.io.netty.handler.ssl.util.InsecureTrustManagerFactory; -import com.couchbase.client.core.env.SecurityConfig; -import com.couchbase.client.java.env.ClusterEnvironment; import com.querydsl.core.types.Predicate; import com.querydsl.core.types.dsl.BooleanExpression;