FLE implementation with property value converter. (#1554)

* FLE Implemenation with Property Value Converter.

Author: mikereiche

spring-data-couchbase/pom.xml

@@ -1,5 +1,6 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
 
     <modelVersion>4.0.0</modelVersion>
 
@@ -32,6 +33,8 @@
         <com.couchbase.mock>1.5.25</com.couchbase.mock>
         <com.squareup.okhttp3>4.8.1</com.squareup.okhttp3>
         <org.awaitility>4.0.3</org.awaitility>
+        <jackson-joda>2.13.4</jackson-joda>
+        <couchbase.encryption>3.1.0</couchbase.encryption>
     </properties>
 
     <dependencyManagement>
@@ -115,6 +118,13 @@
             <optional>true</optional>
         </dependency>
 
+        <dependency>
+            <groupId>com.fasterxml.jackson.datatype</groupId>
+            <artifactId>jackson-datatype-joda</artifactId>
+            <version>${jackson-joda}</version>
+            <scope>test</scope>
+        </dependency>
+
         <dependency>
             <groupId>org.testcontainers</groupId>
             <artifactId>testcontainers</artifactId>
@@ -217,6 +227,13 @@
             <scope>test</scope>
         </dependency>
 
+        <dependency>
+            <groupId>com.couchbase.client</groupId>
+            <artifactId>couchbase-encryption</artifactId>
+            <version>${couchbase.encryption}</version>
+            <scope>test</scope>
+        </dependency>
+
     </dependencies>
 
     <build>
@@ -279,7 +296,9 @@
                         </goals>
                         <configuration>
                             <outputDirectory>target/generated-test-sources</outputDirectory>
-                            <processor>org.springframework.data.couchbase.repository.support.CouchbaseAnnotationProcessor</processor>
+                            <processor>
+                                org.springframework.data.couchbase.repository.support.CouchbaseAnnotationProcessor
+                            </processor>
                         </configuration>
                     </execution>
                 </executions>

spring-data-couchbase/src/main/java/org/springframework/data/couchbase/config/AbstractCouchbaseConfiguration.java

@@ -18,22 +18,28 @@
 
 import static com.couchbase.client.java.ClusterOptions.clusterOptions;
 
-import java.util.Collections;
+import java.util.ArrayList;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Set;
 
 import org.springframework.beans.factory.config.BeanDefinition;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.ClassPathScanningCandidateComponentProvider;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.Role;
+import org.springframework.core.convert.converter.GenericConverter;
 import org.springframework.core.type.filter.AnnotationTypeFilter;
 import org.springframework.data.convert.CustomConversions;
+import org.springframework.data.convert.PropertyValueConverterRegistrar;
+import org.springframework.data.convert.SimplePropertyValueConversions;
 import org.springframework.data.couchbase.CouchbaseClientFactory;
 import org.springframework.data.couchbase.SimpleCouchbaseClientFactory;
 import org.springframework.data.couchbase.core.CouchbaseTemplate;
 import org.springframework.data.couchbase.core.ReactiveCouchbaseTemplate;
 import org.springframework.data.couchbase.core.convert.CouchbaseCustomConversions;
+import org.springframework.data.couchbase.core.convert.CouchbasePropertyValueConverterFactory;
+import org.springframework.data.couchbase.core.convert.CryptoConverter;
 import org.springframework.data.couchbase.core.convert.MappingCouchbaseConverter;
 import org.springframework.data.couchbase.core.convert.translation.JacksonTranslationService;
 import org.springframework.data.couchbase.core.convert.translation.TranslationService;
@@ -149,7 +155,9 @@ public ClusterEnvironment couchbaseClusterEnvironment() {
 		if (!nonShadowedJacksonPresent()) {
 			throw new CouchbaseException("non-shadowed Jackson not present");
 		}
-		builder.jsonSerializer(JacksonJsonSerializer.create(couchbaseObjectMapper()));
+		CryptoManager cryptoManager = cryptoManager();
+		builder.jsonSerializer(JacksonJsonSerializer.create(couchbaseObjectMapper(cryptoManager)));
+		builder.cryptoManager(cryptoManager);
 		configureEnvironment(builder);
 		return builder.build();
 	}
@@ -160,7 +168,6 @@ public ClusterEnvironment couchbaseClusterEnvironment() {
 	 * @param builder the builder that can be customized.
 	 */
 	protected void configureEnvironment(final ClusterEnvironment.Builder builder) {
-
 	}
 
 	@Bean(name = BeanNames.COUCHBASE_TEMPLATE)
@@ -269,6 +276,7 @@ public MappingCouchbaseConverter mappingCouchbaseConverter(CouchbaseMappingConte
 			CouchbaseCustomConversions couchbaseCustomConversions) {
 		MappingCouchbaseConverter converter = new MappingCouchbaseConverter(couchbaseMappingContext, typeKey());
 		converter.setCustomConversions(couchbaseCustomConversions);
+		couchbaseMappingContext.setSimpleTypeHolder(couchbaseCustomConversions.getSimpleTypeHolder());
 		return converter;
 	}
 
@@ -280,8 +288,8 @@ public MappingCouchbaseConverter mappingCouchbaseConverter(CouchbaseMappingConte
 	@Bean
 	public TranslationService couchbaseTranslationService() {
 		final JacksonTranslationService jacksonTranslationService = new JacksonTranslationService();
+		jacksonTranslationService.setObjectMapper(couchbaseObjectMapper(cryptoManager()));
 		jacksonTranslationService.afterPropertiesSet();
-
 		// for sdk3, we need to ask the mapper _it_ uses to ignore extra fields...
 		JacksonTransformers.MAPPER.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
 		return jacksonTranslationService;
@@ -306,12 +314,26 @@ public CouchbaseMappingContext couchbaseMappingContext(CustomConversions customC
 	 *
 	 * @return ObjectMapper
 	 */
+	private ObjectMapper couchbaseObjectMapper() {
+		return couchbaseObjectMapper(cryptoManager());
+	}
+
+	/**
+	 * Creates a {@link ObjectMapper} for the jsonSerializer of the ClusterEnvironment
+	 *
+	 * @param cryptoManager
+	 * @return ObjectMapper
+	 */
 
-	public ObjectMapper couchbaseObjectMapper() {
-		ObjectMapper mapper = new ObjectMapper();
+	ObjectMapper mapper;
+
+	public ObjectMapper couchbaseObjectMapper(CryptoManager cryptoManager) {
+		if (mapper != null) {
+			return mapper;
+		}
+		mapper = new ObjectMapper(); // or use the one from the Java SDK (?) JacksonTransformers.MAPPER
 		mapper.configure(com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
 		mapper.registerModule(new JsonValueModule());
-		CryptoManager cryptoManager = null;
 		if (cryptoManager != null) {
 			mapper.registerModule(new EncryptionModule(cryptoManager));
 		}
@@ -320,7 +342,7 @@ public ObjectMapper couchbaseObjectMapper() {
 
 	/**
 	 * The default blocking transaction manager. It is an implementation of CallbackPreferringTransactionManager
-	 * CallbackPreferrringTransactionmanagers do not play well with test-cases that rely
+	 * CallbackPreferringTransactionManagers do not play well with test-cases that rely
 	 * on @TestTransaction/@BeforeTransaction/@AfterTransaction
 	 *
 	 * @param clientFactory
@@ -341,6 +363,7 @@ CouchbaseCallbackTransactionManager couchbaseTransactionManager(CouchbaseClientF
 	TransactionTemplate couchbaseTransactionTemplate(CouchbaseCallbackTransactionManager couchbaseTransactionManager) {
 		return new TransactionTemplate(couchbaseTransactionManager);
 	}
+
 	/**
 	 * The default TransactionalOperator.
 	 *
@@ -376,14 +399,43 @@ protected boolean autoIndexCreation() {
 	/**
 	 * Register custom Converters in a {@link CustomConversions} object if required. These {@link CustomConversions} will
 	 * be registered with the {@link #mappingCouchbaseConverter(CouchbaseMappingContext, CouchbaseCustomConversions)} )}
-	 * and {@link #couchbaseMappingContext(CustomConversions)}. Returns an empty {@link CustomConversions} instance by
-	 * default.
+	 * and {@link #couchbaseMappingContext(CustomConversions)}.
 	 *
 	 * @return must not be {@literal null}.
 	 */
 	@Bean(name = BeanNames.COUCHBASE_CUSTOM_CONVERSIONS)
 	public CustomConversions customConversions() {
-		return new CouchbaseCustomConversions(Collections.emptyList());
+		return customConversions(cryptoManager());
+	}
+
+	/**
+	 * Register custom Converters in a {@link CustomConversions} object if required. These {@link CustomConversions} will
+	 * be registered with the {@link #mappingCouchbaseConverter(CouchbaseMappingContext, CouchbaseCustomConversions)} )}
+	 * and {@link #couchbaseMappingContext(CustomConversions)}.
+	 *
+	 * @param cryptoManager
+	 * @return must not be {@literal null}.
+	 */
+	public CustomConversions customConversions(CryptoManager cryptoManager) {
+		List<GenericConverter> newConverters = new ArrayList();
+		CustomConversions customConversions = CouchbaseCustomConversions.create(configurationAdapter -> {
+			SimplePropertyValueConversions valueConversions = new SimplePropertyValueConversions();
+			valueConversions.setConverterFactory(new CouchbasePropertyValueConverterFactory(cryptoManager));
+			valueConversions.setValueConverterRegistry(new PropertyValueConverterRegistrar().buildRegistry());
+			configurationAdapter.setPropertyValueConversions(valueConversions);
+			configurationAdapter.registerConverters(newConverters);
+		});
+		return customConversions;
+	}
+
+	@Bean
+	protected CryptoManager cryptoManager() {
+		return null;
+	}
+
+	@Bean
+	protected CryptoConverter cryptoConverter(CryptoManager cryptoManager) {
+		return cryptoManager == null ? null : new CryptoConverter(cryptoManager);
 	}
 
 	/**

spring-data-couchbase/src/main/java/org/springframework/data/couchbase/core/convert/AbstractCouchbaseConverter.java

@@ -18,17 +18,23 @@
 
 import java.util.Collections;
 
+import com.couchbase.client.java.query.QueryScanConsistency;
 import org.springframework.beans.factory.InitializingBean;
 import org.springframework.core.convert.ConversionService;
+import org.springframework.core.convert.TypeDescriptor;
 import org.springframework.core.convert.support.GenericConversionService;
 import org.springframework.data.convert.CustomConversions;
+import org.springframework.data.couchbase.core.mapping.CouchbaseDocument;
+import org.springframework.data.couchbase.core.mapping.CouchbasePersistentProperty;
+import org.springframework.data.mapping.model.ConvertingPropertyAccessor;
 import org.springframework.data.mapping.model.EntityInstantiators;
 
 /**
  * An abstract {@link CouchbaseConverter} that provides the basics for the {@link MappingCouchbaseConverter}.
  *
  * @author Michael Nitschinger
  * @author Mark Paluch
+ * @author Michael Reiche
  */
 public abstract class AbstractCouchbaseConverter implements CouchbaseConverter, InitializingBean {
 
@@ -93,39 +99,70 @@ public void afterPropertiesSet() {
 		conversions.registerConvertersIn(conversionService);
 	}
 
+	/**
+	 * This convertForWriteIfNeeded takes a property and accessor so that the annotations can be accessed (ie. @Encrypted)
+	 *
+	 * @param prop the property to be converted to the class that would actually be stored.
+	 * @param accessor the property accessor
+	 * @return
+	 */
 	@Override
-	public Object convertForWriteIfNeeded(Object value) {
+	public Object convertForWriteIfNeeded(CouchbasePersistentProperty prop, ConvertingPropertyAccessor<Object> accessor,
+			boolean processValueConverter) {
+		Object value = accessor.getProperty(prop, prop.getType());
 		if (value == null) {
 			return null;
 		}
+		if (processValueConverter && conversions.hasValueConverter(prop)) {
+			CouchbaseDocument encrypted = (CouchbaseDocument) conversions.getPropertyValueConversions()
+					.getValueConverter(prop)
+					.write(value, new CouchbaseConversionContext(prop, (MappingCouchbaseConverter) this, accessor));
+			return encrypted;
+		}
+		Class<?> targetClass = this.conversions.getCustomWriteTarget(value.getClass()).orElse(null);
 
-		return this.conversions.getCustomWriteTarget(value.getClass()) //
-				.map(it -> (Object) this.conversionService.convert(value, it)) //
+		boolean canConvert = targetClass == null ? false
+				: this.conversionService.canConvert(new TypeDescriptor(prop.getField()), TypeDescriptor.valueOf(targetClass));
+		if (canConvert) {
+			return this.conversionService.convert(value, new TypeDescriptor(prop.getField()),
+					TypeDescriptor.valueOf(targetClass));
+		}
+
+		Object result = this.conversions.getCustomWriteTarget(prop.getType()) //
+				.map(it -> this.conversionService.convert(value, new TypeDescriptor(prop.getField()),
+						TypeDescriptor.valueOf(it))) //
 				.orElseGet(() -> Enum.class.isAssignableFrom(value.getClass()) ? ((Enum<?>) value).name() : value);
 
+		return result;
+
 	}
 
-	/* TODO needed later
+	/**
+	 * This convertForWriteIfNeed takes only the value to convert. It cannot access the annotations of the Field being
+	 * converted.
+	 *
+	 * @param value the value to be converted to the class that would actually be stored.
+	 * @return
+	 */
 	@Override
-	public Object convertToCouchbaseType(Object value,  TypeInformation<?> typeInformation) {
+	public Object convertForWriteIfNeeded(Object value) {
 		if (value == null) {
 			return null;
 		}
-	
+
 		return this.conversions.getCustomWriteTarget(value.getClass()) //
 				.map(it -> (Object) this.conversionService.convert(value, it)) //
 				.orElseGet(() -> Enum.class.isAssignableFrom(value.getClass()) ? ((Enum<?>) value).name() : value);
-	
-	}
-	
-	@Override
-	public Object convertToCouchbaseType(String source) {
-		return source;
+
 	}
-	*/
 
 	@Override
 	public Class<?> getWriteClassFor(Class<?> clazz) {
 		return this.conversions.getCustomWriteTarget(clazz).orElse(clazz);
 	}
+
+	@Override
+	public CustomConversions getConversions() {
+		return conversions;
+	}
 }

spring-data-couchbase/src/main/java/org/springframework/data/couchbase/core/convert/CouchbaseConversionContext.java

@@ -0,0 +1,66 @@
+/*
+ * Copyright 2022 the original author or authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *        https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.springframework.data.couchbase.core.convert;
+
+import org.springframework.data.convert.ValueConversionContext;
+import org.springframework.data.couchbase.core.mapping.CouchbasePersistentProperty;
+import org.springframework.data.mapping.model.ConvertingPropertyAccessor;
+import org.springframework.data.util.TypeInformation;
+import org.springframework.lang.Nullable;
+
+/**
+ * {@link ValueConversionContext} that allows to delegate read/write to an underlying {@link CouchbaseConverter}.
+ *
+ * @author Michael Reiche
+ * @since 5.0
+ */
+public class CouchbaseConversionContext implements ValueConversionContext<CouchbasePersistentProperty> {
+
+	private final CouchbasePersistentProperty persistentProperty;
+	private final MappingCouchbaseConverter couchbaseConverter;
+	private final ConvertingPropertyAccessor propertyAccessor;
+
+	public CouchbaseConversionContext(CouchbasePersistentProperty persistentProperty,
+			MappingCouchbaseConverter couchbaseConverter, ConvertingPropertyAccessor accessor) {
+
+		this.persistentProperty = persistentProperty;
+		this.couchbaseConverter = couchbaseConverter;
+		this.propertyAccessor = accessor;
+	}
+
+	@Override
+	public CouchbasePersistentProperty getProperty() {
+		return persistentProperty;
+	}
+
+	@Override
+	public <T> T write(@Nullable Object value, TypeInformation<T> target) {
+		return (T) ValueConversionContext.super.write(value, target);
+	}
+
+	@Override
+	public <T> T read(@Nullable Object value, TypeInformation<T> target) {
+		return ValueConversionContext.super.read(value, target);
+	}
+
+	public MappingCouchbaseConverter getConverter() {
+		return couchbaseConverter;
+	}
+
+	public ConvertingPropertyAccessor getAccessor() {
+		return propertyAccessor;
+	}
+}