[ASTERIXDB-3196][CONF] Add cloud properties

- user model changes: yes
- storage format changes: no
- interface changes: yes

Details:

- Add the following cloud properties to allow configuring cloud
  storage which can be used for compute-storage separation:
  -- cloud.storage.scheme
  -- cloud.storage.bucket
  -- cloud.storage.region
  -- cloud.storage.prefix
  -- cloud.storage.endpoint
  -- cloud.storage.anonymous.auth

Change-Id: I1184818476c9b35bb521dc959033eeedd2a836c1
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17594
Reviewed-by: Murtadha Al Hubail <[email protected]>
Reviewed-by: Ali Alsuliman <[email protected]>
Integration-Tests: Jenkins <[email protected]>
Tested-by: Jenkins <[email protected]>

Author: mhubail

asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/CcApplicationContext.java

@@ -37,6 +37,7 @@
 import org.apache.asterix.common.cluster.IGlobalRecoveryManager;
 import org.apache.asterix.common.config.ActiveProperties;
 import org.apache.asterix.common.config.BuildProperties;
+import org.apache.asterix.common.config.CloudProperties;
 import org.apache.asterix.common.config.CompilerProperties;
 import org.apache.asterix.common.config.ExtensionProperties;
 import org.apache.asterix.common.config.ExternalProperties;
@@ -99,6 +100,7 @@ public class CcApplicationContext implements ICcApplicationContext {
     private ExtensionProperties extensionProperties;
     private MessagingProperties messagingProperties;
     private NodeProperties nodeProperties;
+    private final CloudProperties cloudProperties;
     private Supplier<IMetadataBootstrap> metadataBootstrapSupplier;
     private volatile HyracksConnection hcc;
     private volatile ResultSet resultSet;
@@ -140,6 +142,7 @@ public CcApplicationContext(ICCServiceContext ccServiceCtx, HyracksConnection hc
         activeProperties = new ActiveProperties(propertiesAccessor);
         extensionProperties = new ExtensionProperties(propertiesAccessor);
         replicationProperties = new ReplicationProperties(propertiesAccessor);
+        cloudProperties = new CloudProperties(propertiesAccessor);
         this.ftStrategy = ftStrategy;
         this.buildProperties = new BuildProperties(propertiesAccessor);
         this.messagingProperties = new MessagingProperties(propertiesAccessor);
@@ -373,4 +376,9 @@ public IDataPartitioningProvider getDataPartitioningProvider() {
     public boolean isCloudDeployment() {
         return ccServiceCtx.getAppConfig().getBoolean(CLOUD_DEPLOYMENT);
     }
+
+    @Override
+    public CloudProperties getCloudProperties() {
+        return cloudProperties;
+    }
 }

asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java

@@ -42,6 +42,7 @@
 import org.apache.asterix.common.api.IReceptionistFactory;
 import org.apache.asterix.common.config.ActiveProperties;
 import org.apache.asterix.common.config.BuildProperties;
+import org.apache.asterix.common.config.CloudProperties;
 import org.apache.asterix.common.config.CompilerProperties;
 import org.apache.asterix.common.config.ExternalProperties;
 import org.apache.asterix.common.config.MessagingProperties;
@@ -164,6 +165,7 @@ public class NCAppRuntimeContext implements INcApplicationContext {
     private final ICacheManager cacheManager;
     private IConfigValidator configValidator;
     private IDiskWriteRateLimiterProvider diskWriteRateLimiterProvider;
+    private final CloudProperties cloudProperties;
 
     public NCAppRuntimeContext(INCServiceContext ncServiceContext, NCExtensionManager extensionManager,
             IPropertiesFactory propertiesFactory) {
@@ -178,6 +180,7 @@ public NCAppRuntimeContext(INCServiceContext ncServiceContext, NCExtensionManage
         replicationProperties = propertiesFactory.newReplicationProperties();
         messagingProperties = propertiesFactory.newMessagingProperties();
         nodeProperties = propertiesFactory.newNodeProperties();
+        cloudProperties = propertiesFactory.newCloudProperties();
         ncExtensionManager = extensionManager;
         componentProvider = new StorageComponentProvider();
         resourceIdFactory = new GlobalResourceIdFactoryProvider(ncServiceContext).createResourceIdFactory();
@@ -191,7 +194,7 @@ public void initialize(IRecoveryManagerFactory recoveryManagerFactory, IReceptio
             boolean initialRun) throws IOException {
         ioManager = getServiceContext().getIoManager();
         if (isCloudDeployment()) {
-            persistenceIOManager = new CloudIOManager((IOManager) ioManager);
+            persistenceIOManager = new CloudIOManager((IOManager) ioManager, cloudProperties);
         } else {
             persistenceIOManager = ioManager;
         }
@@ -661,4 +664,9 @@ public IDiskWriteRateLimiterProvider getDiskWriteRateLimiterProvider() {
     public boolean isCloudDeployment() {
         return ncServiceContext.getAppConfig().getBoolean(CLOUD_DEPLOYMENT);
     }
+
+    @Override
+    public CloudProperties getCloudProperties() {
+        return cloudProperties;
+    }
 }

asterixdb/asterix-app/src/main/resources/cc-cloud-storage.conf

@@ -20,7 +20,6 @@
 
 import java.net.URI;
 
-import org.apache.asterix.cloud.storage.MockCloudStorageConfiguration;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
@@ -36,15 +35,19 @@ public class CloudUtils {
     private static final Logger LOGGER = LogManager.getLogger();
 
     private static final int MOCK_SERVER_PORT = 8001;
-    private static final String MOCK_SERVER_HOSTNAME = "http://127.0.0.1:" + MOCK_SERVER_PORT;
-    private static final String CLOUD_STORAGE_BUCKET = MockCloudStorageConfiguration.INSTANCE.getContainer();
-    private static final String MOCK_SERVER_REGION = "us-west-2";
+    public static final String MOCK_SERVER_HOSTNAME = "http://127.0.0.1:" + MOCK_SERVER_PORT;
+    public static final String CLOUD_STORAGE_BUCKET = "cloud-storage-container";
+    public static final String MOCK_SERVER_REGION = "us-west-2";
     private static S3Mock s3MockServer;
 
     private CloudUtils() {
         throw new AssertionError("Do not instantiate");
     }
 
+    public static void main(String[] args) {
+        startS3CloudEnvironment();
+    }
+
     public static void startS3CloudEnvironment() {
         // Starting S3 mock server to be used instead of real S3 server
         LOGGER.info("Starting S3 mock server");

asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/CloudUtils.java

@@ -45,7 +45,7 @@ public class CloudStorageTest {
     private static final TestExecutor testExecutor = new TestExecutor();
     private static final String SUITE_TESTS = "testsuite_cloud_storage.xml";
     private static final String ONLY_TESTS = "testsuite_cloud_storage_only.xml";
-    private static final String CONFIG_FILE_NAME = "src/main/resources/cc-cloud-storage.conf";
+    private static final String CONFIG_FILE_NAME = "src/test/resources/cc-cloud-storage.conf";
 
     public CloudStorageTest(TestCaseContext tcCtx) {
         this.tcCtx = tcCtx;

asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/CloudStorageTest.java

@@ -61,4 +61,9 @@ compiler.windowmemory=192KB
 messaging.frame.size=4096
 messaging.frame.count=512
 cloud.deployment=true
-storage.buffercache.pagesize=32KB
+storage.buffercache.pagesize=32KB
+cloud.storage.scheme=s3
+cloud.storage.bucket=cloud-storage-container
+cloud.storage.region=us-west-2
+cloud.storage.endpoint=http://127.0.0.1:8001
+cloud.storage.anonymous.auth=true

asterixdb/asterix-app/src/test/resources/cc-cloud-storage.conf

@@ -10,6 +10,12 @@
     "active\.suspend\.timeout" : 3600,
     "azure.request.timeout" : 120,
     "cloud.deployment" : false,
+    "cloud.storage.anonymous.auth" : false,
+    "cloud.storage.bucket" : "",
+    "cloud.storage.endpoint" : "",
+    "cloud.storage.prefix" : "",
+    "cloud.storage.region" : "",
+    "cloud.storage.scheme" : "",
     "compiler\.arrayindex" : true,
     "compiler.batch.lookup" : true,
     "compiler.cbo" : true,

asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm

@@ -10,6 +10,12 @@
     "active\.suspend\.timeout" : 3600,
     "azure.request.timeout" : 120,
     "cloud.deployment" : false,
+    "cloud.storage.anonymous.auth" : false,
+    "cloud.storage.bucket" : "",
+    "cloud.storage.endpoint" : "",
+    "cloud.storage.prefix" : "",
+    "cloud.storage.region" : "",
+    "cloud.storage.scheme" : "",
     "compiler\.arrayindex" : true,
     "compiler.batch.lookup" : true,
     "compiler.cbo" : true,

asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm

@@ -10,6 +10,12 @@
     "active\.suspend\.timeout" : 3600,
     "azure.request.timeout" : 120,
     "cloud.deployment" : false,
+    "cloud.storage.anonymous.auth" : false,
+    "cloud.storage.bucket" : "",
+    "cloud.storage.endpoint" : "",
+    "cloud.storage.prefix" : "",
+    "cloud.storage.region" : "",
+    "cloud.storage.scheme" : "",
     "compiler\.arrayindex" : true,
     "compiler.batch.lookup" : true,
     "compiler.cbo" : true,

asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm

@@ -36,12 +36,8 @@
 import java.util.Set;
 
 import org.apache.asterix.cloud.clients.CloudClientProvider;
-import org.apache.asterix.cloud.clients.CloudClientProvider.ClientType;
 import org.apache.asterix.cloud.clients.ICloudClient;
-import org.apache.asterix.cloud.clients.ICloudClientCredentialsProvider.CredentialsType;
-import org.apache.asterix.cloud.storage.CloudStorageConfigurationProvider;
-import org.apache.asterix.cloud.storage.ICloudStorageConfiguration;
-import org.apache.asterix.cloud.storage.ICloudStorageConfiguration.ConfigurationType;
+import org.apache.asterix.common.config.CloudProperties;
 import org.apache.commons.io.FileUtils;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
 import org.apache.hyracks.api.io.FileReference;
@@ -62,19 +58,17 @@ public class CloudIOManager extends IOManager {
     private IOManager localIoManager;
 
     private CloudIOManager(List<IODeviceHandle> devices, IFileDeviceResolver deviceComputer, int ioParallelism,
-            int queueSize) throws HyracksDataException {
+            int queueSize, CloudProperties cloudProperties) throws HyracksDataException {
         super(devices, deviceComputer, ioParallelism, queueSize);
-        ICloudStorageConfiguration cloudStorageConfiguration =
-                CloudStorageConfigurationProvider.INSTANCE.getConfiguration(ConfigurationType.FILE);
-        this.bucket = cloudStorageConfiguration.getContainer();
-        cloudClient = CloudClientProvider.getClient(ClientType.S3, CredentialsType.FILE);
+        this.bucket = cloudProperties.getStorageBucket();
+        cloudClient = CloudClientProvider.getClient(cloudProperties);
         int numOfThreads = getIODevices().size() * getIoParallelism();
         writeBufferProvider = new WriteBufferProvider(numOfThreads);
     }
 
-    public CloudIOManager(IOManager ioManager) throws HyracksDataException {
+    public CloudIOManager(IOManager ioManager, CloudProperties cloudProperties) throws HyracksDataException {
         this(ioManager.getIoDevices(), ioManager.getDeviceComputer(), ioManager.getIoParallelism(),
-                ioManager.getQueueSize());
+                ioManager.getQueueSize(), cloudProperties);
         this.localIoManager = ioManager;
     }