add hudi to generate_ctest and run_ctest

jessicahuang523 · jessicahuang523 · commit 0907bacac707 · 2022-11-19T18:44:36.000-06:00
diff --git a/core/add_project.sh b/core/add_project.sh
@@ -48,6 +48,14 @@ function setup_alluxio() {
     mvn clean install -DskipTests -Dcheckstyle.skip -Dlicense.skip -Dfindbugs.skip -Dmaven.javadoc.skip=true
 }
 
+function setup_hudi() {
+    [ ! -d "app/ctest-hudi" ] && git clone https://github.com/jessicahuang523/hudi app/ctest-hudi
+    cd app/ctest-hudi
+    git fetch && git checkout ctest-injection
+    cd hudi-common
+    mvn clean install -DskipTests -Dcheckstyle.skip
+}
+
 function usage() {
     echo "Usage: add_project.sh <main project>"
     exit 1
@@ -64,7 +72,8 @@ function main() {
             hbase) setup_hbase ;;
             zookeeper) setup_zookeeper ;;
             alluxio) setup_alluxio ;;
-            *) echo "Unexpected project: $project - only support hadoop, hbase, zookeeper and alluxio." ;;
+            hudi) setup_hudi ;;
+            *) echo "Unexpected project: $project - only support hadoop, hbase, zookeeper, alluxio and hudi." ;;
         esac
     fi
 }
diff --git a/core/ctest_const.py b/core/ctest_const.py
@@ -12,18 +12,22 @@
 HBASE = "hbase-server"
 ZOOKEEPER = "zookeeper-server"
 ALLUXIO = "alluxio-core"
+HUDI = "hudi-common"
 
 CTEST_HADOOP_DIR = os.path.join(APP_DIR, "ctest-hadoop")
 CTEST_HBASE_DIR = os.path.join(APP_DIR, "ctest-hbase")
 CTEST_ZK_DIR = os.path.join(APP_DIR, "ctest-zookeeper")
 CTEST_ALLUXIO_DIR = os.path.join(APP_DIR, "ctest-alluxio")
+CTEST_HUDI_DIR = os.path.join(APP_DIR, "ctest-hudi")
+
 
 PROJECT_DIR = {
     HCOMMON: CTEST_HADOOP_DIR,
     HDFS: CTEST_HADOOP_DIR,
     HBASE: CTEST_HBASE_DIR,
     ZOOKEEPER: CTEST_ZK_DIR,
     ALLUXIO: CTEST_ALLUXIO_DIR,
+    HUDI: CTEST_HUDI_DIR
 }
 
 
@@ -34,6 +38,7 @@
     HBASE: "hbase-server",
     ZOOKEEPER: "zookeeper-server",
     ALLUXIO: "core",
+    HUDI: "hudi-common"
 }
 
 
@@ -58,6 +63,7 @@
         os.path.join(CTEST_ALLUXIO_DIR, MODULE_SUBDIR[ALLUXIO], "server/worker", SUREFIRE_SUBDIR),
         os.path.join(CTEST_ALLUXIO_DIR, MODULE_SUBDIR[ALLUXIO], "server/master", SUREFIRE_SUBDIR),
     ],
+    HUDI: [os.path.join(CTEST_HUDI_DIR, MODULE_SUBDIR[HUDI], SUREFIRE_SUBDIR)]
 }
 
 # default or deprecate conf path
@@ -74,7 +80,8 @@
     HDFS: os.path.join(DEFAULT_CONF_DIR, HDFS + "-default.tsv"),
     HBASE: os.path.join(DEFAULT_CONF_DIR, HBASE + "-default.tsv"),
     ALLUXIO: os.path.join(DEFAULT_CONF_DIR, ALLUXIO + "-default.tsv"),
-    ZOOKEEPER: os.path.join(DEFAULT_CONF_DIR, ZOOKEEPER + "-default.tsv")
+    ZOOKEEPER: os.path.join(DEFAULT_CONF_DIR, ZOOKEEPER + "-default.tsv"),
+    HUDI: os.path.join(DEFAULT_CONF_DIR, HUDI + "-default.tsv")
 }
 
 
@@ -96,7 +103,10 @@
     ],
     ALLUXIO: [
         os.path.join(CTEST_ALLUXIO_DIR, "core/alluxio-ctest.properties")
-    ]
+    ],
+    HUDI: [
+        os.path.join(CTEST_HUDI_DIR, "hudi-common/core-ctest.yaml")
+    ],
 }
 
 
diff --git a/core/generate_ctest/inject.py b/core/generate_ctest/inject.py
@@ -21,6 +21,13 @@ def inject_config(param_value_pairs):
             for p, v in param_value_pairs.items():
                 file.write(p + "=" + v + "\n")
             file.close()
+    elif project in [HUDI]:
+         for inject_path in INJECTION_PATH[project]:
+             print(">>>>[ctest_core] injecting into file: {}".format(inject_path))
+             file = open(inject_path, "w")
+             for p, v in param_value_pairs.items():
+                 file.write(p + ": " + v + "\n")
+             file.close()
     elif project in [HCOMMON, HDFS, HBASE]:
         conf = ET.Element("configuration")
         for p, v in param_value_pairs.items():
@@ -41,7 +48,7 @@ def inject_config(param_value_pairs):
 
 def clean_conf_file(project):
     print(">>>> cleaning injected configuration from file")
-    if project in [ZOOKEEPER, ALLUXIO]:
+    if project in [ZOOKEEPER, ALLUXIO, HUDI]:
         for inject_path in INJECTION_PATH[project]:
             file = open(inject_path, "w")
             file.write("\n")
diff --git a/core/generate_ctest/main.py b/core/generate_ctest/main.py
@@ -33,7 +33,7 @@ def test_value_pair(test_input):
         mt_file = open(os.path.join(GENCTEST_TR_DIR, project, MT_FILE.format(id=param)), "w")
 
         associated_tests = mapping[param] if param in mapping else []
-        if len(mapping[param]) != 0:
+        if len(associated_tests) != 0:
             for value in values:
                 tr = run_test_seperate(param, value, associated_tests)
                 
diff --git a/core/generate_ctest/program_input.py b/core/generate_ctest/program_input.py
@@ -4,11 +4,11 @@
     # run mode
     "run_mode": "generate_ctest", # string
     # name of the project, i.e. hadoop-common, hadoop-hdfs, see constant.py
-    "project": "hadoop-common", # string
+    "project": "hudi-common", # string
     # path to param -> tests json mapping
-    "mapping_path": "../../data/ctest_mapping/opensource-hadoop-common.json", # string
+    "mapping_path": "../../data/ctest_mapping/opensource-hudi-common.json", # string
     # good values of params tests will be run against
-    "param_value_tsv": "sample-hadoop-common.tsv", # string
+    "param_value_tsv": "hudi-common-generated-values.tsv", # string
     # display the terminal output live, without saving any results
     "display_mode": False, # bool
     # whether to use mvn test or mvn surefire:test
diff --git a/core/run_ctest/inject.py b/core/run_ctest/inject.py
@@ -21,6 +21,13 @@ def inject_config(param_value_pairs):
             for p, v in param_value_pairs.items():
                 file.write(p + "=" + v + "\n")
             file.close()
+    elif project in [HUDI]:
+         for inject_path in INJECTION_PATH[project]:
+             print(">>>>[ctest_core] injecting into file: {}".format(inject_path))
+             file = open(inject_path, "w")
+             for p, v in param_value_pairs.items():
+                 file.write(p + ": " + v + "\n")
+             file.close()
     elif project in [HCOMMON, HDFS, HBASE]:
         conf = ET.Element("configuration")
         for p, v in param_value_pairs.items():
@@ -41,7 +48,7 @@ def inject_config(param_value_pairs):
 
 def clean_conf_file(project):
     print(">>>> cleaning injected configuration from file")
-    if project in [ZOOKEEPER, ALLUXIO]:
+    if project in [ZOOKEEPER, ALLUXIO, HUDI]:
         for inject_path in INJECTION_PATH[project]:
             file = open(inject_path, "w")
             file.write("\n")
diff --git a/core/run_ctest/parse_input.py b/core/run_ctest/parse_input.py
@@ -45,6 +45,8 @@ def parse_conf_file(path):
         # parsing for alluxio and zookeeper conf file format
         if "no default configuration file" in path:
             return {}
+        if project in [HUDI]:
+             return parse_conf_file_yaml(path)
         return parse_conf_file_properties(path)
 
 
@@ -89,6 +91,22 @@ def parse_conf_file_properties(path):
                 conf_map[cur_key] = cur_value
     return conf_map
 
+def parse_conf_file_yaml(path):
+     deprecate_conf = load_deprecate_config_map()
+     conf_map = {}
+     for line in open(path):
+         if line.startswith("#"):
+             continue
+         seg = line.strip("\n").split(": ")
+         if len(seg) == 2:
+             cur_key, cur_value = [x.strip() for x in seg]
+             if cur_key not in conf_map:
+                 if cur_key in deprecate_conf:
+                     print(">>>>[ctest_core] {} in your input conf file is deprecated in the project,".format(cur_key)
+                      + " replaced with {}".format(deprecate_conf[cur_key]))
+                     cur_key = deprecate_conf[cur_key]
+                 conf_map[cur_key] = cur_value
+     return conf_map
 
 def extract_conf_diff(path):
     """get the config diff"""
diff --git a/core/run_ctest/program_input.py b/core/run_ctest/program_input.py
@@ -4,11 +4,11 @@
     # run mode
     "run_mode": "run_ctest", # string
     # name of the project, i.e. hadoop-common, hadoop-hdfs
-    "project": "hadoop-common", # string
+    "project": "hudi-common", # string
     # path to param -> tests json mapping
-    "mapping_path": "../../data/ctest_mapping/opensource-hadoop-common.json", # string
+    "mapping_path": "../../data/ctest_mapping/opensource-hudi-common.json", # string
     # input directory hosting configuration files to be test, target-project-format specific
-    "conf_file_dir": "sample-hadoop-common", # string
+    "conf_file_dir": "hudi-common", # string
     # display the terminal output live, without saving any results
     "display_mode": False, # bool
     # whether to use mvn test or mvn surefire:test
