From 760ede7d1d3d0cdc31a5755aef869791c152446f Mon Sep 17 00:00:00 2001 From: Eric Enzler Date: Thu, 26 Oct 2023 09:55:37 +0200 Subject: [PATCH 01/24] Added Random distributions --- data/templated/example.xml | 4 +- .../templated/procedures/GenericQuery.java | 60 +++++++++++++++++-- 2 files changed, 56 insertions(+), 8 deletions(-) diff --git a/data/templated/example.xml b/data/templated/example.xml index 2e2177859..d16404290 100644 --- a/data/templated/example.xml +++ b/data/templated/example.xml @@ -20,10 +20,10 @@ \ No newline at end of file diff --git a/src/main/java/com/oltpbenchmark/benchmarks/templated/TemplatedBenchmark.java b/src/main/java/com/oltpbenchmark/benchmarks/templated/TemplatedBenchmark.java index 9de6cb151..0c0f44c6b 100644 --- a/src/main/java/com/oltpbenchmark/benchmarks/templated/TemplatedBenchmark.java +++ b/src/main/java/com/oltpbenchmark/benchmarks/templated/TemplatedBenchmark.java @@ -125,25 +125,24 @@ protected List> makeWorkersImpl() { // list of params. List list = new ArrayList<>(); String[] paramsTypes = info.getParamsTypes(); - + ComplexValue[] params = info.getParamsValues(); int paramsLen = params.length; int typesLen = paramsTypes.length; assert (paramsLen % typesLen) == 0; - - int subRange = paramsLen / typesLen; - if(paramsLen == typesLen){ + if (paramsLen == typesLen) { list.add(new GenericQueryOperation(params)); } else { - for(int j = 0; j < paramsLen; j+= subRange){ - ComplexValue[] subset = Arrays.copyOfRange(params, j, j+subRange); + int numSplits = paramsLen / typesLen; + for (int j = 0; j < numSplits; j += 1) { + ComplexValue[] subset = Arrays.copyOfRange(params, j * typesLen, j * typesLen + typesLen); assert subset.length == typesLen; list.add(new GenericQueryOperation(subset)); } } - + generators.put(proc.getClass(), new TraceTransactionGenerator(list)); } @@ -187,11 +186,12 @@ private CustomClassLoader loadQueryTemplates(String file) { b.name(template.getName()); b.query(template.getQuery()); b.paramsTypes(template.getTypes().getType()); - + for (ValuesType paramValue : template.getValues()) { - for(ValueType value: paramValue.getValue()){ - b.addParamsValues(new ComplexValue(value.getDist(),value.getMin(),value.getMax(),value.getSeed(),value.getValue())); - } + for (ValueType value : paramValue.getValue()) { + b.addParamsValues(new ComplexValue(value.getDist(), value.getMin(), value.getMax(), + value.getSeed(), value.getValue())); + } } ParsedQueryTemplate qt = b.build(); @@ -245,7 +245,7 @@ private String getParamsString(List params) { private String buildComplexValueString(List params) { String result = ""; for (ComplexValue param : params) { - result += "new ComplexValue(" + result += "new ComplexValue(" + "\"" + param.getDist() + "\"" + "," + "\"" + param.getMin() + "\"" + "," + "\"" + param.getMax() + "\"" + "," @@ -253,10 +253,9 @@ private String buildComplexValueString(List params) { + "\"" + StringEscapeUtils.escapeJava(param.getValue()) + "\"" + "),"; } - return result.isEmpty() ? "" : result.substring(0,result.length()-1); + return result.isEmpty() ? "" : result.substring(0, result.length() - 1); } - private static class CustomClassLoader extends ClassLoader { private final Map> classes = new HashMap<>(); diff --git a/src/main/java/com/oltpbenchmark/benchmarks/templated/procedures/GenericQuery.java b/src/main/java/com/oltpbenchmark/benchmarks/templated/procedures/GenericQuery.java index 3fa909933..81fa4d319 100644 --- a/src/main/java/com/oltpbenchmark/benchmarks/templated/procedures/GenericQuery.java +++ b/src/main/java/com/oltpbenchmark/benchmarks/templated/procedures/GenericQuery.java @@ -80,93 +80,208 @@ public PreparedStatement getStatement(Connection conn, List params if ((!hasDist && !hasValue) || paramsTypes[i].equalsIgnoreCase("NULL")) { stmt.setNull(i + 1, Types.NULL); - + } else if (hasDist) { String distribution = param.getDist(); - switch (distribution) { - case "uniform": - Random rnd = (Random) param.getGen(); + String paramType = paramsTypes[i].toLowerCase(); + switch (paramType) { + case "integer": + int min = param.getMin().intValue(); + int max = param.getMax().intValue(); + + switch (distribution) { - switch (paramsTypes[i].toLowerCase()) { - case "integer": - int min = param.getMin().intValue(); - int max = param.getMax().intValue(); - stmt.setInt(i+1, rnd.nextInt(min,max)); + case "uniform": + Random uniformRand = (Random) param.getGen(); + stmt.setInt(i + 1, uniformRand.nextInt(min, max)); break; - case "float": - case "real": - float minF = Float.parseFloat(param.getMinString()); - float maxF = Float.parseFloat(param.getMaxString()); - stmt.setFloat(i+1, rnd.nextFloat(minF,maxF)); + case "binomial": + Random binomialRandom = (Random) param.getGen(); + int bVal; + do { + bVal = (int) (min + Math.abs(binomialRandom.nextGaussian()) * max); + } while (bVal > max || bVal < min); + + stmt.setInt(i + 1, bVal); break; - case "bigint": - stmt.setLong(i+1,rnd.nextLong(param.getMin(), param.getMax())); - case "string": - int maxLen = param.getMax().intValue(); - stmt.setString(i + 1, TextGenerator.randomStr(rnd, maxLen)); + case "zipf": + ZipfianGenerator zipfGen = (ZipfianGenerator) param.getGen(); + stmt.setInt(i + 1, zipfGen.nextInt()); break; - case "timestamp": - stmt.setTimestamp(i + 1, new Timestamp(rnd.nextLong(param.getMin(), param.getMax()))); + case "scrambled": + ScrambledZipfianGenerator scramGen = (ScrambledZipfianGenerator) param.getGen(); + stmt.setInt(i + 1, scramGen.nextInt()); break; - case "date": - stmt.setDate(i + 1, new Date(rnd.nextLong(param.getMin(), param.getMax()))); + default: + throw new RuntimeException( + "Distribution: " + distribution + " not supported for type: " + paramType); + } + break; + case "float": + case "real": + float minF = Float.parseFloat(param.getMinString()); + float maxF = Float.parseFloat(param.getMaxString()); + switch (distribution) { + + case "uniform": + Random uniformRand = (Random) param.getGen(); + stmt.setFloat(i + 1, uniformRand.nextFloat(minF, maxF)); break; - case "time": - stmt.setTime(i + 1, new Time(rnd.nextLong(param.getMin(), param.getMax()))); + case "binomial": + Random binomialRandom = (Random) param.getGen(); + float fVal; + do { + fVal = (float) (minF + Math.abs(binomialRandom.nextGaussian()) * maxF); + } while (fVal > maxF || fVal < minF); + + stmt.setFloat(i + 1, fVal); break; default: - throw new RuntimeException("The type '" + paramsTypes[i] + "' is not supported by the '" + distribution + "' distribution"); + throw new RuntimeException( + "Distribution: " + distribution + " not supported for type: " + paramType); } break; - case "binomial": - break; - /* - minI = Integer.parseInt(params.get(index + 1).toString()); - maxI = Integer.parseInt(params.get(index + 2).toString()); - Random rng = (Random) randomGenerators.get(index); - int bVal; - do { - bVal = (int) (minI + Math.abs(rng.nextGaussian()) * maxI); - } while (bVal > maxI || bVal < minI); - stmt.setInt(i + 1, bVal); - j += 2; - break; + case "bigint": + Long minL = param.getMin(); + Long maxL = param.getMax(); + switch (distribution) { - case "zipf": - ZipfianGenerator zipfGen = (ZipfianGenerator) randomGenerators.get(index); - stmt.setInt(i + 1, zipfGen.nextInt()); - zipfGen.n - j += 2; - break; - - - case "scrambled": - ScrambledZipfianGenerator scramZipf = (ScrambledZipfianGenerator) randomGenerators.get(index); - stmt.setInt(i + 1, scramZipf.nextInt()); - j += 2; + case "uniform": + Random uniformRand = (Random) param.getGen(); + stmt.setLong(i + 1, uniformRand.nextLong(minL, maxL)); + break; + case "binomial": + Random binomialRandom = (Random) param.getGen(); + Long lVal; + do { + lVal = Double.valueOf(minL + Math.abs(binomialRandom.nextGaussian()) * maxL) + .longValue(); + } while (lVal > maxL || lVal < minL); + + stmt.setLong(i + 1, lVal); + break; + case "zipf": + ZipfianGenerator zipfGen = (ZipfianGenerator) param.getGen(); + stmt.setLong(i + 1, zipfGen.nextLong()); + break; + case "scrambled": + ScrambledZipfianGenerator scramGen = (ScrambledZipfianGenerator) param.getGen(); + stmt.setLong(i + 1, scramGen.nextLong()); + break; + default: + throw new RuntimeException( + "Distribution: " + distribution + " not supported for type: " + paramType); + } break; + case "varchar": case "string": - maxI = Integer.parseInt(params.get(index + 1).toString()); - Random rand = (Random) randomGenerators.get(index); - String randText = TextGenerator.randomStr(rand, maxI); - stmt.setString(i + 1, randText); - j += 1; + switch (distribution) { + case "uniform": + Random strRandom = (Random) param.getGen(); + stmt.setString(i + 1, TextGenerator.randomStr(strRandom, param.getMax().intValue())); + break; + default: + throw new RuntimeException( + "Distribution: " + distribution + " not supported for type: " + paramType); + } break; - case "datetime": case "timestamp": - stmt.setTimestamp(i + 1, new Timestamp(System.currentTimeMillis())); + Long stampMin = param.getMin(); + Long stampMax = param.getMax(); + switch (distribution) { + case "uniform": + Random uniformRand = (Random) param.getGen(); + stmt.setTimestamp(i + 1, new Timestamp(uniformRand.nextLong(stampMin, stampMax))); + break; + case "binomial": + Random binomialRandom = (Random) param.getGen(); + Long lVal; + do { + lVal = Double.valueOf(stampMin + Math.abs(binomialRandom.nextGaussian()) * stampMax) + .longValue(); + } while (lVal > stampMax || lVal < stampMin); + + stmt.setTimestamp(i + 1, new Timestamp(lVal)); + break; + case "zipf": + ZipfianGenerator zipfGen = (ZipfianGenerator) param.getGen(); + stmt.setTimestamp(i + 1, new Timestamp(zipfGen.nextLong())); + break; + case "scrambled": + ScrambledZipfianGenerator scramGen = (ScrambledZipfianGenerator) param.getGen(); + stmt.setTimestamp(i + 1, new Timestamp(scramGen.nextLong())); + break; + default: + throw new RuntimeException( + "Distribution: " + distribution + " not supported for type: " + paramType); + } break; case "date": - stmt.setDate(i + 1, new Date(System.currentTimeMillis())); + Long dateMin = param.getMin(); + Long dateMax = param.getMax(); + switch (distribution) { + case "uniform": + Random uniformRand = (Random) param.getGen(); + stmt.setDate(i + 1, new Date(uniformRand.nextLong(dateMin, dateMax))); + break; + case "binomial": + Random binomialRandom = (Random) param.getGen(); + Long lVal; + do { + lVal = Double.valueOf(dateMin + Math.abs(binomialRandom.nextGaussian()) * dateMax) + .longValue(); + } while (lVal > dateMax || lVal < dateMin); + + stmt.setDate(i + 1, new Date(lVal)); + break; + case "zipf": + ZipfianGenerator zipfGen = (ZipfianGenerator) param.getGen(); + stmt.setDate(i + 1, new Date(zipfGen.nextLong())); + break; + case "scrambled": + ScrambledZipfianGenerator scramGen = (ScrambledZipfianGenerator) param.getGen(); + stmt.setDate(i + 1, new Date(scramGen.nextLong())); + break; + default: + throw new RuntimeException( + "Distribution: " + distribution + " not supported for type: " + paramType); + } break; case "time": - stmt.setTime(i + 1, new Time(System.currentTimeMillis())); + Long timeMin = param.getMin(); + Long timeMax = param.getMax(); + switch (distribution) { + case "uniform": + Random uniformRand = (Random) param.getGen(); + stmt.setTime(i + 1, new Time(uniformRand.nextLong(timeMin, timeMax))); + break; + case "binomial": + Random binomialRandom = (Random) param.getGen(); + Long lVal; + do { + lVal = Double.valueOf(timeMin + Math.abs(binomialRandom.nextGaussian()) * timeMax) + .longValue(); + } while (lVal > timeMax || lVal < timeMin); + + stmt.setTime(i + 1, new Time(lVal)); + break; + case "zipf": + ZipfianGenerator zipfGen = (ZipfianGenerator) param.getGen(); + stmt.setTime(i + 1, new Time(zipfGen.nextLong())); + break; + case "scrambled": + ScrambledZipfianGenerator scramGen = (ScrambledZipfianGenerator) param.getGen(); + stmt.setTime(i + 1, new Time(scramGen.nextLong())); + break; + default: + throw new RuntimeException( + "Distribution: " + distribution + " not supported for type: " + paramType); + } break; - */ default: - throw new RuntimeException("No suitable distribution found: " + distribution); + throw new RuntimeException( + "Support for distributions for the type: " + paramType + " is current not implemented"); } - } else { try { Object val = param.getValue(); @@ -180,8 +295,8 @@ public PreparedStatement getStatement(Connection conn, List params } } } - System.out.println(stmt.toString()); return stmt; + } public abstract QueryTemplateInfo getQueryTemplateInfo(); From 573967c58b85948f385b502d450a273115a7aec4 Mon Sep 17 00:00:00 2001 From: Eric Enzler Date: Wed, 29 Nov 2023 15:40:16 +0100 Subject: [PATCH 09/24] Renamings and Documentation --- config/postgres/sample_templated_config.xml | 2 +- .../api/templates/TemplateType.java | 63 ++++++------ .../api/templates/TemplatesType.java | 21 ++-- .../api/templates/TypesType.java | 17 ++-- .../api/templates/ValuesType.java | 21 ++-- .../benchmarks/templated/README.md | 47 ++++++++- .../templated/TemplatedBenchmark.java | 22 ++--- .../templated/procedures/GenericQuery.java | 56 +++++------ .../templated/util/ComplexValue.java | 82 --------------- .../templated/util/GenericQueryOperation.java | 7 +- .../templated/util/TemplatedValue.java | 99 +++++++++++++++++++ 11 files changed, 252 insertions(+), 185 deletions(-) delete mode 100644 src/main/java/com/oltpbenchmark/benchmarks/templated/util/ComplexValue.java create mode 100644 src/main/java/com/oltpbenchmark/benchmarks/templated/util/TemplatedValue.java diff --git a/config/postgres/sample_templated_config.xml b/config/postgres/sample_templated_config.xml index 9ab417941..0e09ffc9c 100644 --- a/config/postgres/sample_templated_config.xml +++ b/config/postgres/sample_templated_config.xml @@ -6,7 +6,7 @@ org.postgresql.Driver jdbc:postgresql://localhost:5432/benchbase?sslmode=disable&ApplicationName=templated&reWriteBatchedInserts=true admin - password + admin TRANSACTION_SERIALIZABLE 128 diff --git a/src/main/java/com/oltpbenchmark/api/templates/TemplateType.java b/src/main/java/com/oltpbenchmark/api/templates/TemplateType.java index 6a6cae997..26b6116be 100644 --- a/src/main/java/com/oltpbenchmark/api/templates/TemplateType.java +++ b/src/main/java/com/oltpbenchmark/api/templates/TemplateType.java @@ -15,26 +15,26 @@ * */ - // +// // This file was generated by the JavaTM Architecture for XML Binding(JAXB) Reference Implementation, v2.3.0.1 // See https://javaee.github.io/jaxb-v2/ // Any modifications to this file will be lost upon recompilation of the source schema. // Generated on: 2023.11.16 at 08:29:59 AM UTC // - package com.oltpbenchmark.api.templates; import java.util.ArrayList; import java.util.List; import jakarta.xml.bind.annotation.*; - - /** - *

Java class for templateType complex type. + *

+ * Java class for templateType complex type. * - *

The following schema fragment specifies the expected content contained within this class. + *

+ * The following schema fragment specifies the expected content contained within + * this class. * *

  * <complexType name="templateType">
@@ -55,9 +55,9 @@
  */
 @XmlAccessorType(XmlAccessType.FIELD)
 @XmlType(name = "templateType", propOrder = {
-    "query",
-    "types",
-    "values"
+        "query",
+        "types",
+        "values"
 })
 public class TemplateType {
 
@@ -74,9 +74,9 @@ public class TemplateType {
      * Gets the value of the query property.
      * 
      * @return
-     *     possible object is
-     *     {@link String }
-     *     
+     *         possible object is
+     *         {@link String }
+     * 
      */
     public String getQuery() {
         return query;
@@ -86,9 +86,9 @@ public String getQuery() {
      * Sets the value of the query property.
      * 
      * @param value
-     *     allowed object is
-     *     {@link String }
-     *     
+     *              allowed object is
+     *              {@link String }
+     * 
      */
     public void setQuery(String value) {
         this.query = value;
@@ -98,21 +98,21 @@ public void setQuery(String value) {
      * Gets the value of the types property.
      * 
      * @return
-     *     possible object is
-     *     {@link TypesType }
-     *     
+     *         possible object is
+     *         {@link TypesType }
+     * 
      */
     public TypesType getTypes() {
-        return types;
+        return this.types;
     }
 
     /**
      * Sets the value of the types property.
      * 
      * @param value
-     *     allowed object is
-     *     {@link TypesType }
-     *     
+     *              allowed object is
+     *              {@link TypesType }
+     * 
      */
     public void setTypes(TypesType value) {
         this.types = value;
@@ -129,8 +129,9 @@ public void setTypes(TypesType value) {
      * 
      * 

* For example, to add a new item, do as follows: + * *

-     *    getValues().add(newItem);
+     * getValues().add(newItem);
      * 
* * @@ -141,8 +142,8 @@ public void setTypes(TypesType value) { * */ public List getValues() { - if (values == null) { - values = new ArrayList(); + if (this.values == null) { + this.values = new ArrayList(); } return this.values; } @@ -151,9 +152,9 @@ public List getValues() { * Gets the value of the name property. * * @return - * possible object is - * {@link String } - * + * possible object is + * {@link String } + * */ public String getName() { return name; @@ -163,9 +164,9 @@ public String getName() { * Sets the value of the name property. * * @param value - * allowed object is - * {@link String } - * + * allowed object is + * {@link String } + * */ public void setName(String value) { this.name = value; diff --git a/src/main/java/com/oltpbenchmark/api/templates/TemplatesType.java b/src/main/java/com/oltpbenchmark/api/templates/TemplatesType.java index c0d573cce..40d71291e 100644 --- a/src/main/java/com/oltpbenchmark/api/templates/TemplatesType.java +++ b/src/main/java/com/oltpbenchmark/api/templates/TemplatesType.java @@ -15,26 +15,26 @@ * */ - // +// // This file was generated by the JavaTM Architecture for XML Binding(JAXB) Reference Implementation, v2.3.0.1 // See https://javaee.github.io/jaxb-v2/ // Any modifications to this file will be lost upon recompilation of the source schema. // Generated on: 2023.11.16 at 08:29:59 AM UTC // - package com.oltpbenchmark.api.templates; import java.util.ArrayList; import java.util.List; import jakarta.xml.bind.annotation.*; - - /** - *

Java class for templatesType complex type. + *

+ * Java class for templatesType complex type. * - *

The following schema fragment specifies the expected content contained within this class. + *

+ * The following schema fragment specifies the expected content contained within + * this class. * *

  * <complexType name="templatesType">
@@ -52,7 +52,7 @@
  */
 @XmlAccessorType(XmlAccessType.FIELD)
 @XmlType(name = "templatesType", propOrder = {
-    "template"
+        "template"
 })
 public class TemplatesType {
 
@@ -70,8 +70,9 @@ public class TemplatesType {
      * 
      * 

* For example, to add a new item, do as follows: + * *

-     *    getTemplate().add(newItem);
+     * getTemplate().add(newItem);
      * 
* * @@ -82,8 +83,8 @@ public class TemplatesType { * */ public List getTemplate() { - if (template == null) { - template = new ArrayList(); + if (this.template == null) { + this.template = new ArrayList(); } return this.template; } diff --git a/src/main/java/com/oltpbenchmark/api/templates/TypesType.java b/src/main/java/com/oltpbenchmark/api/templates/TypesType.java index 68405ed61..28ff0f284 100644 --- a/src/main/java/com/oltpbenchmark/api/templates/TypesType.java +++ b/src/main/java/com/oltpbenchmark/api/templates/TypesType.java @@ -22,7 +22,6 @@ // Generated on: 2023.11.16 at 08:29:59 AM UTC // - package com.oltpbenchmark.api.templates; import java.util.ArrayList; @@ -30,9 +29,12 @@ import jakarta.xml.bind.annotation.*; /** - *

Java class for typesType complex type. + *

+ * Java class for typesType complex type. * - *

The following schema fragment specifies the expected content contained within this class. + *

+ * The following schema fragment specifies the expected content contained within + * this class. * *

  * <complexType name="typesType">
@@ -50,7 +52,7 @@
  */
 @XmlAccessorType(XmlAccessType.FIELD)
 @XmlType(name = "typesType", propOrder = {
-    "type"
+        "type"
 })
 public class TypesType {
 
@@ -68,8 +70,9 @@ public class TypesType {
      * 
      * 

* For example, to add a new item, do as follows: + * *

-     *    getType().add(newItem);
+     * getType().add(newItem);
      * 
* * @@ -80,8 +83,8 @@ public class TypesType { * */ public List getType() { - if (type == null) { - type = new ArrayList(); + if (this.type == null) { + this.type = new ArrayList<>(); } return this.type; } diff --git a/src/main/java/com/oltpbenchmark/api/templates/ValuesType.java b/src/main/java/com/oltpbenchmark/api/templates/ValuesType.java index 2c5f97e12..9062e23e7 100644 --- a/src/main/java/com/oltpbenchmark/api/templates/ValuesType.java +++ b/src/main/java/com/oltpbenchmark/api/templates/ValuesType.java @@ -15,26 +15,26 @@ * */ - // +// // This file was generated by the JavaTM Architecture for XML Binding(JAXB) Reference Implementation, v2.3.0.1 // See https://javaee.github.io/jaxb-v2/ // Any modifications to this file will be lost upon recompilation of the source schema. // Generated on: 2023.11.16 at 08:29:59 AM UTC // - package com.oltpbenchmark.api.templates; import java.util.ArrayList; import java.util.List; import jakarta.xml.bind.annotation.*; - - /** - *

Java class for valuesType complex type. + *

+ * Java class for valuesType complex type. * - *

The following schema fragment specifies the expected content contained within this class. + *

+ * The following schema fragment specifies the expected content contained within + * this class. * *

  * <complexType name="valuesType">
@@ -52,7 +52,7 @@
  */
 @XmlAccessorType(XmlAccessType.FIELD)
 @XmlType(name = "valuesType", propOrder = {
-    "value"
+        "value"
 })
 public class ValuesType {
 
@@ -70,8 +70,9 @@ public class ValuesType {
      * 
      * 

* For example, to add a new item, do as follows: + * *

-     *    getValue().add(newItem);
+     * getValue().add(newItem);
      * 
* * @@ -82,8 +83,8 @@ public class ValuesType { * */ public List getValue() { - if (value == null) { - value = new ArrayList(); + if (this.value == null) { + this.value = new ArrayList(); } return this.value; } diff --git a/src/main/java/com/oltpbenchmark/benchmarks/templated/README.md b/src/main/java/com/oltpbenchmark/benchmarks/templated/README.md index 38d919f69..c4004c84b 100644 --- a/src/main/java/com/oltpbenchmark/benchmarks/templated/README.md +++ b/src/main/java/com/oltpbenchmark/benchmarks/templated/README.md @@ -34,7 +34,8 @@ The file path for the XML template has to be defined in the workload configurati An example configuration can be found in [`config/sqlserver/sample_template_config.xml`](../../../../../../../config/sqlserver/sample_templated_config.xml). -> Since the templated benchmark is meant to flexibly support a myriad of different database schemas, it doesn't *currently* support the `init` and `load` phases. +> Since the templated benchmark is meant to flexibly support a myriad of different database schemas, it doesn't _currently_ support the `init` and `load` phases. + The example can be executed if a loaded TPC-C instance is used as JDBC endpoint. @@ -54,3 +55,47 @@ java -jar benchbase.jar -b templated -c config/sqlserver/sample_templated_config ``` > For additional examples, please refer to the build pipeline definition in the [`maven.yml`](../../../../../../../.github/workflows/maven.yml#L423) Github Actions workflow file. + +## Value Distributions + +In order to support more variety in templated queries, it is possible to use a whole distribution of values instead of a single static value in a templated query + +Each time the query is run, a different value will substitute the `?`. + +```xml + + + + +``` + +The distributions are dependent on the type of the value. Currently, the following type-distributions pairs are supported: +| Type | uniform | binomial | zipfian | scrambled zipfian | +|---|:---:|:---:|:---:|:---:| +| INTEGER | X | X | X | X | +| FLOAT / REAL | X | X|- | - | +| BIGINT | X | X | X | X | +| VARCHAR / STRING | X | -| -| -| +| TIMESTAMP | X | X |X |X | +| DATE | X | X| X| X| +| TIME | X |X | X| X| + +The following properties can be set on the value: + +- dist _The distribution of the values_ +- min _The minimum value the generator can produce. Defaults to 0_ +- max _The maximum value the generator can produce. Defaults to 1_ +- seed _A seed for the generator to ensure consistency_ + +For Timestamps, Dates and Times, the min and max values must be converted to a long (milliseconds) diff --git a/src/main/java/com/oltpbenchmark/benchmarks/templated/TemplatedBenchmark.java b/src/main/java/com/oltpbenchmark/benchmarks/templated/TemplatedBenchmark.java index 0c0f44c6b..133f36d92 100644 --- a/src/main/java/com/oltpbenchmark/benchmarks/templated/TemplatedBenchmark.java +++ b/src/main/java/com/oltpbenchmark/benchmarks/templated/TemplatedBenchmark.java @@ -49,7 +49,7 @@ import com.oltpbenchmark.api.templates.ValueType; import com.oltpbenchmark.benchmarks.templated.procedures.GenericQuery; import com.oltpbenchmark.benchmarks.templated.procedures.GenericQuery.QueryTemplateInfo; -import com.oltpbenchmark.benchmarks.templated.util.ComplexValue; +import com.oltpbenchmark.benchmarks.templated.util.TemplatedValue; import com.oltpbenchmark.benchmarks.templated.util.GenericQueryOperation; import com.oltpbenchmark.benchmarks.templated.util.TraceTransactionGenerator; @@ -126,7 +126,7 @@ protected List> makeWorkersImpl() { List list = new ArrayList<>(); String[] paramsTypes = info.getParamsTypes(); - ComplexValue[] params = info.getParamsValues(); + TemplatedValue[] params = info.getParamsValues(); int paramsLen = params.length; int typesLen = paramsTypes.length; @@ -137,7 +137,7 @@ protected List> makeWorkersImpl() { } else { int numSplits = paramsLen / typesLen; for (int j = 0; j < numSplits; j += 1) { - ComplexValue[] subset = Arrays.copyOfRange(params, j * typesLen, j * typesLen + typesLen); + TemplatedValue[] subset = Arrays.copyOfRange(params, j * typesLen, j * typesLen + typesLen); assert subset.length == typesLen; list.add(new GenericQueryOperation(subset)); } @@ -189,7 +189,7 @@ private CustomClassLoader loadQueryTemplates(String file) { for (ValuesType paramValue : template.getValues()) { for (ValueType value : paramValue.getValue()) { - b.addParamsValues(new ComplexValue(value.getDist(), value.getMin(), value.getMax(), + b.addParamsValues(new TemplatedValue(value.getDist(), value.getMin(), value.getMax(), value.getSeed(), value.getValue())); } } @@ -205,20 +205,20 @@ public final class %s extends %s { return ImmutableQueryTemplateInfo.builder() .query(new %s(\"%s\")) .paramsTypes(new String[] {%s}) - .paramsValues(new ComplexValue[] {%s}) + .paramsValues(new TemplatedValue[] {%s}) .build(); } } """.formatted( GenericQuery.class.getPackageName(), - ComplexValue.class.getCanonicalName(), + TemplatedValue.class.getCanonicalName(), qt.getName(), GenericQuery.class.getCanonicalName(), QueryTemplateInfo.class.getCanonicalName(), SQLStmt.class.getCanonicalName(), StringEscapeUtils.escapeJava(qt.getQuery()), getParamsString(qt.getParamsTypes()), - buildComplexValueString(qt.getParamsValues())); + buildTemplatedValueString(qt.getParamsValues())); LOG.debug("Class definition for query template {}:\n {}", qt.getName(), s); final String qualifiedClassName = GenericQuery.class.getPackageName() + "." + qt.getName(); final ISimpleCompiler compiler = compilerFactory.newSimpleCompiler(); @@ -242,10 +242,10 @@ private String getParamsString(List params) { return result.isEmpty() ? "" : result.substring(0, result.length() - 1); } - private String buildComplexValueString(List params) { + private String buildTemplatedValueString(List params) { String result = ""; - for (ComplexValue param : params) { - result += "new ComplexValue(" + for (TemplatedValue param : params) { + result += "new TemplatedValue(" + "\"" + param.getDist() + "\"" + "," + "\"" + param.getMin() + "\"" + "," + "\"" + param.getMax() + "\"" + "," @@ -303,7 +303,7 @@ default List getParamsTypes() { /** Potential query parameter values. */ @Value.Default - default List getParamsValues() { + default List getParamsValues() { return List.of(); } } diff --git a/src/main/java/com/oltpbenchmark/benchmarks/templated/procedures/GenericQuery.java b/src/main/java/com/oltpbenchmark/benchmarks/templated/procedures/GenericQuery.java index 81fa4d319..ad12ed69a 100644 --- a/src/main/java/com/oltpbenchmark/benchmarks/templated/procedures/GenericQuery.java +++ b/src/main/java/com/oltpbenchmark/benchmarks/templated/procedures/GenericQuery.java @@ -33,7 +33,7 @@ import com.oltpbenchmark.api.Procedure; import com.oltpbenchmark.api.SQLStmt; -import com.oltpbenchmark.benchmarks.templated.util.ComplexValue; +import com.oltpbenchmark.benchmarks.templated.util.TemplatedValue; import com.oltpbenchmark.distributions.ScrambledZipfianGenerator; import com.oltpbenchmark.distributions.ZipfianGenerator; import com.oltpbenchmark.util.TextGenerator; @@ -43,7 +43,7 @@ public abstract class GenericQuery extends Procedure { protected static final Logger LOG = LoggerFactory.getLogger(GenericQuery.class); /** Execution method with parameters. */ - public void run(Connection conn, List params) + public void run(Connection conn, List params) throws SQLException { try (PreparedStatement stmt = getStatement(conn, params); ResultSet rs = stmt.executeQuery()) { @@ -67,14 +67,14 @@ public void run(Connection conn) throws SQLException { conn.commit(); } - public PreparedStatement getStatement(Connection conn, List params) throws SQLException { + public PreparedStatement getStatement(Connection conn, List params) throws SQLException { QueryTemplateInfo queryTemplateInfo = this.getQueryTemplateInfo(); PreparedStatement stmt = this.getPreparedStatement(conn, queryTemplateInfo.getQuery()); String[] paramsTypes = queryTemplateInfo.getParamsTypes(); for (int i = 0; i < paramsTypes.length; i++) { - ComplexValue param = params.get(i); + TemplatedValue param = params.get(i); boolean hasDist = param.getDist().length() > 0; boolean hasValue = param.getValue().length() > 0; @@ -92,11 +92,11 @@ public PreparedStatement getStatement(Connection conn, List params switch (distribution) { case "uniform": - Random uniformRand = (Random) param.getGen(); + Random uniformRand = (Random) param.getGenerator(); stmt.setInt(i + 1, uniformRand.nextInt(min, max)); break; case "binomial": - Random binomialRandom = (Random) param.getGen(); + Random binomialRandom = (Random) param.getGenerator(); int bVal; do { bVal = (int) (min + Math.abs(binomialRandom.nextGaussian()) * max); @@ -105,11 +105,11 @@ public PreparedStatement getStatement(Connection conn, List params stmt.setInt(i + 1, bVal); break; case "zipf": - ZipfianGenerator zipfGen = (ZipfianGenerator) param.getGen(); + ZipfianGenerator zipfGen = (ZipfianGenerator) param.getGenerator(); stmt.setInt(i + 1, zipfGen.nextInt()); break; case "scrambled": - ScrambledZipfianGenerator scramGen = (ScrambledZipfianGenerator) param.getGen(); + ScrambledZipfianGenerator scramGen = (ScrambledZipfianGenerator) param.getGenerator(); stmt.setInt(i + 1, scramGen.nextInt()); break; default: @@ -124,11 +124,11 @@ public PreparedStatement getStatement(Connection conn, List params switch (distribution) { case "uniform": - Random uniformRand = (Random) param.getGen(); + Random uniformRand = (Random) param.getGenerator(); stmt.setFloat(i + 1, uniformRand.nextFloat(minF, maxF)); break; case "binomial": - Random binomialRandom = (Random) param.getGen(); + Random binomialRandom = (Random) param.getGenerator(); float fVal; do { fVal = (float) (minF + Math.abs(binomialRandom.nextGaussian()) * maxF); @@ -147,11 +147,11 @@ public PreparedStatement getStatement(Connection conn, List params switch (distribution) { case "uniform": - Random uniformRand = (Random) param.getGen(); + Random uniformRand = (Random) param.getGenerator(); stmt.setLong(i + 1, uniformRand.nextLong(minL, maxL)); break; case "binomial": - Random binomialRandom = (Random) param.getGen(); + Random binomialRandom = (Random) param.getGenerator(); Long lVal; do { lVal = Double.valueOf(minL + Math.abs(binomialRandom.nextGaussian()) * maxL) @@ -161,11 +161,11 @@ public PreparedStatement getStatement(Connection conn, List params stmt.setLong(i + 1, lVal); break; case "zipf": - ZipfianGenerator zipfGen = (ZipfianGenerator) param.getGen(); + ZipfianGenerator zipfGen = (ZipfianGenerator) param.getGenerator(); stmt.setLong(i + 1, zipfGen.nextLong()); break; case "scrambled": - ScrambledZipfianGenerator scramGen = (ScrambledZipfianGenerator) param.getGen(); + ScrambledZipfianGenerator scramGen = (ScrambledZipfianGenerator) param.getGenerator(); stmt.setLong(i + 1, scramGen.nextLong()); break; default: @@ -177,7 +177,7 @@ public PreparedStatement getStatement(Connection conn, List params case "string": switch (distribution) { case "uniform": - Random strRandom = (Random) param.getGen(); + Random strRandom = (Random) param.getGenerator(); stmt.setString(i + 1, TextGenerator.randomStr(strRandom, param.getMax().intValue())); break; default: @@ -190,11 +190,11 @@ public PreparedStatement getStatement(Connection conn, List params Long stampMax = param.getMax(); switch (distribution) { case "uniform": - Random uniformRand = (Random) param.getGen(); + Random uniformRand = (Random) param.getGenerator(); stmt.setTimestamp(i + 1, new Timestamp(uniformRand.nextLong(stampMin, stampMax))); break; case "binomial": - Random binomialRandom = (Random) param.getGen(); + Random binomialRandom = (Random) param.getGenerator(); Long lVal; do { lVal = Double.valueOf(stampMin + Math.abs(binomialRandom.nextGaussian()) * stampMax) @@ -204,11 +204,11 @@ public PreparedStatement getStatement(Connection conn, List params stmt.setTimestamp(i + 1, new Timestamp(lVal)); break; case "zipf": - ZipfianGenerator zipfGen = (ZipfianGenerator) param.getGen(); + ZipfianGenerator zipfGen = (ZipfianGenerator) param.getGenerator(); stmt.setTimestamp(i + 1, new Timestamp(zipfGen.nextLong())); break; case "scrambled": - ScrambledZipfianGenerator scramGen = (ScrambledZipfianGenerator) param.getGen(); + ScrambledZipfianGenerator scramGen = (ScrambledZipfianGenerator) param.getGenerator(); stmt.setTimestamp(i + 1, new Timestamp(scramGen.nextLong())); break; default: @@ -221,11 +221,11 @@ public PreparedStatement getStatement(Connection conn, List params Long dateMax = param.getMax(); switch (distribution) { case "uniform": - Random uniformRand = (Random) param.getGen(); + Random uniformRand = (Random) param.getGenerator(); stmt.setDate(i + 1, new Date(uniformRand.nextLong(dateMin, dateMax))); break; case "binomial": - Random binomialRandom = (Random) param.getGen(); + Random binomialRandom = (Random) param.getGenerator(); Long lVal; do { lVal = Double.valueOf(dateMin + Math.abs(binomialRandom.nextGaussian()) * dateMax) @@ -235,11 +235,11 @@ public PreparedStatement getStatement(Connection conn, List params stmt.setDate(i + 1, new Date(lVal)); break; case "zipf": - ZipfianGenerator zipfGen = (ZipfianGenerator) param.getGen(); + ZipfianGenerator zipfGen = (ZipfianGenerator) param.getGenerator(); stmt.setDate(i + 1, new Date(zipfGen.nextLong())); break; case "scrambled": - ScrambledZipfianGenerator scramGen = (ScrambledZipfianGenerator) param.getGen(); + ScrambledZipfianGenerator scramGen = (ScrambledZipfianGenerator) param.getGenerator(); stmt.setDate(i + 1, new Date(scramGen.nextLong())); break; default: @@ -252,11 +252,11 @@ public PreparedStatement getStatement(Connection conn, List params Long timeMax = param.getMax(); switch (distribution) { case "uniform": - Random uniformRand = (Random) param.getGen(); + Random uniformRand = (Random) param.getGenerator(); stmt.setTime(i + 1, new Time(uniformRand.nextLong(timeMin, timeMax))); break; case "binomial": - Random binomialRandom = (Random) param.getGen(); + Random binomialRandom = (Random) param.getGenerator(); Long lVal; do { lVal = Double.valueOf(timeMin + Math.abs(binomialRandom.nextGaussian()) * timeMax) @@ -266,11 +266,11 @@ public PreparedStatement getStatement(Connection conn, List params stmt.setTime(i + 1, new Time(lVal)); break; case "zipf": - ZipfianGenerator zipfGen = (ZipfianGenerator) param.getGen(); + ZipfianGenerator zipfGen = (ZipfianGenerator) param.getGenerator(); stmt.setTime(i + 1, new Time(zipfGen.nextLong())); break; case "scrambled": - ScrambledZipfianGenerator scramGen = (ScrambledZipfianGenerator) param.getGen(); + ScrambledZipfianGenerator scramGen = (ScrambledZipfianGenerator) param.getGenerator(); stmt.setTime(i + 1, new Time(scramGen.nextLong())); break; default: @@ -311,7 +311,7 @@ public interface QueryTemplateInfo { String[] getParamsTypes(); /** Potential query parameter values. */ - ComplexValue[] getParamsValues(); + TemplatedValue[] getParamsValues(); } } diff --git a/src/main/java/com/oltpbenchmark/benchmarks/templated/util/ComplexValue.java b/src/main/java/com/oltpbenchmark/benchmarks/templated/util/ComplexValue.java deleted file mode 100644 index bf1a5585d..000000000 --- a/src/main/java/com/oltpbenchmark/benchmarks/templated/util/ComplexValue.java +++ /dev/null @@ -1,82 +0,0 @@ -package com.oltpbenchmark.benchmarks.templated.util; - -import java.util.Random; - -import com.oltpbenchmark.distributions.ScrambledZipfianGenerator; -import com.oltpbenchmark.distributions.ZipfianGenerator; - -public class ComplexValue { - String dist; - Long min; - Long max; - Long seed; - String value; - String minS; - String maxS; - - public final Object randGen; - - - public ComplexValue(String dist, String min, String max, String seed, String value) { - this.dist = dist == null ? "" : dist; - this.min = min == null ? 0 : Long.parseLong(min); - this.max = max == null ? 1 : Long.parseLong(max); - this.seed = seed == null ? 0 : Long.parseLong(seed); - this.value = value; - this.randGen = createGen(dist,this.min,this.max,this.seed); - this.minS = min; - this.maxS = max; - } - - private Object createGen(String distribution, Long min, Long max, Long seed) { - - if(distribution == null || distribution.length() < 1) return null; - - switch (distribution) { - case "uniform": - case "binomial": - return new Random(seed); - case "zipf": - return new ZipfianGenerator(new Random(seed),min,max); - case "scrambled": - return new ScrambledZipfianGenerator(min,max); - default: - throw new RuntimeException( - "The distribution: '" + distribution + "' is not supported. Currently supported are 'zipf' | 'scrambled' | 'normal' | 'uniform'"); - } - - } - - - public String getDist(){ - return dist; - } - - public Long getMin(){ - return min; - } - - public Long getMax(){ - return max; - } - - public Long getSeed(){ - return seed; - } - - public String getValue(){ - return value; - } - - public String getMinString(){ - return this.minS; - } - - public String getMaxString(){ - return this.maxS; - } - - public Object getGen(){ - return this.randGen; - } -} diff --git a/src/main/java/com/oltpbenchmark/benchmarks/templated/util/GenericQueryOperation.java b/src/main/java/com/oltpbenchmark/benchmarks/templated/util/GenericQueryOperation.java index 7c570926a..50080c75a 100644 --- a/src/main/java/com/oltpbenchmark/benchmarks/templated/util/GenericQueryOperation.java +++ b/src/main/java/com/oltpbenchmark/benchmarks/templated/util/GenericQueryOperation.java @@ -26,15 +26,14 @@ */ public class GenericQueryOperation extends Operation { - public final List params; - + public final List params; - public GenericQueryOperation(ComplexValue[] params) { + public GenericQueryOperation(TemplatedValue[] params) { super(); this.params = Arrays.asList(params); } - public List getParams() { + public List getParams() { return params; } } diff --git a/src/main/java/com/oltpbenchmark/benchmarks/templated/util/TemplatedValue.java b/src/main/java/com/oltpbenchmark/benchmarks/templated/util/TemplatedValue.java new file mode 100644 index 000000000..7864e3c4d --- /dev/null +++ b/src/main/java/com/oltpbenchmark/benchmarks/templated/util/TemplatedValue.java @@ -0,0 +1,99 @@ +package com.oltpbenchmark.benchmarks.templated.util; + +import java.util.Random; + +import com.oltpbenchmark.distributions.ScrambledZipfianGenerator; +import com.oltpbenchmark.distributions.ZipfianGenerator; + +public class TemplatedValue { + String dist; + Long min; + Long max; + Long seed; + String value; + String minS; + String maxS; + + public final Object generatorObject; + + /** + * + * @param dist The desired value distribution + * @param min Minimum value. Default is 0 + * @param max Maximum value. Default is 1 + * @param seed The seed for the random generator. Default is 0 + * @param value Value that is used if no distribution is given + */ + public TemplatedValue(String dist, String min, String max, String seed, String value) { + this.dist = dist == null ? "" : dist; + this.min = min == null ? 0 : Long.parseLong(min); + this.max = max == null ? 1 : Long.parseLong(max); + this.seed = seed == null ? 0 : Long.parseLong(seed); + this.value = value; + this.generatorObject = createGenerator(dist, this.min, this.max, this.seed); + + this.minS = min; + this.maxS = max; + } + + /** + * + * @param distribution The distribution that drives the generator + * @param min Minimum value the generator will produce + * @param max Maximum value the generator will produce + * @param seed A seed for the generator + * @return + */ + private Object createGenerator(String distribution, Long min, Long max, Long seed) { + + if (distribution == null || distribution.length() < 1) + return null; + + switch (distribution) { + case "uniform": + case "binomial": + return new Random(seed); + case "zipf": + return new ZipfianGenerator(new Random(seed), min, max); + case "scrambled": + return new ScrambledZipfianGenerator(min, max); + default: + throw new RuntimeException( + "The distribution: '" + distribution + + "' is not supported. Currently supported are 'zipf' | 'scrambled' | 'normal' | 'uniform'"); + } + + } + + public String getDist() { + return dist; + } + + public Long getMin() { + return min; + } + + public Long getMax() { + return max; + } + + public Long getSeed() { + return seed; + } + + public String getValue() { + return value; + } + + public String getMinString() { + return this.minS; + } + + public String getMaxString() { + return this.maxS; + } + + public Object getGenerator() { + return this.generatorObject; + } +} From c12e60af88a232049aeca0d8f72f29d8d117f792 Mon Sep 17 00:00:00 2001 From: Eric Enzler Date: Thu, 30 Nov 2023 10:17:37 +0100 Subject: [PATCH 10/24] Enum, More Documentation --- config/postgres/sample_templated_config.xml | 2 +- .../templated/TemplatedBenchmark.java | 2 +- .../templated/procedures/GenericQuery.java | 51 ++++++++++--------- .../templated/util/TemplatedValue.java | 35 +++++++++---- .../templated/util/ValueGenerator.java | 8 +++ 5 files changed, 60 insertions(+), 38 deletions(-) create mode 100644 src/main/java/com/oltpbenchmark/benchmarks/templated/util/ValueGenerator.java diff --git a/config/postgres/sample_templated_config.xml b/config/postgres/sample_templated_config.xml index 0e09ffc9c..9ab417941 100644 --- a/config/postgres/sample_templated_config.xml +++ b/config/postgres/sample_templated_config.xml @@ -6,7 +6,7 @@ org.postgresql.Driver jdbc:postgresql://localhost:5432/benchbase?sslmode=disable&ApplicationName=templated&reWriteBatchedInserts=true admin - admin + password TRANSACTION_SERIALIZABLE 128 diff --git a/src/main/java/com/oltpbenchmark/benchmarks/templated/TemplatedBenchmark.java b/src/main/java/com/oltpbenchmark/benchmarks/templated/TemplatedBenchmark.java index 133f36d92..5dbc2a67f 100644 --- a/src/main/java/com/oltpbenchmark/benchmarks/templated/TemplatedBenchmark.java +++ b/src/main/java/com/oltpbenchmark/benchmarks/templated/TemplatedBenchmark.java @@ -246,7 +246,7 @@ private String buildTemplatedValueString(List params) { String result = ""; for (TemplatedValue param : params) { result += "new TemplatedValue(" - + "\"" + param.getDist() + "\"" + "," + + "\"" + param.getDistribution() + "\"" + "," + "\"" + param.getMin() + "\"" + "," + "\"" + param.getMax() + "\"" + "," + "\"" + param.getSeed() + "\"" + "," diff --git a/src/main/java/com/oltpbenchmark/benchmarks/templated/procedures/GenericQuery.java b/src/main/java/com/oltpbenchmark/benchmarks/templated/procedures/GenericQuery.java index ad12ed69a..f8bac3369 100644 --- a/src/main/java/com/oltpbenchmark/benchmarks/templated/procedures/GenericQuery.java +++ b/src/main/java/com/oltpbenchmark/benchmarks/templated/procedures/GenericQuery.java @@ -34,6 +34,7 @@ import com.oltpbenchmark.api.Procedure; import com.oltpbenchmark.api.SQLStmt; import com.oltpbenchmark.benchmarks.templated.util.TemplatedValue; +import com.oltpbenchmark.benchmarks.templated.util.ValueGenerator; import com.oltpbenchmark.distributions.ScrambledZipfianGenerator; import com.oltpbenchmark.distributions.ZipfianGenerator; import com.oltpbenchmark.util.TextGenerator; @@ -75,14 +76,14 @@ public PreparedStatement getStatement(Connection conn, List para for (int i = 0; i < paramsTypes.length; i++) { TemplatedValue param = params.get(i); - boolean hasDist = param.getDist().length() > 0; + boolean hasDist = param.getDistribution() != null; boolean hasValue = param.getValue().length() > 0; if ((!hasDist && !hasValue) || paramsTypes[i].equalsIgnoreCase("NULL")) { stmt.setNull(i + 1, Types.NULL); } else if (hasDist) { - String distribution = param.getDist(); + ValueGenerator distribution = param.getDistribution(); String paramType = paramsTypes[i].toLowerCase(); switch (paramType) { case "integer": @@ -91,11 +92,11 @@ public PreparedStatement getStatement(Connection conn, List para switch (distribution) { - case "uniform": + case UNIFORM: Random uniformRand = (Random) param.getGenerator(); stmt.setInt(i + 1, uniformRand.nextInt(min, max)); break; - case "binomial": + case BINOMIAL: Random binomialRandom = (Random) param.getGenerator(); int bVal; do { @@ -104,11 +105,11 @@ public PreparedStatement getStatement(Connection conn, List para stmt.setInt(i + 1, bVal); break; - case "zipf": + case ZIPFIAN: ZipfianGenerator zipfGen = (ZipfianGenerator) param.getGenerator(); stmt.setInt(i + 1, zipfGen.nextInt()); break; - case "scrambled": + case SCRAMBLED: ScrambledZipfianGenerator scramGen = (ScrambledZipfianGenerator) param.getGenerator(); stmt.setInt(i + 1, scramGen.nextInt()); break; @@ -123,11 +124,11 @@ public PreparedStatement getStatement(Connection conn, List para float maxF = Float.parseFloat(param.getMaxString()); switch (distribution) { - case "uniform": + case UNIFORM: Random uniformRand = (Random) param.getGenerator(); stmt.setFloat(i + 1, uniformRand.nextFloat(minF, maxF)); break; - case "binomial": + case BINOMIAL: Random binomialRandom = (Random) param.getGenerator(); float fVal; do { @@ -146,11 +147,11 @@ public PreparedStatement getStatement(Connection conn, List para Long maxL = param.getMax(); switch (distribution) { - case "uniform": + case UNIFORM: Random uniformRand = (Random) param.getGenerator(); stmt.setLong(i + 1, uniformRand.nextLong(minL, maxL)); break; - case "binomial": + case BINOMIAL: Random binomialRandom = (Random) param.getGenerator(); Long lVal; do { @@ -160,11 +161,11 @@ public PreparedStatement getStatement(Connection conn, List para stmt.setLong(i + 1, lVal); break; - case "zipf": + case ZIPFIAN: ZipfianGenerator zipfGen = (ZipfianGenerator) param.getGenerator(); stmt.setLong(i + 1, zipfGen.nextLong()); break; - case "scrambled": + case SCRAMBLED: ScrambledZipfianGenerator scramGen = (ScrambledZipfianGenerator) param.getGenerator(); stmt.setLong(i + 1, scramGen.nextLong()); break; @@ -176,7 +177,7 @@ public PreparedStatement getStatement(Connection conn, List para case "varchar": case "string": switch (distribution) { - case "uniform": + case UNIFORM: Random strRandom = (Random) param.getGenerator(); stmt.setString(i + 1, TextGenerator.randomStr(strRandom, param.getMax().intValue())); break; @@ -189,11 +190,11 @@ public PreparedStatement getStatement(Connection conn, List para Long stampMin = param.getMin(); Long stampMax = param.getMax(); switch (distribution) { - case "uniform": + case UNIFORM: Random uniformRand = (Random) param.getGenerator(); stmt.setTimestamp(i + 1, new Timestamp(uniformRand.nextLong(stampMin, stampMax))); break; - case "binomial": + case BINOMIAL: Random binomialRandom = (Random) param.getGenerator(); Long lVal; do { @@ -203,11 +204,11 @@ public PreparedStatement getStatement(Connection conn, List para stmt.setTimestamp(i + 1, new Timestamp(lVal)); break; - case "zipf": + case ZIPFIAN: ZipfianGenerator zipfGen = (ZipfianGenerator) param.getGenerator(); stmt.setTimestamp(i + 1, new Timestamp(zipfGen.nextLong())); break; - case "scrambled": + case SCRAMBLED: ScrambledZipfianGenerator scramGen = (ScrambledZipfianGenerator) param.getGenerator(); stmt.setTimestamp(i + 1, new Timestamp(scramGen.nextLong())); break; @@ -220,11 +221,11 @@ public PreparedStatement getStatement(Connection conn, List para Long dateMin = param.getMin(); Long dateMax = param.getMax(); switch (distribution) { - case "uniform": + case UNIFORM: Random uniformRand = (Random) param.getGenerator(); stmt.setDate(i + 1, new Date(uniformRand.nextLong(dateMin, dateMax))); break; - case "binomial": + case BINOMIAL: Random binomialRandom = (Random) param.getGenerator(); Long lVal; do { @@ -234,11 +235,11 @@ public PreparedStatement getStatement(Connection conn, List para stmt.setDate(i + 1, new Date(lVal)); break; - case "zipf": + case ZIPFIAN: ZipfianGenerator zipfGen = (ZipfianGenerator) param.getGenerator(); stmt.setDate(i + 1, new Date(zipfGen.nextLong())); break; - case "scrambled": + case SCRAMBLED: ScrambledZipfianGenerator scramGen = (ScrambledZipfianGenerator) param.getGenerator(); stmt.setDate(i + 1, new Date(scramGen.nextLong())); break; @@ -251,11 +252,11 @@ public PreparedStatement getStatement(Connection conn, List para Long timeMin = param.getMin(); Long timeMax = param.getMax(); switch (distribution) { - case "uniform": + case UNIFORM: Random uniformRand = (Random) param.getGenerator(); stmt.setTime(i + 1, new Time(uniformRand.nextLong(timeMin, timeMax))); break; - case "binomial": + case BINOMIAL: Random binomialRandom = (Random) param.getGenerator(); Long lVal; do { @@ -265,11 +266,11 @@ public PreparedStatement getStatement(Connection conn, List para stmt.setTime(i + 1, new Time(lVal)); break; - case "zipf": + case ZIPFIAN: ZipfianGenerator zipfGen = (ZipfianGenerator) param.getGenerator(); stmt.setTime(i + 1, new Time(zipfGen.nextLong())); break; - case "scrambled": + case SCRAMBLED: ScrambledZipfianGenerator scramGen = (ScrambledZipfianGenerator) param.getGenerator(); stmt.setTime(i + 1, new Time(scramGen.nextLong())); break; diff --git a/src/main/java/com/oltpbenchmark/benchmarks/templated/util/TemplatedValue.java b/src/main/java/com/oltpbenchmark/benchmarks/templated/util/TemplatedValue.java index 7864e3c4d..cb88a90e4 100644 --- a/src/main/java/com/oltpbenchmark/benchmarks/templated/util/TemplatedValue.java +++ b/src/main/java/com/oltpbenchmark/benchmarks/templated/util/TemplatedValue.java @@ -5,8 +5,13 @@ import com.oltpbenchmark.distributions.ScrambledZipfianGenerator; import com.oltpbenchmark.distributions.ZipfianGenerator; +/** + * This class is used to store information about the values + * used in templated benchmarks. It can hold static values but also + * generators for value distributions + */ public class TemplatedValue { - String dist; + ValueGenerator distribution; Long min; Long max; Long seed; @@ -25,11 +30,11 @@ public class TemplatedValue { * @param value Value that is used if no distribution is given */ public TemplatedValue(String dist, String min, String max, String seed, String value) { - this.dist = dist == null ? "" : dist; this.min = min == null ? 0 : Long.parseLong(min); this.max = max == null ? 1 : Long.parseLong(max); this.seed = seed == null ? 0 : Long.parseLong(seed); this.value = value; + this.generatorObject = createGenerator(dist, this.min, this.max, this.seed); this.minS = min; @@ -46,43 +51,51 @@ public TemplatedValue(String dist, String min, String max, String seed, String v */ private Object createGenerator(String distribution, Long min, Long max, Long seed) { - if (distribution == null || distribution.length() < 1) + if (distribution == null || distribution.equals("null") || distribution.length() < 1) return null; - switch (distribution) { + switch (distribution.toLowerCase()) { case "uniform": + this.distribution = ValueGenerator.UNIFORM; + return new Random(seed); case "binomial": + case "normal": + this.distribution = ValueGenerator.BINOMIAL; return new Random(seed); case "zipf": + case "zipfian": + this.distribution = ValueGenerator.ZIPFIAN; return new ZipfianGenerator(new Random(seed), min, max); case "scrambled": + case "scramzipf": + this.distribution = ValueGenerator.SCRAMBLED; return new ScrambledZipfianGenerator(min, max); default: throw new RuntimeException( "The distribution: '" + distribution - + "' is not supported. Currently supported are 'zipf' | 'scrambled' | 'normal' | 'uniform'"); + + "' is not supported. Currently supported are 'uniform' | 'binomial' | 'zipfian' | 'scrambled'"); } } - public String getDist() { - return dist; + public ValueGenerator getDistribution() { + return this.distribution; } public Long getMin() { - return min; + return this.min; } public Long getMax() { - return max; + return this.max; } public Long getSeed() { - return seed; + return this.seed; } public String getValue() { - return value; + return this.value; } public String getMinString() { diff --git a/src/main/java/com/oltpbenchmark/benchmarks/templated/util/ValueGenerator.java b/src/main/java/com/oltpbenchmark/benchmarks/templated/util/ValueGenerator.java new file mode 100644 index 000000000..2fcec9c7e --- /dev/null +++ b/src/main/java/com/oltpbenchmark/benchmarks/templated/util/ValueGenerator.java @@ -0,0 +1,8 @@ +package com.oltpbenchmark.benchmarks.templated.util; + +public enum ValueGenerator { + UNIFORM, + BINOMIAL, + ZIPFIAN, + SCRAMBLED +} From 51f0f1dee4e8c5742c694937960483d1b9baf4ef Mon Sep 17 00:00:00 2001 From: Eric Enzler Date: Mon, 4 Dec 2023 15:24:01 +0100 Subject: [PATCH 11/24] Renamed autogenerated methods, readme format --- .../api/templates/TemplateType.java | 2 +- .../api/templates/TemplatesType.java | 2 +- .../api/templates/TypesType.java | 2 +- .../api/templates/ValuesType.java | 2 +- .../benchmarks/templated/README.md | 27 +++++++++---------- .../templated/TemplatedBenchmark.java | 8 +++--- 6 files changed, 21 insertions(+), 22 deletions(-) diff --git a/src/main/java/com/oltpbenchmark/api/templates/TemplateType.java b/src/main/java/com/oltpbenchmark/api/templates/TemplateType.java index 26b6116be..d66505f05 100644 --- a/src/main/java/com/oltpbenchmark/api/templates/TemplateType.java +++ b/src/main/java/com/oltpbenchmark/api/templates/TemplateType.java @@ -141,7 +141,7 @@ public void setTypes(TypesType value) { * * */ - public List getValues() { + public List getValuesList() { if (this.values == null) { this.values = new ArrayList(); } diff --git a/src/main/java/com/oltpbenchmark/api/templates/TemplatesType.java b/src/main/java/com/oltpbenchmark/api/templates/TemplatesType.java index 40d71291e..add09d932 100644 --- a/src/main/java/com/oltpbenchmark/api/templates/TemplatesType.java +++ b/src/main/java/com/oltpbenchmark/api/templates/TemplatesType.java @@ -82,7 +82,7 @@ public class TemplatesType { * * */ - public List getTemplate() { + public List getTemplateList() { if (this.template == null) { this.template = new ArrayList(); } diff --git a/src/main/java/com/oltpbenchmark/api/templates/TypesType.java b/src/main/java/com/oltpbenchmark/api/templates/TypesType.java index 28ff0f284..e386ffc0f 100644 --- a/src/main/java/com/oltpbenchmark/api/templates/TypesType.java +++ b/src/main/java/com/oltpbenchmark/api/templates/TypesType.java @@ -82,7 +82,7 @@ public class TypesType { * * */ - public List getType() { + public List getTypeList() { if (this.type == null) { this.type = new ArrayList<>(); } diff --git a/src/main/java/com/oltpbenchmark/api/templates/ValuesType.java b/src/main/java/com/oltpbenchmark/api/templates/ValuesType.java index 9062e23e7..e6b010908 100644 --- a/src/main/java/com/oltpbenchmark/api/templates/ValuesType.java +++ b/src/main/java/com/oltpbenchmark/api/templates/ValuesType.java @@ -82,7 +82,7 @@ public class ValuesType { * * */ - public List getValue() { + public List getValueList() { if (this.value == null) { this.value = new ArrayList(); } diff --git a/src/main/java/com/oltpbenchmark/benchmarks/templated/README.md b/src/main/java/com/oltpbenchmark/benchmarks/templated/README.md index c4004c84b..e8fe5446c 100644 --- a/src/main/java/com/oltpbenchmark/benchmarks/templated/README.md +++ b/src/main/java/com/oltpbenchmark/benchmarks/templated/README.md @@ -81,21 +81,20 @@ Each time the query is run, a different value will substitute the `?`. ``` The distributions are dependent on the type of the value. Currently, the following type-distributions pairs are supported: -| Type | uniform | binomial | zipfian | scrambled zipfian | -|---|:---:|:---:|:---:|:---:| -| INTEGER | X | X | X | X | -| FLOAT / REAL | X | X|- | - | -| BIGINT | X | X | X | X | -| VARCHAR / STRING | X | -| -| -| -| TIMESTAMP | X | X |X |X | -| DATE | X | X| X| X| -| TIME | X |X | X| X| - +| Type | uniform | binomial | zipfian | scrambled zipfian | +| ---------------- | :-----: | :------: | :-----: | :---------------: | +| INTEGER | X | X | X | X | +| FLOAT / REAL | X | X | - | - | +| BIGINT | X | X | X | X | +| VARCHAR / STRING | X | - | - | - | +| TIMESTAMP | X | X | X | X | +| DATE | X | X | X | X | +| TIME | X | X | X | X | The following properties can be set on the value: -- dist _The distribution of the values_ -- min _The minimum value the generator can produce. Defaults to 0_ -- max _The maximum value the generator can produce. Defaults to 1_ -- seed _A seed for the generator to ensure consistency_ +- `dist`: The distribution of the values +- `min`: The minimum value the generator can produce. Defaults to 0 +- `max`: The maximum value the generator can produce. Defaults to 1 +- `seed`: A seed for the generator to ensure consistency For Timestamps, Dates and Times, the min and max values must be converted to a long (milliseconds) diff --git a/src/main/java/com/oltpbenchmark/benchmarks/templated/TemplatedBenchmark.java b/src/main/java/com/oltpbenchmark/benchmarks/templated/TemplatedBenchmark.java index 5dbc2a67f..cf17dfd8c 100644 --- a/src/main/java/com/oltpbenchmark/benchmarks/templated/TemplatedBenchmark.java +++ b/src/main/java/com/oltpbenchmark/benchmarks/templated/TemplatedBenchmark.java @@ -181,14 +181,14 @@ private CustomClassLoader loadQueryTemplates(String file) { JAXBElement result = unmarshaller.unmarshal(streamSource, TemplatesType.class); TemplatesType templates = result.getValue(); - for (TemplateType template : templates.getTemplate()) { + for (TemplateType template : templates.getTemplateList()) { ImmutableParsedQueryTemplate.Builder b = ImmutableParsedQueryTemplate.builder(); b.name(template.getName()); b.query(template.getQuery()); - b.paramsTypes(template.getTypes().getType()); + b.paramsTypes(template.getTypes().getTypeList()); - for (ValuesType paramValue : template.getValues()) { - for (ValueType value : paramValue.getValue()) { + for (ValuesType paramValue : template.getValuesList()) { + for (ValueType value : paramValue.getValueList()) { b.addParamsValues(new TemplatedValue(value.getDist(), value.getMin(), value.getMax(), value.getSeed(), value.getValue())); } From 38747aeae900ee3b79f6b61a93a352ef3d8782cf Mon Sep 17 00:00:00 2001 From: Eric Enzler Date: Thu, 7 Dec 2023 15:27:36 +0100 Subject: [PATCH 12/24] Testing increased, size reduction of massive switch statement --- data/templated/example.xml | 18 +- .../templated/procedures/GenericQuery.java | 168 ++++-------------- .../templated/util/GenericQueryOperation.java | 3 +- .../templated/util/TemplatedValue.java | 64 ++++++- 4 files changed, 105 insertions(+), 148 deletions(-) diff --git a/data/templated/example.xml b/data/templated/example.xml index c48917d4e..9d6ceeacf 100644 --- a/data/templated/example.xml +++ b/data/templated/example.xml @@ -64,19 +64,19 @@ \ No newline at end of file diff --git a/src/main/java/com/oltpbenchmark/benchmarks/templated/procedures/GenericQuery.java b/src/main/java/com/oltpbenchmark/benchmarks/templated/procedures/GenericQuery.java index 5beb6e681..aeaac79d4 100644 --- a/src/main/java/com/oltpbenchmark/benchmarks/templated/procedures/GenericQuery.java +++ b/src/main/java/com/oltpbenchmark/benchmarks/templated/procedures/GenericQuery.java @@ -103,202 +103,104 @@ public PreparedStatement getStatement(Connection conn, List para String paramType = paramsTypes[i].toLowerCase(); switch (paramType) { case "integer": - int min = param.getMin().intValue(); - int max = param.getMax().intValue(); - + int generatedInt; switch (distribution) { - case UNIFORM: - Random uniformRand = (Random) param.getGenerator(); - stmt.setInt(i + 1, uniformRand.nextInt(min, max)); + generatedInt = param.getNextLongUniform().intValue(); break; case BINOMIAL: - Random binomialRandom = (Random) param.getGenerator(); - int bVal; - do { - bVal = (int) (min + Math.abs(binomialRandom.nextGaussian()) * max); - } while (bVal > max || bVal < min); - - stmt.setInt(i + 1, bVal); + generatedInt = param.getNextLongBinomial().intValue(); break; case ZIPFIAN: - ZipfianGenerator zipfGen = (ZipfianGenerator) param.getGenerator(); - stmt.setInt(i + 1, zipfGen.nextInt()); + generatedInt = param.getNextLongZipf().intValue(); break; case SCRAMBLED: - ScrambledZipfianGenerator scramGen = (ScrambledZipfianGenerator) param.getGenerator(); - stmt.setInt(i + 1, scramGen.nextInt()); + generatedInt = param.getNextLongScrambled().intValue(); break; default: - throw new RuntimeException( - "Distribution: " + distribution + " not supported for type: " + paramType); + throw param.createRuntimeException(paramType); } + stmt.setInt(i + 1, generatedInt); break; case "float": case "real": - float minF = Float.parseFloat(param.getMinString()); - float maxF = Float.parseFloat(param.getMaxString()); + float generatedFloat; switch (distribution) { - case UNIFORM: - Random uniformRand = (Random) param.getGenerator(); - stmt.setFloat(i + 1, uniformRand.nextFloat(minF, maxF)); + generatedFloat = param.getNextFloatUniform(); break; case BINOMIAL: - Random binomialRandom = (Random) param.getGenerator(); - float fVal; - do { - fVal = (float) (minF + Math.abs(binomialRandom.nextGaussian()) * maxF); - } while (fVal > maxF || fVal < minF); - - stmt.setFloat(i + 1, fVal); + generatedFloat = param.getNextFloatBinomial(); break; default: - throw new RuntimeException( - "Distribution: " + distribution + " not supported for type: " + paramType); + throw param.createRuntimeException(paramType); } + stmt.setFloat(i + 1, generatedFloat); break; case "bigint": - Long minL = param.getMin(); - Long maxL = param.getMax(); + Long generatedLong; switch (distribution) { case UNIFORM: - Random uniformRand = (Random) param.getGenerator(); - stmt.setLong(i + 1, uniformRand.nextLong(minL, maxL)); + generatedLong = param.getNextLongUniform(); break; case BINOMIAL: - Random binomialRandom = (Random) param.getGenerator(); - Long lVal; - do { - lVal = Double.valueOf(minL + Math.abs(binomialRandom.nextGaussian()) * maxL) - .longValue(); - } while (lVal > maxL || lVal < minL); - - stmt.setLong(i + 1, lVal); + generatedLong = param.getNextLongBinomial(); break; case ZIPFIAN: - ZipfianGenerator zipfGen = (ZipfianGenerator) param.getGenerator(); - stmt.setLong(i + 1, zipfGen.nextLong()); + generatedLong = param.getNextLongZipf(); break; case SCRAMBLED: - ScrambledZipfianGenerator scramGen = (ScrambledZipfianGenerator) param.getGenerator(); - stmt.setLong(i + 1, scramGen.nextLong()); + generatedLong = param.getNextLongScrambled(); break; default: - throw new RuntimeException( - "Distribution: " + distribution + " not supported for type: " + paramType); + throw param.createRuntimeException(paramType); } + stmt.setLong(i+1, generatedLong); break; case "varchar": case "string": switch (distribution) { case UNIFORM: - Random strRandom = (Random) param.getGenerator(); - stmt.setString(i + 1, TextGenerator.randomStr(strRandom, param.getMax().intValue())); + stmt.setString(i + 1,param.getNextString()); break; default: - throw new RuntimeException( - "Distribution: " + distribution + " not supported for type: " + paramType); + throw param.createRuntimeException(paramType); } break; case "timestamp": - Long stampMin = param.getMin(); - Long stampMax = param.getMax(); - switch (distribution) { - case UNIFORM: - Random uniformRand = (Random) param.getGenerator(); - stmt.setTimestamp(i + 1, new Timestamp(uniformRand.nextLong(stampMin, stampMax))); - break; - case BINOMIAL: - Random binomialRandom = (Random) param.getGenerator(); - Long lVal; - do { - lVal = Double.valueOf(stampMin + Math.abs(binomialRandom.nextGaussian()) * stampMax) - .longValue(); - } while (lVal > stampMax || lVal < stampMin); - - stmt.setTimestamp(i + 1, new Timestamp(lVal)); - break; - case ZIPFIAN: - ZipfianGenerator zipfGen = (ZipfianGenerator) param.getGenerator(); - stmt.setTimestamp(i + 1, new Timestamp(zipfGen.nextLong())); - break; - case SCRAMBLED: - ScrambledZipfianGenerator scramGen = (ScrambledZipfianGenerator) param.getGenerator(); - stmt.setTimestamp(i + 1, new Timestamp(scramGen.nextLong())); - break; - default: - throw new RuntimeException( - "Distribution: " + distribution + " not supported for type: " + paramType); - } - break; case "date": - Long dateMin = param.getMin(); - Long dateMax = param.getMax(); - switch (distribution) { - case UNIFORM: - Random uniformRand = (Random) param.getGenerator(); - stmt.setDate(i + 1, new Date(uniformRand.nextLong(dateMin, dateMax))); - break; - case BINOMIAL: - Random binomialRandom = (Random) param.getGenerator(); - Long lVal; - do { - lVal = Double.valueOf(dateMin + Math.abs(binomialRandom.nextGaussian()) * dateMax) - .longValue(); - } while (lVal > dateMax || lVal < dateMin); - - stmt.setDate(i + 1, new Date(lVal)); - break; - case ZIPFIAN: - ZipfianGenerator zipfGen = (ZipfianGenerator) param.getGenerator(); - stmt.setDate(i + 1, new Date(zipfGen.nextLong())); - break; - case SCRAMBLED: - ScrambledZipfianGenerator scramGen = (ScrambledZipfianGenerator) param.getGenerator(); - stmt.setDate(i + 1, new Date(scramGen.nextLong())); - break; - default: - throw new RuntimeException( - "Distribution: " + distribution + " not supported for type: " + paramType); - } - break; case "time": - Long timeMin = param.getMin(); - Long timeMax = param.getMax(); + Long generatedTimestamp; switch (distribution) { case UNIFORM: - Random uniformRand = (Random) param.getGenerator(); - stmt.setTime(i + 1, new Time(uniformRand.nextLong(timeMin, timeMax))); + generatedTimestamp = param.getNextLongUniform(); break; case BINOMIAL: - Random binomialRandom = (Random) param.getGenerator(); - Long lVal; - do { - lVal = Double.valueOf(timeMin + Math.abs(binomialRandom.nextGaussian()) * timeMax) - .longValue(); - } while (lVal > timeMax || lVal < timeMin); - - stmt.setTime(i + 1, new Time(lVal)); + generatedTimestamp = param.getNextLongBinomial(); break; case ZIPFIAN: - ZipfianGenerator zipfGen = (ZipfianGenerator) param.getGenerator(); - stmt.setTime(i + 1, new Time(zipfGen.nextLong())); + generatedTimestamp = param.getNextLongZipf(); break; case SCRAMBLED: - ScrambledZipfianGenerator scramGen = (ScrambledZipfianGenerator) param.getGenerator(); - stmt.setTime(i + 1, new Time(scramGen.nextLong())); + generatedTimestamp = param.getNextLongScrambled(); break; default: - throw new RuntimeException( - "Distribution: " + distribution + " not supported for type: " + paramType); + throw param.createRuntimeException(paramType); + } + if(paramType.equals("timestamp")){ + stmt.setTimestamp(i+1, new Timestamp(generatedTimestamp)); + } else if(paramType.equals("date")){ + stmt.setDate(i+1, new Date(generatedTimestamp)); + } else { + stmt.setTime(i+1,new Time(generatedTimestamp)); } break; default: throw new RuntimeException( "Support for distributions for the type: " + paramType + " is current not implemented"); } + } else { try { Object val = param.getValue(); diff --git a/src/main/java/com/oltpbenchmark/benchmarks/templated/util/GenericQueryOperation.java b/src/main/java/com/oltpbenchmark/benchmarks/templated/util/GenericQueryOperation.java index 50080c75a..29daa77c3 100644 --- a/src/main/java/com/oltpbenchmark/benchmarks/templated/util/GenericQueryOperation.java +++ b/src/main/java/com/oltpbenchmark/benchmarks/templated/util/GenericQueryOperation.java @@ -17,6 +17,7 @@ package com.oltpbenchmark.benchmarks.templated.util; import java.util.Arrays; +import java.util.Collections; import java.util.List; import com.oltpbenchmark.api.Operation; @@ -30,7 +31,7 @@ public class GenericQueryOperation extends Operation { public GenericQueryOperation(TemplatedValue[] params) { super(); - this.params = Arrays.asList(params); + this.params = Collections.unmodifiableList(Arrays.asList(params)); } public List getParams() { diff --git a/src/main/java/com/oltpbenchmark/benchmarks/templated/util/TemplatedValue.java b/src/main/java/com/oltpbenchmark/benchmarks/templated/util/TemplatedValue.java index cb88a90e4..45d39bffb 100644 --- a/src/main/java/com/oltpbenchmark/benchmarks/templated/util/TemplatedValue.java +++ b/src/main/java/com/oltpbenchmark/benchmarks/templated/util/TemplatedValue.java @@ -4,6 +4,7 @@ import com.oltpbenchmark.distributions.ScrambledZipfianGenerator; import com.oltpbenchmark.distributions.ZipfianGenerator; +import com.oltpbenchmark.util.TextGenerator; /** * This class is used to store information about the values @@ -23,20 +24,21 @@ public class TemplatedValue { /** * - * @param dist The desired value distribution + * @param distribution The desired value distribution * @param min Minimum value. Default is 0 * @param max Maximum value. Default is 1 * @param seed The seed for the random generator. Default is 0 * @param value Value that is used if no distribution is given */ - public TemplatedValue(String dist, String min, String max, String seed, String value) { + public TemplatedValue(String distribution, String min, String max, String seed, String value) { this.min = min == null ? 0 : Long.parseLong(min); this.max = max == null ? 1 : Long.parseLong(max); this.seed = seed == null ? 0 : Long.parseLong(seed); this.value = value; - this.generatorObject = createGenerator(dist, this.min, this.max, this.seed); + this.generatorObject = createGenerator(distribution, this.min, this.max, this.seed); + //String values are kept for the construction of Floating Point bounds this.minS = min; this.maxS = max; } @@ -51,9 +53,10 @@ public TemplatedValue(String dist, String min, String max, String seed, String v */ private Object createGenerator(String distribution, Long min, Long max, Long seed) { - if (distribution == null || distribution.equals("null") || distribution.length() < 1) + if (distribution == null || distribution.equals("null") || distribution.length() < 1){ return null; - + } + switch (distribution.toLowerCase()) { case "uniform": this.distribution = ValueGenerator.UNIFORM; @@ -109,4 +112,55 @@ public String getMaxString() { public Object getGenerator() { return this.generatorObject; } + + public Long getNextLongBinomial(){ + Random binomialGenerator = (Random) this.generatorObject; + Long generatedValue; + do { + generatedValue = Double.valueOf(this.min + Math.abs(binomialGenerator.nextGaussian()) * this.max).longValue(); + } while (generatedValue > max || generatedValue < min); + + return generatedValue; + } + + public Long getNextLongUniform(){ + Random uniformGenerator = (Random) this.generatorObject; + return uniformGenerator.nextLong(this.min, this.max); + } + + public Long getNextLongZipf(){ + ZipfianGenerator zipfianGenerator = (ZipfianGenerator) this.generatorObject; + return zipfianGenerator.nextLong(); + } + + public Long getNextLongScrambled(){ + ScrambledZipfianGenerator scrambledGenerator = (ScrambledZipfianGenerator) this.generatorObject; + return scrambledGenerator.nextLong(); + } + + public String getNextString(){ + Random stringGenerator = (Random) this.generatorObject; + return TextGenerator.randomStr(stringGenerator, this.max.intValue()); + } + + public Float getNextFloatUniform(){ + Random floatGenerator = (Random) this.generatorObject; + return floatGenerator.nextFloat(Float.parseFloat(this.minS), Float.parseFloat(this.maxS)); + } + + public Float getNextFloatBinomial(){ + Float minF = Float.parseFloat(this.minS); + Float maxF = Float.parseFloat(this.maxS); + Random floatGenerator = (Random) this.generatorObject; + Float generatedFloat; + do { + generatedFloat = (float) (minF + Math.abs(floatGenerator.nextGaussian()) * maxF); + } while (generatedFloat > maxF || generatedFloat < minF); + + return generatedFloat; + } + + public RuntimeException createRuntimeException(String paramType){ + return new RuntimeException("Distribution: " + this.distribution + " not supported for type: " + paramType); + } } From 961d5c41521281af5cb3b4ae79ae11d4489f3715 Mon Sep 17 00:00:00 2001 From: Eric Enzler Date: Wed, 17 Jan 2024 14:07:47 +0100 Subject: [PATCH 13/24] Lightweight constructor added --- .../templated/TemplatedBenchmark.java | 66 +++++++++++-------- .../templated/procedures/GenericQuery.java | 2 +- .../templated/util/TemplatedValue.java | 39 +++++++---- 3 files changed, 63 insertions(+), 44 deletions(-) diff --git a/src/main/java/com/oltpbenchmark/benchmarks/templated/TemplatedBenchmark.java b/src/main/java/com/oltpbenchmark/benchmarks/templated/TemplatedBenchmark.java index 11851bc4e..9e9d32397 100644 --- a/src/main/java/com/oltpbenchmark/benchmarks/templated/TemplatedBenchmark.java +++ b/src/main/java/com/oltpbenchmark/benchmarks/templated/TemplatedBenchmark.java @@ -190,13 +190,16 @@ private CustomClassLoader loadQueryTemplates(String file) { for (ValuesType paramValue : template.getValuesList()) { for (ValueType value : paramValue.getValueList()) { - b.addParamsValues( - new TemplatedValue( - value.getDist(), - value.getMin(), - value.getMax(), - value.getSeed(), - value.getValue())); + /* Lightweight constructor used if no distribution is present */ + if (value.getDist() == null + || value.getDist().length() < 1 + || value.getDist().equals("null")) { + b.addParamsValues(new TemplatedValue(value.getValue())); + } else { + b.addParamsValues( + new TemplatedValue( + value.getDist(), value.getMin(), value.getMax(), value.getSeed())); + } } } @@ -253,28 +256,33 @@ private String getParamsString(List params) { private String buildTemplatedValueString(List params) { String result = ""; for (TemplatedValue param : params) { - result += - "new TemplatedValue(" - + "\"" - + param.getDistribution() - + "\"" - + "," - + "\"" - + param.getMin() - + "\"" - + "," - + "\"" - + param.getMax() - + "\"" - + "," - + "\"" - + param.getSeed() - + "\"" - + "," - + "\"" - + StringEscapeUtils.escapeJava(param.getValue()) - + "\"" - + "),"; + if (param.getDistribution() != null) { + result += + "new TemplatedValue(" + + "\"" + + param.getDistribution() + + "\"" + + "," + + "\"" + + param.getMin() + + "\"" + + "," + + "\"" + + param.getMax() + + "\"" + + "," + + "\"" + + param.getSeed() + + "\"" + + "),"; + } else { + result += + "new TemplatedValue(" + + "\"" + + StringEscapeUtils.escapeJava(param.getValue()) + + "\"" + + "),"; + } } return result.isEmpty() ? "" : result.substring(0, result.length() - 1); } diff --git a/src/main/java/com/oltpbenchmark/benchmarks/templated/procedures/GenericQuery.java b/src/main/java/com/oltpbenchmark/benchmarks/templated/procedures/GenericQuery.java index ee9c6962e..da622c8a3 100644 --- a/src/main/java/com/oltpbenchmark/benchmarks/templated/procedures/GenericQuery.java +++ b/src/main/java/com/oltpbenchmark/benchmarks/templated/procedures/GenericQuery.java @@ -91,7 +91,7 @@ public PreparedStatement getStatement(Connection conn, List para TemplatedValue param = params.get(i); boolean hasDist = param.getDistribution() != null; - boolean hasValue = param.getValue().length() > 0; + boolean hasValue = param.getValue() != null; if ((!hasDist && !hasValue) || paramsTypes[i].equalsIgnoreCase("NULL")) { stmt.setNull(i + 1, Types.NULL); diff --git a/src/main/java/com/oltpbenchmark/benchmarks/templated/util/TemplatedValue.java b/src/main/java/com/oltpbenchmark/benchmarks/templated/util/TemplatedValue.java index 5a86c0fe6..dc5ca2ef1 100644 --- a/src/main/java/com/oltpbenchmark/benchmarks/templated/util/TemplatedValue.java +++ b/src/main/java/com/oltpbenchmark/benchmarks/templated/util/TemplatedValue.java @@ -17,8 +17,14 @@ public class TemplatedValue { String value; String minS; String maxS; + Object generatorObject; - public final Object generatorObject; + /** + * @param value Value that is used if no distribution is given + */ + public TemplatedValue(String value) { + this.value = value; + } /** * @param distribution The desired value distribution @@ -27,11 +33,24 @@ public class TemplatedValue { * @param seed The seed for the random generator. Default is 0 * @param value Value that is used if no distribution is given */ - public TemplatedValue(String distribution, String min, String max, String seed, String value) { - this.min = min == null ? 0 : Long.parseLong(min); - this.max = max == null ? 1 : Long.parseLong(max); - this.seed = seed == null ? 0 : Long.parseLong(seed); - this.value = value; + public TemplatedValue(String distribution, String min, String max, String seed) { + try { + this.min = Long.parseLong(min); + } catch (Exception e) { + this.min = 0L; + } + + try { + this.max = Long.parseLong(max); + } catch (Exception e) { + this.max = 1L; + } + + try { + this.seed = Long.parseLong(seed); + } catch (Exception e) { + this.seed = 0L; + } this.generatorObject = createGenerator(distribution, this.min, this.max, this.seed); @@ -97,14 +116,6 @@ public String getValue() { return this.value; } - public String getMinString() { - return this.minS; - } - - public String getMaxString() { - return this.maxS; - } - public Object getGenerator() { return this.generatorObject; } From 5454f526370fffbcde30bee665b971f860976211 Mon Sep 17 00:00:00 2001 From: Eric Enzler Date: Tue, 23 Jan 2024 13:48:37 +0100 Subject: [PATCH 14/24] Insert statment modification --- .../cockroachdb/sample_templated_config.xml | 4 +--- config/mariadb/sample_templated_config.xml | 4 +--- config/mysql/sample_templated_config.xml | 4 +--- config/oracle/sample_templated_config.xml | 4 +--- config/postgres/sample_templated_config.xml | 4 +--- config/sqlite/sample_templated_config.xml | 4 +--- config/sqlserver/sample_templated_config.xml | 4 +--- data/templated/example.xml | 24 +++++++++++-------- 8 files changed, 21 insertions(+), 31 deletions(-) diff --git a/config/cockroachdb/sample_templated_config.xml b/config/cockroachdb/sample_templated_config.xml index 8f0e31630..b7dca724e 100644 --- a/config/cockroachdb/sample_templated_config.xml +++ b/config/cockroachdb/sample_templated_config.xml @@ -24,7 +24,7 @@ 100 - 20,20,10,10,10,10,10 + 20,20,10,10,10,10,10,10 @@ -51,10 +51,8 @@ DeleteItem - diff --git a/config/mariadb/sample_templated_config.xml b/config/mariadb/sample_templated_config.xml index 1f74ac859..ac836639f 100644 --- a/config/mariadb/sample_templated_config.xml +++ b/config/mariadb/sample_templated_config.xml @@ -25,7 +25,7 @@ 100 - 20,20,10,10,10,10,10 + 20,20,10,10,10,10,10,10 @@ -52,10 +52,8 @@ DeleteItem - diff --git a/config/mysql/sample_templated_config.xml b/config/mysql/sample_templated_config.xml index c58efb0de..db54cd1d8 100644 --- a/config/mysql/sample_templated_config.xml +++ b/config/mysql/sample_templated_config.xml @@ -25,7 +25,7 @@ 100 - 20,20,10,10,10,10,10 + 20,20,10,10,10,10,10,10 @@ -52,10 +52,8 @@ DeleteItem - diff --git a/config/oracle/sample_templated_config.xml b/config/oracle/sample_templated_config.xml index 57a828487..a10605fee 100644 --- a/config/oracle/sample_templated_config.xml +++ b/config/oracle/sample_templated_config.xml @@ -25,7 +25,7 @@ 100 - 20,20,10,10,10,10,10 + 20,20,10,10,10,10,10,10 @@ -52,10 +52,8 @@ DeleteItem - diff --git a/config/postgres/sample_templated_config.xml b/config/postgres/sample_templated_config.xml index 25cc68739..51a701d58 100644 --- a/config/postgres/sample_templated_config.xml +++ b/config/postgres/sample_templated_config.xml @@ -25,7 +25,7 @@ 100 - 20,20,10,10,10,10,10 + 20,20,10,10,10,10,10,10 @@ -52,10 +52,8 @@ DeleteItem - diff --git a/config/sqlite/sample_templated_config.xml b/config/sqlite/sample_templated_config.xml index 8b2c8ecbf..a3b5b5f7f 100644 --- a/config/sqlite/sample_templated_config.xml +++ b/config/sqlite/sample_templated_config.xml @@ -22,7 +22,7 @@ 100 - 20,20,10,10,10,10,10 + 20,20,10,10,10,10,10,10 @@ -49,10 +49,8 @@ DeleteItem - diff --git a/config/sqlserver/sample_templated_config.xml b/config/sqlserver/sample_templated_config.xml index 1899addae..37a418d44 100644 --- a/config/sqlserver/sample_templated_config.xml +++ b/config/sqlserver/sample_templated_config.xml @@ -25,7 +25,7 @@ 100 - 20,20,10,10,10,10,10 + 20,20,10,10,10,10,10,10 @@ -52,10 +52,8 @@ DeleteItem - diff --git a/data/templated/example.xml b/data/templated/example.xml index f48489f04..7d82b726c 100644 --- a/data/templated/example.xml +++ b/data/templated/example.xml @@ -79,23 +79,27 @@ - \ No newline at end of file From 06d3f39ddd9790c7f55677b28ada57bd93abae42 Mon Sep 17 00:00:00 2001 From: Eric Enzler Date: Wed, 24 Jan 2024 08:22:07 +0100 Subject: [PATCH 15/24] Min-Max check --- .../benchmarks/templated/util/TemplatedValue.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/oltpbenchmark/benchmarks/templated/util/TemplatedValue.java b/src/main/java/com/oltpbenchmark/benchmarks/templated/util/TemplatedValue.java index dc5ca2ef1..81897b1af 100644 --- a/src/main/java/com/oltpbenchmark/benchmarks/templated/util/TemplatedValue.java +++ b/src/main/java/com/oltpbenchmark/benchmarks/templated/util/TemplatedValue.java @@ -29,7 +29,7 @@ public TemplatedValue(String value) { /** * @param distribution The desired value distribution * @param min Minimum value. Default is 0 - * @param max Maximum value. Default is 1 + * @param max Maximum value. Default is min + 1 * @param seed The seed for the random generator. Default is 0 * @param value Value that is used if no distribution is given */ @@ -43,9 +43,11 @@ public TemplatedValue(String distribution, String min, String max, String seed) try { this.max = Long.parseLong(max); } catch (Exception e) { - this.max = 1L; + this.max = this.min + 1L; } + assert this.max > this.min; + try { this.seed = Long.parseLong(seed); } catch (Exception e) { From 949c93db0f044eb264ab90fe7429f0d7eae92c8b Mon Sep 17 00:00:00 2001 From: Eric Enzler Date: Wed, 24 Jan 2024 08:41:08 +0100 Subject: [PATCH 16/24] Changed distributions due to FK contraints --- data/templated/example.xml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/data/templated/example.xml b/data/templated/example.xml index 7d82b726c..138ef57db 100644 --- a/data/templated/example.xml +++ b/data/templated/example.xml @@ -9,11 +9,11 @@ INTEGER - 3 - 4 + + - 5 + 6 @@ -53,8 +53,8 @@ FLOAT - 10 - 10.49 + + 10.50 @@ -92,10 +92,10 @@ VARCHAR - - - - + 1 + 1 + 1 + 1 1 From 19fad84b91c37d19e795c6c43ad5a89022d5a2eb Mon Sep 17 00:00:00 2001 From: Eric Enzler Date: Wed, 24 Jan 2024 08:48:14 +0100 Subject: [PATCH 17/24] Typo --- data/templated/example.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/templated/example.xml b/data/templated/example.xml index 138ef57db..96d9f76c9 100644 --- a/data/templated/example.xml +++ b/data/templated/example.xml @@ -54,7 +54,7 @@ - + 10.50 From c9dc450e4be1bfc4301e669bcf3541c8841bf84c Mon Sep 17 00:00:00 2001 From: Eric Enzler Date: Wed, 24 Jan 2024 08:57:49 +0100 Subject: [PATCH 18/24] Typo 2, build checked --- data/templated/example.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/templated/example.xml b/data/templated/example.xml index 96d9f76c9..2e16a8be6 100644 --- a/data/templated/example.xml +++ b/data/templated/example.xml @@ -9,7 +9,7 @@ INTEGER - + @@ -54,7 +54,7 @@ - + 10.50 From ee8f4c110d8a5e1347c295c743f726b74bcf1e30 Mon Sep 17 00:00:00 2001 From: Eric Enzler Date: Thu, 25 Jan 2024 14:27:47 +0100 Subject: [PATCH 19/24] Enums, removal of aliases, nits --- data/templated/example.xml | 19 +++++++--- .../templated/procedures/GenericQuery.java | 35 +++++++++---------- .../templated/util/TemplatedValue.java | 32 ++++++++++------- .../templated/util/ValueGenerator.java | 2 +- .../oltpbenchmark/util/JDBCSupportedType.java | 21 +++++++++++ 5 files changed, 72 insertions(+), 37 deletions(-) create mode 100644 src/main/java/com/oltpbenchmark/util/JDBCSupportedType.java diff --git a/data/templated/example.xml b/data/templated/example.xml index 2e16a8be6..ec8425936 100644 --- a/data/templated/example.xml +++ b/data/templated/example.xml @@ -10,11 +10,11 @@ - + - 6 + diff --git a/src/main/java/com/oltpbenchmark/benchmarks/templated/procedures/GenericQuery.java b/src/main/java/com/oltpbenchmark/benchmarks/templated/procedures/GenericQuery.java index da622c8a3..813aba8a9 100644 --- a/src/main/java/com/oltpbenchmark/benchmarks/templated/procedures/GenericQuery.java +++ b/src/main/java/com/oltpbenchmark/benchmarks/templated/procedures/GenericQuery.java @@ -20,6 +20,7 @@ import com.oltpbenchmark.api.SQLStmt; import com.oltpbenchmark.benchmarks.templated.util.TemplatedValue; import com.oltpbenchmark.benchmarks.templated.util.ValueGenerator; +import com.oltpbenchmark.util.JDBCSupportedType; import java.sql.Connection; import java.sql.Date; import java.sql.PreparedStatement; @@ -95,18 +96,17 @@ public PreparedStatement getStatement(Connection conn, List para if ((!hasDist && !hasValue) || paramsTypes[i].equalsIgnoreCase("NULL")) { stmt.setNull(i + 1, Types.NULL); - } else if (hasDist) { ValueGenerator distribution = param.getDistribution(); - String paramType = paramsTypes[i].toLowerCase(); + JDBCSupportedType paramType = JDBCSupportedType.valueOf(paramsTypes[i].toUpperCase()); switch (paramType) { - case "integer": + case INTEGER: int generatedInt; switch (distribution) { case UNIFORM: generatedInt = param.getNextLongUniform().intValue(); break; - case BINOMIAL: + case NORMAL: generatedInt = param.getNextLongBinomial().intValue(); break; case ZIPFIAN: @@ -120,14 +120,14 @@ public PreparedStatement getStatement(Connection conn, List para } stmt.setInt(i + 1, generatedInt); break; - case "float": - case "real": + case FLOAT: + case REAL: float generatedFloat; switch (distribution) { case UNIFORM: generatedFloat = param.getNextFloatUniform(); break; - case BINOMIAL: + case NORMAL: generatedFloat = param.getNextFloatBinomial(); break; default: @@ -135,13 +135,13 @@ public PreparedStatement getStatement(Connection conn, List para } stmt.setFloat(i + 1, generatedFloat); break; - case "bigint": + case BIGINT: Long generatedLong; switch (distribution) { case UNIFORM: generatedLong = param.getNextLongUniform(); break; - case BINOMIAL: + case NORMAL: generatedLong = param.getNextLongBinomial(); break; case ZIPFIAN: @@ -155,8 +155,7 @@ public PreparedStatement getStatement(Connection conn, List para } stmt.setLong(i + 1, generatedLong); break; - case "varchar": - case "string": + case VARCHAR: switch (distribution) { case UNIFORM: stmt.setString(i + 1, param.getNextString()); @@ -165,15 +164,15 @@ public PreparedStatement getStatement(Connection conn, List para throw param.createRuntimeException(paramType); } break; - case "timestamp": - case "date": - case "time": + case TIMESTAMP: + case DATE: + case TIME: Long generatedTimestamp; switch (distribution) { case UNIFORM: generatedTimestamp = param.getNextLongUniform(); break; - case BINOMIAL: + case NORMAL: generatedTimestamp = param.getNextLongBinomial(); break; case ZIPFIAN: @@ -185,9 +184,9 @@ public PreparedStatement getStatement(Connection conn, List para default: throw param.createRuntimeException(paramType); } - if (paramType.equals("timestamp")) { + if (paramType == JDBCSupportedType.TIMESTAMP) { stmt.setTimestamp(i + 1, new Timestamp(generatedTimestamp)); - } else if (paramType.equals("date")) { + } else if (paramType == JDBCSupportedType.DATE) { stmt.setDate(i + 1, new Date(generatedTimestamp)); } else { stmt.setTime(i + 1, new Time(generatedTimestamp)); @@ -197,7 +196,7 @@ public PreparedStatement getStatement(Connection conn, List para throw new RuntimeException( "Support for distributions for the type: " + paramType - + " is current not implemented"); + + " is not currently implemented"); } } else { diff --git a/src/main/java/com/oltpbenchmark/benchmarks/templated/util/TemplatedValue.java b/src/main/java/com/oltpbenchmark/benchmarks/templated/util/TemplatedValue.java index 81897b1af..e630ec53b 100644 --- a/src/main/java/com/oltpbenchmark/benchmarks/templated/util/TemplatedValue.java +++ b/src/main/java/com/oltpbenchmark/benchmarks/templated/util/TemplatedValue.java @@ -2,6 +2,7 @@ import com.oltpbenchmark.distributions.ScrambledZipfianGenerator; import com.oltpbenchmark.distributions.ZipfianGenerator; +import com.oltpbenchmark.util.JDBCSupportedType; import com.oltpbenchmark.util.TextGenerator; import java.util.Random; @@ -23,7 +24,11 @@ public class TemplatedValue { * @param value Value that is used if no distribution is given */ public TemplatedValue(String value) { - this.value = value; + if (value.equals("null") || value.length() < 1) { + this.value = null; + } else { + this.value = value; + } } /** @@ -59,6 +64,8 @@ public TemplatedValue(String distribution, String min, String max, String seed) // String values are kept for the construction of Floating Point bounds this.minS = min; this.maxS = max; + + this.value = null; } /** @@ -74,27 +81,26 @@ private Object createGenerator(String distribution, Long min, Long max, Long see return null; } - switch (distribution.toLowerCase()) { - case "uniform": + this.distribution = ValueGenerator.valueOf(distribution.toUpperCase()); + + switch (this.distribution) { + case UNIFORM: this.distribution = ValueGenerator.UNIFORM; return new Random(seed); - case "binomial": - case "normal": - this.distribution = ValueGenerator.BINOMIAL; + case NORMAL: + this.distribution = ValueGenerator.NORMAL; return new Random(seed); - case "zipf": - case "zipfian": + case ZIPFIAN: this.distribution = ValueGenerator.ZIPFIAN; return new ZipfianGenerator(new Random(seed), min, max); - case "scrambled": - case "scramzipf": + case SCRAMBLED: this.distribution = ValueGenerator.SCRAMBLED; return new ScrambledZipfianGenerator(min, max); default: throw new RuntimeException( "The distribution: '" + distribution - + "' is not supported. Currently supported are 'uniform' | 'binomial' | 'zipfian' | 'scrambled'"); + + "' is not supported. Currently supported are 'uniform' | 'normal' | 'zipfian' | 'scrambled'"); } } @@ -171,8 +177,8 @@ public Float getNextFloatBinomial() { return generatedFloat; } - public RuntimeException createRuntimeException(String paramType) { + public RuntimeException createRuntimeException(JDBCSupportedType paramType) { return new RuntimeException( - "Distribution: " + this.distribution + " not supported for type: " + paramType); + "Distribution: " + this.distribution + " not supported for type: " + paramType.toString()); } } diff --git a/src/main/java/com/oltpbenchmark/benchmarks/templated/util/ValueGenerator.java b/src/main/java/com/oltpbenchmark/benchmarks/templated/util/ValueGenerator.java index 5f554b989..c49b663b7 100644 --- a/src/main/java/com/oltpbenchmark/benchmarks/templated/util/ValueGenerator.java +++ b/src/main/java/com/oltpbenchmark/benchmarks/templated/util/ValueGenerator.java @@ -2,7 +2,7 @@ public enum ValueGenerator { UNIFORM, - BINOMIAL, + NORMAL, ZIPFIAN, SCRAMBLED } diff --git a/src/main/java/com/oltpbenchmark/util/JDBCSupportedType.java b/src/main/java/com/oltpbenchmark/util/JDBCSupportedType.java new file mode 100644 index 000000000..edd7bad39 --- /dev/null +++ b/src/main/java/com/oltpbenchmark/util/JDBCSupportedType.java @@ -0,0 +1,21 @@ +package com.oltpbenchmark.util; + +public enum JDBCSupportedType { + BIGINT, + BIT, + BLOB, + CHAR, + CLOB, + DATE, + DECIMAL, + DOUBLE, + FLOAT, + INTEGER, + NUMERIC, + REAL, + SMALLINT, + TIME, + TIMESTAMP, + TINYINT, + VARCHAR +} From 0235de774e6b6909a2ee5369f32617f96096825e Mon Sep 17 00:00:00 2001 From: Eric Enzler Date: Thu, 25 Jan 2024 14:48:00 +0100 Subject: [PATCH 20/24] Adapted description --- .../benchmarks/templated/README.md | 22 ++++++++++++++++--- .../templated/procedures/GenericQuery.java | 5 +++-- .../oltpbenchmark/util/JDBCSupportedType.java | 1 + 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/oltpbenchmark/benchmarks/templated/README.md b/src/main/java/com/oltpbenchmark/benchmarks/templated/README.md index e8fe5446c..2f474000f 100644 --- a/src/main/java/com/oltpbenchmark/benchmarks/templated/README.md +++ b/src/main/java/com/oltpbenchmark/benchmarks/templated/README.md @@ -81,12 +81,12 @@ Each time the query is run, a different value will substitute the `?`. ``` The distributions are dependent on the type of the value. Currently, the following type-distributions pairs are supported: -| Type | uniform | binomial | zipfian | scrambled zipfian | +| Type | uniform | binomial | zipfian | scrambled | | ---------------- | :-----: | :------: | :-----: | :---------------: | | INTEGER | X | X | X | X | | FLOAT / REAL | X | X | - | - | | BIGINT | X | X | X | X | -| VARCHAR / STRING | X | - | - | - | +| VARCHAR | X | - | - | - | | TIMESTAMP | X | X | X | X | | DATE | X | X | X | X | | TIME | X | X | X | X | @@ -97,4 +97,20 @@ The following properties can be set on the value: - `max`: The maximum value the generator can produce. Defaults to 1 - `seed`: A seed for the generator to ensure consistency -For Timestamps, Dates and Times, the min and max values must be converted to a long (milliseconds) +*NOTE:* For `Timestamp`, `Date`, and `Time` types, the min and max values must be converted to UNIX time (milliseconds). +The value is interpreted as the milliseconds since January 1, 1970, 00:00:00 GMT/UTC. +```xml + +``` +To get the current UNIX time in milliseconds, use the following bash command: +```bash +date +%s%3N +``` +or use a web services like [www.currentmillis.com](www.currentmillis.com) + + diff --git a/src/main/java/com/oltpbenchmark/benchmarks/templated/procedures/GenericQuery.java b/src/main/java/com/oltpbenchmark/benchmarks/templated/procedures/GenericQuery.java index 813aba8a9..1afb00466 100644 --- a/src/main/java/com/oltpbenchmark/benchmarks/templated/procedures/GenericQuery.java +++ b/src/main/java/com/oltpbenchmark/benchmarks/templated/procedures/GenericQuery.java @@ -94,11 +94,12 @@ public PreparedStatement getStatement(Connection conn, List para boolean hasDist = param.getDistribution() != null; boolean hasValue = param.getValue() != null; - if ((!hasDist && !hasValue) || paramsTypes[i].equalsIgnoreCase("NULL")) { + JDBCSupportedType paramType = JDBCSupportedType.valueOf(paramsTypes[i].toUpperCase()); + + if ((!hasDist && !hasValue) || paramType == JDBCSupportedType.NULL) { stmt.setNull(i + 1, Types.NULL); } else if (hasDist) { ValueGenerator distribution = param.getDistribution(); - JDBCSupportedType paramType = JDBCSupportedType.valueOf(paramsTypes[i].toUpperCase()); switch (paramType) { case INTEGER: int generatedInt; diff --git a/src/main/java/com/oltpbenchmark/util/JDBCSupportedType.java b/src/main/java/com/oltpbenchmark/util/JDBCSupportedType.java index edd7bad39..fb777dbc2 100644 --- a/src/main/java/com/oltpbenchmark/util/JDBCSupportedType.java +++ b/src/main/java/com/oltpbenchmark/util/JDBCSupportedType.java @@ -12,6 +12,7 @@ public enum JDBCSupportedType { FLOAT, INTEGER, NUMERIC, + NULL, REAL, SMALLINT, TIME, From ad94565962b52709c305a36b383668a6d8e3357a Mon Sep 17 00:00:00 2001 From: Eric Enzler Date: Wed, 20 Mar 2024 16:51:54 +0100 Subject: [PATCH 21/24] Templated Values with type information --- data/templated/example.xml | 10 +- .../benchmarks/templated/README.md | 14 +- .../templated/TemplatedBenchmark.java | 16 +- .../templated/procedures/GenericQuery.java | 10 +- .../templated/util/TemplatedValue.java | 154 ++++++++++++++---- 5 files changed, 151 insertions(+), 53 deletions(-) diff --git a/data/templated/example.xml b/data/templated/example.xml index ec8425936..be91ed493 100644 --- a/data/templated/example.xml +++ b/data/templated/example.xml @@ -10,10 +10,10 @@ - + - + @@ -54,7 +54,7 @@ - + 10.50 @@ -106,9 +106,9 @@ 1 1 1 - + - + \ No newline at end of file diff --git a/src/main/java/com/oltpbenchmark/benchmarks/templated/README.md b/src/main/java/com/oltpbenchmark/benchmarks/templated/README.md index 2f474000f..cbe6d939e 100644 --- a/src/main/java/com/oltpbenchmark/benchmarks/templated/README.md +++ b/src/main/java/com/oltpbenchmark/benchmarks/templated/README.md @@ -73,7 +73,7 @@ Each time the query is run, a different value will substitute the `?`. - + @@ -93,17 +93,17 @@ The distributions are dependent on the type of the value. Currently, the followi The following properties can be set on the value: - `dist`: The distribution of the values -- `min`: The minimum value the generator can produce. Defaults to 0 -- `max`: The maximum value the generator can produce. Defaults to 1 -- `seed`: A seed for the generator to ensure consistency +- `min`: The minimum value the generator can produce. +- `max`: The maximum value the generator can produce. +- `seed`: A seed for the generator to ensure consistency. Default is 0 -*NOTE:* For `Timestamp`, `Date`, and `Time` types, the min and max values must be converted to UNIX time (milliseconds). +*NOTE:* For `Timestamp`, `Date`, and `Time` types, the min and max values must be converted to UNIX time (milliseconds) OR given as strings in the following format: (yyyy-MM-dd HH:mm:ss). The value is interpreted as the milliseconds since January 1, 1970, 00:00:00 GMT/UTC. ```xml ``` diff --git a/src/main/java/com/oltpbenchmark/benchmarks/templated/TemplatedBenchmark.java b/src/main/java/com/oltpbenchmark/benchmarks/templated/TemplatedBenchmark.java index 176e62a8f..38ae2c533 100644 --- a/src/main/java/com/oltpbenchmark/benchmarks/templated/TemplatedBenchmark.java +++ b/src/main/java/com/oltpbenchmark/benchmarks/templated/TemplatedBenchmark.java @@ -184,11 +184,14 @@ private CustomClassLoader loadQueryTemplates(String file) { for (TemplateType template : templates.getTemplateList()) { ImmutableParsedQueryTemplate.Builder b = ImmutableParsedQueryTemplate.builder(); + List templateTypes = template.getTypes().getTypeList(); + b.name(template.getName()); b.query(template.getQuery()); - b.paramsTypes(template.getTypes().getTypeList()); + b.paramsTypes(templateTypes); for (ValuesType paramValue : template.getValuesList()) { + int typeIndex = 0; for (ValueType value : paramValue.getValueList()) { /* Lightweight constructor used if no distribution is present */ if (value.getDist() == null @@ -198,8 +201,13 @@ private CustomClassLoader loadQueryTemplates(String file) { } else { b.addParamsValues( new TemplatedValue( - value.getDist(), value.getMin(), value.getMax(), value.getSeed())); + value.getDist(), + value.getMin(), + value.getMax(), + value.getSeed(), + templateTypes.get(typeIndex))); } + typeIndex++; } } @@ -274,6 +282,10 @@ private String buildTemplatedValueString(List params) { + "\"" + param.getSeed() + "\"" + + "," + + "\"" + + param.getValueType() + + "\"" + "),"; } else { result += diff --git a/src/main/java/com/oltpbenchmark/benchmarks/templated/procedures/GenericQuery.java b/src/main/java/com/oltpbenchmark/benchmarks/templated/procedures/GenericQuery.java index 1afb00466..9c5ea67d0 100644 --- a/src/main/java/com/oltpbenchmark/benchmarks/templated/procedures/GenericQuery.java +++ b/src/main/java/com/oltpbenchmark/benchmarks/templated/procedures/GenericQuery.java @@ -117,7 +117,7 @@ public PreparedStatement getStatement(Connection conn, List para generatedInt = param.getNextLongScrambled().intValue(); break; default: - throw param.createRuntimeException(paramType); + throw param.createRuntimeException(); } stmt.setInt(i + 1, generatedInt); break; @@ -132,7 +132,7 @@ public PreparedStatement getStatement(Connection conn, List para generatedFloat = param.getNextFloatBinomial(); break; default: - throw param.createRuntimeException(paramType); + throw param.createRuntimeException(); } stmt.setFloat(i + 1, generatedFloat); break; @@ -152,7 +152,7 @@ public PreparedStatement getStatement(Connection conn, List para generatedLong = param.getNextLongScrambled(); break; default: - throw param.createRuntimeException(paramType); + throw param.createRuntimeException(); } stmt.setLong(i + 1, generatedLong); break; @@ -162,7 +162,7 @@ public PreparedStatement getStatement(Connection conn, List para stmt.setString(i + 1, param.getNextString()); break; default: - throw param.createRuntimeException(paramType); + throw param.createRuntimeException(); } break; case TIMESTAMP: @@ -183,7 +183,7 @@ public PreparedStatement getStatement(Connection conn, List para generatedTimestamp = param.getNextLongScrambled(); break; default: - throw param.createRuntimeException(paramType); + throw param.createRuntimeException(); } if (paramType == JDBCSupportedType.TIMESTAMP) { stmt.setTimestamp(i + 1, new Timestamp(generatedTimestamp)); diff --git a/src/main/java/com/oltpbenchmark/benchmarks/templated/util/TemplatedValue.java b/src/main/java/com/oltpbenchmark/benchmarks/templated/util/TemplatedValue.java index e630ec53b..0be811da0 100644 --- a/src/main/java/com/oltpbenchmark/benchmarks/templated/util/TemplatedValue.java +++ b/src/main/java/com/oltpbenchmark/benchmarks/templated/util/TemplatedValue.java @@ -4,6 +4,8 @@ import com.oltpbenchmark.distributions.ZipfianGenerator; import com.oltpbenchmark.util.JDBCSupportedType; import com.oltpbenchmark.util.TextGenerator; +import java.text.ParseException; +import java.text.SimpleDateFormat; import java.util.Random; /** @@ -11,13 +13,15 @@ * hold static values but also generators for value distributions */ public class TemplatedValue { + static SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); ValueGenerator distribution; Long min; Long max; Long seed; String value; - String minS; - String maxS; + JDBCSupportedType valueType; + Float minF; + Float maxF; Object generatorObject; /** @@ -33,41 +37,99 @@ public TemplatedValue(String value) { /** * @param distribution The desired value distribution - * @param min Minimum value. Default is 0 - * @param max Maximum value. Default is min + 1 + * @param min Minimum value + * @param max Maximum value * @param seed The seed for the random generator. Default is 0 * @param value Value that is used if no distribution is given */ - public TemplatedValue(String distribution, String min, String max, String seed) { - try { - this.min = Long.parseLong(min); - } catch (Exception e) { - this.min = 0L; - } + public TemplatedValue( + String distribution, String min, String max, String seed, String valueType) { + this.valueType = JDBCSupportedType.valueOf(valueType.toUpperCase()); - try { - this.max = Long.parseLong(max); - } catch (Exception e) { - this.max = this.min + 1L; + switch (this.valueType) { + case DATE: + case TIME: + case TIMESTAMP: + this.min = parseDateTime(min); + this.max = parseDateTime(max); + assert this.max > this.min; + break; + case FLOAT: + case REAL: + this.minF = parseBoundaryFloat(min); + this.maxF = parseBoundaryFloat(max); + assert this.maxF > this.minF; + break; + default: + this.min = parseBoundary(min); + this.max = parseBoundary(max); + assert this.max > this.min; + break; } - assert this.max > this.min; - + // Parse seed with fallback of 0L try { this.seed = Long.parseLong(seed); - } catch (Exception e) { + } catch (NumberFormatException e) { this.seed = 0L; } - this.generatorObject = createGenerator(distribution, this.min, this.max, this.seed); + assert this.seed >= 0L; - // String values are kept for the construction of Floating Point bounds - this.minS = min; - this.maxS = max; + this.generatorObject = createGenerator(distribution, this.min, this.max, this.seed); this.value = null; } + /** + * @param boundary A string signifying a numerical bondary + * @return The numerical value parsed to a Long + */ + private Long parseBoundary(String boundary) { + Long lBound; + try { + lBound = Long.parseLong(boundary); + } catch (NumberFormatException e) { + throw new RuntimeException( + String.format("Error occurred while trying to parse %s to a Long", boundary)); + } + return lBound; + } + + /** + * @param boundary A string signifying a numerical bondary + * @return The numerical value parsed to a float + */ + private Float parseBoundaryFloat(String boundary) { + Float fBound; + try { + fBound = Float.parseFloat(boundary); + } catch (NumberFormatException e) { + throw new RuntimeException( + String.format("Error occurred while trying to parse %s to a Float", boundary)); + } + return fBound; + } + + /** + * @param timeSource A String in a specific time format (yyyy-MM-dd HH:mm:ss) or UNIX timestamp + * @return The UNIX timestamp of the value + */ + private Long parseDateTime(String timeSource) { + long timestamp; + try { + timestamp = Long.parseLong(timeSource); + } catch (NumberFormatException ex) { + try { + timestamp = dateFormat.parse(timeSource).getTime(); + } catch (ParseException e) { + throw new RuntimeException( + String.format("Error occurred while trying to parse date: %s", timeSource)); + } + } + return timestamp; + } + /** * @param distribution The distribution that drives the generator * @param min Minimum value the generator will produce @@ -108,27 +170,35 @@ public ValueGenerator getDistribution() { return this.distribution; } - public Long getMin() { - return this.min; + public String getMin() { + return (this.min != null) ? this.min.toString() : this.minF.toString(); } - public Long getMax() { - return this.max; + public String getMax() { + return (this.max != null) ? this.max.toString() : this.maxF.toString(); } - public Long getSeed() { - return this.seed; + public String getSeed() { + return this.seed.toString(); } public String getValue() { return this.value; } + public String getValueType() { + return this.valueType.toString(); + } + public Object getGenerator() { return this.generatorObject; } public Long getNextLongBinomial() { + assert !this.valueType.equals(JDBCSupportedType.REAL) + && !this.valueType.equals(JDBCSupportedType.FLOAT) + && !this.valueType.equals(JDBCSupportedType.VARCHAR) + && !this.valueType.equals(JDBCSupportedType.CHAR); Random binomialGenerator = (Random) this.generatorObject; Long generatedValue; do { @@ -141,44 +211,60 @@ public Long getNextLongBinomial() { } public Long getNextLongUniform() { + assert !this.valueType.equals(JDBCSupportedType.REAL) + && !this.valueType.equals(JDBCSupportedType.FLOAT) + && !this.valueType.equals(JDBCSupportedType.VARCHAR) + && !this.valueType.equals(JDBCSupportedType.CHAR); Random uniformGenerator = (Random) this.generatorObject; return uniformGenerator.nextLong(this.min, this.max); } public Long getNextLongZipf() { + assert !this.valueType.equals(JDBCSupportedType.REAL) + && !this.valueType.equals(JDBCSupportedType.FLOAT) + && !this.valueType.equals(JDBCSupportedType.VARCHAR) + && !this.valueType.equals(JDBCSupportedType.CHAR); ZipfianGenerator zipfianGenerator = (ZipfianGenerator) this.generatorObject; return zipfianGenerator.nextLong(); } public Long getNextLongScrambled() { + assert !this.valueType.equals(JDBCSupportedType.REAL) + && !this.valueType.equals(JDBCSupportedType.FLOAT) + && !this.valueType.equals(JDBCSupportedType.VARCHAR) + && !this.valueType.equals(JDBCSupportedType.CHAR); ScrambledZipfianGenerator scrambledGenerator = (ScrambledZipfianGenerator) this.generatorObject; return scrambledGenerator.nextLong(); } public String getNextString() { + assert this.valueType.equals(JDBCSupportedType.VARCHAR) + || this.valueType.equals(JDBCSupportedType.CHAR); Random stringGenerator = (Random) this.generatorObject; return TextGenerator.randomStr(stringGenerator, this.max.intValue()); } public Float getNextFloatUniform() { + assert this.valueType.equals(JDBCSupportedType.REAL) + || this.valueType.equals(JDBCSupportedType.FLOAT); Random floatGenerator = (Random) this.generatorObject; - return floatGenerator.nextFloat(Float.parseFloat(this.minS), Float.parseFloat(this.maxS)); + return floatGenerator.nextFloat(this.minF, this.maxF); } public Float getNextFloatBinomial() { - Float minF = Float.parseFloat(this.minS); - Float maxF = Float.parseFloat(this.maxS); + assert this.valueType.equals(JDBCSupportedType.REAL) + || this.valueType.equals(JDBCSupportedType.FLOAT); Random floatGenerator = (Random) this.generatorObject; Float generatedFloat; do { - generatedFloat = (float) (minF + Math.abs(floatGenerator.nextGaussian()) * maxF); - } while (generatedFloat > maxF || generatedFloat < minF); + generatedFloat = (float) (this.minF + Math.abs(floatGenerator.nextGaussian()) * this.maxF); + } while (generatedFloat > this.maxF || generatedFloat < this.minF); return generatedFloat; } - public RuntimeException createRuntimeException(JDBCSupportedType paramType) { + public RuntimeException createRuntimeException() { return new RuntimeException( - "Distribution: " + this.distribution + " not supported for type: " + paramType.toString()); + "Distribution: " + this.distribution + " not supported for type: " + getValueType()); } } From 556ff15266183bf9b08301f42ad3114661d2ac0d Mon Sep 17 00:00:00 2001 From: Eric Enzler Date: Thu, 21 Mar 2024 10:12:04 +0100 Subject: [PATCH 22/24] README cleanup + Assert pattern --- data/templated/example.xml | 4 +-- .../benchmarks/templated/README.md | 34 +++++++++++++----- .../templated/util/TemplatedValue.java | 36 ++++++++++--------- 3 files changed, 48 insertions(+), 26 deletions(-) diff --git a/data/templated/example.xml b/data/templated/example.xml index be91ed493..e4c428b14 100644 --- a/data/templated/example.xml +++ b/data/templated/example.xml @@ -67,7 +67,7 @@ INTEGER - 2.1 + 2