Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -462,13 +462,19 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
checkEvaluation(AddMonths(Literal.create(null, DateType), Literal(1)), null)
checkEvaluation(AddMonths(Literal.create(null, DateType), Literal.create(null, IntegerType)),
null)
// Valid range of DateType is [0001-01-01, 9999-12-31]
val maxMonthInterval = 10000 * 12
checkEvaluation(
AddMonths(Literal(Date.valueOf("2015-01-30")), Literal(Int.MinValue)), -938165455)
AddMonths(Literal(Date.valueOf("0001-01-01")), Literal(maxMonthInterval)), 2933261)
checkEvaluation(
AddMonths(Literal(Date.valueOf("2016-02-28")), positiveIntLit), 1014213)
checkEvaluation(
AddMonths(Literal(Date.valueOf("2016-02-28")), negativeIntLit), -980528)
checkConsistencyBetweenInterpretedAndCodegen(AddMonths, DateType, IntegerType)
AddMonths(Literal(Date.valueOf("9999-12-31")), Literal(-1 * maxMonthInterval)), -719529)
// Test evaluation results between Interpreted mode and Codegen mode
forAll (
LiteralGenerator.randomGen(DateType),
LiteralGenerator.monthIntervalLiterGen
) { (l1: Literal, l2: Literal) =>
cmpInterpretWithCodegen(EmptyRow, AddMonths(l1, l2))
}
}

test("months_between") {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -398,7 +398,7 @@ trait ExpressionEvalHelper extends GeneratorDrivenPropertyChecks with PlanTestBa
}
}

private def cmpInterpretWithCodegen(inputRow: InternalRow, expr: Expression): Unit = {
def cmpInterpretWithCodegen(inputRow: InternalRow, expr: Expression): Unit = {
val interpret = try {
evaluateWithoutCodegen(expr, inputRow)
} catch {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@
package org.apache.spark.sql.catalyst.expressions

import java.sql.{Date, Timestamp}
import java.time.{Duration, Instant, LocalDate}
import java.util.concurrent.TimeUnit

import org.scalacheck.{Arbitrary, Gen}

Expand Down Expand Up @@ -100,23 +102,44 @@ object LiteralGenerator {
lazy val booleanLiteralGen: Gen[Literal] =
for { b <- Arbitrary.arbBool.arbitrary } yield Literal.create(b, BooleanType)

lazy val dateLiteralGen: Gen[Literal] =
for { d <- Arbitrary.arbInt.arbitrary } yield Literal.create(new Date(d), DateType)
lazy val dateLiteralGen: Gen[Literal] = {
// Valid range for DateType is [0001-01-01, 9999-12-31]
val minDay = LocalDate.of(1, 1, 1).toEpochDay
val maxDay = LocalDate.of(9999, 12, 31).toEpochDay
for { day <- Gen.choose(minDay, maxDay) }
yield Literal.create(new Date(day * DateTimeUtils.MILLIS_PER_DAY), DateType)
}

lazy val timestampLiteralGen: Gen[Literal] = {
// Catalyst's Timestamp type stores number of microseconds since epoch in
// a variable of Long type. To prevent arithmetic overflow of Long on
// conversion from milliseconds to microseconds, the range of random milliseconds
// since epoch is restricted here.
val maxMillis = Long.MaxValue / DateTimeUtils.MICROS_PER_MILLIS
val minMillis = Long.MinValue / DateTimeUtils.MICROS_PER_MILLIS
// Valid range for TimestampType is [0001-01-01T00:00:00.000000Z, 9999-12-31T23:59:59.999999Z]
val minMillis = Instant.parse("0001-01-01T00:00:00.000000Z").toEpochMilli
val maxMillis = Instant.parse("9999-12-31T23:59:59.999999Z").toEpochMilli
for { millis <- Gen.choose(minMillis, maxMillis) }
yield Literal.create(new Timestamp(millis), TimestampType)
}

lazy val calendarIntervalLiterGen: Gen[Literal] =
for { m <- Arbitrary.arbInt.arbitrary; s <- Arbitrary.arbLong.arbitrary}
yield Literal.create(new CalendarInterval(m, s), CalendarIntervalType)
// Valid range for DateType and TimestampType is [0001-01-01, 9999-12-31]
private val maxIntervalInMonths: Int = 10000 * 12

lazy val monthIntervalLiterGen: Gen[Literal] = {
for { months <- Gen.choose(-1 * maxIntervalInMonths, maxIntervalInMonths) }
yield Literal.create(months, IntegerType)
}

lazy val calendarIntervalLiterGen: Gen[Literal] = {
val maxDurationInSec = Duration.between(
Instant.parse("0001-01-01T00:00:00.000000Z"),
Instant.parse("9999-12-31T23:59:59.999999Z")).getSeconds
val maxMicros = TimeUnit.SECONDS.toMicros(maxDurationInSec)
for {
months <- Gen.choose(-1 * maxIntervalInMonths, maxIntervalInMonths)
micros <- Gen.choose(-1 * maxMicros, maxMicros)
} yield Literal.create(new CalendarInterval(months, micros), CalendarIntervalType)
}


// Sometimes, it would be quite expensive when unlimited value is used,
Expand Down