diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala index 01b21feab0dd..f653bf41c162 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala @@ -491,7 +491,7 @@ object CatalogColumnStat extends Logging { dataType match { case BooleanType => s.toBoolean case DateType if version == 1 => DateTimeUtils.fromJavaDate(java.sql.Date.valueOf(s)) - case DateType => DateFormatter().parse(s) + case DateType => DateFormatter(ZoneOffset.UTC).parse(s) case TimestampType if version == 1 => DateTimeUtils.fromJavaTimestamp(java.sql.Timestamp.valueOf(s)) case TimestampType => getTimestampFormatter().parse(s) @@ -516,7 +516,7 @@ object CatalogColumnStat extends Logging { */ def toExternalString(v: Any, colName: String, dataType: DataType): String = { val externalValue = dataType match { - case DateType => DateFormatter().format(v.asInstanceOf[Int]) + case DateType => DateFormatter(ZoneOffset.UTC).format(v.asInstanceOf[Int]) case TimestampType => getTimestampFormatter().format(v.asInstanceOf[Long]) case BooleanType | _: IntegralType | FloatType | DoubleType => v case _: DecimalType => v.asInstanceOf[Decimal].toJavaBigDecimal diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityGenerator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityGenerator.scala index 9ca94501f5c5..05cb91d10868 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityGenerator.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityGenerator.scala @@ -45,7 +45,10 @@ class UnivocityGenerator( options.timestampFormat, options.zoneId, options.locale) - private val dateFormatter = DateFormatter(options.dateFormat, options.locale) + private val dateFormatter = DateFormatter( + options.dateFormat, + options.zoneId, + options.locale) private def makeConverter(dataType: DataType): ValueConverter = dataType match { case DateType => diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityParser.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityParser.scala index 39a08ec06c6a..661525a65294 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityParser.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityParser.scala @@ -78,7 +78,10 @@ class UnivocityParser( options.timestampFormat, options.zoneId, options.locale) - private val dateFormatter = DateFormatter(options.dateFormat, options.locale) + private val dateFormatter = DateFormatter( + options.dateFormat, + options.zoneId, + options.locale) // Retrieve the raw record string. private def getCurrentInput: UTF8String = { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala index d1943f02f85e..d8edabf3ed35 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala @@ -106,7 +106,7 @@ object Cast { * * Cast.castToTimestamp */ def needsTimeZone(from: DataType, to: DataType): Boolean = (from, to) match { - case (StringType, TimestampType) => true + case (StringType, TimestampType | DateType) => true case (DateType, TimestampType) => true case (TimestampType, StringType) => true case (TimestampType, DateType) => true @@ -287,7 +287,7 @@ case class Cast(child: Expression, dataType: DataType, timeZoneId: Option[String // [[func]] assumes the input is no longer null because eval already does the null check. @inline private[this] def buildCast[T](a: Any, func: T => Any): Any = func(a.asInstanceOf[T]) - private lazy val dateFormatter = DateFormatter() + private lazy val dateFormatter = DateFormatter(zoneId) private lazy val timestampFormatter = TimestampFormatter.getFractionFormatter(zoneId) private val failOnIntegralTypeOverflow = SQLConf.get.failOnIntegralTypeOverflow @@ -469,7 +469,7 @@ case class Cast(child: Expression, dataType: DataType, timeZoneId: Option[String // DateConverter private[this] def castToDate(from: DataType): Any => Any = from match { case StringType => - buildCast[UTF8String](_, s => DateTimeUtils.stringToDate(s).orNull) + buildCast[UTF8String](_, s => DateTimeUtils.stringToDate(s, zoneId).orNull) case TimestampType => // throw valid precision more than seconds, according to Hive. // Timestamp.nanos is in 0 to 999,999,999, no more than a second. @@ -1056,28 +1056,35 @@ case class Cast(child: Expression, dataType: DataType, timeZoneId: Option[String private[this] def castToDateCode( from: DataType, - ctx: CodegenContext): CastFunction = from match { - case StringType => - val intOpt = ctx.freshVariable("intOpt", classOf[Option[Integer]]) - (c, evPrim, evNull) => code""" - scala.Option $intOpt = - org.apache.spark.sql.catalyst.util.DateTimeUtils.stringToDate($c); - if ($intOpt.isDefined()) { - $evPrim = ((Integer) $intOpt.get()).intValue(); - } else { - $evNull = true; - } - """ - case TimestampType => + ctx: CodegenContext): CastFunction = { + def getZoneId() = { val zoneIdClass = classOf[ZoneId] - val zid = JavaCode.global( + JavaCode.global( ctx.addReferenceObj("zoneId", zoneId, zoneIdClass.getName), zoneIdClass) - (c, evPrim, evNull) => - code"""$evPrim = - org.apache.spark.sql.catalyst.util.DateTimeUtils.microsToEpochDays($c, $zid);""" - case _ => - (c, evPrim, evNull) => code"$evNull = true;" + } + from match { + case StringType => + val intOpt = ctx.freshVariable("intOpt", classOf[Option[Integer]]) + val zid = getZoneId() + (c, evPrim, evNull) => + code""" + scala.Option $intOpt = + org.apache.spark.sql.catalyst.util.DateTimeUtils.stringToDate($c, $zid); + if ($intOpt.isDefined()) { + $evPrim = ((Integer) $intOpt.get()).intValue(); + } else { + $evNull = true; + } + """ + case TimestampType => + val zid = getZoneId() + (c, evPrim, evNull) => + code"""$evPrim = + org.apache.spark.sql.catalyst.util.DateTimeUtils.microsToEpochDays($c, $zid);""" + case _ => + (c, evPrim, evNull) => code"$evNull = true;" + } } private[this] def changePrecision(d: ExprValue, decimalType: DecimalType, diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala index 89a6d23b1d73..13dd31f124d5 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala @@ -76,9 +76,7 @@ case class CurrentDate(timeZoneId: Option[String] = None) override def withTimeZone(timeZoneId: String): TimeZoneAwareExpression = copy(timeZoneId = Option(timeZoneId)) - override def eval(input: InternalRow): Any = { - localDateToDays(LocalDate.now(zoneId)) - } + override def eval(input: InternalRow): Any = currentDate(zoneId) override def prettyName: String = "current_date" } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala index 9cef3ecadc54..4793b5942a79 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala @@ -371,7 +371,9 @@ case class Literal (value: Any, dataType: DataType) extends LeafExpression { case _ => v + "D" } case (v: Decimal, t: DecimalType) => v + "BD" - case (v: Int, DateType) => s"DATE '${DateFormatter().format(v)}'" + case (v: Int, DateType) => + val formatter = DateFormatter(DateTimeUtils.getZoneId(SQLConf.get.sessionLocalTimeZone)) + s"DATE '${formatter.format(v)}'" case (v: Long, TimestampType) => val formatter = TimestampFormatter.getFractionFormatter( DateTimeUtils.getZoneId(SQLConf.get.sessionLocalTimeZone)) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonGenerator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonGenerator.scala index 3378040d1b64..3ee7e484690d 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonGenerator.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonGenerator.scala @@ -81,7 +81,10 @@ private[sql] class JacksonGenerator( options.timestampFormat, options.zoneId, options.locale) - private val dateFormatter = DateFormatter(options.dateFormat, options.locale) + private val dateFormatter = DateFormatter( + options.dateFormat, + options.zoneId, + options.locale) private def makeWriter(dataType: DataType): ValueWriter = dataType match { case NullType => diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala index 19bc5bf3b29e..b534b5a3d2d6 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala @@ -59,7 +59,10 @@ class JacksonParser( options.timestampFormat, options.zoneId, options.locale) - private val dateFormatter = DateFormatter(options.dateFormat, options.locale) + private val dateFormatter = DateFormatter( + options.dateFormat, + options.zoneId, + options.locale) /** * Create a converter which converts the JSON documents held by the `JsonParser` diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala index 9335be5b239b..869f30e06554 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala @@ -1734,7 +1734,8 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging } try { valueType match { - case "DATE" => toLiteral(stringToDate, DateType) + case "DATE" => + toLiteral(stringToDate(_, getZoneId(SQLConf.get.sessionLocalTimeZone)), DateType) case "TIMESTAMP" => val zoneId = getZoneId(SQLConf.get.sessionLocalTimeZone) toLiteral(stringToTimestamp(_, zoneId), TimestampType) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala index 4940aa83a301..7f982b019c8d 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala @@ -17,9 +17,11 @@ package org.apache.spark.sql.catalyst.util -import java.time.LocalDate +import java.time.{LocalDate, ZoneId} import java.util.Locale +import DateTimeUtils.{convertSpecialDate, localDateToDays} + sealed trait DateFormatter extends Serializable { def parse(s: String): Int // returns days since epoch def format(days: Int): String @@ -27,14 +29,18 @@ sealed trait DateFormatter extends Serializable { class Iso8601DateFormatter( pattern: String, + zoneId: ZoneId, locale: Locale) extends DateFormatter with DateTimeFormatterHelper { @transient private lazy val formatter = getOrCreateFormatter(pattern, locale) override def parse(s: String): Int = { - val localDate = LocalDate.parse(s, formatter) - DateTimeUtils.localDateToDays(localDate) + val specialDate = convertSpecialDate(s.trim, zoneId) + specialDate.getOrElse { + val localDate = LocalDate.parse(s, formatter) + localDateToDays(localDate) + } } override def format(days: Int): String = { @@ -46,11 +52,13 @@ object DateFormatter { val defaultPattern: String = "uuuu-MM-dd" val defaultLocale: Locale = Locale.US - def apply(format: String, locale: Locale): DateFormatter = { - new Iso8601DateFormatter(format, locale) + def apply(format: String, zoneId: ZoneId, locale: Locale): DateFormatter = { + new Iso8601DateFormatter(format, zoneId, locale) } - def apply(format: String): DateFormatter = apply(format, defaultLocale) + def apply(format: String, zoneId: ZoneId): DateFormatter = { + apply(format, zoneId, defaultLocale) + } - def apply(): DateFormatter = apply(defaultPattern) + def apply(zoneId: ZoneId): DateFormatter = apply(defaultPattern, zoneId) } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala index a82471aae652..e3dc89c57203 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala @@ -24,7 +24,6 @@ import java.time.temporal.{ChronoField, ChronoUnit, IsoFields} import java.util.{Locale, TimeZone} import java.util.concurrent.TimeUnit._ -import scala.util.Try import scala.util.control.NonFatal import org.apache.spark.sql.types.Decimal @@ -379,7 +378,7 @@ object DateTimeUtils { * `yyyy-[m]m-[d]d *` * `yyyy-[m]m-[d]dT*` */ - def stringToDate(s: UTF8String): Option[SQLDate] = { + def stringToDate(s: UTF8String, zoneId: ZoneId): Option[SQLDate] = { if (s == null) { return None } @@ -387,6 +386,8 @@ object DateTimeUtils { var i = 0 var currentSegmentValue = 0 val bytes = s.trim.getBytes + val specialDate = convertSpecialDate(bytes, zoneId) + if (specialDate.isDefined) return specialDate var j = 0 while (j < bytes.length && (i < 3 && !(bytes(j) == ' ' || bytes(j) == 'T'))) { val b = bytes(j) @@ -855,6 +856,8 @@ object DateTimeUtils { def currentTimestamp(): SQLTimestamp = instantToMicros(Instant.now()) + def currentDate(zoneId: ZoneId): SQLDate = localDateToDays(LocalDate.now(zoneId)) + private def today(zoneId: ZoneId): ZonedDateTime = { Instant.now().atZone(zoneId).`with`(LocalTime.MIDNIGHT) } @@ -915,4 +918,28 @@ object DateTimeUtils { None } } + + /** + * Converts notational shorthands that are converted to ordinary dates. + * @param input - a trimmed string + * @param zoneId - zone identifier used to get the current date. + * @return some of days since the epoch if the conversion completed successfully otherwise None. + */ + def convertSpecialDate(input: String, zoneId: ZoneId): Option[SQLDate] = { + extractSpecialValue(input, zoneId).flatMap { + case "epoch" => Some(0) + case "now" | "today" => Some(currentDate(zoneId)) + case "tomorrow" => Some(Math.addExact(currentDate(zoneId), 1)) + case "yesterday" => Some(Math.subtractExact(currentDate(zoneId), 1)) + case _ => None + } + } + + private def convertSpecialDate(bytes: Array[Byte], zoneId: ZoneId): Option[SQLDate] = { + if (bytes.length > 0 && Character.isAlphabetic(bytes(0))) { + convertSpecialDate(new String(bytes, StandardCharsets.UTF_8), zoneId) + } else { + None + } + } } diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/HashExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/HashExpressionsSuite.scala index b5cfaf8f4b0f..f90c98be0b3f 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/HashExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/HashExpressionsSuite.scala @@ -174,7 +174,7 @@ class HashExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { test("hive-hash for date type") { def checkHiveHashForDateType(dateString: String, expected: Long): Unit = { checkHiveHash( - DateTimeUtils.stringToDate(UTF8String.fromString(dateString)).get, + DateTimeUtils.stringToDate(UTF8String.fromString(dateString), ZoneOffset.UTC).get, DateType, expected) } diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala index 31fefd613f9c..1da8efe4ef42 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala @@ -19,7 +19,7 @@ package org.apache.spark.sql.catalyst.util import java.sql.{Date, Timestamp} import java.text.SimpleDateFormat -import java.time.{LocalDateTime, LocalTime, ZoneId} +import java.time.{LocalDate, LocalDateTime, LocalTime, ZoneId, ZoneOffset} import java.util.{Locale, TimeZone} import java.util.concurrent.TimeUnit @@ -120,28 +120,32 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers { checkFromToJavaDate(new Date(df2.parse("1776-07-04 18:30:00 UTC").getTime)) } + private def toDate(s: String, zoneId: ZoneId = ZoneOffset.UTC): Option[SQLDate] = { + stringToDate(UTF8String.fromString(s), zoneId) + } + test("string to date") { - assert(stringToDate(UTF8String.fromString("2015-01-28")).get === days(2015, 1, 28)) - assert(stringToDate(UTF8String.fromString("2015")).get === days(2015, 1, 1)) - assert(stringToDate(UTF8String.fromString("0001")).get === days(1, 1, 1)) - assert(stringToDate(UTF8String.fromString("2015-03")).get === days(2015, 3, 1)) + assert(toDate("2015-01-28").get === days(2015, 1, 28)) + assert(toDate("2015").get === days(2015, 1, 1)) + assert(toDate("0001").get === days(1, 1, 1)) + assert(toDate("2015-03").get === days(2015, 3, 1)) Seq("2015-03-18", "2015-03-18 ", " 2015-03-18", " 2015-03-18 ", "2015-03-18 123142", "2015-03-18T123123", "2015-03-18T").foreach { s => - assert(stringToDate(UTF8String.fromString(s)).get === days(2015, 3, 18)) + assert(toDate(s).get === days(2015, 3, 18)) } - assert(stringToDate(UTF8String.fromString("2015-03-18X")).isEmpty) - assert(stringToDate(UTF8String.fromString("2015/03/18")).isEmpty) - assert(stringToDate(UTF8String.fromString("2015.03.18")).isEmpty) - assert(stringToDate(UTF8String.fromString("20150318")).isEmpty) - assert(stringToDate(UTF8String.fromString("2015-031-8")).isEmpty) - assert(stringToDate(UTF8String.fromString("02015-03-18")).isEmpty) - assert(stringToDate(UTF8String.fromString("015-03-18")).isEmpty) - assert(stringToDate(UTF8String.fromString("015")).isEmpty) - assert(stringToDate(UTF8String.fromString("02015")).isEmpty) - assert(stringToDate(UTF8String.fromString("1999 08 01")).isEmpty) - assert(stringToDate(UTF8String.fromString("1999-08 01")).isEmpty) - assert(stringToDate(UTF8String.fromString("1999 08")).isEmpty) + assert(toDate("2015-03-18X").isEmpty) + assert(toDate("2015/03/18").isEmpty) + assert(toDate("2015.03.18").isEmpty) + assert(toDate("20150318").isEmpty) + assert(toDate("2015-031-8").isEmpty) + assert(toDate("02015-03-18").isEmpty) + assert(toDate("015-03-18").isEmpty) + assert(toDate("015").isEmpty) + assert(toDate("02015").isEmpty) + assert(toDate("1999 08 01").isEmpty) + assert(toDate("1999-08 01").isEmpty) + assert(toDate("1999 08").isEmpty) } private def toTimestamp(str: String, zoneId: ZoneId): Option[SQLTimestamp] = { @@ -264,12 +268,10 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers { test("SPARK-15379: special invalid date string") { // Test stringToDate - assert(stringToDate( - UTF8String.fromString("2015-02-29 00:00:00")).isEmpty) - assert(stringToDate( - UTF8String.fromString("2015-04-31 00:00:00")).isEmpty) - assert(stringToDate(UTF8String.fromString("2015-02-29")).isEmpty) - assert(stringToDate(UTF8String.fromString("2015-04-31")).isEmpty) + assert(toDate("2015-02-29 00:00:00").isEmpty) + assert(toDate("2015-04-31 00:00:00").isEmpty) + assert(toDate("2015-02-29").isEmpty) + assert(toDate("2015-04-31").isEmpty) // Test stringToTimestamp @@ -586,4 +588,16 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers { toTimestamp(" tomorrow CET ", zoneId).get should be (today + MICROS_PER_DAY +- tolerance) } } + + test("special date values") { + DateTimeTestUtils.outstandingZoneIds.foreach { zoneId => + assert(toDate("epoch", zoneId).get === 0) + val today = localDateToDays(LocalDate.now(zoneId)) + assert(toDate("YESTERDAY", zoneId).get === today - 1) + assert(toDate(" Now ", zoneId).get === today) + assert(toDate("now UTC", zoneId) === None) // "now" does not accept time zones + assert(toDate("today", zoneId).get === today) + assert(toDate("tomorrow CET ", zoneId).get === today + 1) + } + } } diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/UnsafeArraySuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/UnsafeArraySuite.scala index 0b9e023b0b45..41adf845a6fa 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/UnsafeArraySuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/UnsafeArraySuite.scala @@ -17,7 +17,7 @@ package org.apache.spark.sql.catalyst.util -import java.time.ZoneId +import java.time.{ZoneId, ZoneOffset} import org.apache.spark.{SparkConf, SparkFunSuite} import org.apache.spark.serializer.{JavaSerializer, KryoSerializer} @@ -38,8 +38,8 @@ class UnsafeArraySuite extends SparkFunSuite { val doubleArray = Array(1.1, 2.2, 3.3) val stringArray = Array("1", "10", "100") val dateArray = Array( - DateTimeUtils.stringToDate(UTF8String.fromString("1970-1-1")).get, - DateTimeUtils.stringToDate(UTF8String.fromString("2016-7-26")).get) + DateTimeUtils.stringToDate(UTF8String.fromString("1970-1-1"), ZoneOffset.UTC).get, + DateTimeUtils.stringToDate(UTF8String.fromString("2016-7-26"), ZoneOffset.UTC).get) private def defaultZoneId = ZoneId.systemDefault() val timestampArray = Array( DateTimeUtils.stringToTimestamp( diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/DateFormatterSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/DateFormatterSuite.scala index 1f0eff2e5b11..291d40a9e84d 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/DateFormatterSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/DateFormatterSuite.scala @@ -17,18 +17,19 @@ package org.apache.spark.sql.util -import java.time.LocalDate +import java.time.{LocalDate, ZoneOffset} import org.apache.spark.SparkFunSuite import org.apache.spark.sql.catalyst.plans.SQLHelper import org.apache.spark.sql.catalyst.util._ +import org.apache.spark.sql.catalyst.util.DateTimeUtils.{getZoneId, localDateToDays} import org.apache.spark.sql.internal.SQLConf class DateFormatterSuite extends SparkFunSuite with SQLHelper { test("parsing dates") { DateTimeTestUtils.outstandingTimezonesIds.foreach { timeZone => withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> timeZone) { - val formatter = DateFormatter() + val formatter = DateFormatter(getZoneId(timeZone)) val daysSinceEpoch = formatter.parse("2018-12-02") assert(daysSinceEpoch === 17867) } @@ -38,7 +39,7 @@ class DateFormatterSuite extends SparkFunSuite with SQLHelper { test("format dates") { DateTimeTestUtils.outstandingTimezonesIds.foreach { timeZone => withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> timeZone) { - val formatter = DateFormatter() + val formatter = DateFormatter(getZoneId(timeZone)) val date = formatter.format(17867) assert(date === "2018-12-02") } @@ -58,7 +59,7 @@ class DateFormatterSuite extends SparkFunSuite with SQLHelper { "5010-11-17").foreach { date => DateTimeTestUtils.outstandingTimezonesIds.foreach { timeZone => withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> timeZone) { - val formatter = DateFormatter() + val formatter = DateFormatter(getZoneId(timeZone)) val days = formatter.parse(date) val formatted = formatter.format(days) assert(date === formatted) @@ -81,7 +82,7 @@ class DateFormatterSuite extends SparkFunSuite with SQLHelper { 1110657).foreach { days => DateTimeTestUtils.outstandingTimezonesIds.foreach { timeZone => withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> timeZone) { - val formatter = DateFormatter() + val formatter = DateFormatter(getZoneId(timeZone)) val date = formatter.format(days) val parsed = formatter.parse(date) assert(days === parsed) @@ -91,13 +92,29 @@ class DateFormatterSuite extends SparkFunSuite with SQLHelper { } test("parsing date without explicit day") { - val formatter = DateFormatter("yyyy MMM") + val formatter = DateFormatter("yyyy MMM", ZoneOffset.UTC) val daysSinceEpoch = formatter.parse("2018 Dec") assert(daysSinceEpoch === LocalDate.of(2018, 12, 1).toEpochDay) } test("formatting negative years with default pattern") { val epochDays = LocalDate.of(-99, 1, 1).toEpochDay.toInt - assert(DateFormatter().format(epochDays) === "-0099-01-01") + assert(DateFormatter(ZoneOffset.UTC).format(epochDays) === "-0099-01-01") + } + + test("special date values") { + DateTimeTestUtils.outstandingTimezonesIds.foreach { timeZone => + withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> timeZone) { + val zoneId = getZoneId(timeZone) + val formatter = DateFormatter(zoneId) + + assert(formatter.parse("EPOCH") === 0) + val today = localDateToDays(LocalDate.now(zoneId)) + assert(formatter.parse("Yesterday") === today - 1) + assert(formatter.parse("now") === today) + assert(formatter.parse("today ") === today) + assert(formatter.parse("tomorrow UTC") === today + 1) + } + } } } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala index eec8d70b5adf..75abac4cfd1d 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala @@ -77,9 +77,9 @@ object HiveResult { TimestampType, BinaryType) - private lazy val dateFormatter = DateFormatter() - private lazy val timestampFormatter = TimestampFormatter.getFractionFormatter( - DateTimeUtils.getZoneId(SQLConf.get.sessionLocalTimeZone)) + private lazy val zoneId = DateTimeUtils.getZoneId(SQLConf.get.sessionLocalTimeZone) + private lazy val dateFormatter = DateFormatter(zoneId) + private lazy val timestampFormatter = TimestampFormatter.getFractionFormatter(zoneId) /** Hive outputs fields of structs slightly differently than top level attributes. */ private def toHiveStructString(a: (Any, DataType)): String = a match { diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningUtils.scala index 1e47d53b7e97..fdad43b23c5a 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningUtils.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningUtils.scala @@ -130,7 +130,7 @@ object PartitioningUtils { Map.empty[String, String] } - val dateFormatter = DateFormatter() + val dateFormatter = DateFormatter(zoneId) val timestampFormatter = TimestampFormatter(timestampPartitionPattern, zoneId) // First, we need to parse every partition's path and see if we can find partition values. val (partitionValues, optDiscoveredBasePaths) = paths.map { path => @@ -492,7 +492,7 @@ object PartitioningUtils { // We need to check that we can cast the raw string since we later can use Cast to get // the partition values with the right DataType (see // org.apache.spark.sql.execution.datasources.PartitioningAwareFileIndex.inferPartitioning) - val dateValue = Cast(Literal(raw), DateType).eval() + val dateValue = Cast(Literal(raw), DateType, Some(zoneId.getId)).eval() // Disallow DateType if the cast returned null require(dateValue != null) Literal.create(dateValue, DateType) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRelation.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRelation.scala index 3cd5cb164792..f5a474ddf390 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRelation.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRelation.scala @@ -186,7 +186,7 @@ private[sql] object JDBCRelation extends Logging { } columnType match { case _: NumericType => value.toLong - case DateType => parse(stringToDate).toLong + case DateType => parse(stringToDate(_, getZoneId(timeZoneId))).toLong case TimestampType => parse(stringToTimestamp(_, getZoneId(timeZoneId))) } } @@ -197,7 +197,9 @@ private[sql] object JDBCRelation extends Logging { timeZoneId: String): String = { def dateTimeToString(): String = { val dateTimeStr = columnType match { - case DateType => DateFormatter().format(value.toInt) + case DateType => + val dateFormatter = DateFormatter(DateTimeUtils.getZoneId(timeZoneId)) + dateFormatter.format(value.toInt) case TimestampType => val timestampFormatter = TimestampFormatter.getFractionFormatter( DateTimeUtils.getZoneId(timeZoneId)) diff --git a/sql/core/src/test/resources/sql-tests/inputs/pgSQL/date.sql b/sql/core/src/test/resources/sql-tests/inputs/pgSQL/date.sql index b9a6b998e52f..d3cd46e4e6b8 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/pgSQL/date.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/pgSQL/date.sql @@ -208,20 +208,19 @@ SELECT date '5874898-01-01'; -- out of range SELECT f1 - date '2000-01-01' AS `Days From 2K` FROM DATE_TBL; --- [SPARK-28141] Date type can not accept special values --- SELECT f1 - date 'epoch' AS "Days From Epoch" FROM DATE_TBL; +SELECT f1 - date 'epoch' AS `Days From Epoch` FROM DATE_TBL; --- SELECT date 'yesterday' - date 'today' AS "One day"; +SELECT date 'yesterday' - date 'today' AS `One day`; --- SELECT date 'today' - date 'tomorrow' AS "One day"; +SELECT date 'today' - date 'tomorrow' AS `One day`; --- SELECT date 'yesterday' - date 'tomorrow' AS "Two days"; +SELECT date 'yesterday' - date 'tomorrow' AS `Two days`; --- SELECT date 'tomorrow' - date 'today' AS "One day"; +SELECT date 'tomorrow' - date 'today' AS `One day`; --- SELECT date 'today' - date 'yesterday' AS "One day"; +SELECT date 'today' - date 'yesterday' AS `One day`; --- SELECT date 'tomorrow' - date 'yesterday' AS "Two days"; +SELECT date 'tomorrow' - date 'yesterday' AS `Two days`; -- [SPARK-28017] Enhance date EXTRACT -- @@ -290,7 +289,7 @@ SELECT DATE_TRUNC('DECADE', DATE '1993-12-25'); -- 1990-01-01 SELECT DATE_TRUNC('DECADE', DATE '0004-12-25'); -- 0001-01-01 BC SELECT DATE_TRUNC('DECADE', TO_DATE('0002-12-31 BC', 'yyyy-MM-dd G')); -- 0011-01-01 BC --- [SPARK-28141] Date type can not accept special values +-- [SPARK-29006] Support special date/timestamp values `infinity`/`-infinity` -- -- test infinity -- diff --git a/sql/core/src/test/resources/sql-tests/results/pgSQL/date.sql.out b/sql/core/src/test/resources/sql-tests/results/pgSQL/date.sql.out index 083832007d61..29fcf61bd5b7 100644 --- a/sql/core/src/test/resources/sql-tests/results/pgSQL/date.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/pgSQL/date.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 91 +-- Number of queries: 98 -- !query 0 @@ -502,352 +502,422 @@ struct -- !query 47 -SELECT EXTRACT(EPOCH FROM DATE '1970-01-01') +SELECT f1 - date 'epoch' AS `Days From Epoch` FROM DATE_TBL -- !query 47 schema -struct +struct -- !query 47 output -0 +-4585 +-4650 +11048 +11049 +11050 +24934 +25300 +25667 +9554 +9555 +9556 +9557 +9920 +9921 +9922 -- !query 48 -SELECT EXTRACT(EPOCH FROM TIMESTAMP '1970-01-01') +SELECT date 'yesterday' - date 'today' AS `One day` -- !query 48 schema -struct +struct -- !query 48 output -0 +-1 -- !query 49 -SELECT EXTRACT(CENTURY FROM TO_DATE('0101-12-31 BC', 'yyyy-MM-dd G')) +SELECT date 'today' - date 'tomorrow' AS `One day` -- !query 49 schema -struct +struct -- !query 49 output --2 +-1 -- !query 50 -SELECT EXTRACT(CENTURY FROM TO_DATE('0100-12-31 BC', 'yyyy-MM-dd G')) +SELECT date 'yesterday' - date 'tomorrow' AS `Two days` -- !query 50 schema -struct +struct -- !query 50 output --1 +-2 -- !query 51 -SELECT EXTRACT(CENTURY FROM TO_DATE('0001-12-31 BC', 'yyyy-MM-dd G')) +SELECT date 'tomorrow' - date 'today' AS `One day` -- !query 51 schema -struct +struct -- !query 51 output --1 +1 -- !query 52 -SELECT EXTRACT(CENTURY FROM DATE '0001-01-01') +SELECT date 'today' - date 'yesterday' AS `One day` -- !query 52 schema -struct +struct -- !query 52 output 1 -- !query 53 -SELECT EXTRACT(CENTURY FROM DATE '0001-01-01 AD') +SELECT date 'tomorrow' - date 'yesterday' AS `Two days` -- !query 53 schema -struct +struct -- !query 53 output -1 +2 -- !query 54 -SELECT EXTRACT(CENTURY FROM DATE '1900-12-31') +SELECT EXTRACT(EPOCH FROM DATE '1970-01-01') -- !query 54 schema -struct +struct -- !query 54 output -19 +0 -- !query 55 -SELECT EXTRACT(CENTURY FROM DATE '1901-01-01') +SELECT EXTRACT(EPOCH FROM TIMESTAMP '1970-01-01') -- !query 55 schema -struct +struct -- !query 55 output -20 +0 -- !query 56 -SELECT EXTRACT(CENTURY FROM DATE '2000-12-31') +SELECT EXTRACT(CENTURY FROM TO_DATE('0101-12-31 BC', 'yyyy-MM-dd G')) -- !query 56 schema -struct +struct -- !query 56 output -20 +-2 -- !query 57 -SELECT EXTRACT(CENTURY FROM DATE '2001-01-01') +SELECT EXTRACT(CENTURY FROM TO_DATE('0100-12-31 BC', 'yyyy-MM-dd G')) -- !query 57 schema -struct +struct -- !query 57 output -21 +-1 -- !query 58 -SELECT EXTRACT(CENTURY FROM CURRENT_DATE)>=21 AS True +SELECT EXTRACT(CENTURY FROM TO_DATE('0001-12-31 BC', 'yyyy-MM-dd G')) -- !query 58 schema -struct +struct -- !query 58 output -true +-1 -- !query 59 -SELECT EXTRACT(MILLENNIUM FROM TO_DATE('0001-12-31 BC', 'yyyy-MM-dd G')) +SELECT EXTRACT(CENTURY FROM DATE '0001-01-01') -- !query 59 schema -struct +struct -- !query 59 output --1 +1 -- !query 60 -SELECT EXTRACT(MILLENNIUM FROM DATE '0001-01-01 AD') +SELECT EXTRACT(CENTURY FROM DATE '0001-01-01 AD') -- !query 60 schema -struct +struct -- !query 60 output 1 -- !query 61 -SELECT EXTRACT(MILLENNIUM FROM DATE '1000-12-31') +SELECT EXTRACT(CENTURY FROM DATE '1900-12-31') -- !query 61 schema -struct +struct -- !query 61 output -1 +19 -- !query 62 -SELECT EXTRACT(MILLENNIUM FROM DATE '1001-01-01') +SELECT EXTRACT(CENTURY FROM DATE '1901-01-01') -- !query 62 schema -struct +struct -- !query 62 output -2 +20 -- !query 63 -SELECT EXTRACT(MILLENNIUM FROM DATE '2000-12-31') +SELECT EXTRACT(CENTURY FROM DATE '2000-12-31') -- !query 63 schema -struct +struct -- !query 63 output -2 +20 -- !query 64 -SELECT EXTRACT(MILLENNIUM FROM DATE '2001-01-01') +SELECT EXTRACT(CENTURY FROM DATE '2001-01-01') -- !query 64 schema -struct +struct -- !query 64 output -3 +21 -- !query 65 -SELECT EXTRACT(MILLENNIUM FROM CURRENT_DATE) +SELECT EXTRACT(CENTURY FROM CURRENT_DATE)>=21 AS True -- !query 65 schema -struct +struct -- !query 65 output -3 +true -- !query 66 -SELECT EXTRACT(DECADE FROM DATE '1994-12-25') +SELECT EXTRACT(MILLENNIUM FROM TO_DATE('0001-12-31 BC', 'yyyy-MM-dd G')) -- !query 66 schema -struct +struct -- !query 66 output -199 +-1 -- !query 67 -SELECT EXTRACT(DECADE FROM DATE '0010-01-01') +SELECT EXTRACT(MILLENNIUM FROM DATE '0001-01-01 AD') -- !query 67 schema -struct +struct -- !query 67 output 1 -- !query 68 -SELECT EXTRACT(DECADE FROM DATE '0009-12-31') +SELECT EXTRACT(MILLENNIUM FROM DATE '1000-12-31') -- !query 68 schema -struct +struct -- !query 68 output -0 +1 -- !query 69 -SELECT EXTRACT(DECADE FROM TO_DATE('0001-01-01 BC', 'yyyy-MM-dd G')) +SELECT EXTRACT(MILLENNIUM FROM DATE '1001-01-01') -- !query 69 schema -struct +struct -- !query 69 output -0 +2 -- !query 70 -SELECT EXTRACT(DECADE FROM TO_DATE('0002-12-31 BC', 'yyyy-MM-dd G')) +SELECT EXTRACT(MILLENNIUM FROM DATE '2000-12-31') -- !query 70 schema -struct +struct -- !query 70 output --1 +2 -- !query 71 -SELECT EXTRACT(DECADE FROM TO_DATE('0011-01-01 BC', 'yyyy-MM-dd G')) +SELECT EXTRACT(MILLENNIUM FROM DATE '2001-01-01') -- !query 71 schema -struct +struct -- !query 71 output --1 +3 -- !query 72 -SELECT EXTRACT(DECADE FROM TO_DATE('0012-12-31 BC', 'yyyy-MM-dd G')) +SELECT EXTRACT(MILLENNIUM FROM CURRENT_DATE) -- !query 72 schema -struct +struct -- !query 72 output --2 +3 -- !query 73 -SELECT EXTRACT(CENTURY FROM NOW())>=21 AS True +SELECT EXTRACT(DECADE FROM DATE '1994-12-25') -- !query 73 schema -struct +struct -- !query 73 output -true +199 -- !query 74 -SELECT EXTRACT(CENTURY FROM TIMESTAMP '1970-03-20 04:30:00.00000') +SELECT EXTRACT(DECADE FROM DATE '0010-01-01') -- !query 74 schema -struct +struct -- !query 74 output -20 +1 -- !query 75 -SELECT DATE_TRUNC('MILLENNIUM', TIMESTAMP '1970-03-20 04:30:00.00000') +SELECT EXTRACT(DECADE FROM DATE '0009-12-31') -- !query 75 schema -struct +struct -- !query 75 output -1001-01-01 00:07:02 +0 -- !query 76 -SELECT DATE_TRUNC('MILLENNIUM', DATE '1970-03-20') +SELECT EXTRACT(DECADE FROM TO_DATE('0001-01-01 BC', 'yyyy-MM-dd G')) -- !query 76 schema -struct +struct -- !query 76 output -1001-01-01 00:07:02 +0 -- !query 77 -SELECT DATE_TRUNC('CENTURY', TIMESTAMP '1970-03-20 04:30:00.00000') +SELECT EXTRACT(DECADE FROM TO_DATE('0002-12-31 BC', 'yyyy-MM-dd G')) -- !query 77 schema -struct +struct -- !query 77 output -1901-01-01 00:00:00 +-1 -- !query 78 -SELECT DATE_TRUNC('CENTURY', DATE '1970-03-20') +SELECT EXTRACT(DECADE FROM TO_DATE('0011-01-01 BC', 'yyyy-MM-dd G')) -- !query 78 schema -struct +struct -- !query 78 output -1901-01-01 00:00:00 +-1 -- !query 79 -SELECT DATE_TRUNC('CENTURY', DATE '2004-08-10') +SELECT EXTRACT(DECADE FROM TO_DATE('0012-12-31 BC', 'yyyy-MM-dd G')) -- !query 79 schema -struct +struct -- !query 79 output -2001-01-01 00:00:00 +-2 -- !query 80 -SELECT DATE_TRUNC('CENTURY', DATE '0002-02-04') +SELECT EXTRACT(CENTURY FROM NOW())>=21 AS True -- !query 80 schema -struct +struct -- !query 80 output -0001-01-01 00:07:02 +true -- !query 81 -SELECT DATE_TRUNC('CENTURY', TO_DATE('0055-08-10 BC', 'yyyy-MM-dd G')) +SELECT EXTRACT(CENTURY FROM TIMESTAMP '1970-03-20 04:30:00.00000') -- !query 81 schema -struct +struct -- !query 81 output --0099-01-01 00:07:02 +20 -- !query 82 -SELECT DATE_TRUNC('DECADE', DATE '1993-12-25') +SELECT DATE_TRUNC('MILLENNIUM', TIMESTAMP '1970-03-20 04:30:00.00000') -- !query 82 schema -struct +struct -- !query 82 output -1990-01-01 00:00:00 +1001-01-01 00:07:02 -- !query 83 -SELECT DATE_TRUNC('DECADE', DATE '0004-12-25') +SELECT DATE_TRUNC('MILLENNIUM', DATE '1970-03-20') -- !query 83 schema -struct +struct -- !query 83 output -0000-01-01 00:07:02 +1001-01-01 00:07:02 -- !query 84 -SELECT DATE_TRUNC('DECADE', TO_DATE('0002-12-31 BC', 'yyyy-MM-dd G')) +SELECT DATE_TRUNC('CENTURY', TIMESTAMP '1970-03-20 04:30:00.00000') -- !query 84 schema -struct +struct -- !query 84 output --0010-01-01 00:07:02 +1901-01-01 00:00:00 -- !query 85 -select make_date(2013, 7, 15) +SELECT DATE_TRUNC('CENTURY', DATE '1970-03-20') -- !query 85 schema -struct +struct -- !query 85 output -2013-07-15 +1901-01-01 00:00:00 -- !query 86 -select make_date(-44, 3, 15) +SELECT DATE_TRUNC('CENTURY', DATE '2004-08-10') -- !query 86 schema -struct +struct -- !query 86 output --0044-03-15 +2001-01-01 00:00:00 -- !query 87 -select make_date(2013, 2, 30) +SELECT DATE_TRUNC('CENTURY', DATE '0002-02-04') -- !query 87 schema -struct +struct -- !query 87 output -NULL +0001-01-01 00:07:02 -- !query 88 -select make_date(2013, 13, 1) +SELECT DATE_TRUNC('CENTURY', TO_DATE('0055-08-10 BC', 'yyyy-MM-dd G')) -- !query 88 schema -struct +struct -- !query 88 output -NULL +-0099-01-01 00:07:02 -- !query 89 -select make_date(2013, 11, -1) +SELECT DATE_TRUNC('DECADE', DATE '1993-12-25') -- !query 89 schema -struct +struct -- !query 89 output -NULL +1990-01-01 00:00:00 -- !query 90 -DROP TABLE DATE_TBL +SELECT DATE_TRUNC('DECADE', DATE '0004-12-25') -- !query 90 schema -struct<> +struct -- !query 90 output +0000-01-01 00:07:02 + + +-- !query 91 +SELECT DATE_TRUNC('DECADE', TO_DATE('0002-12-31 BC', 'yyyy-MM-dd G')) +-- !query 91 schema +struct +-- !query 91 output +-0010-01-01 00:07:02 + + +-- !query 92 +select make_date(2013, 7, 15) +-- !query 92 schema +struct +-- !query 92 output +2013-07-15 + + +-- !query 93 +select make_date(-44, 3, 15) +-- !query 93 schema +struct +-- !query 93 output +-0044-03-15 + + +-- !query 94 +select make_date(2013, 2, 30) +-- !query 94 schema +struct +-- !query 94 output +NULL + + +-- !query 95 +select make_date(2013, 13, 1) +-- !query 95 schema +struct +-- !query 95 output +NULL + + +-- !query 96 +select make_date(2013, 11, -1) +-- !query 96 schema +struct +-- !query 96 output +NULL + + +-- !query 97 +DROP TABLE DATE_TBL +-- !query 97 schema +struct<> +-- !query 97 output diff --git a/sql/core/src/test/scala/org/apache/spark/sql/CsvFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/CsvFunctionsSuite.scala index 1094d7d23e5e..d34e50518348 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/CsvFunctionsSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/CsvFunctionsSuite.scala @@ -17,7 +17,7 @@ package org.apache.spark.sql -import java.sql.Timestamp +import java.sql.{Date, Timestamp} import java.text.SimpleDateFormat import java.util.Locale @@ -191,4 +191,13 @@ class CsvFunctionsSuite extends QueryTest with SharedSparkSession { assert(readback(0).getAs[Row](0).getAs[Timestamp](0).getTime >= 0) } } + + test("special date values") { + Seq("now", "today", "epoch", "tomorrow", "yesterday").foreach { specialValue => + val input = Seq(specialValue).toDS() + val readback = input.select(from_csv($"value", lit("d date"), + Map.empty[String, String].asJava)).collect() + assert(readback(0).getAs[Row](0).getAs[Date](0).getTime >= 0) + } + } } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala index c61c8109ee8e..e55d2bbe00e6 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala @@ -17,7 +17,7 @@ package org.apache.spark.sql -import java.sql.Timestamp +import java.sql.{Date, Timestamp} import java.text.SimpleDateFormat import java.util.Locale @@ -618,4 +618,13 @@ class JsonFunctionsSuite extends QueryTest with SharedSparkSession { assert(readback(0).getAs[Row](0).getAs[Timestamp](0).getTime >= 0) } } + + test("special date values") { + Seq("now", "today", "epoch", "tomorrow", "yesterday").foreach { specialValue => + val input = Seq(s"""{"d": "$specialValue"}""").toDS() + val readback = input.select(from_json($"value", lit("d date"), + Map.empty[String, String].asJava)).collect() + assert(readback(0).getAs[Row](0).getAs[Date](0).getTime >= 0) + } + } } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetPartitionDiscoverySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetPartitionDiscoverySuite.scala index 0a85e3cdeaf1..138336dc7e33 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetPartitionDiscoverySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetPartitionDiscoverySuite.scala @@ -58,7 +58,7 @@ abstract class ParquetPartitionDiscoverySuite val defaultPartitionName = ExternalCatalogUtils.DEFAULT_PARTITION_NAME val timeZoneId = ZoneId.systemDefault() - val df = DateFormatter() + val df = DateFormatter(timeZoneId) val tf = TimestampFormatter(timestampPartitionPattern, timeZoneId) protected override def beforeAll(): Unit = {