Skip to content

Commit 9f41436

Browse files
author
zuotingbing
committed
Fix code review
1 parent 2a542e4 commit 9f41436

File tree

2 files changed

+24
-34
lines changed

2 files changed

+24
-34
lines changed

sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveTable.scala

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -104,8 +104,9 @@ case class InsertIntoHiveTable(
104104
inputPathName.substring(0, inputPathName.indexOf(stagingDir) + stagingDir.length)
105105
}
106106

107-
// SPARK-20594: The staging directory should be a child directory starts with "." to avoid
108-
// being deleted if we set hive.exec.stagingdir under the table directory.
107+
// SPARK-20594: This is a walk-around fix to resolve a Hive bug. Hive requires that the
108+
// staging directory needs to avoid being deleted when users set hive.exec.stagingdir
109+
// under the table directory.
109110
if (FileUtils.isSubDir(new Path(stagingPathName), inputPath, fs)
110111
&& !stagingPathName.stripPrefix(inputPathName).stripPrefix(File.separator).startsWith(".")) {
111112
logDebug(s"The staging dir '$stagingPathName' should be a child directory starts " +

sql/hive/src/test/scala/org/apache/spark/sql/hive/InsertIntoHiveTableSuite.scala

Lines changed: 21 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -495,44 +495,33 @@ class InsertIntoHiveTableSuite extends QueryTest with TestHiveSingleton with Bef
495495
}
496496
}
497497

498-
private def dropTables(tableNames: String*): Unit = {
499-
tableNames.foreach { name =>
500-
sql(s"DROP TABLE IF EXISTS $name")
501-
}
502-
}
503-
504498
test(
505-
"""SPARK-20594: The staging directory should be appended with ".hive-staging"
506-
|to avoid being deleted if we set hive.exec.stagingdir under the table directory
507-
|without start with "."""".stripMargin) {
499+
"""SPARK-20594: This is a walk-around fix to resolve a Hive bug. Hive requires that the
500+
|staging directory needs to avoid being deleted when users set hive.exec.stagingdir
501+
|under the table directory.""".stripMargin) {
508502

509-
dropTables("test_table", "test_table1")
503+
withTable("test_table", "test_table1") {
504+
spark.range(1).write.saveAsTable("test_table")
510505

511-
sql("CREATE TABLE test_table (key int, value string)")
512-
513-
// Add some data.
514-
testData.write.mode(SaveMode.Append).insertInto("test_table")
506+
// Make sure the table has also been updated.
507+
checkAnswer(
508+
sql("SELECT * FROM test_table"),
509+
Row(0)
510+
)
515511

516-
// Make sure the table has also been updated.
517-
checkAnswer(
518-
sql("SELECT * FROM test_table"),
519-
testData.collect().toSeq
520-
)
512+
sql("CREATE TABLE test_table1 (key int)")
521513

522-
sql("CREATE TABLE test_table1 (key int, value string)")
514+
// Set hive.exec.stagingdir under the table directory without start with ".".
515+
sql("set hive.exec.stagingdir=./test")
523516

524-
// Set hive.exec.stagingdir under the table directory without start with ".".
525-
sql("set hive.exec.stagingdir=./test")
517+
// Now overwrite.
518+
sql("INSERT OVERWRITE TABLE test_table1 SELECT * FROM test_table")
526519

527-
// Now overwrite.
528-
sql("INSERT OVERWRITE TABLE test_table1 SELECT * FROM test_table")
529-
530-
// Make sure the table has also been updated.
531-
checkAnswer(
532-
sql("SELECT * FROM test_table1"),
533-
testData.collect().toSeq
534-
)
535-
536-
dropTables("test_table", "test_table1")
520+
// Make sure the table has also been updated.
521+
checkAnswer(
522+
sql("SELECT * FROM test_table1"),
523+
Row(0)
524+
)
525+
}
537526
}
538527
}

0 commit comments

Comments
 (0)