From 3a7067ee9264e44a7d953989c525ea5f16b611d2 Mon Sep 17 00:00:00 2001 From: windpiger Date: Wed, 1 Mar 2017 18:21:10 +0800 Subject: [PATCH 1/4] [SPARK-19577][SQL]refresh table after alter the location --- .../spark/sql/execution/command/ddl.scala | 2 ++ .../sql/hive/execution/HiveDDLSuite.scala | 31 +++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala index 82cbb4aa4744..08d0b502e04f 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala @@ -754,6 +754,8 @@ case class AlterTableSetLocationCommand( // No partition spec is specified, so we set the location for the table itself catalog.alterTable(table.withNewStorage(locationUri = Some(location))) } + + sparkSession.catalog.refreshTable(table.identifier.table) Seq.empty[Row] } } diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala index 792ac1e25949..f2c23862526a 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala @@ -1587,4 +1587,35 @@ class HiveDDLSuite } } } + + test("refresh table after alter the location") { + withTable("t", "t1") { + withTempDir { dir => + spark.sql( + """ + |CREATE TABLE t(a string) + |USING parquet + """.stripMargin) + + spark.sql("INSERT INTO TABLE t SELECT 1") + checkAnswer(spark.table("t"), Row("1") :: Nil) + spark.sql(s"ALTER TABLE t SET LOCATION '$dir'") + checkAnswer(spark.table("t"), Nil) + } + + withTempDir { dir => + spark.sql( + """ + |CREATE TABLE t1(a string, b string) + |USING parquet + |PARTITIONED BY(b) + """.stripMargin) + + spark.sql("INSERT INTO TABLE t1 PARTITION(b=1) SELECT 2") + checkAnswer(spark.table("t1"), Row("2", "1") :: Nil) + spark.sql(s"ALTER TABLE t1 PARTITION(b=1)SET LOCATION '$dir'") + checkAnswer(spark.table("t1"), Nil) + } + } + } } From 9050349fff162ea9c0c506f4021fc4585fe09026 Mon Sep 17 00:00:00 2001 From: windpiger Date: Wed, 1 Mar 2017 18:28:22 +0800 Subject: [PATCH 2/4] add hive table test --- .../spark/sql/execution/command/ddl.scala | 4 ++- .../sql/hive/execution/HiveDDLSuite.scala | 30 ++++++++++++++++++- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala index 08d0b502e04f..b45bc10b3319 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala @@ -755,7 +755,9 @@ case class AlterTableSetLocationCommand( catalog.alterTable(table.withNewStorage(locationUri = Some(location))) } - sparkSession.catalog.refreshTable(table.identifier.table) + if (DDLUtils.isDatasourceTable(table)) { + sparkSession.catalog.refreshTable(table.identifier.table) + } Seq.empty[Row] } } diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala index f2c23862526a..4a26850d54c2 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala @@ -1589,7 +1589,7 @@ class HiveDDLSuite } test("refresh table after alter the location") { - withTable("t", "t1") { + withTable("t", "t1", "t2", "t3") { withTempDir { dir => spark.sql( """ @@ -1616,6 +1616,34 @@ class HiveDDLSuite spark.sql(s"ALTER TABLE t1 PARTITION(b=1)SET LOCATION '$dir'") checkAnswer(spark.table("t1"), Nil) } + + + withTempDir { dir => + spark.sql( + """ + |CREATE TABLE t2(a string) + |USING hive + """.stripMargin) + + spark.sql("INSERT INTO TABLE t2 SELECT 1") + checkAnswer(spark.table("t2"), Row("1") :: Nil) + spark.sql(s"ALTER TABLE t2 SET LOCATION '$dir'") + checkAnswer(spark.table("t2"), Nil) + } + + withTempDir { dir => + spark.sql( + """ + |CREATE TABLE t3(a string, b string) + |USING hive + |PARTITIONED BY(b) + """.stripMargin) + + spark.sql("INSERT INTO TABLE t3 PARTITION(b=1) SELECT 2") + checkAnswer(spark.table("t3"), Row("2", "1") :: Nil) + spark.sql(s"ALTER TABLE t3 PARTITION(b=1)SET LOCATION '$dir'") + checkAnswer(spark.table("t3"), Nil) + } } } } From dccac9a02e6191d09782d8a97d7d9a4ab0edc92e Mon Sep 17 00:00:00 2001 From: windpiger Date: Wed, 1 Mar 2017 22:20:03 +0800 Subject: [PATCH 3/4] add more test --- .../scala/org/apache/spark/sql/execution/command/ddl.scala | 4 +--- .../org/apache/spark/sql/hive/execution/HiveDDLSuite.scala | 7 +------ 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala index b45bc10b3319..d39c368a3a19 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala @@ -755,9 +755,7 @@ case class AlterTableSetLocationCommand( catalog.alterTable(table.withNewStorage(locationUri = Some(location))) } - if (DDLUtils.isDatasourceTable(table)) { - sparkSession.catalog.refreshTable(table.identifier.table) - } + sparkSession.sessionState.catalog.refreshTable(table.identifier) Seq.empty[Row] } } diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala index 4a26850d54c2..dccce7864361 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala @@ -1588,7 +1588,7 @@ class HiveDDLSuite } } - test("refresh table after alter the location") { + test("refresh non-cached table after alter the location") { withTable("t", "t1", "t2", "t3") { withTempDir { dir => spark.sql( @@ -1596,7 +1596,6 @@ class HiveDDLSuite |CREATE TABLE t(a string) |USING parquet """.stripMargin) - spark.sql("INSERT INTO TABLE t SELECT 1") checkAnswer(spark.table("t"), Row("1") :: Nil) spark.sql(s"ALTER TABLE t SET LOCATION '$dir'") @@ -1610,21 +1609,18 @@ class HiveDDLSuite |USING parquet |PARTITIONED BY(b) """.stripMargin) - spark.sql("INSERT INTO TABLE t1 PARTITION(b=1) SELECT 2") checkAnswer(spark.table("t1"), Row("2", "1") :: Nil) spark.sql(s"ALTER TABLE t1 PARTITION(b=1)SET LOCATION '$dir'") checkAnswer(spark.table("t1"), Nil) } - withTempDir { dir => spark.sql( """ |CREATE TABLE t2(a string) |USING hive """.stripMargin) - spark.sql("INSERT INTO TABLE t2 SELECT 1") checkAnswer(spark.table("t2"), Row("1") :: Nil) spark.sql(s"ALTER TABLE t2 SET LOCATION '$dir'") @@ -1638,7 +1634,6 @@ class HiveDDLSuite |USING hive |PARTITIONED BY(b) """.stripMargin) - spark.sql("INSERT INTO TABLE t3 PARTITION(b=1) SELECT 2") checkAnswer(spark.table("t3"), Row("2", "1") :: Nil) spark.sql(s"ALTER TABLE t3 PARTITION(b=1)SET LOCATION '$dir'") From be98a0fabc9244ccb9e376ac8e7aef5125675c9b Mon Sep 17 00:00:00 2001 From: windpiger Date: Thu, 2 Mar 2017 13:20:15 +0800 Subject: [PATCH 4/4] add more test --- .../sql/execution/command/DDLSuite.scala | 24 ++++ .../sql/hive/execution/HiveDDLSuite.scala | 109 ++++++++++-------- 2 files changed, 86 insertions(+), 47 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala index b44f20e367f0..e2c635af776d 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala @@ -1952,4 +1952,28 @@ class DDLSuite extends QueryTest with SharedSQLContext with BeforeAndAfterEach { } } } + + Seq(true, false).foreach { shouldCache => + val testName = if (shouldCache) "cached" else "non-cached" + test(s"refresh $testNames table after alter the location") { + withTable("t", "t1", "t2", "t3") { + withTempDir { dir => + spark.sql( + """ + |CREATE TABLE t(a string) + |USING parquet + """.stripMargin) + spark.sql("INSERT INTO TABLE t SELECT 1") + if (shouldCache) { + spark.catalog.cacheTable("t") + } + checkAnswer(spark.table("t"), Row("1") :: Nil) + spark.sql(s"ALTER TABLE t SET LOCATION '$dir'") + checkAnswer(spark.table("t"), Nil) + } + + // TODO: partition table tests + } + } + } } diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala index dccce7864361..6eee19ac98d4 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala @@ -1588,56 +1588,71 @@ class HiveDDLSuite } } - test("refresh non-cached table after alter the location") { - withTable("t", "t1", "t2", "t3") { - withTempDir { dir => - spark.sql( - """ - |CREATE TABLE t(a string) - |USING parquet - """.stripMargin) - spark.sql("INSERT INTO TABLE t SELECT 1") - checkAnswer(spark.table("t"), Row("1") :: Nil) - spark.sql(s"ALTER TABLE t SET LOCATION '$dir'") - checkAnswer(spark.table("t"), Nil) - } + Seq(true, false).foreach { shouldCache => + val testName = if (shouldCache) "cached" else "non-cached" + test(s"refresh $testNames table after alter the location") { + withTable("t", "t1", "t2", "t3") { + withTempDir { dir => + spark.sql( + """ + |CREATE TABLE t(a string) + |USING parquet + """.stripMargin) + spark.sql("INSERT INTO TABLE t SELECT 1") + if (shouldCache) { + spark.catalog.cacheTable("t") + } + checkAnswer(spark.table("t"), Row("1") :: Nil) + spark.sql(s"ALTER TABLE t SET LOCATION '$dir'") + checkAnswer(spark.table("t"), Nil) + } - withTempDir { dir => - spark.sql( - """ - |CREATE TABLE t1(a string, b string) - |USING parquet - |PARTITIONED BY(b) - """.stripMargin) - spark.sql("INSERT INTO TABLE t1 PARTITION(b=1) SELECT 2") - checkAnswer(spark.table("t1"), Row("2", "1") :: Nil) - spark.sql(s"ALTER TABLE t1 PARTITION(b=1)SET LOCATION '$dir'") - checkAnswer(spark.table("t1"), Nil) - } + withTempDir { dir => + spark.sql( + """ + |CREATE TABLE t1(a string, b string) + |USING parquet + |PARTITIONED BY(b) + """.stripMargin) + spark.sql("INSERT INTO TABLE t1 PARTITION(b=1) SELECT 2") + if (shouldCache) { + spark.catalog.cacheTable("t") + } + checkAnswer(spark.table("t1"), Row("2", "1") :: Nil) + spark.sql(s"ALTER TABLE t1 PARTITION(b=1)SET LOCATION '$dir'") + checkAnswer(spark.table("t1"), Nil) + } - withTempDir { dir => - spark.sql( - """ - |CREATE TABLE t2(a string) - |USING hive - """.stripMargin) - spark.sql("INSERT INTO TABLE t2 SELECT 1") - checkAnswer(spark.table("t2"), Row("1") :: Nil) - spark.sql(s"ALTER TABLE t2 SET LOCATION '$dir'") - checkAnswer(spark.table("t2"), Nil) - } + withTempDir { dir => + spark.sql( + """ + |CREATE TABLE t2(a string) + |USING hive + """.stripMargin) + spark.sql("INSERT INTO TABLE t2 SELECT 1") + if (shouldCache) { + spark.catalog.cacheTable("t") + } + checkAnswer(spark.table("t2"), Row("1") :: Nil) + spark.sql(s"ALTER TABLE t2 SET LOCATION '$dir'") + checkAnswer(spark.table("t2"), Nil) + } - withTempDir { dir => - spark.sql( - """ - |CREATE TABLE t3(a string, b string) - |USING hive - |PARTITIONED BY(b) - """.stripMargin) - spark.sql("INSERT INTO TABLE t3 PARTITION(b=1) SELECT 2") - checkAnswer(spark.table("t3"), Row("2", "1") :: Nil) - spark.sql(s"ALTER TABLE t3 PARTITION(b=1)SET LOCATION '$dir'") - checkAnswer(spark.table("t3"), Nil) + withTempDir { dir => + spark.sql( + """ + |CREATE TABLE t3(a string, b string) + |USING hive + |PARTITIONED BY(b) + """.stripMargin) + spark.sql("INSERT INTO TABLE t3 PARTITION(b=1) SELECT 2") + if (shouldCache) { + spark.catalog.cacheTable("t") + } + checkAnswer(spark.table("t3"), Row("2", "1") :: Nil) + spark.sql(s"ALTER TABLE t3 PARTITION(b=1)SET LOCATION '$dir'") + checkAnswer(spark.table("t3"), Nil) + } } } }