From f7235a23d843a4f21da226f5af76f30bcd0d28b5 Mon Sep 17 00:00:00 2001 From: thomastechs Date: Thu, 4 Feb 2016 21:53:13 +0530 Subject: [PATCH 1/2] Update DataFrame.scala for the bug fix 13197 Editing the select method to allow the select of columns with . in it. --- .../src/main/scala/org/apache/spark/sql/DataFrame.scala | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala index 518f9dcf94a7..a2875c7e4d33 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala @@ -718,7 +718,11 @@ class DataFrame private[sql]( * @since 1.3.0 */ @scala.annotation.varargs - def select(col: String, cols: String*): DataFrame = select((col +: cols).map(Column(_)) : _*) + def select(col: String, cols: String*): DataFrame = { + val output = queryExecution.analyzed.output + val resolver = sqlContext.analyzer.resolver + select(output.filter(x=>(col +: cols).exists(y=>resolver(x.name,y))).map(Column(_)) : _*) + } /** * Selects a set of SQL expressions. This is a variant of `select` that accepts From 3482d277aca7b45ece562ad8baef7a86d00d4cfc Mon Sep 17 00:00:00 2001 From: thomastechs Date: Thu, 4 Feb 2016 21:56:54 +0530 Subject: [PATCH 2/2] Update DataFrameSuite.scala for testcases -SPARK-13197 fix Added testcases for SPARK-13197 fix - selecting the dataframe with columns containing . in it --- .../test/scala/org/apache/spark/sql/DataFrameSuite.scala | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala index c02133ffc854..771b249cb4bf 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala @@ -1297,4 +1297,10 @@ class DataFrameSuite extends QueryTest with SharedSQLContext { Seq(1 -> "a").toDF("i", "j").filter($"i".cast(StringType) === "1"), Row(1, "a")) } + + test("SPARK-13197: Support select from the table with dot character in column name") { + val df = Seq((50, 100)).toDF("a_b", "a.c") + checkAnswer(df.select("a_b"),Row(50)) + checkAnswer(df.select("a.c"),Row(100)) + } }