From 2c985943f81470b3d7fe3c59d98ab276ffe80e64 Mon Sep 17 00:00:00 2001 From: HyukjinKwon Date: Mon, 7 Oct 2019 17:37:32 +0900 Subject: [PATCH 1/6] Investiagte Arrow 0.15 with R Arrow optiomization --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 29777a5962bc..82d9fe5342ce 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Apache Spark +# Spark Spark is a unified analytics engine for large-scale data processing. It provides high-level APIs in Scala, Java, Python, and R, and an optimized engine that From f30f4de97f7b843a643892c83061134a79305e60 Mon Sep 17 00:00:00 2001 From: HyukjinKwon Date: Mon, 7 Oct 2019 17:54:16 +0900 Subject: [PATCH 2/6] Changes in R --- R/README.md | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/R/README.md b/R/README.md index 31174c73526f..a857b3dd9c4a 100644 --- a/R/README.md +++ b/R/README.md @@ -1,4 +1,4 @@ -# R on Spark +# R on Apache Spark SparkR is an R package that provides a light-weight frontend to use Spark from R. diff --git a/README.md b/README.md index 82d9fe5342ce..29777a5962bc 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Spark +# Apache Spark Spark is a unified analytics engine for large-scale data processing. It provides high-level APIs in Scala, Java, Python, and R, and an optimized engine that From 34da71dd28919d6099c86a8ee0e543d798524cee Mon Sep 17 00:00:00 2001 From: HyukjinKwon Date: Tue, 8 Oct 2019 12:18:08 +0900 Subject: [PATCH 3/6] Test From bea530f6df1fe1d696e7cf414381a4c251c7bf6e Mon Sep 17 00:00:00 2001 From: HyukjinKwon Date: Tue, 8 Oct 2019 12:54:58 +0900 Subject: [PATCH 4/6] ARROW_PRE_0_15_IPC_FORMAT=1 --- R/README.md | 2 +- appveyor.yml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/R/README.md b/R/README.md index a857b3dd9c4a..31174c73526f 100644 --- a/R/README.md +++ b/R/README.md @@ -1,4 +1,4 @@ -# R on Apache Spark +# R on Spark SparkR is an R package that provides a light-weight frontend to use Spark from R. diff --git a/appveyor.yml b/appveyor.yml index 00c688ba18eb..d16f39fb2fc3 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -57,6 +57,8 @@ build_script: environment: NOT_CRAN: true + # See SPARK-29378. + ARROW_PRE_0_15_IPC_FORMAT: 1 # See SPARK-27848. Currently installing some dependent packages causes # "(converted from warning) unable to identify current timezone 'C':" for an unknown reason. # This environment variable works around to test SparkR against a higher version. From e1a680755475d0a4cfd60d5352b9af4dcf573dd7 Mon Sep 17 00:00:00 2001 From: HyukjinKwon Date: Wed, 9 Oct 2019 10:37:32 +0900 Subject: [PATCH 5/6] Use 0.14.1 for now --- appveyor.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index d16f39fb2fc3..17c86ea9ae22 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -42,7 +42,9 @@ install: # Install maven and dependencies - ps: .\dev\appveyor-install-dependencies.ps1 # Required package for R unit tests - - cmd: R -e "install.packages(c('knitr', 'rmarkdown', 'e1071', 'survival', 'arrow'), repos='https://cloud.r-project.org/')" + - cmd: R -e "install.packages(c('knitr', 'rmarkdown', 'e1071', 'survival'), repos='https://cloud.r-project.org/')" + # Uses Arrow R 0.14.1 for now. 0.15.0 seems not working for now. See SPARK-29378. + - cmd: R -e "install.packages('https://cran.r-project.org/src/contrib/Archive/arrow/arrow_0.14.1.tar.gz', repos=NULL, type='source')" # Here, we use the fixed version of testthat. For more details, please see SPARK-22817. # As of devtools 2.1.0, it requires testthat higher then 2.1.1 as a dependency. SparkR test requires testthat 1.0.2. # Therefore, we don't use devtools but installs it directly from the archive including its dependencies. @@ -57,8 +59,6 @@ build_script: environment: NOT_CRAN: true - # See SPARK-29378. - ARROW_PRE_0_15_IPC_FORMAT: 1 # See SPARK-27848. Currently installing some dependent packages causes # "(converted from warning) unable to identify current timezone 'C':" for an unknown reason. # This environment variable works around to test SparkR against a higher version. From 564ec38d9a47d7bd0bceef8f471ac7d2433b66f4 Mon Sep 17 00:00:00 2001 From: HyukjinKwon Date: Wed, 9 Oct 2019 16:02:40 +0900 Subject: [PATCH 6/6] dependencies --- appveyor.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 17c86ea9ae22..b36175a787ae 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -43,7 +43,8 @@ install: - ps: .\dev\appveyor-install-dependencies.ps1 # Required package for R unit tests - cmd: R -e "install.packages(c('knitr', 'rmarkdown', 'e1071', 'survival'), repos='https://cloud.r-project.org/')" - # Uses Arrow R 0.14.1 for now. 0.15.0 seems not working for now. See SPARK-29378. + # Use Arrow R 0.14.1 for now. 0.15.0 seems not working for now. See SPARK-29378. + - cmd: R -e "install.packages(c('assertthat', 'bit64', 'fs', 'purrr', 'R6', 'tidyselect'), repos='https://cloud.r-project.org/')" - cmd: R -e "install.packages('https://cran.r-project.org/src/contrib/Archive/arrow/arrow_0.14.1.tar.gz', repos=NULL, type='source')" # Here, we use the fixed version of testthat. For more details, please see SPARK-22817. # As of devtools 2.1.0, it requires testthat higher then 2.1.1 as a dependency. SparkR test requires testthat 1.0.2.