From 6aa1ab7d60f7a130fa189c47b6f473944b6d98d4 Mon Sep 17 00:00:00 2001 From: Venkata Ramana Gollamudi Date: Thu, 27 Nov 2014 10:52:27 +0530 Subject: [PATCH 1/2] Fix run-examples script error during multiple jars --- bin/run-example | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/bin/run-example b/bin/run-example index 3d932509426f..db805f7cbd7f 100755 --- a/bin/run-example +++ b/bin/run-example @@ -35,17 +35,29 @@ else fi if [ -f "$FWDIR/RELEASE" ]; then - export SPARK_EXAMPLES_JAR="`ls "$FWDIR"/lib/spark-examples-*hadoop*.jar`" -elif [ -e "$EXAMPLES_DIR"/target/scala-$SPARK_SCALA_VERSION/spark-examples-*hadoop*.jar ]; then - export SPARK_EXAMPLES_JAR="`ls "$EXAMPLES_DIR"/target/scala-$SPARK_SCALA_VERSION/spark-examples-*hadoop*.jar`" + JAR_PATH="${FWDIR}/lib" +else + JAR_PATH="${EXAMPLES_DIR}/target/scala-${SPARK_SCALA_VERSION}" fi -if [[ -z "$SPARK_EXAMPLES_JAR" ]]; then +JAR_COUNT="`ls ${JAR_PATH}/spark-examples-*hadoop*.jar 2>>/dev/null | wc -l`" + +if [ "$JAR_COUNT" -eq "0" ]; then echo "Failed to find Spark examples assembly in $FWDIR/lib or $FWDIR/examples/target" 1>&2 echo "You need to build Spark before running this program" 1>&2 exit 1 fi +if [ "$JAR_COUNT" -gt "1" ]; then + JARS_LIST="`ls -t ${JAR_PATH}/spark-examples-*hadoop*.jar`" + echo "Found multiple Spark examples assembly jars in ${JAR_PATH}" 1>&2 + echo "$JARS_LIST" 1>&2 + echo "Please remove all but one jar." 1>&2 + exit 1 +fi + +export SPARK_EXAMPLES_JAR="`ls ${JAR_PATH}/spark-examples-*hadoop*.jar 2>>/dev/null`" + EXAMPLE_MASTER=${MASTER:-"local[*]"} if [[ ! $EXAMPLE_CLASS == org.apache.spark.examples* ]]; then From fa7f48170cf5e8c3e39d691b5c8f1bea8af2fb42 Mon Sep 17 00:00:00 2001 From: Venkata Ramana Gollamudi Date: Fri, 9 Jan 2015 11:26:20 +0530 Subject: [PATCH 2/2] Fixed review comments, avoiding ls output scanning. --- bin/compute-classpath.sh | 27 +++++++++++++++------------ bin/run-example | 21 ++++++++++++--------- 2 files changed, 27 insertions(+), 21 deletions(-) diff --git a/bin/compute-classpath.sh b/bin/compute-classpath.sh index a31ea73d3ce1..5e3a72de473e 100755 --- a/bin/compute-classpath.sh +++ b/bin/compute-classpath.sh @@ -72,22 +72,25 @@ else assembly_folder="$ASSEMBLY_DIR" fi -num_jars="$(ls "$assembly_folder" | grep "spark-assembly.*hadoop.*\.jar$" | wc -l)" -if [ "$num_jars" -eq "0" ]; then - echo "Failed to find Spark assembly in $assembly_folder" - echo "You need to build Spark before running this program." - exit 1 -fi +num_jars=0 + +for f in ${assembly_folder}/spark-assembly*hadoop*.jar; do + if [[ ! -e "$f" ]]; then + echo "Failed to find Spark assembly in $assembly_folder" 1>&2 + echo "You need to build Spark before running this program." 1>&2 + exit 1 + fi + ASSEMBLY_JAR="$f" + num_jars=$((num_jars+1)) +done + if [ "$num_jars" -gt "1" ]; then - jars_list=$(ls "$assembly_folder" | grep "spark-assembly.*hadoop.*.jar$") - echo "Found multiple Spark assembly jars in $assembly_folder:" - echo "$jars_list" - echo "Please remove all but one jar." + echo "Found multiple Spark assembly jars in $assembly_folder:" 1>&2 + ls ${assembly_folder}/spark-assembly*hadoop*.jar 1>&2 + echo "Please remove all but one jar." 1>&2 exit 1 fi -ASSEMBLY_JAR="$(ls "$assembly_folder"/spark-assembly*hadoop*.jar 2>/dev/null)" - # Verify that versions of java used to build the jars and run Spark are compatible jar_error_check=$("$JAR_CMD" -tf "$ASSEMBLY_JAR" nonexistent/class/path 2>&1) if [[ "$jar_error_check" =~ "invalid CEN header" ]]; then diff --git a/bin/run-example b/bin/run-example index db805f7cbd7f..4ca72b7278ee 100755 --- a/bin/run-example +++ b/bin/run-example @@ -40,23 +40,26 @@ else JAR_PATH="${EXAMPLES_DIR}/target/scala-${SPARK_SCALA_VERSION}" fi -JAR_COUNT="`ls ${JAR_PATH}/spark-examples-*hadoop*.jar 2>>/dev/null | wc -l`" +JAR_COUNT=0 -if [ "$JAR_COUNT" -eq "0" ]; then - echo "Failed to find Spark examples assembly in $FWDIR/lib or $FWDIR/examples/target" 1>&2 - echo "You need to build Spark before running this program" 1>&2 - exit 1 -fi +for f in ${JAR_PATH}/spark-examples-*hadoop*.jar; do + if [[ ! -e "$f" ]]; then + echo "Failed to find Spark examples assembly in $FWDIR/lib or $FWDIR/examples/target" 1>&2 + echo "You need to build Spark before running this program" 1>&2 + exit 1 + fi + SPARK_EXAMPLES_JAR="$f" + JAR_COUNT=$((JAR_COUNT+1)) +done if [ "$JAR_COUNT" -gt "1" ]; then - JARS_LIST="`ls -t ${JAR_PATH}/spark-examples-*hadoop*.jar`" echo "Found multiple Spark examples assembly jars in ${JAR_PATH}" 1>&2 - echo "$JARS_LIST" 1>&2 + ls ${JAR_PATH}/spark-examples-*hadoop*.jar 1>&2 echo "Please remove all but one jar." 1>&2 exit 1 fi -export SPARK_EXAMPLES_JAR="`ls ${JAR_PATH}/spark-examples-*hadoop*.jar 2>>/dev/null`" +export SPARK_EXAMPLES_JAR EXAMPLE_MASTER=${MASTER:-"local[*]"}