From 361e96225bad6497009d75c74778336105854a6c Mon Sep 17 00:00:00 2001 From: Daniel Darabos Date: Tue, 8 Jul 2014 15:07:30 +0200 Subject: [PATCH 1/3] Catch all errors during serialization in DAGScheduler. --- .../main/scala/org/apache/spark/scheduler/DAGScheduler.scala | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala index 81c136d970312..bdf070db5e161 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala @@ -768,6 +768,10 @@ class DAGScheduler( abortStage(stage, "Task not serializable: " + e.toString) runningStages -= stage return + case e: Throwable => // Other exceptions, such as IllegalArgumentException from Kryo. + abortStage(stage, "Task serialization failed: " + e.toString) + runningStages -= stage + return } logInfo("Submitting " + tasks.size + " missing tasks from " + stage + " (" + stage.rdd + ")") From 52c22ba6b0ae28bcdbb5d7b636c3992cf06afdc1 Mon Sep 17 00:00:00 2001 From: Daniel Darabos Date: Tue, 8 Jul 2014 19:03:30 +0200 Subject: [PATCH 2/3] Only catch NonFatal exceptions. --- .../main/scala/org/apache/spark/scheduler/DAGScheduler.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala index bdf070db5e161..4d728266ffa58 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala @@ -26,6 +26,7 @@ import scala.concurrent.Await import scala.concurrent.duration._ import scala.language.postfixOps import scala.reflect.ClassTag +import scala.util.control.NonFatal import akka.actor._ import akka.actor.OneForOneStrategy @@ -768,7 +769,7 @@ class DAGScheduler( abortStage(stage, "Task not serializable: " + e.toString) runningStages -= stage return - case e: Throwable => // Other exceptions, such as IllegalArgumentException from Kryo. + case NonFatal(e) => // Other exceptions, such as IllegalArgumentException from Kryo. abortStage(stage, "Task serialization failed: " + e.toString) runningStages -= stage return From 3aceaad656f6606276725b22b20c2d89fd8fafeb Mon Sep 17 00:00:00 2001 From: Daniel Darabos Date: Tue, 8 Jul 2014 19:03:50 +0200 Subject: [PATCH 3/3] Print full stack trace for miscellaneous exceptions during serialization. --- .../main/scala/org/apache/spark/scheduler/DAGScheduler.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala index 4d728266ffa58..f72bfde572c96 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala @@ -770,7 +770,7 @@ class DAGScheduler( runningStages -= stage return case NonFatal(e) => // Other exceptions, such as IllegalArgumentException from Kryo. - abortStage(stage, "Task serialization failed: " + e.toString) + abortStage(stage, s"Task serialization failed: $e\n${e.getStackTraceString}") runningStages -= stage return }