Skip to content

Commit 8dc5218

Browse files
jiangxb1987jiangxb1987
committed
add comments
1 parent 244bbae commit 8dc5218

File tree

1 file changed

+128
-16
lines changed

1 file changed

+128
-16
lines changed

‎core/src/main/scala/org/apache/spark/deploy/JsonProtocol.scala‎

Lines changed: 128 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -21,26 +21,62 @@ import org.json4s.JsonAST.JObject
2121
import org.json4s.JsonDSL._
2222

2323
import org.apache.spark.deploy.DeployMessages.{MasterStateResponse, WorkerStateResponse}
24-
import org.apache.spark.deploy.master.{ApplicationInfo, DriverInfo, WorkerInfo}
24+
import org.apache.spark.deploy.master._
25+
import org.apache.spark.deploy.master.RecoveryState.MasterState
2526
import org.apache.spark.deploy.worker.ExecutorRunner
2627

2728
private[deploy] object JsonProtocol {
28-
def writeWorkerInfo(obj: WorkerInfo): JObject = {
29-
("id" -> obj.id) ~
30-
("host" -> obj.host) ~
31-
("port" -> obj.port) ~
32-
("address" -> obj.hostPort) ~
33-
("webuiaddress" -> obj.webUiAddress) ~
34-
("cores" -> obj.cores) ~
35-
("coresused" -> obj.coresUsed) ~
36-
("coresfree" -> obj.coresFree) ~
37-
("memory" -> obj.memory) ~
38-
("memoryused" -> obj.memoryUsed) ~
39-
("memoryfree" -> obj.memoryFree) ~
40-
("state" -> obj.state.toString) ~
41-
("lastheartbeat" -> obj.lastHeartbeat)
42-
}
29+
/**
30+
* Export the [[WorkerInfo]] to a Json object, a [[WorkerInfo]] consists of the information of a
31+
* worker.
32+
*
33+
* @return a Json object containing the following fields:
34+
* `id` a string identifier of the worker
35+
* `host` the host that the worker is running on
36+
* `port` the port that the worker is bound to
37+
* `address` ${host}:${port}
38+
* `webuiaddress` the address used in web UI
39+
* `cores` total cores of the worker
40+
* `coresused` allocated cores of the worker
41+
* `coresfree` free cores of the worker
42+
* `memory` total memory of the worker
43+
* `memoryused` allocated memory of the worker
44+
* `memoryfree` free memory of the worker
45+
* `state` state of the worker, see [[WorkerState]]
46+
* `lastheartbeat` time in milliseconds that the latest heart beat message from the
47+
* worker is received.
48+
*/
49+
def writeWorkerInfo(obj: WorkerInfo): JObject = {
50+
("id" -> obj.id) ~
51+
("host" -> obj.host) ~
52+
("port" -> obj.port) ~
53+
("address" -> obj.hostPort) ~
54+
("webuiaddress" -> obj.webUiAddress) ~
55+
("cores" -> obj.cores) ~
56+
("coresused" -> obj.coresUsed) ~
57+
("coresfree" -> obj.coresFree) ~
58+
("memory" -> obj.memory) ~
59+
("memoryused" -> obj.memoryUsed) ~
60+
("memoryfree" -> obj.memoryFree) ~
61+
("state" -> obj.state.toString) ~
62+
("lastheartbeat" -> obj.lastHeartbeat)
63+
}
4364

65+
/**
66+
* Export the [[ApplicationInfo]] to a Json object, an [[ApplicationInfo]] consists of the
67+
* information of an application.
68+
*
69+
* @return a Json object containing the following fields:
70+
* `id` a string identifier of the application
71+
* `starttime` time in milliseconds that the application starts
72+
* `name` a name describes the application
73+
* `cores` total cores granted to the application
74+
* `user` name of the user who submitted the application
75+
* `memoryperslave` minimal memory in MB required to each executor
76+
* `submitdate` time in Date that the application is submitted
77+
* `state` state of the application, see [[ApplicationState]]
78+
* `duration` time in milliseconds that the application has been running
79+
*/
4480
def writeApplicationInfo(obj: ApplicationInfo): JObject = {
4581
("starttime" -> obj.startTime) ~
4682
("id" -> obj.id) ~
@@ -53,6 +89,17 @@ private[deploy] object JsonProtocol {
5389
("duration" -> obj.duration)
5490
}
5591

92+
/**
93+
* Export the [[ApplicationDescription]] to a Json object, an [[ApplicationDescription]] consists
94+
* of the description of an application.
95+
*
96+
* @return a Json object containing the following fields:
97+
* `name` a name describes the application
98+
* `cores` max cores can be allocated to the application, 0 means unlimited
99+
* `memoryperslave` minimal memory in MB required to each executor
100+
* `user` name of the user who submitted the application
101+
* `command` the command string that submitted the application
102+
*/
56103
def writeApplicationDescription(obj: ApplicationDescription): JObject = {
57104
("name" -> obj.name) ~
58105
("cores" -> obj.maxCores.getOrElse(0)) ~
@@ -61,13 +108,38 @@ private[deploy] object JsonProtocol {
61108
("command" -> obj.command.toString)
62109
}
63110

111+
/**
112+
* Export the [[ExecutorRunner]] to a Json object, an [[ExecutorRunner]] consists of the
113+
* information of an executor.
114+
*
115+
* @return a Json object containing the following fields:
116+
* `id` a integer identifier of the executor
117+
* `memory` memory in MB allocated to the executor
118+
* `appid` a string identifier of the application that the executor is working for
119+
* `appdesc` a Json object of the [[ApplicationDescription]] of the application that the
120+
* executor is working for
121+
*/
64122
def writeExecutorRunner(obj: ExecutorRunner): JObject = {
65123
("id" -> obj.execId) ~
66124
("memory" -> obj.memory) ~
67125
("appid" -> obj.appId) ~
68126
("appdesc" -> writeApplicationDescription(obj.appDesc))
69127
}
70128

129+
/**
130+
* Export the [[DriverInfo]] to a Json object, a [[DriverInfo]] consists of the information of a
131+
* driver.
132+
*
133+
* @return a Json object containing the following fields:
134+
* `id` a string identifier of the driver
135+
* `starttime` time in milliseconds that the driver starts
136+
* `state` state of the driver, see [[DriverState]]
137+
* `cores` cores allocated to the driver
138+
* `memory` memory in MB allocated to the driver
139+
* `submitdate` time in Date that the driver is created
140+
* `worker` identifier of the worker that the driver is running on
141+
* `mainclass` main class of the command string that started the driver
142+
*/
71143
def writeDriverInfo(obj: DriverInfo): JObject = {
72144
("id" -> obj.id) ~
73145
("starttime" -> obj.startTime.toString) ~
@@ -79,6 +151,29 @@ private[deploy] object JsonProtocol {
79151
("mainclass" -> obj.desc.command.arguments(2))
80152
}
81153

154+
/**
155+
* Export the [[MasterStateResponse]] to a Json object, a [[MasterStateResponse]] consists the
156+
* information of a master node.
157+
*
158+
* @return a Json object containing the following fields:
159+
* `url` the url of the master node
160+
* `workers` a list of Json objects of [[WorkerInfo]] of the workers allocated to the
161+
* master
162+
* `aliveworkers` size of alive workers allocated to the master
163+
* `cores` total cores available of the master
164+
* `coresused` cores used by the master
165+
* `memory` total memory available of the master
166+
* `memoryused` memory used by the master
167+
* `activeapps` a list of Json objects of [[ApplicationInfo]] of the active applications
168+
* running on the master
169+
* `completedapps` a list of Json objects of [[ApplicationInfo]] of the completed
170+
* applications from the master
171+
* `activedrivers` a list of Json objects of [[DriverInfo]] of the active drivers of the
172+
* master
173+
* `completeddrivers` a list of Json objects of [[DriverInfo]] of the completed drivers
174+
* of the master
175+
* `status` status of the master, see [[MasterState]]
176+
*/
82177
def writeMasterState(obj: MasterStateResponse): JObject = {
83178
val aliveWorkers = obj.workers.filter(_.isAlive())
84179
("url" -> obj.uri) ~
@@ -95,6 +190,23 @@ private[deploy] object JsonProtocol {
95190
("status" -> obj.status.toString)
96191
}
97192

193+
/**
194+
* Export the [[WorkerStateResponse]] to a Json object, a [[WorkerStateResponse]] consists the
195+
* information of a worker node.
196+
*
197+
* @return a Json object containing the following fields:
198+
* `id` a string identifier of the worker node
199+
* `masterurl` url of the master node of the worker
200+
* `masterwebuiurl` the address used in web UI of the master node of the worker
201+
* `cores` total cores of the worker
202+
* `coreused` used cores of the worker
203+
* `memory` total memory of the worker
204+
* `memoryused` used memory of the worker
205+
* `executors` a list of Json objects of [[ExecutorRunner]] of the executors running on
206+
* the worker
207+
* `finishedexecutors` a list of Json objects of [[ExecutorRunner]] of the finished
208+
* executors of the worker
209+
*/
98210
def writeWorkerState(obj: WorkerStateResponse): JObject = {
99211
("id" -> obj.workerId) ~
100212
("masterurl" -> obj.masterUrl) ~

0 commit comments

Comments
 (0)