Skip to content

Commit ebcd845

Browse files
author
Steve Vaughan Jr
committed
Update mapreduce to follow yarn pattern
Use dependencies in the mapreduce project for the purpose of assembly plugin and packaging. This moves all assembly dependencies added for parallel builds up to the module.
1 parent 79bb860 commit ebcd845

File tree

2 files changed

+29
-139
lines changed

2 files changed

+29
-139
lines changed

‎hadoop-mapreduce-project/pom.xml‎

Lines changed: 24 additions & 132 deletions
Original file line numberDiff line numberDiff line change
@@ -41,118 +41,51 @@
4141
<module>hadoop-mapreduce-examples</module>
4242
</modules>
4343

44+
<!--
45+
Do not add 3rd party dependencies here, add them to the POM of the leaf module.
46+
The dependencies in this module are for the assembly plugin, packaging purposes.
47+
-->
4448
<dependencies>
45-
<dependency>
46-
<groupId>com.google.protobuf</groupId>
47-
<artifactId>protobuf-java</artifactId>
48-
</dependency>
49-
<dependency>
50-
<groupId>org.apache.avro</groupId>
51-
<artifactId>avro</artifactId>
52-
<exclusions>
53-
<exclusion>
54-
<groupId>org.eclipse.jetty</groupId>
55-
<artifactId>jetty-server</artifactId>
56-
</exclusion>
57-
<exclusion>
58-
<groupId>org.apache.ant</groupId>
59-
<artifactId>ant</artifactId>
60-
</exclusion>
61-
<exclusion>
62-
<groupId>io.netty</groupId>
63-
<artifactId>netty</artifactId>
64-
</exclusion>
65-
<exclusion>
66-
<groupId>org.apache.velocity</groupId>
67-
<artifactId>velocity</artifactId>
68-
</exclusion>
69-
<exclusion>
70-
<groupId>org.slf4j</groupId>
71-
<artifactId>slf4j-api</artifactId>
72-
</exclusion>
73-
<exclusion>
74-
<artifactId>paranamer-ant</artifactId>
75-
<groupId>com.thoughtworks.paranamer</groupId>
76-
</exclusion>
77-
<exclusion>
78-
<groupId>org.xerial.snappy</groupId>
79-
<artifactId>snappy-java</artifactId>
80-
</exclusion>
81-
</exclusions>
82-
</dependency>
8349
<dependency>
8450
<groupId>org.apache.hadoop</groupId>
85-
<artifactId>hadoop-common</artifactId>
86-
<scope>provided</scope>
87-
</dependency>
88-
89-
<dependency>
90-
<groupId>org.slf4j</groupId>
91-
<artifactId>slf4j-api</artifactId>
92-
</dependency>
93-
<dependency>
94-
<groupId>org.slf4j</groupId>
95-
<artifactId>slf4j-log4j12</artifactId>
51+
<artifactId>hadoop-mapreduce-client-app</artifactId>
52+
<version>${project.version}</version>
9653
</dependency>
9754
<dependency>
9855
<groupId>org.apache.hadoop</groupId>
99-
<artifactId>hadoop-annotations</artifactId>
100-
</dependency>
101-
<dependency>
102-
<groupId>org.mockito</groupId>
103-
<artifactId>mockito-core</artifactId>
104-
<scope>test</scope>
56+
<artifactId>hadoop-mapreduce-client-common</artifactId>
57+
<version>${project.version}</version>
10558
</dependency>
10659
<dependency>
10760
<groupId>org.apache.hadoop</groupId>
108-
<artifactId>hadoop-common</artifactId>
109-
<type>test-jar</type>
110-
<scope>test</scope>
61+
<artifactId>hadoop-mapreduce-client-core</artifactId>
62+
<version>${project.version}</version>
11163
</dependency>
11264
<dependency>
11365
<groupId>org.apache.hadoop</groupId>
114-
<artifactId>hadoop-hdfs</artifactId>
115-
<scope>test</scope>
116-
</dependency>
117-
<dependency>
118-
<groupId>com.google.inject</groupId>
119-
<artifactId>guice</artifactId>
120-
</dependency>
121-
<dependency>
122-
<groupId>com.sun.jersey</groupId>
123-
<artifactId>jersey-server</artifactId>
66+
<artifactId>hadoop-mapreduce-client-hs</artifactId>
67+
<version>${project.version}</version>
12468
</dependency>
12569
<dependency>
126-
<groupId>com.sun.jersey.contribs</groupId>
127-
<artifactId>jersey-guice</artifactId>
128-
</dependency>
129-
<dependency>
130-
<groupId>com.google.inject.extensions</groupId>
131-
<artifactId>guice-servlet</artifactId>
132-
</dependency>
133-
<dependency>
134-
<groupId>junit</groupId>
135-
<artifactId>junit</artifactId>
136-
</dependency>
137-
<dependency>
138-
<groupId>io.netty</groupId>
139-
<artifactId>netty</artifactId>
70+
<groupId>org.apache.hadoop</groupId>
71+
<artifactId>hadoop-mapreduce-client-jobclient</artifactId>
72+
<version>${project.version}</version>
14073
</dependency>
14174
<dependency>
142-
<groupId>commons-io</groupId>
143-
<artifactId>commons-io</artifactId>
75+
<groupId>org.apache.hadoop</groupId>
76+
<artifactId>hadoop-mapreduce-client-nativetask</artifactId>
77+
<version>${project.version}</version>
14478
</dependency>
14579
<dependency>
146-
<groupId>org.hsqldb</groupId>
147-
<artifactId>hsqldb</artifactId>
148-
<scope>compile</scope>
149-
<classifier>jdk8</classifier>
80+
<groupId>org.apache.hadoop</groupId>
81+
<artifactId>hadoop-mapreduce-client-shuffle</artifactId>
82+
<version>${project.version}</version>
15083
</dependency>
15184
<dependency>
152-
<groupId>${leveldbjni.group}</groupId>
153-
<artifactId>leveldbjni-all</artifactId>
85+
<groupId>org.apache.hadoop</groupId>
86+
<artifactId>hadoop-mapreduce-examples</artifactId>
87+
<version>${project.version}</version>
15488
</dependency>
155-
15689
</dependencies>
15790

15891
<build>
@@ -274,47 +207,6 @@
274207
<artifactId>hadoop-assemblies</artifactId>
275208
<version>${project.version}</version>
276209
</dependency>
277-
<!-- Add dependencies for parallel builds -->
278-
<dependency>
279-
<groupId>org.apache.hadoop</groupId>
280-
<artifactId>hadoop-mapreduce-client-app</artifactId>
281-
<version>${project.version}</version>
282-
</dependency>
283-
<dependency>
284-
<groupId>org.apache.hadoop</groupId>
285-
<artifactId>hadoop-mapreduce-client-common</artifactId>
286-
<version>${project.version}</version>
287-
</dependency>
288-
<dependency>
289-
<groupId>org.apache.hadoop</groupId>
290-
<artifactId>hadoop-mapreduce-client-core</artifactId>
291-
<version>${project.version}</version>
292-
</dependency>
293-
<dependency>
294-
<groupId>org.apache.hadoop</groupId>
295-
<artifactId>hadoop-mapreduce-client-hs</artifactId>
296-
<version>${project.version}</version>
297-
</dependency>
298-
<dependency>
299-
<groupId>org.apache.hadoop</groupId>
300-
<artifactId>hadoop-mapreduce-client-jobclient</artifactId>
301-
<version>${project.version}</version>
302-
</dependency>
303-
<dependency>
304-
<groupId>org.apache.hadoop</groupId>
305-
<artifactId>hadoop-mapreduce-client-nativetask</artifactId>
306-
<version>${project.version}</version>
307-
</dependency>
308-
<dependency>
309-
<groupId>org.apache.hadoop</groupId>
310-
<artifactId>hadoop-mapreduce-client-shuffle</artifactId>
311-
<version>${project.version}</version>
312-
</dependency>
313-
<dependency>
314-
<groupId>org.apache.hadoop</groupId>
315-
<artifactId>hadoop-mapreduce-examples</artifactId>
316-
<version>${project.version}</version>
317-
</dependency>
318210
</dependencies>
319211
<configuration>
320212
<tarLongFileMode>gnu</tarLongFileMode>

‎hadoop-yarn-project/pom.xml‎

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,11 @@
8181
<groupId>org.apache.hadoop</groupId>
8282
<artifactId>hadoop-yarn-services-core</artifactId>
8383
</dependency>
84+
<dependency>
85+
<groupId>org.apache.hadoop</groupId>
86+
<artifactId>hadoop-yarn-applications-catalog-webapp</artifactId>
87+
<type>war</type>
88+
</dependency>
8489
</dependencies>
8590

8691
<build>
@@ -142,13 +147,6 @@
142147
<artifactId>hadoop-assemblies</artifactId>
143148
<version>${project.version}</version>
144149
</dependency>
145-
<!-- Add dependencies for parallel builds -->
146-
<dependency>
147-
<groupId>org.apache.hadoop</groupId>
148-
<artifactId>hadoop-yarn-applications-catalog-webapp</artifactId>
149-
<version>${project.version}</version>
150-
<type>war</type>
151-
</dependency>
152150
</dependencies>
153151
<configuration>
154152
<tarLongFileMode>gnu</tarLongFileMode>

0 commit comments

Comments
 (0)