Skip to content

Commit 1ec5c67

Browse files
brahmareddybattulabrahmareddybattula
committed
HADOOP-16492. Support HuaweiCloud Object Storage as a Hadoop Backend File System. Contributed by zhongjun.
1 parent ec22850 commit 1ec5c67

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

52 files changed

+13227
-0
lines changed

‎hadoop-cloud-storage-project/hadoop-cloud-storage/pom.xml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -133,5 +133,10 @@
133133
<artifactId>hadoop-cos</artifactId>
134134
<scope>compile</scope>
135135
</dependency>
136+
<dependency>
137+
<groupId>org.apache.hadoop</groupId>
138+
<artifactId>hadoop-huaweicloud</artifactId>
139+
<scope>compile</scope>
140+
</dependency>
136141
</dependencies>
137142
</project>

‎hadoop-cloud-storage-project/hadoop-huaweicloud/dev-support/findbugs-exclude.xml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
<!--
2+
Licensed to the Apache Software Foundation (ASF) under one or more
3+
contributor license agreements. See the NOTICE file distributed with
4+
this work for additional information regarding copyright ownership.
5+
The ASF licenses this file to You under the Apache License, Version 2.0
6+
(the "License"); you may not use this file except in compliance with
7+
the License. You may obtain a copy of the License at
8+
9+
http://www.apache.org/licenses/LICENSE-2.0
10+
11+
Unless required by applicable law or agreed to in writing, software
12+
distributed under the License is distributed on an "AS IS" BASIS,
13+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
See the License for the specific language governing permissions and
15+
limitations under the License.
16+
-->
17+
<FindBugsFilter>
18+
</FindBugsFilter>

‎hadoop-cloud-storage-project/hadoop-huaweicloud/pom.xml‎

Lines changed: 180 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,180 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--Licensed under the Apache License, Version 2.0 (the "License");
3+
you may not use this file except in compliance with the License.
4+
You may obtain a copy of the License at
5+
http://www.apache.org/licenses/LICENSE-2.0
6+
Unless required by applicable law or agreed to in writing, software
7+
distributed under the License is distributed on an "AS IS" BASIS,
8+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
9+
See the License for the specific language governing permissions and
10+
limitations under the License. See accompanying LICENSE file.-->
11+
<project xmlns="http://maven.apache.org/POM/4.0.0"
12+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
13+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
14+
<modelVersion>4.0.0</modelVersion>
15+
<parent>
16+
<groupId>org.apache.hadoop</groupId>
17+
<artifactId>hadoop-project</artifactId>
18+
<version>3.4.0-SNAPSHOT</version>
19+
<relativePath>../../hadoop-project</relativePath>
20+
</parent>
21+
<artifactId>hadoop-huaweicloud</artifactId>
22+
<version>3.4.0-SNAPSHOT</version>
23+
<name>Apache Hadoop OBS support</name>
24+
<description>
25+
This module contains code to support integration with OBS.
26+
It also declares the dependencies needed to work with OBS services.
27+
</description>
28+
<packaging>jar</packaging>
29+
<properties>
30+
<file.encoding>UTF-8</file.encoding>
31+
<downloadSources>true</downloadSources>
32+
<esdk.version>3.20.4.2</esdk.version>
33+
</properties>
34+
35+
<profiles>
36+
<profile>
37+
<id>tests-off</id>
38+
<activation>
39+
<file>
40+
<missing>src/test/resources/auth-keys.xml</missing>
41+
</file>
42+
</activation>
43+
<properties>
44+
<maven.test.skip>true</maven.test.skip>
45+
</properties>
46+
</profile>
47+
<profile>
48+
<id>tests-on</id>
49+
<activation>
50+
<file>
51+
<exists>src/test/resources/auth-keys.xml</exists>
52+
</file>
53+
</activation>
54+
<properties>
55+
<maven.test.skip>false</maven.test.skip>
56+
</properties>
57+
</profile>
58+
</profiles>
59+
60+
<build>
61+
<plugins>
62+
<plugin>
63+
<groupId>org.codehaus.mojo</groupId>
64+
<artifactId>findbugs-maven-plugin</artifactId>
65+
<configuration>
66+
<findbugsXmlOutput>true</findbugsXmlOutput>
67+
<xmlOutput>true</xmlOutput>
68+
<excludeFilterFile>${basedir}/dev-support/findbugs-exclude.xml
69+
</excludeFilterFile>
70+
<effort>Max</effort>
71+
</configuration>
72+
</plugin>
73+
<plugin>
74+
<groupId>org.apache.maven.plugins</groupId>
75+
<artifactId>maven-surefire-plugin</artifactId>
76+
<configuration>
77+
<forkedProcessTimeoutInSeconds>3600</forkedProcessTimeoutInSeconds>
78+
</configuration>
79+
</plugin>
80+
<plugin>
81+
<groupId>org.apache.maven.plugins</groupId>
82+
<artifactId>maven-dependency-plugin</artifactId>
83+
<executions>
84+
<execution>
85+
<id>deplist</id>
86+
<phase>compile</phase>
87+
<goals>
88+
<goal>list</goal>
89+
</goals>
90+
<configuration>
91+
<outputFile>${project.basedir}/target/hadoop-cloud-storage-deps/${project.artifactId}.cloud-storage-optional.txt</outputFile>
92+
</configuration>
93+
</execution>
94+
</executions>
95+
</plugin>
96+
</plugins>
97+
</build>
98+
<dependencies>
99+
<dependency>
100+
<groupId>org.apache.hadoop</groupId>
101+
<artifactId>hadoop-common</artifactId>
102+
<scope>provided</scope>
103+
<exclusions>
104+
<exclusion>
105+
<groupId>jdk.tools</groupId>
106+
<artifactId>jdk.tools</artifactId>
107+
</exclusion>
108+
</exclusions>
109+
</dependency>
110+
<dependency>
111+
<groupId>org.apache.hadoop</groupId>
112+
<artifactId>hadoop-common</artifactId>
113+
<scope>test</scope>
114+
<type>test-jar</type>
115+
</dependency>
116+
<dependency>
117+
<groupId>junit</groupId>
118+
<artifactId>junit</artifactId>
119+
<version>4.12</version>
120+
<scope>test</scope>
121+
</dependency>
122+
<dependency>
123+
<groupId>org.mockito</groupId>
124+
<artifactId>mockito-all</artifactId>
125+
<version>1.10.19</version>
126+
<scope>test</scope>
127+
</dependency>
128+
<dependency>
129+
<groupId>org.apache.hadoop</groupId>
130+
<artifactId>hadoop-mapreduce-client-jobclient</artifactId>
131+
<scope>test</scope>
132+
</dependency>
133+
<dependency>
134+
<groupId>org.apache.hadoop</groupId>
135+
<artifactId>hadoop-yarn-server-tests</artifactId>
136+
<scope>test</scope>
137+
<type>test-jar</type>
138+
</dependency>
139+
<dependency>
140+
<groupId>org.apache.hadoop</groupId>
141+
<artifactId>hadoop-mapreduce-examples</artifactId>
142+
<scope>test</scope>
143+
<type>jar</type>
144+
</dependency>
145+
<dependency>
146+
<groupId>org.apache.hadoop</groupId>
147+
<artifactId>hadoop-distcp</artifactId>
148+
<scope>test</scope>
149+
</dependency>
150+
<dependency>
151+
<groupId>org.apache.hadoop</groupId>
152+
<artifactId>hadoop-distcp</artifactId>
153+
<scope>test</scope>
154+
<type>test-jar</type>
155+
</dependency>
156+
<dependency>
157+
<groupId>com.huaweicloud</groupId>
158+
<artifactId>esdk-obs-java</artifactId>
159+
<version>${esdk.version}</version>
160+
<exclusions>
161+
<exclusion>
162+
<artifactId>okio</artifactId>
163+
<groupId>com.squareup.okio</groupId>
164+
</exclusion>
165+
</exclusions>
166+
</dependency>
167+
<dependency>
168+
<groupId>org.powermock</groupId>
169+
<artifactId>powermock-api-mockito</artifactId>
170+
<version>1.7.4</version>
171+
<scope>test</scope>
172+
</dependency>
173+
<dependency>
174+
<groupId>org.powermock</groupId>
175+
<artifactId>powermock-module-junit4</artifactId>
176+
<version>1.7.4</version>
177+
<scope>test</scope>
178+
</dependency>
179+
</dependencies>
180+
</project>

‎hadoop-cloud-storage-project/hadoop-huaweicloud/src/main/java/org/apache/hadoop/fs/obs/BasicSessionCredential.java‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
package org.apache.hadoop.fs.obs;
20+
21+
/**
22+
* Interface class for getting basic session credential.
23+
*/
24+
public interface BasicSessionCredential {
25+
/**
26+
* Get OBS access key.
27+
*
28+
* @return OBS access key
29+
*/
30+
String getOBSAccessKeyId();
31+
32+
/**
33+
* Get OBS secret key.
34+
*
35+
* @return OBS secret key
36+
*/
37+
String getOBSSecretKey();
38+
39+
/**
40+
* Get session token.
41+
*
42+
* @return session token
43+
*/
44+
String getSessionToken();
45+
}

0 commit comments

Comments
 (0)