Skip to content

Commit 219e286

Browse files
xiaoyuyaoxiaoyuyao
authored
HDDS-1608. Support Ozone Prefix ACLs in OM metadata table. Contributed by Xiaoyu Yao. (#875)
1 parent 9122b9b commit 219e286

File tree

7 files changed

+360
-8
lines changed

7 files changed

+360
-8
lines changed

‎hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
2525
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
2626
import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo;
27+
import org.apache.hadoop.ozone.om.helpers.OmPrefixInfo;
2728
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
2829
import org.apache.hadoop.ozone.om.helpers.S3SecretValue;
2930
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.VolumeList;
@@ -262,6 +263,12 @@ List<OmVolumeArgs> listVolumes(String userName, String prefix,
262263

263264
Table<String, String> getS3Table();
264265

266+
/**
267+
* Gets the Ozone prefix path to its acl mapping table.
268+
* @return Table.
269+
*/
270+
Table<String, OmPrefixInfo> getPrefixTable();
271+
265272
/**
266273
* Returns the DB key name of a multipart upload key in OM metadata store.
267274
*

‎hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/codec/OmPrefixInfoCodec.java‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
/**
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
* <p>
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
* <p>
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
package org.apache.hadoop.ozone.om.codec;
19+
20+
import com.google.common.base.Preconditions;
21+
import com.google.protobuf.InvalidProtocolBufferException;
22+
import org.apache.hadoop.ozone.om.helpers.OmPrefixInfo;
23+
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PrefixInfo;
24+
25+
import org.apache.hadoop.utils.db.Codec;
26+
27+
import java.io.IOException;
28+
29+
/**
30+
* Codec to encode PrefixAcl as byte array.
31+
*/
32+
public class OmPrefixInfoCodec implements Codec<OmPrefixInfo> {
33+
34+
@Override
35+
public byte[] toPersistedFormat(OmPrefixInfo object) throws IOException {
36+
Preconditions
37+
.checkNotNull(object, "Null object can't be converted to byte array.");
38+
return object.getProtobuf().toByteArray();
39+
}
40+
41+
@Override
42+
public OmPrefixInfo fromPersistedFormat(byte[] rawData) throws IOException {
43+
Preconditions
44+
.checkNotNull(rawData,
45+
"Null byte array can't converted to real object.");
46+
try {
47+
return OmPrefixInfo.getFromProtobuf(PrefixInfo.parseFrom(rawData));
48+
} catch (InvalidProtocolBufferException e) {
49+
throw new IllegalArgumentException(
50+
"Can't encode the the raw data from the byte array", e);
51+
}
52+
}
53+
}

‎hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmPrefixInfo.java‎

Lines changed: 170 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,170 @@
1+
/**
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
* <p>
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
* <p>
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
package org.apache.hadoop.ozone.om.helpers;
20+
21+
import com.google.common.base.Preconditions;
22+
import org.apache.hadoop.ozone.OzoneAcl;
23+
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PrefixInfo;
24+
import org.apache.hadoop.ozone.protocolPB.OMPBHelper;
25+
26+
import java.util.HashMap;
27+
import java.util.LinkedList;
28+
import java.util.List;
29+
import java.util.Map;
30+
import java.util.Objects;
31+
import java.util.stream.Collectors;
32+
33+
/**
34+
* Wrapper class for Ozone prefix path info, currently mainly target for ACL but
35+
* can be extended for other OzFS optimizations in future.
36+
*/
37+
// TODO: support Auditable interface
38+
public final class OmPrefixInfo extends WithMetadata {
39+
40+
private String name;
41+
private List<OzoneAcl> acls;
42+
43+
public OmPrefixInfo(String name, List<OzoneAcl> acls,
44+
Map<String, String> metadata) {
45+
this.name = name;
46+
this.acls = acls;
47+
this.metadata = metadata;
48+
}
49+
50+
/**
51+
* Returns the ACL's associated with this prefix.
52+
* @return {@literal List<OzoneAcl>}
53+
*/
54+
public List<OzoneAcl> getAcls() {
55+
return acls;
56+
}
57+
58+
/**
59+
* Returns the name of the prefix path.
60+
* @return name of the prefix path.
61+
*/
62+
public String getName() {
63+
return name;
64+
}
65+
66+
/**
67+
* Returns new builder class that builds a OmPrefixInfo.
68+
*
69+
* @return Builder
70+
*/
71+
public static OmPrefixInfo.Builder newBuilder() {
72+
return new OmPrefixInfo.Builder();
73+
}
74+
75+
/**
76+
* Builder for OmPrefixInfo.
77+
*/
78+
public static class Builder {
79+
private String name;
80+
private List<OzoneAcl> acls;
81+
private Map<String, String> metadata;
82+
83+
public Builder() {
84+
//Default values
85+
this.acls = new LinkedList<>();
86+
this.metadata = new HashMap<>();
87+
}
88+
89+
public Builder setAcls(List<OzoneAcl> listOfAcls) {
90+
this.acls = listOfAcls;
91+
return this;
92+
}
93+
94+
public Builder setName(String n) {
95+
this.name = n;
96+
return this;
97+
}
98+
99+
public OmPrefixInfo.Builder addMetadata(String key, String value) {
100+
metadata.put(key, value);
101+
return this;
102+
}
103+
104+
public OmPrefixInfo.Builder addAllMetadata(
105+
Map<String, String> additionalMetadata) {
106+
if (additionalMetadata != null) {
107+
metadata.putAll(additionalMetadata);
108+
}
109+
return this;
110+
}
111+
112+
/**
113+
* Constructs the OmPrefixInfo.
114+
* @return instance of OmPrefixInfo.
115+
*/
116+
public OmPrefixInfo build() {
117+
Preconditions.checkNotNull(name);
118+
Preconditions.checkNotNull(acls);
119+
return new OmPrefixInfo(name, acls, metadata);
120+
}
121+
}
122+
123+
/**
124+
* Creates PrefixInfo protobuf from OmPrefixInfo.
125+
*/
126+
public PrefixInfo getProtobuf() {
127+
PrefixInfo.Builder pib = PrefixInfo.newBuilder().setName(name)
128+
.addAllAcls(acls.stream().map(OMPBHelper::convertOzoneAcl)
129+
.collect(Collectors.toList()))
130+
.addAllMetadata(KeyValueUtil.toProtobuf(metadata));
131+
return pib.build();
132+
}
133+
134+
/**
135+
* Parses PrefixInfo protobuf and creates OmPrefixInfo.
136+
* @param prefixInfo
137+
* @return instance of OmPrefixInfo
138+
*/
139+
public static OmPrefixInfo getFromProtobuf(PrefixInfo prefixInfo) {
140+
OmPrefixInfo.Builder opib = OmPrefixInfo.newBuilder()
141+
.setName(prefixInfo.getName())
142+
.setAcls(prefixInfo.getAclsList().stream().map(
143+
OMPBHelper::convertOzoneAcl).collect(Collectors.toList()));
144+
if (prefixInfo.getMetadataList() != null) {
145+
opib.addAllMetadata(KeyValueUtil
146+
.getFromProtobuf(prefixInfo.getMetadataList()));
147+
}
148+
return opib.build();
149+
}
150+
151+
@Override
152+
public boolean equals(Object o) {
153+
if (this == o) {
154+
return true;
155+
}
156+
if (o == null || getClass() != o.getClass()) {
157+
return false;
158+
}
159+
OmPrefixInfo that = (OmPrefixInfo) o;
160+
return name.equals(that.name) &&
161+
Objects.equals(acls, that.acls) &&
162+
Objects.equals(metadata, that.metadata);
163+
}
164+
165+
@Override
166+
public int hashCode() {
167+
return Objects.hash(name);
168+
}
169+
}
170+

‎hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -459,11 +459,18 @@ message BucketArgs {
459459
repeated hadoop.hdds.KeyValue metadata = 7;
460460
}
461461

462+
message PrefixInfo {
463+
required string name = 1;
464+
repeated OzoneAclInfo acls = 2;
465+
repeated hadoop.hdds.KeyValue metadata = 3;
466+
}
467+
462468
message OzoneObj {
463469
enum ObjectType {
464470
VOLUME = 1;
465471
BUCKET = 2;
466472
KEY = 3;
473+
PREFIX = 4;
467474
}
468475

469476
enum StoreType {

‎hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/codec/TestOmPrefixInfoCodec.java‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
/**
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with this
4+
* work for additional information regarding copyright ownership. The ASF
5+
* licenses this file to you under the Apache License, Version 2.0 (the
6+
* "License"); you may not use this file except in compliance with the License.
7+
* You may obtain a copy of the License at
8+
* <p>
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
* <p>
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
13+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14+
* License for the specific language governing permissions and limitations under
15+
* the License.
16+
*/
17+
18+
package org.apache.hadoop.ozone.om.codec;
19+
20+
import org.apache.hadoop.ozone.OzoneAcl;
21+
import org.apache.hadoop.ozone.om.helpers.OmPrefixInfo;
22+
import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLIdentityType;
23+
import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType;
24+
25+
import org.apache.hadoop.test.GenericTestUtils;
26+
import org.junit.Before;
27+
import org.junit.Rule;
28+
import org.junit.Test;
29+
import org.junit.rules.ExpectedException;
30+
31+
import java.io.IOException;
32+
import java.nio.charset.StandardCharsets;
33+
import java.util.LinkedList;
34+
import java.util.List;
35+
36+
import static org.junit.Assert.assertTrue;
37+
import static org.junit.Assert.fail;
38+
39+
/**
40+
* This class test OmPrefixInfoCodec.
41+
*/
42+
public class TestOmPrefixInfoCodec {
43+
44+
@Rule
45+
public ExpectedException thrown = ExpectedException.none();
46+
47+
48+
private OmPrefixInfoCodec codec;
49+
50+
@Before
51+
public void setUp() {
52+
codec = new OmPrefixInfoCodec();
53+
}
54+
55+
@Test
56+
public void testCodecWithIncorrectValues() throws Exception {
57+
try {
58+
codec.fromPersistedFormat("random".getBytes(StandardCharsets.UTF_8));
59+
fail("testCodecWithIncorrectValues failed");
60+
} catch (IllegalArgumentException ex) {
61+
GenericTestUtils.assertExceptionContains("Can't encode the the raw " +
62+
"data from the byte array", ex);
63+
}
64+
}
65+
66+
@Test
67+
public void testCodecWithNullDataFromTable() throws Exception {
68+
thrown.expect(NullPointerException.class);
69+
codec.fromPersistedFormat(null);
70+
}
71+
72+
73+
@Test
74+
public void testCodecWithNullDataFromUser() throws Exception {
75+
thrown.expect(NullPointerException.class);
76+
codec.toPersistedFormat(null);
77+
}
78+
79+
@Test
80+
public void testToAndFromPersistedFormat() throws IOException {
81+
82+
List<OzoneAcl> acls = new LinkedList<>();
83+
OzoneAcl ozoneAcl = new OzoneAcl(ACLIdentityType.USER,
84+
"hive", ACLType.ALL);
85+
acls.add(ozoneAcl);
86+
OmPrefixInfo opiSave = OmPrefixInfo.newBuilder()
87+
.setName("/user/hive/warehouse")
88+
.setAcls(acls)
89+
.addMetadata("id", "100")
90+
.build();
91+
92+
OmPrefixInfo opiLoad = codec.fromPersistedFormat(
93+
codec.toPersistedFormat(opiSave));
94+
95+
assertTrue("Load saved prefix info should match",
96+
opiLoad.equals(opiSave));
97+
}
98+
}

‎hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/TestCSMMetrics.java‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -58,13 +58,13 @@
5858
import org.junit.Test;
5959
import org.junit.Assert;
6060

61-
/**
62-
* This class tests the metrics of ContainerStateMachine.
63-
*/
64-
public class TestCSMMetrics {
65-
static final String TEST_DIR =
66-
GenericTestUtils.getTestDir("dfs").getAbsolutePath()
67-
+ File.separator;
61+
/**
62+
* This class tests the metrics of ContainerStateMachine.
63+
*/
64+
public class TestCSMMetrics {
65+
static final String TEST_DIR =
66+
GenericTestUtils.getTestDir("dfs").getAbsolutePath()
67+
+ File.separator;
6868
@FunctionalInterface
6969
interface CheckedBiFunction<LEFT, RIGHT, OUT, THROWABLE extends Throwable> {
7070
OUT apply(LEFT left, RIGHT right) throws THROWABLE;

0 commit comments

Comments
 (0)