Skip to content

RANGER-5406: Support export policies in a segmented manner #741

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
yunyezhang-work wants to merge 1 commit into
base: ranger-2.3
Choose a base branch
from
Open

RANGER-5406: Support export policies in a segmented manner #741

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions agents-common/src/main/java/org/apache/ranger/plugin/util/SearchFilter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,8 @@ public class SearchFilter {
public static final String CREATE_TIME = "createTime"; // sort
public static final String UPDATE_TIME = "updateTime"; // sort
public static final String START_INDEX = "startIndex";
public static final String BEGIN_INDEX = "beginIndex";
public static final String OFFSET_INDEX = "offsetIndex";
Copy link
Contributor

@vyommani vyommani Nov 27, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think OFFSET is more meaning full than OFFSET_INDEX, offset is not index. What do you think ?

public static final String PAGE_SIZE = "pageSize";
public static final String SORT_BY = "sortBy";
public static final String RESOURCE_SIGNATURE = "resourceSignature:"; // search
Expand Down Expand Up @@ -98,6 +100,8 @@ public class SearchFilter {
private Map<String, String> params;
private int startIndex;
private int maxRows = Integer.MAX_VALUE;
private int beginIndex = -1;
private int offsetIndex = -1;
Copy link
Contributor

@vyommani vyommani Nov 27, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Since you've added new fields to the SearchFilter class, don't forget to modify the copy constructor (public SearchFilter(SearchFilter other)) accordingly to ensure the new attributes are properly copied.

private boolean getCount = true;
private String sortBy;
private String sortType;
Expand Down Expand Up @@ -182,6 +186,22 @@ public void setStartIndex(int startIndex) {
this.startIndex = startIndex;
}

public int getBeginIndex() {
return beginIndex;
}

public void setBeginIndex(int beginIndex) {
this.beginIndex = beginIndex;
Copy link
Contributor

@vyommani vyommani Nov 27, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we should validate that beginIndex >= 0. What’s your opinion?

}

public int getOffsetIndex() {
return offsetIndex;
}

public void setOffsetIndex(int offsetIndex) {
this.offsetIndex = offsetIndex;
}

public int getMaxRows() {
return maxRows;
}
Expand Down
17 changes: 17 additions & 0 deletions security-admin/src/main/java/org/apache/ranger/common/RangerSearchUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -175,13 +175,30 @@ public SearchFilter extractCommonCriteriasForFilter(HttpServletRequest request,
"Invalid value for parameter startIndex", MessageEnums.INVALID_INPUT_DATA, null,
SearchFilter.START_INDEX);
startIndex = startIndex < 0 ? 0 : startIndex;
logger.info("==> setStartIndex=" + startIndex);
ret.setStartIndex(startIndex);

int pageSize = restErrorUtil.parseInt(request.getParameter(SearchFilter.PAGE_SIZE),
configUtil.getDefaultMaxRows(), "Invalid value for parameter pageSize",
MessageEnums.INVALID_INPUT_DATA, null, SearchFilter.PAGE_SIZE);
logger.info("==> setMaxRows=" + pageSize);
logger.info("==> DefaultMaxRows=" + configUtil.getDefaultMaxRows());
ret.setMaxRows(validatePageSize(pageSize));

int beginIndex = restErrorUtil.parseInt(request.getParameter(SearchFilter.BEGIN_INDEX), 0,
"Invalid value for parameter beginIndex", MessageEnums.INVALID_INPUT_DATA, null,
SearchFilter.BEGIN_INDEX);
beginIndex = beginIndex < 0 ? startIndex : beginIndex;
logger.info("==> setBeginIndex=" + beginIndex);
ret.setBeginIndex(beginIndex);

int offsetSize = restErrorUtil.parseInt(request.getParameter(SearchFilter.OFFSET_INDEX), 0,
"Invalid value for parameter offset", MessageEnums.INVALID_INPUT_DATA, null,
SearchFilter.OFFSET_INDEX);
logger.info("==> setOffsetIndex=" + offsetSize);
offsetSize = offsetSize < 0 ? pageSize : offsetSize;
ret.setOffsetIndex(offsetSize);

ret.setGetCount(restErrorUtil.parseBoolean(request.getParameter("getCount"), true));
String sortBy = restErrorUtil.validateString(request.getParameter(SearchFilter.SORT_BY),
StringUtil.VALIDATION_ALPHA, "Invalid value for parameter sortBy", MessageEnums.INVALID_INPUT_DATA,
Expand Down
50 changes: 50 additions & 0 deletions security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2604,6 +2604,8 @@ private List<RangerPolicy> getAllFilteredPolicyList(SearchFilter filter,
List<String> serviceTypeList = null;
List<String> serviceNameInServiceTypeList = new ArrayList<String>();
boolean isServiceExists = false;
int begin = -1;
int offset = -1;

if (request.getParameter(PARAM_SERVICE_NAME) != null){
serviceNames = request.getParameter(PARAM_SERVICE_NAME);
Expand All @@ -2623,6 +2625,9 @@ private List<RangerPolicy> getAllFilteredPolicyList(SearchFilter filter,
List<RangerPolicy> policyListByServiceName = new ArrayList<RangerPolicy>();

if (filter != null) {
begin = filter.getBeginIndex();
offset = filter.getOffsetIndex();
LOG.info("==> beginIndex: " + begin + " offsetIndex: " + offset);
filter.setStartIndex(0);
filter.setMaxRows(Integer.MAX_VALUE);

Expand Down Expand Up @@ -2690,6 +2695,12 @@ private List<RangerPolicy> getAllFilteredPolicyList(SearchFilter filter,
policyLists.addAll(orderedPolicies.values());
}
}
LOG.info("<==policyLists size:" + policyLists.size());

if(begin>=0 && offset >0) {
policyLists = cutRangerPolicyList(policyLists, filter);
LOG.info("<==policyLists size after cut:" + policyLists.size());
}
return policyLists;
}

Expand Down Expand Up @@ -3889,6 +3900,45 @@ private RangerPolicyList toRangerPolicyList(List<RangerPolicy> policyList, Searc
return ret;
}

private List<RangerPolicy> cutRangerPolicyList(List<RangerPolicy> policyList, SearchFilter filter) {
Copy link
Contributor

@kumaab kumaab Nov 26, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested name: getRangerPoliciesInRange

List<RangerPolicy> retList = null;
if (CollectionUtils.isNotEmpty(policyList)) {
int totalCount = policyList.size();
int startIndex = filter.getBeginIndex();
int pageSize = filter.getOffsetIndex();
int toIndex = Math.min(startIndex + pageSize, totalCount);
LOG.info("==>totalCount: " + totalCount + " startIndex: " + startIndex + " pageSize: " +pageSize + " toIndex: " + toIndex);
Copy link
Contributor

@kumaab kumaab Nov 26, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Avoid string concatenation, use String.format()

String sortType = filter.getSortType();
String sortBy = filter.getSortBy();

if (StringUtils.isNotEmpty(sortBy) && StringUtils.isNotEmpty(sortType)) {
// By default policyList is sorted by policyId in asc order, So handling only desc case.
if (SearchFilter.POLICY_ID.equalsIgnoreCase(sortBy)) {
if (SORT_ORDER.DESC.name().equalsIgnoreCase(sortType)) {
policyList.sort(this.getPolicyComparator(sortBy, sortType));
}
} else if (SearchFilter.POLICY_NAME.equalsIgnoreCase(sortBy)) {
if (SORT_ORDER.ASC.name().equalsIgnoreCase(sortType)) {
policyList.sort(this.getPolicyComparator(sortBy, sortType));
} else if (SORT_ORDER.DESC.name().equalsIgnoreCase(sortType)) {
policyList.sort(this.getPolicyComparator(sortBy, sortType));
} else {
LOG.info("Invalid or Unsupported sortType : " + sortType);
}
} else {
LOG.info("Invalid or Unsupported sortBy property : " + sortBy);
Copy link
Contributor

@kumaab kumaab Nov 26, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Avoid string concat, check all references.

See: https://cwiki.apache.org/confluence/display/RANGER/Apache+Ranger+Java+Style+Guide

}
}

retList = new ArrayList<RangerPolicy>();
for (int i = startIndex; i < toIndex; i++) {
retList.add(policyList.get(i));
}

}
return retList;
}

private Comparator<RangerPolicy> getPolicyComparator(String sortBy, String sortType) {
Comparator<RangerPolicy> rangerPolComparator = (RangerPolicy me, RangerPolicy other) -> {
int ret = 0;
Expand Down