From 4b936ee9b8c2586d31956f0b085c48a3f0949165 Mon Sep 17 00:00:00 2001 From: Rakesh Gupta Date: Thu, 13 Aug 2026 20:46:53 +0530 Subject: [PATCH 1/2] RANGER-3905: Ranger metrics on audit throughput for each of the Ranger Service plugins --- .../plugin/model/RangerAuditMetrics.java | 143 +++++ .../model/RangerAuditMetricsByDays.java | 148 +++++ .../model/RangerAuditMetricsByHours.java | 145 +++++ .../ranger/plugin/util/SearchFilter.java | 3 + .../ranger/common/RangerSearchUtil.java | 2 + .../apache/ranger/rest/AuditMetricsREST.java | 210 +++++++ .../security/context/RangerAPIList.java | 9 + .../security/context/RangerAPIMapping.java | 6 + .../ranger/solr/SolrAccessAuditsService.java | 541 ++++++++++++++++++ .../ranger/view/RangerAuditMetricsList.java | 65 +++ .../view/RangerAuditMetricsListByDays.java | 65 +++ .../view/RangerAuditMetricsListByHours.java | 65 +++ .../ranger/rest/TestAuditMetricsREST.java | 397 +++++++++++++ .../solr/SolrAccessAuditsServiceTest.java | 297 ++++++++++ 14 files changed, 2096 insertions(+) create mode 100644 agents-common/src/main/java/org/apache/ranger/plugin/model/RangerAuditMetrics.java create mode 100644 agents-common/src/main/java/org/apache/ranger/plugin/model/RangerAuditMetricsByDays.java create mode 100644 agents-common/src/main/java/org/apache/ranger/plugin/model/RangerAuditMetricsByHours.java create mode 100644 security-admin/src/main/java/org/apache/ranger/rest/AuditMetricsREST.java create mode 100644 security-admin/src/main/java/org/apache/ranger/view/RangerAuditMetricsList.java create mode 100644 security-admin/src/main/java/org/apache/ranger/view/RangerAuditMetricsListByDays.java create mode 100644 security-admin/src/main/java/org/apache/ranger/view/RangerAuditMetricsListByHours.java create mode 100644 security-admin/src/test/java/org/apache/ranger/rest/TestAuditMetricsREST.java diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerAuditMetrics.java b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerAuditMetrics.java new file mode 100644 index 00000000000..6445d522335 --- /dev/null +++ b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerAuditMetrics.java @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.plugin.model; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; + +@JsonAutoDetect(fieldVisibility = Visibility.ANY) +@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown = true) +public class RangerAuditMetrics extends RangerBaseModelObject implements java.io.Serializable { + private static final long serialVersionUID = 1L; + + private String serviceType; + private String serviceName; + private String appId; + private String clusterName; + private String clientIP; + private Long numberOfAudits; + + /** + * @param + */ + public RangerAuditMetrics() { + } + + /** + * @param serviceType + * @param serviceName + */ + public RangerAuditMetrics(String serviceType, String serviceName, String appId, String clusterName, String clientIP, Long numberOfAudits) { + super(); + setServiceType(serviceType); + setServiceName(serviceName); + setAppId(appId); + setClusterName(clusterName); + setClientIP(clientIP); + setNumberOfAudits(numberOfAudits); + } + + /** + * @return the serviceType + */ + public String getServiceType() { + return serviceType; + } + + /** + * @param serviceType to set + */ + public void setServiceType(String serviceType) { + this.serviceType = serviceType; + } + + /** + * @return the serviceName + */ + public String getServiceName() { + return serviceName; + } + + /** + * @param serviceName to set + */ + public void setServiceName(String serviceName) { + this.serviceName = serviceName; + } + + public String getAppId() { + return appId; + } + + public void setAppId(String appId) { + this.appId = appId; + } + + public String getClusterName() { + return clusterName; + } + + public void setClusterName(String clusterName) { + this.clusterName = clusterName; + } + + public String getClientIP() { + return clientIP; + } + + public void setClientIP(String clientIP) { + if (clientIP == null) { + this.clientIP = ""; + } else { + this.clientIP = clientIP; + } + } + + public Long getNumberOfAudits() { + return numberOfAudits; + } + + public void setNumberOfAudits(Long numberOfAudits) { + this.numberOfAudits = numberOfAudits; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + toString(sb); + return sb.toString(); + } + + public StringBuilder toString(StringBuilder sb) { + sb.append("RangerAuditMetrics={ "); + super.toString(sb); + sb.append("serviceType={").append(serviceType).append("} "); + sb.append("serviceName={").append(serviceName).append("} "); + sb.append("appId={").append(appId).append("} "); + sb.append("clusterName={").append(clusterName).append("} "); + sb.append("clientIP={").append(clientIP).append("} "); + sb.append("numberOfAudits={").append(numberOfAudits).append("} "); + sb.append("}"); + return sb; + } +} diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerAuditMetricsByDays.java b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerAuditMetricsByDays.java new file mode 100644 index 00000000000..e6ff110d686 --- /dev/null +++ b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerAuditMetricsByDays.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.plugin.model; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; + +@JsonAutoDetect(getterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE, fieldVisibility = Visibility.ANY) +@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown = true) +public class RangerAuditMetricsByDays extends RangerBaseModelObject implements java.io.Serializable { + private static final long serialVersionUID = 1L; + + private String serviceType; + private String serviceName; + private String appId; + private String clusterName; + private String clientIP; + private Long auditDate; + private Long numberOfAudits; + + /** + * @param + */ + public RangerAuditMetricsByDays() { + this(null, null, null, null, null, null, null); + } + + /** + * @param serviceType + * @param serviceName + * @param appId + * @param clusterName + * @param clientIP + * @param auditDate + * @param numberOfAudits + */ + public RangerAuditMetricsByDays(String serviceType, String serviceName, String appId, String clusterName, String clientIP, Long auditDate, Long numberOfAudits) { + super(); + setServiceType(serviceType); + setServiceName(serviceName); + setAppId(appId); + setClusterName(clusterName); + setClientIP(clientIP); + setAuditDate(auditDate); + setNumberOfAudits(numberOfAudits); + } + + public String getServiceType() { + return serviceType; + } + + public void setServiceType(String serviceType) { + this.serviceType = serviceType; + } + + public String getServiceName() { + return serviceName; + } + + public void setServiceName(String serviceName) { + this.serviceName = serviceName; + } + + public String getAppId() { + return appId; + } + + public void setAppId(String appId) { + this.appId = appId; + } + + public String getClusterName() { + return clusterName; + } + + public void setClusterName(String clusterName) { + this.clusterName = clusterName; + } + + public String getClientIP() { + return clientIP; + } + + public void setClientIP(String clientIP) { + if (clientIP == null) { + this.clientIP = ""; + } else { + this.clientIP = clientIP; + } + } + + public Long getNumberOfAudits() { + return numberOfAudits; + } + + public void setNumberOfAudits(Long numberOfAudits) { + this.numberOfAudits = numberOfAudits; + } + + public Long getAuditDate() { + return auditDate; + } + + public void setAuditDate(Long auditDate) { + this.auditDate = auditDate; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + toString(sb); + return sb.toString(); + } + + public StringBuilder toString(StringBuilder sb) { + sb.append("RangerAuditMetricsByDays={ "); + super.toString(sb); + sb.append("serviceType={").append(serviceType).append("} "); + sb.append("serviceName={").append(serviceName).append("} "); + sb.append("appId={").append(appId).append("} "); + sb.append("clusterName={").append(clusterName).append("} "); + sb.append("clientIP={").append(clientIP).append("} "); + sb.append("auditDate={").append(auditDate).append("} "); + sb.append("numberOfAudits={").append(numberOfAudits).append("} "); + sb.append("}"); + return sb; + } +} diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerAuditMetricsByHours.java b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerAuditMetricsByHours.java new file mode 100644 index 00000000000..141036ec2e5 --- /dev/null +++ b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerAuditMetricsByHours.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.plugin.model; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; + +@JsonAutoDetect(fieldVisibility = Visibility.ANY) +@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown = true) +public class RangerAuditMetricsByHours extends RangerBaseModelObject implements java.io.Serializable { + private static final long serialVersionUID = 1L; + + private String serviceType; + private String serviceName; + private String appId; + private String clusterName; + private String clientIP; + private Integer hours; + private Long numberOfAudits; + + /** + * @param + */ + public RangerAuditMetricsByHours() { + this(null, null, null, null, null, null, null); + } + + /** + * @param serviceType + * @param serviceName + * @param hours + * @param numberOfAudits + */ + public RangerAuditMetricsByHours(String serviceType, String serviceName, String appId, String clusterName, String clientIP, Integer hours, Long numberOfAudits) { + super(); + setServiceType(serviceType); + setServiceName(serviceName); + setAppId(appId); + setClusterName(clusterName); + setClientIP(clientIP); + setHours(hours); + setNumberOfAudits(numberOfAudits); + } + + public String getServiceType() { + return serviceType; + } + + public void setServiceType(String serviceType) { + this.serviceType = serviceType; + } + + public String getServiceName() { + return serviceName; + } + + public void setServiceName(String serviceName) { + this.serviceName = serviceName; + } + + public Integer getHours() { + return hours; + } + + public void setHours(Integer hours) { + this.hours = hours; + } + + public String getAppId() { + return appId; + } + + public void setAppId(String appId) { + this.appId = appId; + } + + public String getClusterName() { + return clusterName; + } + + public void setClusterName(String clusterName) { + this.clusterName = clusterName; + } + + public String getClientIP() { + return clientIP; + } + + public void setClientIP(String clientIP) { + if (clientIP == null) { + this.clientIP = ""; + } else { + this.clientIP = clientIP; + } + } + + public Long getNumberOfAudits() { + return numberOfAudits; + } + + public void setNumberOfAudits(Long numberOfAudits) { + this.numberOfAudits = numberOfAudits; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + toString(sb); + return sb.toString(); + } + + public StringBuilder toString(StringBuilder sb) { + sb.append("RangerAuditMetricsByHours={ "); + super.toString(sb); + sb.append("serviceType={").append(serviceType).append("} "); + sb.append("serviceName={").append(serviceName).append("} "); + sb.append("appId={").append(appId).append("} "); + sb.append("clusterName={").append(clusterName).append("} "); + sb.append("clientIP={").append(clientIP).append("} "); + sb.append("hours={").append(hours).append("} "); + sb.append("numberOfAudits={").append(numberOfAudits).append("} "); + sb.append("}"); + return sb; + } +} diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/SearchFilter.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/SearchFilter.java index 9db01a3f59c..084a857176d 100755 --- a/agents-common/src/main/java/org/apache/ranger/plugin/util/SearchFilter.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/SearchFilter.java @@ -95,6 +95,9 @@ public class SearchFilter { public static final String SERVICE_NAME_PARTIAL = "serviceNamePartial"; + public static final String CLIENT_IP = "clientIP"; + public static final String APP_ID = "appId"; + public static final String PLUGIN_HOST_NAME = "pluginHostName"; public static final String PLUGIN_APP_TYPE = "pluginAppType"; public static final String PLUGIN_ENTITY_TYPE = "pluginEntityType"; diff --git a/security-admin/src/main/java/org/apache/ranger/common/RangerSearchUtil.java b/security-admin/src/main/java/org/apache/ranger/common/RangerSearchUtil.java index 4a69b57707c..c562eb8f8ad 100755 --- a/security-admin/src/main/java/org/apache/ranger/common/RangerSearchUtil.java +++ b/security-admin/src/main/java/org/apache/ranger/common/RangerSearchUtil.java @@ -116,6 +116,8 @@ public SearchFilter getSearchFilter(@Nonnull HttpServletRequest request, List e : request.getParameterMap().entrySet()) { String name = e.getKey(); diff --git a/security-admin/src/main/java/org/apache/ranger/rest/AuditMetricsREST.java b/security-admin/src/main/java/org/apache/ranger/rest/AuditMetricsREST.java new file mode 100644 index 00000000000..8a992b88a1b --- /dev/null +++ b/security-admin/src/main/java/org/apache/ranger/rest/AuditMetricsREST.java @@ -0,0 +1,210 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.rest; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.ranger.authorization.hadoop.config.RangerAdminConfig; +import org.apache.ranger.common.MessageEnums; +import org.apache.ranger.common.RESTErrorUtil; +import org.apache.ranger.common.RangerSearchUtil; +import org.apache.ranger.plugin.model.RangerAuditMetrics; +import org.apache.ranger.plugin.model.RangerAuditMetricsByDays; +import org.apache.ranger.plugin.model.RangerAuditMetricsByHours; +import org.apache.ranger.plugin.util.SearchFilter; +import org.apache.ranger.security.context.RangerAPIList; +import org.apache.ranger.solr.SolrAccessAuditsService; +import org.apache.ranger.view.RangerAuditMetricsList; +import org.apache.ranger.view.RangerAuditMetricsListByDays; +import org.apache.ranger.view.RangerAuditMetricsListByHours; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Scope; +import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Propagation; +import org.springframework.transaction.annotation.Transactional; + +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.DefaultValue; +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; +import javax.ws.rs.WebApplicationException; +import javax.ws.rs.core.Context; + +import java.util.Collections; +import java.util.List; + +@Path("audit") +@Component +@Scope("request") +@Transactional(propagation = Propagation.REQUIRES_NEW) +public class AuditMetricsREST { + private static final Logger LOG = LoggerFactory.getLogger(AuditMetricsREST.class); + + private static final String PROP_AUDIT_METRICS_MAX_SUPPORTED_DAYS = "ranger.audit.metrics.max.supported.days"; + private static final Integer PROP_AUDIT_METRICS_MAX_SUPPORTED_DAYS_DEFAULT = 90; + private static final RangerAdminConfig config = RangerAdminConfig.getInstance(); + + @Autowired + RESTErrorUtil restErrorUtil; + + @Autowired + RangerSearchUtil searchUtil; + + @Autowired + SolrAccessAuditsService solrAccessAuditsService; + + @GET + @Path("/metrics/servicetype/{servicetype}/servicename/{servicename}") + @Produces("application/json") + @PreAuthorize("@rangerPreAuthSecurityHandler.isAPIAccessible(\"" + RangerAPIList.GET_LATEST_AUDIT_METRICS + "\")") + public RangerAuditMetrics getLatestAuditMetrics(@PathParam("servicetype") String serviceType, @PathParam("servicename") String serviceName, + @QueryParam("timezone") String timezone) { + LOG.debug("==> AuditMetricsREST.getLatestAuditMetrics(serviceType={} serviceName={})", serviceType, serviceName); + RangerAuditMetrics ret; + try { + ret = solrAccessAuditsService.getLatestAuditMetrics(serviceType, serviceName, timezone); + } catch (WebApplicationException excp) { + throw excp; + } catch (Throwable excp) { + LOG.error("getLatestAuditMetrics for serviceType={}, serviceName={} failed", serviceType, serviceName, excp); + + throw restErrorUtil.createRESTException(excp.getMessage()); + } + LOG.debug("<== AuditMetricsREST.getLatestAuditMetrics(serviceType={} serviceName={}): {}", serviceType, serviceName, ret); + return ret; + } + + @GET + @Path("/metrics/{id}") + @Produces("application/json") + @PreAuthorize("@rangerPreAuthSecurityHandler.isAPIAccessible(\"" + RangerAPIList.GET_AUDIT_METRICS + "\")") + public RangerAuditMetrics getAuditMetrics(@PathParam("id") Long id, @QueryParam("timezone") String timezone) { + LOG.debug("==> AuditMetricsREST.getAuditMetrics(id={})", id); + RangerAuditMetrics ret; + try { + ret = solrAccessAuditsService.getAuditMetrics(id, timezone); + } catch (WebApplicationException excp) { + throw excp; + } catch (Throwable excp) { + LOG.error("getAuditMetrics({}) failed", id, excp); + + throw restErrorUtil.createRESTException(excp.getMessage()); + } + LOG.debug("<== AuditMetricsREST.getAuditMetrics(id={}):{}", id, ret); + return ret; + } + + @GET + @Path("/metrics") + @Produces("application/json") + @PreAuthorize("@rangerPreAuthSecurityHandler.isAPIAccessible(\"" + RangerAPIList.GET_ALL_LATEST_AUDIT_METRICS + "\")") + public RangerAuditMetricsList getAllLatestRangerAuditMetrics(@Context HttpServletRequest request, @QueryParam("timezone") String timezone) { + LOG.debug("==> AuditMetricsREST.getAllLatestAuditMetrics()"); + RangerAuditMetricsList ret = new RangerAuditMetricsList(); + List rangerAuditMetrics; + + SearchFilter filter = searchUtil.getSearchFilter(request, Collections.emptyList()); + + try { + rangerAuditMetrics = solrAccessAuditsService.getLatestAuditMetricsList(filter, timezone); + } catch (WebApplicationException excp) { + throw excp; + } catch (Throwable excp) { + LOG.error("getAllLatestRangerAuditMetrics failed", excp); + throw restErrorUtil.createRESTException(excp.getMessage()); + } + + if (CollectionUtils.isNotEmpty(rangerAuditMetrics)) { + ret = new RangerAuditMetricsList(rangerAuditMetrics); + } + + LOG.debug("<== AuditMetricsREST.getAllLatestRangerAuditMetrics()"); + + return ret; + } + + @GET + @Path("/dailymetrics") + @Produces("application/json") + @PreAuthorize("@rangerPreAuthSecurityHandler.isAPIAccessible(\"" + RangerAPIList.GET_DAILY_AUDIT_METRICS + "\")") + public RangerAuditMetricsListByHours getDailyAuditMetrics(@Context HttpServletRequest request, @QueryParam("timezone") String timezone) { + LOG.debug("==> AuditMetricsREST.getDailyAuditMetrics()"); + RangerAuditMetricsListByHours ret = new RangerAuditMetricsListByHours(); + List rangerAuditMetricsByHours; + + SearchFilter filter = searchUtil.getSearchFilter(request, Collections.emptyList()); + try { + rangerAuditMetricsByHours = solrAccessAuditsService.getAuditMetricsByHours(filter, timezone); + } catch (WebApplicationException excp) { + throw excp; + } catch (Throwable excp) { + LOG.error("getDailyAuditMetrics failed", excp); + throw restErrorUtil.createRESTException(excp.getMessage()); + } + + if (CollectionUtils.isNotEmpty(rangerAuditMetricsByHours)) { + ret = new RangerAuditMetricsListByHours(rangerAuditMetricsByHours); + } + + LOG.debug("<== AuditMetricsREST.getDailyAuditMetrics()"); + + return ret; + } + + @GET + @Path("/daysmetrics") + @Produces("application/json") + @PreAuthorize("@rangerPreAuthSecurityHandler.isAPIAccessible(\"" + RangerAPIList.GET_DAYS_AUDIT_METRICS + "\")") + public RangerAuditMetricsListByDays getDaysAuditMetrics(@Context HttpServletRequest request, @DefaultValue("7") @QueryParam("olderThanInDays") Integer olderThanInDays, + @QueryParam("timezone") String timezone) { + LOG.debug("==> AuditMetricsREST.getDaysAuditMetrics()"); + + Integer maxAllowedDays = config.getInt(PROP_AUDIT_METRICS_MAX_SUPPORTED_DAYS, PROP_AUDIT_METRICS_MAX_SUPPORTED_DAYS_DEFAULT); + + if (olderThanInDays <= 0 || olderThanInDays > maxAllowedDays) { + throw restErrorUtil.createRESTException("Invalid parameter: olderThanInDays must be between 1 and " + maxAllowedDays, MessageEnums.INVALID_INPUT_DATA); + } + + RangerAuditMetricsListByDays ret = new RangerAuditMetricsListByDays(); + List rangerAuditMetricsByDays; + SearchFilter filter = searchUtil.getSearchFilter(request, Collections.emptyList()); + try { + rangerAuditMetricsByDays = solrAccessAuditsService.getAuditMetricsByDays(olderThanInDays, filter, timezone); + } catch (WebApplicationException excp) { + throw excp; + } catch (Throwable excp) { + LOG.error("getDaysAuditMetrics failed", excp); + throw restErrorUtil.createRESTException(excp.getMessage()); + } + + if (CollectionUtils.isNotEmpty(rangerAuditMetricsByDays)) { + ret = new RangerAuditMetricsListByDays(rangerAuditMetricsByDays); + } + + LOG.debug("<== AuditMetricsREST.getDaysAuditMetrics()"); + + return ret; + } +} diff --git a/security-admin/src/main/java/org/apache/ranger/security/context/RangerAPIList.java b/security-admin/src/main/java/org/apache/ranger/security/context/RangerAPIList.java index 27d076a4805..ae210322a95 100755 --- a/security-admin/src/main/java/org/apache/ranger/security/context/RangerAPIList.java +++ b/security-admin/src/main/java/org/apache/ranger/security/context/RangerAPIList.java @@ -258,6 +258,15 @@ public class RangerAPIList { public static final String UPDATE_DATASET_GRANTS = "GdsREST.updateDataSetGrants"; public static final String EVALUATE_SHARED_RESOURCES = "GdsREST.evaluateSharedResources"; + /** + * List of APIs for AuditMetricsREST + */ + public static final String GET_LATEST_AUDIT_METRICS = "AuditMetricsREST.getLatestAuditMetrics"; + public static final String GET_AUDIT_METRICS = "AuditMetricsREST.getAuditMetrics"; + public static final String GET_ALL_LATEST_AUDIT_METRICS = "AuditMetricsREST.getAllLatestRangerAuditMetrics"; + public static final String GET_DAILY_AUDIT_METRICS = "AuditMetricsREST.getDailyAuditMetrics"; + public static final String GET_DAYS_AUDIT_METRICS = "AuditMetricsREST.getDaysAuditMetrics"; + /** * List of APIs for PublicAPIsv2 */ diff --git a/security-admin/src/main/java/org/apache/ranger/security/context/RangerAPIMapping.java b/security-admin/src/main/java/org/apache/ranger/security/context/RangerAPIMapping.java index 997e5fd35b8..84dd26c5517 100644 --- a/security-admin/src/main/java/org/apache/ranger/security/context/RangerAPIMapping.java +++ b/security-admin/src/main/java/org/apache/ranger/security/context/RangerAPIMapping.java @@ -427,6 +427,12 @@ private void mapAuditWithAPIs() { apiAssociatedWithAudit.add(RangerAPIList.GET_AUTH_SESSION); apiAssociatedWithAudit.add(RangerAPIList.GET_AUTH_SESSIONS); + apiAssociatedWithAudit.add(RangerAPIList.GET_LATEST_AUDIT_METRICS); + apiAssociatedWithAudit.add(RangerAPIList.GET_AUDIT_METRICS); + apiAssociatedWithAudit.add(RangerAPIList.GET_ALL_LATEST_AUDIT_METRICS); + apiAssociatedWithAudit.add(RangerAPIList.GET_DAILY_AUDIT_METRICS); + apiAssociatedWithAudit.add(RangerAPIList.GET_DAYS_AUDIT_METRICS); + rangerAPIMappingWithUI.put(TAB_AUDIT, apiAssociatedWithAudit); for (String api : apiAssociatedWithAudit) { diff --git a/security-admin/src/main/java/org/apache/ranger/solr/SolrAccessAuditsService.java b/security-admin/src/main/java/org/apache/ranger/solr/SolrAccessAuditsService.java index 3ad9b4a89ee..95c8c85aac8 100644 --- a/security-admin/src/main/java/org/apache/ranger/solr/SolrAccessAuditsService.java +++ b/security-admin/src/main/java/org/apache/ranger/solr/SolrAccessAuditsService.java @@ -19,6 +19,7 @@ package org.apache.ranger.solr; +import org.apache.commons.lang3.StringUtils; import org.apache.ranger.AccessAuditsService; import org.apache.ranger.audit.provider.MiscUtil; import org.apache.ranger.common.MessageEnums; @@ -26,14 +27,21 @@ import org.apache.ranger.common.SearchCriteria; import org.apache.ranger.entity.XXService; import org.apache.ranger.entity.XXServiceDef; +import org.apache.ranger.plugin.model.RangerAuditMetrics; +import org.apache.ranger.plugin.model.RangerAuditMetricsByDays; +import org.apache.ranger.plugin.model.RangerAuditMetricsByHours; import org.apache.ranger.plugin.util.JsonUtilsV2; +import org.apache.ranger.plugin.util.SearchFilter; import org.apache.ranger.view.VXAccessAudit; import org.apache.ranger.view.VXAccessAuditList; import org.apache.ranger.view.VXLong; import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.client.solrj.util.ClientUtils; import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrDocumentList; +import org.apache.solr.common.util.NamedList; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -41,14 +49,33 @@ import org.springframework.stereotype.Service; import java.io.UnsupportedEncodingException; +import java.time.DateTimeException; +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.time.format.DateTimeParseException; import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; import java.util.List; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; @Service @Scope("singleton") public class SolrAccessAuditsService extends AccessAuditsService { private static final Logger LOGGER = LoggerFactory.getLogger(SolrAccessAuditsService.class); + private static final long MISSING_REPO_TYPE_SENTINEL = -1L; + + private static final String FACET_REPO = "per_repo"; + private static final String FACET_AGENT = "per_agent"; + private static final String FACET_CLUSTER = "per_cluster"; + private static final String FACET_CLIIP = "per_cliip"; + private static final String BUCKET_VAL = "val"; + private static final String FACETS_KEY = "facets"; + + private final Map repoTypeByServiceType = new ConcurrentHashMap<>(); @Autowired SolrMgr solrMgr; @@ -126,6 +153,520 @@ public VXLong getXAccessAuditSearchCount(SearchCriteria searchCriteria) { return vXLong; } + public RangerAuditMetrics getLatestAuditMetrics(String serviceType, String serviceName) { + return getLatestAuditMetrics(serviceType, serviceName, null); + } + + public RangerAuditMetrics getLatestAuditMetrics(String serviceType, String serviceName, String timezone) { + SearchFilter filter = buildSearchFilter(serviceType, serviceName, null, null, null); + + SolrQuery query = buildMetricsQuery(); + applyAuditMetricsFilters(query, filter); + addLatestMetricsRangeFilter(query); + applyTimezone(query, timezone); + + QueryResponse response = runMetricsQuery(query, "latest audit metrics"); + long count = response.getResults() != null ? response.getResults().getNumFound() : 0L; + return buildAuditMetrics(serviceType, serviceName, null, null, null, count); + } + + public RangerAuditMetrics getAuditMetrics(Long serviceId) { + return getAuditMetrics(serviceId, null); + } + + public RangerAuditMetrics getAuditMetrics(Long serviceId, String timezone) { + if (serviceId == null) { + throw restErrorUtil.createRESTException("AuditMetrics id is required"); + } + + if (daoManager == null || daoManager.getXXService() == null) { + throw restErrorUtil.createRESTException("Service lookup is not available"); + } + + XXService service = daoManager.getXXService().getById(serviceId); + if (service == null) { + throw restErrorUtil.createRESTException("AuditMetrics with Id: " + serviceId + " does not exist"); + } + + String serviceName = service.getName(); + String serviceType = resolveServiceType(service); + + RangerAuditMetrics metric = getLatestAuditMetrics(serviceType, serviceName, timezone); + metric.setId(serviceId); + return metric; + } + + public List getLatestAuditMetricsList(SearchFilter filter) { + return getLatestAuditMetricsList(filter, null); + } + + public List getLatestAuditMetricsList(SearchFilter filter, String timezone) { + SolrQuery query = buildMetricsQuery(); + query.set("json.facet", buildAuditMetricsListFacet()); + + applyAuditMetricsFilters(query, filter); + addLatestMetricsRangeFilter(query); + applyTimezone(query, timezone); + + return extractAuditMetricsList(runMetricsQuery(query, "audit metrics list"), filter); + } + + public List getAuditMetricsByDays(int olderThanInDays, SearchFilter filter) { + return getAuditMetricsByDays(olderThanInDays, filter, null); + } + + public List getAuditMetricsByDays(int olderThanInDays, SearchFilter filter, String timezone) { + SolrQuery query = buildMetricsQuery(); + query.set("json.facet", buildAuditAccessMetricsFacet(olderThanInDays)); + + applyAuditMetricsFilters(query, filter); + applyTimezone(query, timezone); + + return extractAuditMetricsByDays(runMetricsQuery(query, "audit metrics by days"), filter); + } + + public List getAuditMetricsByHours(SearchFilter filter) { + return getAuditMetricsByHours(filter, null); + } + + public List getAuditMetricsByHours(SearchFilter filter, String timezone) { + SolrQuery query = buildMetricsQuery(); + query.set("json.facet", buildAuditMetricsByHourFacet()); + + applyAuditMetricsFilters(query, filter); + addTodayMetricsRangeFilter(query); + applyTimezone(query, timezone); + + return extractAuditMetricsByHours(runMetricsQuery(query, "audit metrics by hours"), filter, timezone); + } + + private SolrQuery buildMetricsQuery() { + SolrQuery query = new SolrQuery(); + query.setQuery("*:*"); + query.setRows(0); + return query; + } + + private QueryResponse runMetricsQuery(SolrQuery query, String context) { + SolrClient solrClient = solrMgr.getSolrClient(); + if (solrClient == null) { + LOGGER.warn("Solr client is null, so not running the query."); + throw restErrorUtil.createRESTException("Error connecting to search engine", MessageEnums.ERROR_SYSTEM); + } + + QueryResponse response; + try { + response = solrUtil.runQuery(solrClient, query); + } catch (Throwable e) { + LOGGER.error("Error running Solr query for {}.", context, e); + throw restErrorUtil.createRESTException("Error running Solr query, please check solr configs. " + e.getMessage(), MessageEnums.ERROR_SYSTEM); + } + + if (response == null || response.getStatus() != 0) { + LOGGER.error("Error running Solr query for {}. Query = {}, response = {}", context, query, response); + throw restErrorUtil.createRESTException("Unable to connect to Audit store !!", MessageEnums.ERROR_SYSTEM); + } + + return response; + } + + private String buildAuditAccessMetricsFacet(int olderThanInDays) { + // If olderThanInDays is 7, we go back 6 days from today to include today + int daysBack = olderThanInDays - 1; + + return String.format("{per_day:{type:range,field:evtTime,start:\"NOW-%dDAYS/DAY\",end:\"NOW\",gap:\"+1DAY\",mincount:1}}", daysBack); + } + + private String buildAuditMetricsListFacet() { + return "{per_repo:{type:terms,field:repo,limit:-1,sort:\"count desc\"," + + "facet:{per_agent:{type:terms,field:agent,limit:-1,missing:true," + + "facet:{per_cliip:{type:terms,field:cliIP,limit:-1,missing:true," + + "facet:{per_cluster:{type:terms,field:cluster,limit:-1,missing:true" + + "}}}}}}}}"; + } + + private String buildAuditMetricsByHourFacet() { + return "{per_hour:{type:range,field:evtTime,start:\"NOW/DAY\",end:\"NOW\",gap:\"+1HOUR\"}}"; + } + + private void addLatestMetricsRangeFilter(SolrQuery query) { + if (query == null) { + return; + } + + query.addFilterQuery("evtTime:[NOW-1DAY TO NOW]"); + } + + private void addTodayMetricsRangeFilter(SolrQuery query) { + if (query == null) { + return; + } + + query.addFilterQuery("evtTime:[NOW/DAY TO NOW]"); + } + + private SearchFilter buildSearchFilter(String serviceType, String serviceName, String appId, String clusterName, String clientIP) { + SearchFilter filter = new SearchFilter(); + filter.setParam(SearchFilter.SERVICE_TYPE, serviceType); + filter.setParam(SearchFilter.SERVICE_NAME, serviceName); + filter.setParam(SearchFilter.APP_ID, appId); + filter.setParam(SearchFilter.CLUSTER_NAME, clusterName); + filter.setParam(SearchFilter.CLIENT_IP, clientIP); + return filter; + } + + private void applyAuditMetricsFilters(SolrQuery query, SearchFilter filter) { + if (query == null || filter == null) { + return; + } + + String serviceName = filter.getParam(SearchFilter.SERVICE_NAME); + addFilterQuery(query, "repo", serviceName); + + String serviceType = filter.getParam(SearchFilter.SERVICE_TYPE); + if (StringUtils.isNotBlank(serviceType)) { + long repoType = resolveRepoType(serviceType); + if (repoType == MISSING_REPO_TYPE_SENTINEL) { + query.addFilterQuery("repoType:-1"); + } else { + query.addFilterQuery("repoType:" + repoType); + } + } + + addFilterQuery(query, "cluster", filter.getParam(SearchFilter.CLUSTER_NAME)); + addFilterQuery(query, "cliIP", filter.getParam(SearchFilter.CLIENT_IP)); + addFilterQuery(query, "agent", filter.getParam(SearchFilter.APP_ID)); + } + + private void addFilterQuery(SolrQuery query, String field, String value) { + if (query == null || StringUtils.isBlank(value)) { + return; + } + + String escapedValue = ClientUtils.escapeQueryChars(value.trim().toLowerCase()); + query.addFilterQuery(field + ":" + escapedValue); + } + + private void applyTimezone(SolrQuery query, String timezone) { + if (query == null || StringUtils.isBlank(timezone)) { + return; + } + + query.set("TZ", timezone.trim()); + } + + private long resolveRepoType(String serviceType) { + if (StringUtils.isBlank(serviceType)) { + return MISSING_REPO_TYPE_SENTINEL; + } + + String cacheKey = serviceType.trim().toLowerCase(); + Long cached = repoTypeByServiceType.get(cacheKey); + if (cached != null) { + return cached; + } + + long resolved = MISSING_REPO_TYPE_SENTINEL; + if (daoManager != null && daoManager.getXXServiceDef() != null) { + XXServiceDef serviceDef = daoManager.getXXServiceDef().findByName(serviceType); + if (serviceDef != null && serviceDef.getId() != null) { + resolved = serviceDef.getId(); + } + } + + repoTypeByServiceType.put(cacheKey, resolved); + return resolved; + } + + private List extractAuditMetricsList(QueryResponse response, SearchFilter filter) { + NamedList responseList = response.getResponse(); + if (responseList == null || !(responseList.get(FACETS_KEY) instanceof NamedList)) { + return Collections.emptyList(); + } + + NamedList facets = (NamedList) responseList.get(FACETS_KEY); + List repoBuckets = extractBuckets(facets.get(FACET_REPO)); + + if (repoBuckets == null || repoBuckets.isEmpty()) { + return Collections.emptyList(); + } + + String serviceTypeFilter = filter != null ? filter.getParam(SearchFilter.SERVICE_TYPE) : null; + List metrics = new ArrayList<>(); + + processRepoBuckets(repoBuckets, metrics, serviceTypeFilter); + + return metrics; + } + + private void processRepoBuckets(List repoBuckets, List metrics, String serviceTypeFilter) { + for (Object repoBucket : repoBuckets) { + Object repoVal = getBucketValue(repoBucket, BUCKET_VAL); + if (repoVal == null) { + continue; + } + + String serviceName = repoVal.toString(); + List agentBuckets = extractBuckets(getBucketValue(repoBucket, FACET_AGENT)); + + if (agentBuckets == null || agentBuckets.isEmpty()) { + metrics.add(buildAuditMetrics(serviceTypeFilter, serviceName, null, null, null, bucketCount(repoBucket))); + } else { + processAgentBuckets(agentBuckets, metrics, serviceTypeFilter, serviceName); + } + } + } + + private void processAgentBuckets(List agentBuckets, List metrics, String serviceTypeFilter, String serviceName) { + for (Object agentBucket : agentBuckets) { + String appId = bucketValToString(getBucketValue(agentBucket, BUCKET_VAL)); + if (appId == null) { + continue; + } + + List cliIpBuckets = extractBuckets(getBucketValue(agentBucket, FACET_CLIIP)); + + if (cliIpBuckets == null || cliIpBuckets.isEmpty()) { + metrics.add(buildAuditMetrics(serviceTypeFilter, serviceName, appId, null, null, bucketCount(agentBucket))); + } else { + processCliIpBuckets(cliIpBuckets, metrics, serviceTypeFilter, serviceName, appId); + } + } + } + + private void processCliIpBuckets(List cliIpBuckets, List metrics, String serviceTypeFilter, String serviceName, String appId) { + for (Object cliIpBucket : cliIpBuckets) { + String clientIP = bucketValToString(getBucketValue(cliIpBucket, BUCKET_VAL)); + List clusterBuckets = extractBuckets(getBucketValue(cliIpBucket, FACET_CLUSTER)); + + if (clusterBuckets == null || clusterBuckets.isEmpty()) { + metrics.add(buildAuditMetrics(serviceTypeFilter, serviceName, appId, null, clientIP, bucketCount(cliIpBucket))); + } else { + processClusterBuckets(clusterBuckets, metrics, serviceTypeFilter, serviceName, appId, clientIP); + } + } + } + + private void processClusterBuckets(List clusterBuckets, List metrics, String serviceTypeFilter, String serviceName, String appId, String clientIP) { + for (Object clusterBucket : clusterBuckets) { + String clusterName = bucketValToString(getBucketValue(clusterBucket, BUCKET_VAL)); + metrics.add(buildAuditMetrics(serviceTypeFilter, serviceName, appId, clusterName, clientIP, bucketCount(clusterBucket))); + } + } + + private long bucketCount(Object bucketObj) { + Object countObj = getBucketValue(bucketObj, "count"); + return countObj instanceof Number ? ((Number) countObj).longValue() : 0L; + } + + private String bucketValToString(Object value) { + if (value == null) { + return null; + } + + String ret = value.toString(); + return StringUtils.isBlank(ret) ? null : ret; + } + + private RangerAuditMetrics buildAuditMetrics(String serviceType, String serviceName, String appId, String clusterName, String clientIP, long count) { + RangerAuditMetrics metric = new RangerAuditMetrics(); + metric.setServiceName(serviceName); + metric.setServiceType(resolveServiceType(serviceType, serviceName)); + metric.setAppId(appId); + metric.setClusterName(clusterName); + metric.setClientIP(clientIP); + metric.setNumberOfAudits(count); + + Long serviceId = resolveServiceId(serviceName); + if (serviceId != null) { + metric.setId(serviceId); + } + + return metric; + } + + private Long resolveServiceId(String serviceName) { + if (StringUtils.isBlank(serviceName) || daoManager == null || daoManager.getXXService() == null) { + return null; + } + + XXService service = daoManager.getXXService().findByName(serviceName); + return service != null ? service.getId() : null; + } + + private String resolveServiceType(XXService service) { + if (service == null || daoManager == null || daoManager.getXXServiceDef() == null) { + return null; + } + + XXServiceDef serviceDef = daoManager.getXXServiceDef().getById(service.getType()); + return serviceDef != null ? serviceDef.getName() : null; + } + + private String resolveServiceType(String serviceType, String serviceName) { + if (StringUtils.isNotBlank(serviceType)) { + return serviceType; + } + + if (StringUtils.isBlank(serviceName) || daoManager == null || daoManager.getXXService() == null) { + return null; + } + + XXService service = daoManager.getXXService().findByName(serviceName); + return resolveServiceType(service); + } + + private List extractAuditMetricsByDays(QueryResponse response, SearchFilter filter) { + NamedList responseList = response.getResponse(); + if (responseList == null) { + return Collections.emptyList(); + } + + Object facetsObj = responseList.get(FACETS_KEY); + if (!(facetsObj instanceof NamedList)) { + return Collections.emptyList(); + } + + NamedList facets = (NamedList) facetsObj; + Object perDayObj = facets.get("per_day"); + List buckets = extractBuckets(perDayObj); + + if (buckets == null || buckets.isEmpty()) { + return Collections.emptyList(); + } + + List metrics = new ArrayList<>(); + String serviceType = filter != null ? filter.getParam(SearchFilter.SERVICE_TYPE) : null; + String serviceName = filter != null ? filter.getParam(SearchFilter.SERVICE_NAME) : null; + String appId = filter != null ? filter.getParam(SearchFilter.APP_ID) : null; + String clusterName = filter != null ? filter.getParam(SearchFilter.CLUSTER_NAME) : null; + String clientIP = filter != null ? filter.getParam(SearchFilter.CLIENT_IP) : null; + + for (Object bucketObj : buckets) { + Object dateValue = getBucketValue(bucketObj, BUCKET_VAL); + Long auditDate = parseAuditDate(dateValue); + if (auditDate == null) { + continue; + } + + Object countObj = getBucketValue(bucketObj, "count"); + long count = countObj instanceof Number ? ((Number) countObj).longValue() : 0L; + + RangerAuditMetricsByDays metric = new RangerAuditMetricsByDays(serviceType, serviceName, appId, clusterName, clientIP, auditDate, count); + metrics.add(metric); + } + + return metrics; + } + + private List extractAuditMetricsByHours(QueryResponse response, SearchFilter filter, String timezone) { + NamedList responseList = response.getResponse(); + if (responseList == null) { + return Collections.emptyList(); + } + + Object facetsObj = responseList.get(FACETS_KEY); + if (!(facetsObj instanceof NamedList)) { + return Collections.emptyList(); + } + + NamedList facets = (NamedList) facetsObj; + Object perHourObj = facets.get("per_hour"); + List buckets = extractBuckets(perHourObj); + + if (buckets == null || buckets.isEmpty()) { + return Collections.emptyList(); + } + + List metrics = new ArrayList<>(); + String serviceType = filter != null ? filter.getParam(SearchFilter.SERVICE_TYPE) : null; + String serviceName = filter != null ? filter.getParam(SearchFilter.SERVICE_NAME) : null; + String appId = filter != null ? filter.getParam(SearchFilter.APP_ID) : null; + String clusterName = filter != null ? filter.getParam(SearchFilter.CLUSTER_NAME) : null; + String clientIP = filter != null ? filter.getParam(SearchFilter.CLIENT_IP) : null; + + for (Object bucketObj : buckets) { + Object dateValue = getBucketValue(bucketObj, BUCKET_VAL); + Long auditDate = parseAuditDate(dateValue); + if (auditDate == null) { + continue; + } + + Object countObj = getBucketValue(bucketObj, "count"); + long count = countObj instanceof Number ? ((Number) countObj).longValue() : 0L; + + ZoneId zoneId = resolveZoneId(timezone); + ZonedDateTime dateTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(auditDate), zoneId); + int hour = dateTime.getHour(); + + RangerAuditMetricsByHours metric = new RangerAuditMetricsByHours(serviceType, serviceName, appId, clusterName, clientIP, hour, count); + metrics.add(metric); + } + + return metrics; + } + + private ZoneId resolveZoneId(String timezone) { + if (StringUtils.isBlank(timezone)) { + return ZoneOffset.UTC; + } + + try { + return ZoneId.of(timezone.trim()); + } catch (DateTimeException e) { + LOGGER.warn("Invalid timezone '{}', using UTC", timezone, e); + return ZoneOffset.UTC; + } + } + + private List extractBuckets(Object perDayObj) { + if (perDayObj instanceof NamedList) { + Object bucketsObj = ((NamedList) perDayObj).get("buckets"); + if (bucketsObj instanceof List) { + return (List) bucketsObj; + } + } else if (perDayObj instanceof Map) { + Object bucketsObj = ((Map) perDayObj).get("buckets"); + if (bucketsObj instanceof List) { + return (List) bucketsObj; + } + } + + return Collections.emptyList(); + } + + private Object getBucketValue(Object bucketObj, String key) { + if (bucketObj instanceof NamedList) { + return ((NamedList) bucketObj).get(key); + } + + if (bucketObj instanceof Map) { + return ((Map) bucketObj).get(key); + } + + return null; + } + + private Long parseAuditDate(Object value) { + if (value instanceof Date) { + return ((Date) value).getTime(); + } + + if (value instanceof Number) { + return ((Number) value).longValue(); + } + + if (value instanceof String) { + try { + return Instant.parse((String) value).toEpochMilli(); + } catch (DateTimeParseException e) { + LOGGER.warn("Unable to parse audit metric date value: {}", value, e); + } + } + + return null; + } + /** * @param doc * @return diff --git a/security-admin/src/main/java/org/apache/ranger/view/RangerAuditMetricsList.java b/security-admin/src/main/java/org/apache/ranger/view/RangerAuditMetricsList.java new file mode 100644 index 00000000000..b63e61e586d --- /dev/null +++ b/security-admin/src/main/java/org/apache/ranger/view/RangerAuditMetricsList.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.view; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; +import com.fasterxml.jackson.annotation.JsonInclude; +import org.apache.ranger.common.view.VList; +import org.apache.ranger.plugin.model.RangerAuditMetrics; + +import java.util.ArrayList; +import java.util.List; + +@JsonAutoDetect(getterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE, fieldVisibility = Visibility.ANY) +@JsonInclude(JsonInclude.Include.NON_EMPTY) +public class RangerAuditMetricsList extends VList { + private static final long serialVersionUID = 1L; + + List rangerAuditMetricsList = new ArrayList<>(); + + public RangerAuditMetricsList() { + super(); + } + + public RangerAuditMetricsList(List objList) { + super(objList); + this.rangerAuditMetricsList = objList; + } + + public List getRangerAuditMetricsList() { + return rangerAuditMetricsList; + } + + public void setRangerAuditMetricsList(List auditMetrics) { + this.rangerAuditMetricsList = auditMetrics; + } + + @Override + public int getListSize() { + if (rangerAuditMetricsList != null) { + return rangerAuditMetricsList.size(); + } + return 0; + } + + @Override + public List getList() { + return rangerAuditMetricsList; + } +} diff --git a/security-admin/src/main/java/org/apache/ranger/view/RangerAuditMetricsListByDays.java b/security-admin/src/main/java/org/apache/ranger/view/RangerAuditMetricsListByDays.java new file mode 100644 index 00000000000..1bf3b8b71c5 --- /dev/null +++ b/security-admin/src/main/java/org/apache/ranger/view/RangerAuditMetricsListByDays.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.view; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; +import com.fasterxml.jackson.annotation.JsonInclude; +import org.apache.ranger.common.view.VList; +import org.apache.ranger.plugin.model.RangerAuditMetricsByDays; + +import java.util.ArrayList; +import java.util.List; + +@JsonAutoDetect(getterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE, fieldVisibility = Visibility.ANY) +@JsonInclude(JsonInclude.Include.NON_EMPTY) +public class RangerAuditMetricsListByDays extends VList { + private static final long serialVersionUID = 1L; + + List auditMetricsByDays = new ArrayList<>(); + + public RangerAuditMetricsListByDays() { + super(); + } + + public RangerAuditMetricsListByDays(List objList) { + super(objList); + this.auditMetricsByDays = objList; + } + + public List getAuditMetricsListByUnit() { + return auditMetricsByDays; + } + + public void setAuditMetricsListByDays(List auditMetricsByDays) { + this.auditMetricsByDays = auditMetricsByDays; + } + + @Override + public int getListSize() { + if (auditMetricsByDays != null) { + return auditMetricsByDays.size(); + } + return 0; + } + + @Override + public List getList() { + return auditMetricsByDays; + } +} diff --git a/security-admin/src/main/java/org/apache/ranger/view/RangerAuditMetricsListByHours.java b/security-admin/src/main/java/org/apache/ranger/view/RangerAuditMetricsListByHours.java new file mode 100644 index 00000000000..e729ca6ec37 --- /dev/null +++ b/security-admin/src/main/java/org/apache/ranger/view/RangerAuditMetricsListByHours.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.view; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; +import com.fasterxml.jackson.annotation.JsonInclude; +import org.apache.ranger.common.view.VList; +import org.apache.ranger.plugin.model.RangerAuditMetricsByHours; + +import java.util.ArrayList; +import java.util.List; + +@JsonAutoDetect(getterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE, fieldVisibility = Visibility.ANY) +@JsonInclude(JsonInclude.Include.NON_EMPTY) +public class RangerAuditMetricsListByHours extends VList { + private static final long serialVersionUID = 1L; + + List auditMetricsByHours = new ArrayList<>(); + + public RangerAuditMetricsListByHours() { + super(); + } + + public RangerAuditMetricsListByHours(List objList) { + super(objList); + this.auditMetricsByHours = objList; + } + + public List getAuditMetricsListByHours() { + return auditMetricsByHours; + } + + public void setAuditMetricsListByHours(List auditMetricsByHours) { + this.auditMetricsByHours = auditMetricsByHours; + } + + @Override + public int getListSize() { + if (auditMetricsByHours != null) { + return auditMetricsByHours.size(); + } + return 0; + } + + @Override + public List getList() { + return auditMetricsByHours; + } +} diff --git a/security-admin/src/test/java/org/apache/ranger/rest/TestAuditMetricsREST.java b/security-admin/src/test/java/org/apache/ranger/rest/TestAuditMetricsREST.java new file mode 100644 index 00000000000..bf31dfcb324 --- /dev/null +++ b/security-admin/src/test/java/org/apache/ranger/rest/TestAuditMetricsREST.java @@ -0,0 +1,397 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.rest; + +import org.apache.ranger.common.MessageEnums; +import org.apache.ranger.common.RESTErrorUtil; +import org.apache.ranger.common.RangerSearchUtil; +import org.apache.ranger.plugin.model.RangerAuditMetrics; +import org.apache.ranger.plugin.model.RangerAuditMetricsByDays; +import org.apache.ranger.plugin.model.RangerAuditMetricsByHours; +import org.apache.ranger.plugin.util.SearchFilter; +import org.apache.ranger.solr.SolrAccessAuditsService; +import org.apache.ranger.view.RangerAuditMetricsList; +import org.apache.ranger.view.RangerAuditMetricsListByDays; +import org.apache.ranger.view.RangerAuditMetricsListByHours; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestMethodOrder; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.WebApplicationException; + +import java.util.Collections; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.contains; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * @generated by Cursor + * @description : Unit Test cases for AuditMetricsREST + */ +@ExtendWith(MockitoExtension.class) +@TestMethodOrder(MethodOrderer.MethodName.class) +class TestAuditMetricsREST { + @Mock + RESTErrorUtil restErrorUtil; + + @Mock + RangerSearchUtil searchUtil; + + @Mock + SolrAccessAuditsService solrAccessAuditsService; + + @Mock + HttpServletRequest httpServletRequest; + + @InjectMocks + AuditMetricsREST auditMetricsREST; + + private RangerAuditMetrics testAuditMetrics; + private RangerAuditMetricsByHours testAuditMetricsByHours; + private RangerAuditMetricsByDays testAuditMetricsByDays; + private SearchFilter testSearchFilter; + + @BeforeEach + void setUp() { + testAuditMetrics = createTestAuditMetrics(); + testAuditMetricsByHours = createTestAuditMetricsByHours(); + testAuditMetricsByDays = createTestAuditMetricsByDays(); + testSearchFilter = new SearchFilter(); + } + + @Test + void testGetLatestAuditMetrics_Success() { + String serviceType = "hive"; + String serviceName = "test-service"; + String timezone = "Asia/Kolkata"; + when(solrAccessAuditsService.getLatestAuditMetrics(serviceType, serviceName, timezone)).thenReturn(testAuditMetrics); + + RangerAuditMetrics result = auditMetricsREST.getLatestAuditMetrics(serviceType, serviceName, timezone); + + assertNotNull(result); + assertEquals(testAuditMetrics.getId(), result.getId()); + assertEquals(testAuditMetrics.getServiceType(), result.getServiceType()); + assertEquals(testAuditMetrics.getNumberOfAudits(), result.getNumberOfAudits()); + verify(solrAccessAuditsService, times(1)).getLatestAuditMetrics(serviceType, serviceName, timezone); + } + + @Test + void testGetLatestAuditMetrics_WebApplicationException() { + String serviceType = "hive"; + String serviceName = "test-service"; + WebApplicationException webException = new WebApplicationException(); + when(solrAccessAuditsService.getLatestAuditMetrics(serviceType, serviceName, null)).thenThrow(webException); + + WebApplicationException exception = assertThrows(WebApplicationException.class, () -> + auditMetricsREST.getLatestAuditMetrics(serviceType, serviceName, null)); + + assertEquals(webException, exception); + verify(restErrorUtil, never()).createRESTException(anyString()); + } + + @Test + void testGetLatestAuditMetrics_GeneralException() { + String serviceType = "hive"; + String serviceName = "test-service"; + when(solrAccessAuditsService.getLatestAuditMetrics(serviceType, serviceName, null)) + .thenThrow(new RuntimeException("General exception")); + when(restErrorUtil.createRESTException(anyString())).thenReturn(new WebApplicationException()); + + assertThrows(WebApplicationException.class, () -> + auditMetricsREST.getLatestAuditMetrics(serviceType, serviceName, null)); + + verify(restErrorUtil, times(1)).createRESTException("General exception"); + } + + @Test + void testGetAuditMetrics_Success() { + Long id = 1L; + String timezone = "UTC"; + when(solrAccessAuditsService.getAuditMetrics(id, timezone)).thenReturn(testAuditMetrics); + + RangerAuditMetrics result = auditMetricsREST.getAuditMetrics(id, timezone); + + assertNotNull(result); + assertEquals(testAuditMetrics.getId(), result.getId()); + assertEquals(testAuditMetrics.getServiceName(), result.getServiceName()); + verify(solrAccessAuditsService, times(1)).getAuditMetrics(id, timezone); + } + + @Test + void testGetAuditMetrics_WebApplicationException() { + Long id = 1L; + WebApplicationException webException = new WebApplicationException(); + when(solrAccessAuditsService.getAuditMetrics(id, null)).thenThrow(webException); + + WebApplicationException exception = assertThrows(WebApplicationException.class, () -> + auditMetricsREST.getAuditMetrics(id, null)); + + assertEquals(webException, exception); + verify(restErrorUtil, never()).createRESTException(anyString()); + } + + @Test + void testGetAuditMetrics_GeneralException() { + Long id = 1L; + when(solrAccessAuditsService.getAuditMetrics(id, null)).thenThrow(new RuntimeException("General exception")); + when(restErrorUtil.createRESTException(anyString())).thenReturn(new WebApplicationException()); + + assertThrows(WebApplicationException.class, () -> auditMetricsREST.getAuditMetrics(id, null)); + + verify(restErrorUtil, times(1)).createRESTException("General exception"); + } + + @Test + void testGetAllLatestRangerAuditMetrics_Success() { + List metricsList = Collections.singletonList(testAuditMetrics); + String timezone = "Asia/Kolkata"; + when(searchUtil.getSearchFilter(eq(httpServletRequest), eq(Collections.emptyList()))).thenReturn(testSearchFilter); + when(solrAccessAuditsService.getLatestAuditMetricsList(testSearchFilter, timezone)).thenReturn(metricsList); + + RangerAuditMetricsList result = auditMetricsREST.getAllLatestRangerAuditMetrics(httpServletRequest, timezone); + + assertNotNull(result); + assertEquals(1, result.getListSize()); + assertEquals(testAuditMetrics, result.getRangerAuditMetricsList().get(0)); + verify(solrAccessAuditsService, times(1)).getLatestAuditMetricsList(testSearchFilter, timezone); + } + + @Test + void testGetAllLatestRangerAuditMetrics_WebApplicationException() { + when(searchUtil.getSearchFilter(eq(httpServletRequest), eq(Collections.emptyList()))).thenReturn(testSearchFilter); + WebApplicationException webException = new WebApplicationException(); + when(solrAccessAuditsService.getLatestAuditMetricsList(testSearchFilter, null)).thenThrow(webException); + + WebApplicationException exception = assertThrows(WebApplicationException.class, () -> + auditMetricsREST.getAllLatestRangerAuditMetrics(httpServletRequest, null)); + + assertEquals(webException, exception); + verify(restErrorUtil, never()).createRESTException(anyString()); + } + + @Test + void testGetAllLatestRangerAuditMetrics_EmptyList() { + when(searchUtil.getSearchFilter(eq(httpServletRequest), eq(Collections.emptyList()))).thenReturn(testSearchFilter); + when(solrAccessAuditsService.getLatestAuditMetricsList(testSearchFilter, null)).thenReturn(Collections.emptyList()); + + RangerAuditMetricsList result = auditMetricsREST.getAllLatestRangerAuditMetrics(httpServletRequest, null); + + assertNotNull(result); + assertEquals(0, result.getListSize()); + } + + @Test + void testGetAllLatestRangerAuditMetrics_GeneralException() { + when(searchUtil.getSearchFilter(eq(httpServletRequest), eq(Collections.emptyList()))).thenReturn(testSearchFilter); + when(solrAccessAuditsService.getLatestAuditMetricsList(testSearchFilter, null)) + .thenThrow(new RuntimeException("General exception")); + when(restErrorUtil.createRESTException(anyString())).thenReturn(new WebApplicationException()); + + assertThrows(WebApplicationException.class, () -> + auditMetricsREST.getAllLatestRangerAuditMetrics(httpServletRequest, null)); + + verify(restErrorUtil, times(1)).createRESTException("General exception"); + } + + @Test + void testGetDailyAuditMetrics_Success() { + List metricsList = Collections.singletonList(testAuditMetricsByHours); + String timezone = "Asia/Kolkata"; + when(searchUtil.getSearchFilter(eq(httpServletRequest), eq(Collections.emptyList()))).thenReturn(testSearchFilter); + when(solrAccessAuditsService.getAuditMetricsByHours(testSearchFilter, timezone)).thenReturn(metricsList); + + RangerAuditMetricsListByHours result = auditMetricsREST.getDailyAuditMetrics(httpServletRequest, timezone); + + assertNotNull(result); + assertEquals(1, result.getListSize()); + verify(solrAccessAuditsService, times(1)).getAuditMetricsByHours(testSearchFilter, timezone); + } + + @Test + void testGetDailyAuditMetrics_WebApplicationException() { + when(searchUtil.getSearchFilter(eq(httpServletRequest), eq(Collections.emptyList()))).thenReturn(testSearchFilter); + WebApplicationException webException = new WebApplicationException(); + when(solrAccessAuditsService.getAuditMetricsByHours(testSearchFilter, null)).thenThrow(webException); + + WebApplicationException exception = assertThrows(WebApplicationException.class, () -> + auditMetricsREST.getDailyAuditMetrics(httpServletRequest, null)); + + assertEquals(webException, exception); + verify(restErrorUtil, never()).createRESTException(anyString()); + } + + @Test + void testGetDailyAuditMetrics_GeneralException() { + when(searchUtil.getSearchFilter(eq(httpServletRequest), eq(Collections.emptyList()))).thenReturn(testSearchFilter); + when(solrAccessAuditsService.getAuditMetricsByHours(testSearchFilter, null)) + .thenThrow(new RuntimeException("Daily metrics failed")); + when(restErrorUtil.createRESTException(anyString())).thenReturn(new WebApplicationException()); + + assertThrows(WebApplicationException.class, () -> + auditMetricsREST.getDailyAuditMetrics(httpServletRequest, null)); + + verify(restErrorUtil, times(1)).createRESTException("Daily metrics failed"); + } + + @Test + void testGetDailyAuditMetrics_EmptyList() { + when(searchUtil.getSearchFilter(eq(httpServletRequest), eq(Collections.emptyList()))).thenReturn(testSearchFilter); + when(solrAccessAuditsService.getAuditMetricsByHours(testSearchFilter, null)).thenReturn(Collections.emptyList()); + + RangerAuditMetricsListByHours result = auditMetricsREST.getDailyAuditMetrics(httpServletRequest, null); + + assertNotNull(result); + assertEquals(0, result.getListSize()); + } + + @Test + void testGetDaysAuditMetrics_Success() { + Integer olderThanInDays = 7; + String timezone = "Asia/Kolkata"; + List metricsList = Collections.singletonList(testAuditMetricsByDays); + when(searchUtil.getSearchFilter(eq(httpServletRequest), eq(Collections.emptyList()))).thenReturn(testSearchFilter); + when(solrAccessAuditsService.getAuditMetricsByDays(olderThanInDays, testSearchFilter, timezone)).thenReturn(metricsList); + + RangerAuditMetricsListByDays result = auditMetricsREST.getDaysAuditMetrics(httpServletRequest, olderThanInDays, timezone); + + assertNotNull(result); + assertEquals(1, result.getListSize()); + verify(solrAccessAuditsService, times(1)).getAuditMetricsByDays(olderThanInDays, testSearchFilter, timezone); + } + + @Test + void testGetDaysAuditMetrics_WebApplicationException() { + Integer olderThanInDays = 7; + when(searchUtil.getSearchFilter(eq(httpServletRequest), eq(Collections.emptyList()))).thenReturn(testSearchFilter); + WebApplicationException webException = new WebApplicationException(); + when(solrAccessAuditsService.getAuditMetricsByDays(olderThanInDays, testSearchFilter, null)).thenThrow(webException); + + WebApplicationException exception = assertThrows(WebApplicationException.class, () -> + auditMetricsREST.getDaysAuditMetrics(httpServletRequest, olderThanInDays, null)); + + assertEquals(webException, exception); + verify(restErrorUtil, never()).createRESTException(anyString()); + } + + @Test + void testGetDaysAuditMetrics_GeneralException() { + Integer olderThanInDays = 7; + when(searchUtil.getSearchFilter(eq(httpServletRequest), eq(Collections.emptyList()))).thenReturn(testSearchFilter); + when(solrAccessAuditsService.getAuditMetricsByDays(olderThanInDays, testSearchFilter, null)) + .thenThrow(new RuntimeException("Days metrics failed")); + when(restErrorUtil.createRESTException(anyString())).thenReturn(new WebApplicationException()); + + assertThrows(WebApplicationException.class, () -> + auditMetricsREST.getDaysAuditMetrics(httpServletRequest, olderThanInDays, null)); + + verify(restErrorUtil, times(1)).createRESTException("Days metrics failed"); + } + + @Test + void testGetDaysAuditMetrics_EmptyList() { + Integer olderThanInDays = 7; + when(searchUtil.getSearchFilter(eq(httpServletRequest), eq(Collections.emptyList()))).thenReturn(testSearchFilter); + when(solrAccessAuditsService.getAuditMetricsByDays(olderThanInDays, testSearchFilter, null)) + .thenReturn(Collections.emptyList()); + + RangerAuditMetricsListByDays result = auditMetricsREST.getDaysAuditMetrics(httpServletRequest, olderThanInDays, null); + + assertNotNull(result); + assertEquals(0, result.getListSize()); + } + + @Test + void testGetDaysAuditMetrics_invalidOlderThanInDays_zero() { + when(restErrorUtil.createRESTException(anyString(), eq(MessageEnums.INVALID_INPUT_DATA))) + .thenReturn(new WebApplicationException()); + + assertThrows(WebApplicationException.class, () -> + auditMetricsREST.getDaysAuditMetrics(httpServletRequest, 0, null)); + + verify(restErrorUtil, times(1)).createRESTException( + contains("olderThanInDays must be between 1 and"), + eq(MessageEnums.INVALID_INPUT_DATA)); + verify(solrAccessAuditsService, never()).getAuditMetricsByDays(anyInt(), any(), any()); + } + + @Test + void testGetDaysAuditMetrics_invalidOlderThanInDays_exceedsMax() { + when(restErrorUtil.createRESTException(anyString(), eq(MessageEnums.INVALID_INPUT_DATA))) + .thenReturn(new WebApplicationException()); + + assertThrows(WebApplicationException.class, () -> + auditMetricsREST.getDaysAuditMetrics(httpServletRequest, 91, null)); + + verify(restErrorUtil, times(1)).createRESTException( + contains("olderThanInDays must be between 1 and"), + eq(MessageEnums.INVALID_INPUT_DATA)); + verify(solrAccessAuditsService, never()).getAuditMetricsByDays(anyInt(), any(), any()); + } + + private RangerAuditMetrics createTestAuditMetrics() { + RangerAuditMetrics metrics = new RangerAuditMetrics(); + metrics.setId(1L); + metrics.setServiceType("hive"); + metrics.setServiceName("test-service"); + metrics.setClusterName("cluster1"); + metrics.setClientIP("127.0.0.1"); + metrics.setNumberOfAudits(20L); + return metrics; + } + + private RangerAuditMetricsByHours createTestAuditMetricsByHours() { + RangerAuditMetricsByHours metrics = new RangerAuditMetricsByHours(); + metrics.setId(1L); + metrics.setServiceType("hive"); + metrics.setServiceName("test-service"); + metrics.setClusterName("cluster1"); + metrics.setClientIP("127.0.0.1"); + metrics.setNumberOfAudits(20L); + return metrics; + } + + private RangerAuditMetricsByDays createTestAuditMetricsByDays() { + RangerAuditMetricsByDays metrics = new RangerAuditMetricsByDays(); + metrics.setId(1L); + metrics.setServiceType("hive"); + metrics.setServiceName("test-service"); + metrics.setClusterName("cluster1"); + metrics.setClientIP("127.0.0.1"); + metrics.setNumberOfAudits(20L); + return metrics; + } +} diff --git a/security-admin/src/test/java/org/apache/ranger/solr/SolrAccessAuditsServiceTest.java b/security-admin/src/test/java/org/apache/ranger/solr/SolrAccessAuditsServiceTest.java index 80b73ed9eb7..01d165432f6 100644 --- a/security-admin/src/test/java/org/apache/ranger/solr/SolrAccessAuditsServiceTest.java +++ b/security-admin/src/test/java/org/apache/ranger/solr/SolrAccessAuditsServiceTest.java @@ -25,11 +25,17 @@ import org.apache.ranger.db.XXServiceDefDao; import org.apache.ranger.entity.XXService; import org.apache.ranger.entity.XXServiceDef; +import org.apache.ranger.plugin.model.RangerAuditMetrics; +import org.apache.ranger.plugin.model.RangerAuditMetricsByDays; +import org.apache.ranger.plugin.model.RangerAuditMetricsByHours; +import org.apache.ranger.plugin.util.SearchFilter; import org.apache.ranger.view.VXAccessAuditList; import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrDocumentList; +import org.apache.solr.common.util.NamedList; import org.junit.jupiter.api.MethodOrderer; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestMethodOrder; @@ -39,11 +45,18 @@ import javax.ws.rs.WebApplicationException; import java.lang.reflect.Field; +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; import java.util.Map; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; @@ -194,6 +207,290 @@ public void getXAccessAuditSearchCount_returnsConstant() { assertEquals(100L, service.getXAccessAuditSearchCount(new SearchCriteria()).getValue()); } + @Test + public void getLatestAuditMetrics_returnsSolrCount() throws Throwable { + SolrAccessAuditsService service = createMetricsService(); + SolrClient client = mock(SolrClient.class); + when(service.solrMgr.getSolrClient()).thenReturn(client); + + QueryResponse response = mock(QueryResponse.class); + SolrDocumentList results = new SolrDocumentList(); + results.setNumFound(42); + when(response.getResults()).thenReturn(results); + when(response.getStatus()).thenReturn(0); + when(service.solrUtil.runQuery(eq(client), any(SolrQuery.class))).thenReturn(response); + + RangerAuditMetrics metrics = service.getLatestAuditMetrics("hdfs", "dev_hdfs", null); + + assertEquals("dev_hdfs", metrics.getServiceName()); + assertEquals("hdfs", metrics.getServiceType()); + assertEquals(42L, metrics.getNumberOfAudits()); + assertEquals("", metrics.getClientIP()); + } + + @Test + public void getLatestAuditMetrics_whenSolrClientNull_throws() { + SolrAccessAuditsService service = createMetricsService(); + when(service.solrMgr.getSolrClient()).thenReturn(null); + + assertThrows(WebApplicationException.class, () -> service.getLatestAuditMetrics("hdfs", "dev_hdfs", null)); + } + + @Test + public void getAuditMetrics_whenIdNull_throws() { + SolrAccessAuditsService service = createMetricsService(); + + assertThrows(WebApplicationException.class, () -> service.getAuditMetrics(null, null)); + } + + @Test + public void getAuditMetrics_whenServiceNotFound_throws() { + SolrAccessAuditsService service = createMetricsService(); + RangerDaoManager daoManager = mock(RangerDaoManager.class); + XXServiceDao svcDao = mock(XXServiceDao.class); + when(daoManager.getXXService()).thenReturn(svcDao); + when(svcDao.getById(99L)).thenReturn(null); + injectField(service, AccessAuditsService.class, "daoManager", daoManager); + + assertThrows(WebApplicationException.class, () -> service.getAuditMetrics(99L, null)); + } + + @Test + public void getAuditMetrics_success_setsServiceId() throws Throwable { + SolrAccessAuditsService service = createMetricsService(); + SolrClient client = mock(SolrClient.class); + when(service.solrMgr.getSolrClient()).thenReturn(client); + + QueryResponse response = mock(QueryResponse.class); + SolrDocumentList results = new SolrDocumentList(); + results.setNumFound(10); + when(response.getResults()).thenReturn(results); + when(response.getStatus()).thenReturn(0); + when(service.solrUtil.runQuery(eq(client), any(SolrQuery.class))).thenReturn(response); + + RangerDaoManager daoManager = mock(RangerDaoManager.class); + XXServiceDao svcDao = mock(XXServiceDao.class); + XXServiceDefDao sdDao = mock(XXServiceDefDao.class); + XXService xxService = mock(XXService.class); + XXServiceDef sd = mock(XXServiceDef.class); + when(daoManager.getXXService()).thenReturn(svcDao); + when(daoManager.getXXServiceDef()).thenReturn(sdDao); + when(svcDao.getById(5L)).thenReturn(xxService); + when(xxService.getName()).thenReturn("dev_hdfs"); + when(xxService.getType()).thenReturn(1L); + when(sdDao.getById(1L)).thenReturn(sd); + when(sd.getName()).thenReturn("hdfs"); + injectField(service, AccessAuditsService.class, "daoManager", daoManager); + + RangerAuditMetrics metrics = service.getAuditMetrics(5L, null); + + assertEquals(5L, metrics.getId()); + assertEquals("dev_hdfs", metrics.getServiceName()); + assertEquals(10L, metrics.getNumberOfAudits()); + } + + @Test + public void getLatestAuditMetricsList_includesAgentCliIpAndCluster() throws Throwable { + SolrAccessAuditsService service = createMetricsServiceWithDao("dev_hdfs", 1L, "hdfs", 10L); + QueryResponse response = buildFacetQueryResponse("per_repo", buildFullMetricsFacetResponse("dev_hdfs", "hdfs", "172.18.0.16", "ABC cluster", 7L)); + stubSolrQuery(service, response); + + List metrics = service.getLatestAuditMetricsList(new SearchFilter(), null); + + assertEquals(1, metrics.size()); + RangerAuditMetrics metric = metrics.get(0); + assertEquals("dev_hdfs", metric.getServiceName()); + assertEquals("hdfs", metric.getAppId()); + assertEquals("172.18.0.16", metric.getClientIP()); + assertEquals("ABC cluster", metric.getClusterName()); + assertEquals(7L, metric.getNumberOfAudits()); + assertEquals(1L, metric.getId()); + } + + @Test + public void getLatestAuditMetricsList_missingCliIp_usesEmptyClientIP() throws Throwable { + SolrAccessAuditsService service = createMetricsServiceWithDao("dev_hdfs", 1L, "hdfs", 10L); + QueryResponse response = buildFacetQueryResponse("per_repo", buildFullMetricsFacetResponse("dev_hdfs", "hdfs", null, "ABC cluster", 3L)); + stubSolrQuery(service, response); + + List metrics = service.getLatestAuditMetricsList(new SearchFilter(), null); + + assertEquals(1, metrics.size()); + assertEquals("", metrics.get(0).getClientIP()); + assertEquals("ABC cluster", metrics.get(0).getClusterName()); + assertEquals("hdfs", metrics.get(0).getAppId()); + } + + @Test + public void getLatestAuditMetricsList_missingCluster_omitsClusterName() throws Throwable { + SolrAccessAuditsService service = createMetricsServiceWithDao("dev_hdfs", 1L, "hdfs", 10L); + QueryResponse response = buildFacetQueryResponse("per_repo", buildFullMetricsFacetResponse("dev_hdfs", "hdfs", "172.18.0.16", null, 4L)); + stubSolrQuery(service, response); + + List metrics = service.getLatestAuditMetricsList(new SearchFilter(), null); + + assertEquals(1, metrics.size()); + assertEquals("172.18.0.16", metrics.get(0).getClientIP()); + assertNull(metrics.get(0).getClusterName()); + } + + @Test + public void getLatestAuditMetricsList_emptyFacets_returnsEmptyList() throws Throwable { + SolrAccessAuditsService service = createMetricsService(); + QueryResponse response = buildFacetQueryResponse("per_repo", facetBuckets(Collections.emptyList())); + stubSolrQuery(service, response); + + List metrics = service.getLatestAuditMetricsList(new SearchFilter(), null); + + assertTrue(metrics.isEmpty()); + } + + @Test + public void getAuditMetricsByHours_usesTimezoneForHour() throws Throwable { + SolrAccessAuditsService service = createMetricsService(); + long epochMillis = Instant.parse("2026-08-11T12:27:43.634Z").toEpochMilli(); + QueryResponse response = buildFacetQueryResponse("per_hour", buildHourlyFacetResponse(epochMillis, 15L)); + stubSolrQuery(service, response); + + List metricsUtc = service.getAuditMetricsByHours(new SearchFilter(), null); + List metricsIst = service.getAuditMetricsByHours(new SearchFilter(), "Asia/Kolkata"); + + assertEquals(1, metricsUtc.size()); + assertEquals(1, metricsIst.size()); + assertEquals(12, metricsUtc.get(0).getHours()); + assertEquals(17, metricsIst.get(0).getHours()); + assertEquals(15L, metricsUtc.get(0).getNumberOfAudits()); + } + + @Test + public void getAuditMetricsByHours_invalidTimezone_fallsBackToUtcHour() throws Throwable { + SolrAccessAuditsService service = createMetricsService(); + long epochMillis = Instant.parse("2026-08-11T12:27:43.634Z").toEpochMilli(); + QueryResponse response = buildFacetQueryResponse("per_hour", buildHourlyFacetResponse(epochMillis, 8L)); + stubSolrQuery(service, response); + + List metrics = service.getAuditMetricsByHours(new SearchFilter(), "Invalid/Zone"); + + assertEquals(1, metrics.size()); + assertEquals(12, metrics.get(0).getHours()); + assertEquals(8L, metrics.get(0).getNumberOfAudits()); + } + + @Test + public void getAuditMetricsByDays_returnsDailyBuckets() throws Throwable { + SolrAccessAuditsService service = createMetricsService(); + long auditDate = ZonedDateTime.of(2026, 8, 11, 0, 0, 0, 0, ZoneId.of("UTC")).toInstant().toEpochMilli(); + QueryResponse response = buildFacetQueryResponse("per_day", buildDailyFacetResponse(auditDate, 25L)); + stubSolrQuery(service, response); + + SearchFilter filter = new SearchFilter(); + filter.setParam(SearchFilter.SERVICE_NAME, "dev_hdfs"); + filter.setParam(SearchFilter.SERVICE_TYPE, "hdfs"); + + List metrics = service.getAuditMetricsByDays(7, filter, "UTC"); + + assertEquals(1, metrics.size()); + assertEquals(auditDate, metrics.get(0).getAuditDate()); + assertEquals(25L, metrics.get(0).getNumberOfAudits()); + assertEquals("dev_hdfs", metrics.get(0).getServiceName()); + assertEquals("hdfs", metrics.get(0).getServiceType()); + } + + @Test + public void runMetricsQuery_whenSolrThrows_wrapsException() throws Throwable { + SolrAccessAuditsService service = createMetricsService(); + SolrClient client = mock(SolrClient.class); + when(service.solrMgr.getSolrClient()).thenReturn(client); + when(service.solrUtil.runQuery(eq(client), any(SolrQuery.class))).thenThrow(new RuntimeException("Solr down")); + + assertThrows(WebApplicationException.class, () -> service.getLatestAuditMetrics("hdfs", "dev_hdfs", null)); + } + + private SolrAccessAuditsService createMetricsService() { + SolrAccessAuditsService service = new SolrAccessAuditsService(); + service.solrMgr = mock(SolrMgr.class); + service.solrUtil = mock(SolrUtil.class); + setRestErrorUtil(service); + return service; + } + + private SolrAccessAuditsService createMetricsServiceWithDao(String serviceName, Long serviceId, String serviceType, Long serviceDefId) { + SolrAccessAuditsService service = createMetricsService(); + RangerDaoManager daoManager = mock(RangerDaoManager.class); + XXServiceDao svcDao = mock(XXServiceDao.class); + XXServiceDefDao sdDao = mock(XXServiceDefDao.class); + XXService xxService = mock(XXService.class); + XXServiceDef sd = mock(XXServiceDef.class); + + when(daoManager.getXXService()).thenReturn(svcDao); + when(daoManager.getXXServiceDef()).thenReturn(sdDao); + when(svcDao.findByName(serviceName)).thenReturn(xxService); + when(xxService.getId()).thenReturn(serviceId); + when(xxService.getType()).thenReturn(serviceDefId); + when(sdDao.getById(serviceDefId)).thenReturn(sd); + when(sd.getName()).thenReturn(serviceType); + injectField(service, AccessAuditsService.class, "daoManager", daoManager); + return service; + } + + private void stubSolrQuery(SolrAccessAuditsService service, QueryResponse response) throws Throwable { + SolrClient client = mock(SolrClient.class); + when(service.solrMgr.getSolrClient()).thenReturn(client); + when(response.getStatus()).thenReturn(0); + when(service.solrUtil.runQuery(eq(client), any(SolrQuery.class))).thenReturn(response); + } + + private QueryResponse buildFacetQueryResponse(String facetKey, Map facetValue) { + QueryResponse response = mock(QueryResponse.class); + NamedList responseList = new NamedList<>(); + NamedList facets = new NamedList<>(); + facets.add(facetKey, facetValue); + responseList.add("facets", facets); + when(response.getResponse()).thenReturn(responseList); + return response; + } + + private Map buildFullMetricsFacetResponse(String serviceName, String appId, String clientIP, + String clusterName, long count) { + Map clusterBucket = facetBucket(clusterName, count, null); + Map cliIpBucket = facetBucket(clientIP, count, nestedFacet("per_cluster", clusterBucket)); + Map agentBucket = facetBucket(appId, count, nestedFacet("per_cliip", cliIpBucket)); + Map repoBucket = facetBucket(serviceName, count, nestedFacet("per_agent", agentBucket)); + return facetBuckets(Collections.singletonList(repoBucket)); + } + + private Map buildHourlyFacetResponse(long epochMillis, long count) { + Map hourBucket = facetBucket(epochMillis, count, null); + return facetBuckets(Collections.singletonList(hourBucket)); + } + + private Map buildDailyFacetResponse(long auditDate, long count) { + Map dayBucket = facetBucket(auditDate, count, null); + return facetBuckets(Collections.singletonList(dayBucket)); + } + + private Map facetBucket(Object val, long count, Map nestedFacet) { + Map bucket = new HashMap<>(); + bucket.put("val", val); + bucket.put("count", count); + if (nestedFacet != null) { + bucket.putAll(nestedFacet); + } + return bucket; + } + + private Map nestedFacet(String facetName, Map childBucket) { + Map nested = new HashMap<>(); + nested.put(facetName, facetBuckets(Collections.singletonList(childBucket))); + return nested; + } + + private Map facetBuckets(List> buckets) { + Map facet = new HashMap<>(); + facet.put("buckets", buckets); + return facet; + } + private static void setRestErrorUtil(AccessAuditsService svc) { try { Field f = AccessAuditsService.class.getDeclaredField("restErrorUtil"); From 748a4a4cc4f61bd1418aab4d8a0b285140770046 Mon Sep 17 00:00:00 2001 From: Rakesh Gupta Date: Fri, 14 Aug 2026 16:57:25 +0530 Subject: [PATCH 2/2] RANGER-3905: Ranger metrics on audit throughput for each of the Ranger Service plugins - fix review comment --- .../plugin/model/RangerAuditMetrics.java | 10 +- .../model/RangerAuditMetricsByDays.java | 10 +- .../model/RangerAuditMetricsByHours.java | 10 +- .../ranger/plugin/util/SearchFilter.java | 2 +- .../ranger/common/RangerSearchUtil.java | 2 +- .../ranger/solr/SolrAccessAuditsService.java | 494 +-------------- .../ranger/solr/SolrAuditMetricsHelper.java | 570 ++++++++++++++++++ .../view/RangerAuditMetricsListByDays.java | 2 +- .../solr/SolrAccessAuditsServiceTest.java | 253 +------- .../solr/SolrAuditMetricsHelperTest.java | 312 ++++++++++ 10 files changed, 914 insertions(+), 751 deletions(-) create mode 100644 security-admin/src/main/java/org/apache/ranger/solr/SolrAuditMetricsHelper.java create mode 100644 security-admin/src/test/java/org/apache/ranger/solr/SolrAuditMetricsHelperTest.java diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerAuditMetrics.java b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerAuditMetrics.java index 6445d522335..c30e05b3eda 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerAuditMetrics.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerAuditMetrics.java @@ -22,10 +22,10 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.fasterxml.jackson.annotation.JsonInclude; @JsonAutoDetect(fieldVisibility = Visibility.ANY) -@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) +@JsonInclude(JsonInclude.Include.NON_EMPTY) @JsonIgnoreProperties(ignoreUnknown = true) public class RangerAuditMetrics extends RangerBaseModelObject implements java.io.Serializable { private static final long serialVersionUID = 1L; @@ -106,11 +106,7 @@ public String getClientIP() { } public void setClientIP(String clientIP) { - if (clientIP == null) { - this.clientIP = ""; - } else { - this.clientIP = clientIP; - } + this.clientIP = clientIP; } public Long getNumberOfAudits() { diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerAuditMetricsByDays.java b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerAuditMetricsByDays.java index e6ff110d686..d4e94892314 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerAuditMetricsByDays.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerAuditMetricsByDays.java @@ -22,10 +22,10 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.fasterxml.jackson.annotation.JsonInclude; @JsonAutoDetect(getterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE, fieldVisibility = Visibility.ANY) -@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) +@JsonInclude(JsonInclude.Include.NON_EMPTY) @JsonIgnoreProperties(ignoreUnknown = true) public class RangerAuditMetricsByDays extends RangerBaseModelObject implements java.io.Serializable { private static final long serialVersionUID = 1L; @@ -102,11 +102,7 @@ public String getClientIP() { } public void setClientIP(String clientIP) { - if (clientIP == null) { - this.clientIP = ""; - } else { - this.clientIP = clientIP; - } + this.clientIP = clientIP; } public Long getNumberOfAudits() { diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerAuditMetricsByHours.java b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerAuditMetricsByHours.java index 141036ec2e5..f9a4f443094 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerAuditMetricsByHours.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerAuditMetricsByHours.java @@ -22,10 +22,10 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.fasterxml.jackson.annotation.JsonInclude; @JsonAutoDetect(fieldVisibility = Visibility.ANY) -@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) +@JsonInclude(JsonInclude.Include.NON_EMPTY) @JsonIgnoreProperties(ignoreUnknown = true) public class RangerAuditMetricsByHours extends RangerBaseModelObject implements java.io.Serializable { private static final long serialVersionUID = 1L; @@ -107,11 +107,7 @@ public String getClientIP() { } public void setClientIP(String clientIP) { - if (clientIP == null) { - this.clientIP = ""; - } else { - this.clientIP = clientIP; - } + this.clientIP = clientIP; } public Long getNumberOfAudits() { diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/SearchFilter.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/SearchFilter.java index 084a857176d..7099dc78244 100755 --- a/agents-common/src/main/java/org/apache/ranger/plugin/util/SearchFilter.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/SearchFilter.java @@ -96,7 +96,7 @@ public class SearchFilter { public static final String SERVICE_NAME_PARTIAL = "serviceNamePartial"; public static final String CLIENT_IP = "clientIP"; - public static final String APP_ID = "appId"; + public static final String PLUGIN_APP_ID = "pluginAppId"; public static final String PLUGIN_HOST_NAME = "pluginHostName"; public static final String PLUGIN_APP_TYPE = "pluginAppType"; diff --git a/security-admin/src/main/java/org/apache/ranger/common/RangerSearchUtil.java b/security-admin/src/main/java/org/apache/ranger/common/RangerSearchUtil.java index c562eb8f8ad..12cf34f47a8 100755 --- a/security-admin/src/main/java/org/apache/ranger/common/RangerSearchUtil.java +++ b/security-admin/src/main/java/org/apache/ranger/common/RangerSearchUtil.java @@ -117,7 +117,7 @@ public SearchFilter getSearchFilter(@Nonnull HttpServletRequest request, List e : request.getParameterMap().entrySet()) { String name = e.getKey(); diff --git a/security-admin/src/main/java/org/apache/ranger/solr/SolrAccessAuditsService.java b/security-admin/src/main/java/org/apache/ranger/solr/SolrAccessAuditsService.java index 95c8c85aac8..4e6a8510e7d 100644 --- a/security-admin/src/main/java/org/apache/ranger/solr/SolrAccessAuditsService.java +++ b/security-admin/src/main/java/org/apache/ranger/solr/SolrAccessAuditsService.java @@ -19,7 +19,6 @@ package org.apache.ranger.solr; -import org.apache.commons.lang3.StringUtils; import org.apache.ranger.AccessAuditsService; import org.apache.ranger.audit.provider.MiscUtil; import org.apache.ranger.common.MessageEnums; @@ -36,12 +35,9 @@ import org.apache.ranger.view.VXAccessAuditList; import org.apache.ranger.view.VXLong; import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.response.QueryResponse; -import org.apache.solr.client.solrj.util.ClientUtils; import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrDocumentList; -import org.apache.solr.common.util.NamedList; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -49,33 +45,14 @@ import org.springframework.stereotype.Service; import java.io.UnsupportedEncodingException; -import java.time.DateTimeException; -import java.time.Instant; -import java.time.ZoneId; -import java.time.ZoneOffset; -import java.time.ZonedDateTime; -import java.time.format.DateTimeParseException; import java.util.ArrayList; -import java.util.Collections; -import java.util.Date; import java.util.List; import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; @Service @Scope("singleton") public class SolrAccessAuditsService extends AccessAuditsService { private static final Logger LOGGER = LoggerFactory.getLogger(SolrAccessAuditsService.class); - private static final long MISSING_REPO_TYPE_SENTINEL = -1L; - - private static final String FACET_REPO = "per_repo"; - private static final String FACET_AGENT = "per_agent"; - private static final String FACET_CLUSTER = "per_cluster"; - private static final String FACET_CLIIP = "per_cliip"; - private static final String BUCKET_VAL = "val"; - private static final String FACETS_KEY = "facets"; - - private final Map repoTypeByServiceType = new ConcurrentHashMap<>(); @Autowired SolrMgr solrMgr; @@ -83,6 +60,9 @@ public class SolrAccessAuditsService extends AccessAuditsService { @Autowired SolrUtil solrUtil; + @Autowired + SolrAuditMetricsHelper auditMetricsHelper; + public VXAccessAuditList searchXAccessAudits(SearchCriteria searchCriteria) { // Make call to Solr SolrClient solrClient = solrMgr.getSolrClient(); @@ -158,16 +138,7 @@ public RangerAuditMetrics getLatestAuditMetrics(String serviceType, String servi } public RangerAuditMetrics getLatestAuditMetrics(String serviceType, String serviceName, String timezone) { - SearchFilter filter = buildSearchFilter(serviceType, serviceName, null, null, null); - - SolrQuery query = buildMetricsQuery(); - applyAuditMetricsFilters(query, filter); - addLatestMetricsRangeFilter(query); - applyTimezone(query, timezone); - - QueryResponse response = runMetricsQuery(query, "latest audit metrics"); - long count = response.getResults() != null ? response.getResults().getNumFound() : 0L; - return buildAuditMetrics(serviceType, serviceName, null, null, null, count); + return auditMetricsHelper.getLatestAuditMetrics(serviceType, serviceName, timezone); } public RangerAuditMetrics getAuditMetrics(Long serviceId) { @@ -189,9 +160,9 @@ public RangerAuditMetrics getAuditMetrics(Long serviceId, String timezone) { } String serviceName = service.getName(); - String serviceType = resolveServiceType(service); + String serviceType = auditMetricsHelper.resolveServiceType(service); - RangerAuditMetrics metric = getLatestAuditMetrics(serviceType, serviceName, timezone); + RangerAuditMetrics metric = auditMetricsHelper.getLatestAuditMetrics(serviceType, serviceName, timezone); metric.setId(serviceId); return metric; } @@ -201,14 +172,7 @@ public List getLatestAuditMetricsList(SearchFilter filter) { } public List getLatestAuditMetricsList(SearchFilter filter, String timezone) { - SolrQuery query = buildMetricsQuery(); - query.set("json.facet", buildAuditMetricsListFacet()); - - applyAuditMetricsFilters(query, filter); - addLatestMetricsRangeFilter(query); - applyTimezone(query, timezone); - - return extractAuditMetricsList(runMetricsQuery(query, "audit metrics list"), filter); + return auditMetricsHelper.getLatestAuditMetricsList(filter, timezone); } public List getAuditMetricsByDays(int olderThanInDays, SearchFilter filter) { @@ -216,13 +180,7 @@ public List getAuditMetricsByDays(int olderThanInDays, } public List getAuditMetricsByDays(int olderThanInDays, SearchFilter filter, String timezone) { - SolrQuery query = buildMetricsQuery(); - query.set("json.facet", buildAuditAccessMetricsFacet(olderThanInDays)); - - applyAuditMetricsFilters(query, filter); - applyTimezone(query, timezone); - - return extractAuditMetricsByDays(runMetricsQuery(query, "audit metrics by days"), filter); + return auditMetricsHelper.getAuditMetricsByDays(olderThanInDays, filter, timezone); } public List getAuditMetricsByHours(SearchFilter filter) { @@ -230,441 +188,7 @@ public List getAuditMetricsByHours(SearchFilter filte } public List getAuditMetricsByHours(SearchFilter filter, String timezone) { - SolrQuery query = buildMetricsQuery(); - query.set("json.facet", buildAuditMetricsByHourFacet()); - - applyAuditMetricsFilters(query, filter); - addTodayMetricsRangeFilter(query); - applyTimezone(query, timezone); - - return extractAuditMetricsByHours(runMetricsQuery(query, "audit metrics by hours"), filter, timezone); - } - - private SolrQuery buildMetricsQuery() { - SolrQuery query = new SolrQuery(); - query.setQuery("*:*"); - query.setRows(0); - return query; - } - - private QueryResponse runMetricsQuery(SolrQuery query, String context) { - SolrClient solrClient = solrMgr.getSolrClient(); - if (solrClient == null) { - LOGGER.warn("Solr client is null, so not running the query."); - throw restErrorUtil.createRESTException("Error connecting to search engine", MessageEnums.ERROR_SYSTEM); - } - - QueryResponse response; - try { - response = solrUtil.runQuery(solrClient, query); - } catch (Throwable e) { - LOGGER.error("Error running Solr query for {}.", context, e); - throw restErrorUtil.createRESTException("Error running Solr query, please check solr configs. " + e.getMessage(), MessageEnums.ERROR_SYSTEM); - } - - if (response == null || response.getStatus() != 0) { - LOGGER.error("Error running Solr query for {}. Query = {}, response = {}", context, query, response); - throw restErrorUtil.createRESTException("Unable to connect to Audit store !!", MessageEnums.ERROR_SYSTEM); - } - - return response; - } - - private String buildAuditAccessMetricsFacet(int olderThanInDays) { - // If olderThanInDays is 7, we go back 6 days from today to include today - int daysBack = olderThanInDays - 1; - - return String.format("{per_day:{type:range,field:evtTime,start:\"NOW-%dDAYS/DAY\",end:\"NOW\",gap:\"+1DAY\",mincount:1}}", daysBack); - } - - private String buildAuditMetricsListFacet() { - return "{per_repo:{type:terms,field:repo,limit:-1,sort:\"count desc\"," - + "facet:{per_agent:{type:terms,field:agent,limit:-1,missing:true," - + "facet:{per_cliip:{type:terms,field:cliIP,limit:-1,missing:true," - + "facet:{per_cluster:{type:terms,field:cluster,limit:-1,missing:true" - + "}}}}}}}}"; - } - - private String buildAuditMetricsByHourFacet() { - return "{per_hour:{type:range,field:evtTime,start:\"NOW/DAY\",end:\"NOW\",gap:\"+1HOUR\"}}"; - } - - private void addLatestMetricsRangeFilter(SolrQuery query) { - if (query == null) { - return; - } - - query.addFilterQuery("evtTime:[NOW-1DAY TO NOW]"); - } - - private void addTodayMetricsRangeFilter(SolrQuery query) { - if (query == null) { - return; - } - - query.addFilterQuery("evtTime:[NOW/DAY TO NOW]"); - } - - private SearchFilter buildSearchFilter(String serviceType, String serviceName, String appId, String clusterName, String clientIP) { - SearchFilter filter = new SearchFilter(); - filter.setParam(SearchFilter.SERVICE_TYPE, serviceType); - filter.setParam(SearchFilter.SERVICE_NAME, serviceName); - filter.setParam(SearchFilter.APP_ID, appId); - filter.setParam(SearchFilter.CLUSTER_NAME, clusterName); - filter.setParam(SearchFilter.CLIENT_IP, clientIP); - return filter; - } - - private void applyAuditMetricsFilters(SolrQuery query, SearchFilter filter) { - if (query == null || filter == null) { - return; - } - - String serviceName = filter.getParam(SearchFilter.SERVICE_NAME); - addFilterQuery(query, "repo", serviceName); - - String serviceType = filter.getParam(SearchFilter.SERVICE_TYPE); - if (StringUtils.isNotBlank(serviceType)) { - long repoType = resolveRepoType(serviceType); - if (repoType == MISSING_REPO_TYPE_SENTINEL) { - query.addFilterQuery("repoType:-1"); - } else { - query.addFilterQuery("repoType:" + repoType); - } - } - - addFilterQuery(query, "cluster", filter.getParam(SearchFilter.CLUSTER_NAME)); - addFilterQuery(query, "cliIP", filter.getParam(SearchFilter.CLIENT_IP)); - addFilterQuery(query, "agent", filter.getParam(SearchFilter.APP_ID)); - } - - private void addFilterQuery(SolrQuery query, String field, String value) { - if (query == null || StringUtils.isBlank(value)) { - return; - } - - String escapedValue = ClientUtils.escapeQueryChars(value.trim().toLowerCase()); - query.addFilterQuery(field + ":" + escapedValue); - } - - private void applyTimezone(SolrQuery query, String timezone) { - if (query == null || StringUtils.isBlank(timezone)) { - return; - } - - query.set("TZ", timezone.trim()); - } - - private long resolveRepoType(String serviceType) { - if (StringUtils.isBlank(serviceType)) { - return MISSING_REPO_TYPE_SENTINEL; - } - - String cacheKey = serviceType.trim().toLowerCase(); - Long cached = repoTypeByServiceType.get(cacheKey); - if (cached != null) { - return cached; - } - - long resolved = MISSING_REPO_TYPE_SENTINEL; - if (daoManager != null && daoManager.getXXServiceDef() != null) { - XXServiceDef serviceDef = daoManager.getXXServiceDef().findByName(serviceType); - if (serviceDef != null && serviceDef.getId() != null) { - resolved = serviceDef.getId(); - } - } - - repoTypeByServiceType.put(cacheKey, resolved); - return resolved; - } - - private List extractAuditMetricsList(QueryResponse response, SearchFilter filter) { - NamedList responseList = response.getResponse(); - if (responseList == null || !(responseList.get(FACETS_KEY) instanceof NamedList)) { - return Collections.emptyList(); - } - - NamedList facets = (NamedList) responseList.get(FACETS_KEY); - List repoBuckets = extractBuckets(facets.get(FACET_REPO)); - - if (repoBuckets == null || repoBuckets.isEmpty()) { - return Collections.emptyList(); - } - - String serviceTypeFilter = filter != null ? filter.getParam(SearchFilter.SERVICE_TYPE) : null; - List metrics = new ArrayList<>(); - - processRepoBuckets(repoBuckets, metrics, serviceTypeFilter); - - return metrics; - } - - private void processRepoBuckets(List repoBuckets, List metrics, String serviceTypeFilter) { - for (Object repoBucket : repoBuckets) { - Object repoVal = getBucketValue(repoBucket, BUCKET_VAL); - if (repoVal == null) { - continue; - } - - String serviceName = repoVal.toString(); - List agentBuckets = extractBuckets(getBucketValue(repoBucket, FACET_AGENT)); - - if (agentBuckets == null || agentBuckets.isEmpty()) { - metrics.add(buildAuditMetrics(serviceTypeFilter, serviceName, null, null, null, bucketCount(repoBucket))); - } else { - processAgentBuckets(agentBuckets, metrics, serviceTypeFilter, serviceName); - } - } - } - - private void processAgentBuckets(List agentBuckets, List metrics, String serviceTypeFilter, String serviceName) { - for (Object agentBucket : agentBuckets) { - String appId = bucketValToString(getBucketValue(agentBucket, BUCKET_VAL)); - if (appId == null) { - continue; - } - - List cliIpBuckets = extractBuckets(getBucketValue(agentBucket, FACET_CLIIP)); - - if (cliIpBuckets == null || cliIpBuckets.isEmpty()) { - metrics.add(buildAuditMetrics(serviceTypeFilter, serviceName, appId, null, null, bucketCount(agentBucket))); - } else { - processCliIpBuckets(cliIpBuckets, metrics, serviceTypeFilter, serviceName, appId); - } - } - } - - private void processCliIpBuckets(List cliIpBuckets, List metrics, String serviceTypeFilter, String serviceName, String appId) { - for (Object cliIpBucket : cliIpBuckets) { - String clientIP = bucketValToString(getBucketValue(cliIpBucket, BUCKET_VAL)); - List clusterBuckets = extractBuckets(getBucketValue(cliIpBucket, FACET_CLUSTER)); - - if (clusterBuckets == null || clusterBuckets.isEmpty()) { - metrics.add(buildAuditMetrics(serviceTypeFilter, serviceName, appId, null, clientIP, bucketCount(cliIpBucket))); - } else { - processClusterBuckets(clusterBuckets, metrics, serviceTypeFilter, serviceName, appId, clientIP); - } - } - } - - private void processClusterBuckets(List clusterBuckets, List metrics, String serviceTypeFilter, String serviceName, String appId, String clientIP) { - for (Object clusterBucket : clusterBuckets) { - String clusterName = bucketValToString(getBucketValue(clusterBucket, BUCKET_VAL)); - metrics.add(buildAuditMetrics(serviceTypeFilter, serviceName, appId, clusterName, clientIP, bucketCount(clusterBucket))); - } - } - - private long bucketCount(Object bucketObj) { - Object countObj = getBucketValue(bucketObj, "count"); - return countObj instanceof Number ? ((Number) countObj).longValue() : 0L; - } - - private String bucketValToString(Object value) { - if (value == null) { - return null; - } - - String ret = value.toString(); - return StringUtils.isBlank(ret) ? null : ret; - } - - private RangerAuditMetrics buildAuditMetrics(String serviceType, String serviceName, String appId, String clusterName, String clientIP, long count) { - RangerAuditMetrics metric = new RangerAuditMetrics(); - metric.setServiceName(serviceName); - metric.setServiceType(resolveServiceType(serviceType, serviceName)); - metric.setAppId(appId); - metric.setClusterName(clusterName); - metric.setClientIP(clientIP); - metric.setNumberOfAudits(count); - - Long serviceId = resolveServiceId(serviceName); - if (serviceId != null) { - metric.setId(serviceId); - } - - return metric; - } - - private Long resolveServiceId(String serviceName) { - if (StringUtils.isBlank(serviceName) || daoManager == null || daoManager.getXXService() == null) { - return null; - } - - XXService service = daoManager.getXXService().findByName(serviceName); - return service != null ? service.getId() : null; - } - - private String resolveServiceType(XXService service) { - if (service == null || daoManager == null || daoManager.getXXServiceDef() == null) { - return null; - } - - XXServiceDef serviceDef = daoManager.getXXServiceDef().getById(service.getType()); - return serviceDef != null ? serviceDef.getName() : null; - } - - private String resolveServiceType(String serviceType, String serviceName) { - if (StringUtils.isNotBlank(serviceType)) { - return serviceType; - } - - if (StringUtils.isBlank(serviceName) || daoManager == null || daoManager.getXXService() == null) { - return null; - } - - XXService service = daoManager.getXXService().findByName(serviceName); - return resolveServiceType(service); - } - - private List extractAuditMetricsByDays(QueryResponse response, SearchFilter filter) { - NamedList responseList = response.getResponse(); - if (responseList == null) { - return Collections.emptyList(); - } - - Object facetsObj = responseList.get(FACETS_KEY); - if (!(facetsObj instanceof NamedList)) { - return Collections.emptyList(); - } - - NamedList facets = (NamedList) facetsObj; - Object perDayObj = facets.get("per_day"); - List buckets = extractBuckets(perDayObj); - - if (buckets == null || buckets.isEmpty()) { - return Collections.emptyList(); - } - - List metrics = new ArrayList<>(); - String serviceType = filter != null ? filter.getParam(SearchFilter.SERVICE_TYPE) : null; - String serviceName = filter != null ? filter.getParam(SearchFilter.SERVICE_NAME) : null; - String appId = filter != null ? filter.getParam(SearchFilter.APP_ID) : null; - String clusterName = filter != null ? filter.getParam(SearchFilter.CLUSTER_NAME) : null; - String clientIP = filter != null ? filter.getParam(SearchFilter.CLIENT_IP) : null; - - for (Object bucketObj : buckets) { - Object dateValue = getBucketValue(bucketObj, BUCKET_VAL); - Long auditDate = parseAuditDate(dateValue); - if (auditDate == null) { - continue; - } - - Object countObj = getBucketValue(bucketObj, "count"); - long count = countObj instanceof Number ? ((Number) countObj).longValue() : 0L; - - RangerAuditMetricsByDays metric = new RangerAuditMetricsByDays(serviceType, serviceName, appId, clusterName, clientIP, auditDate, count); - metrics.add(metric); - } - - return metrics; - } - - private List extractAuditMetricsByHours(QueryResponse response, SearchFilter filter, String timezone) { - NamedList responseList = response.getResponse(); - if (responseList == null) { - return Collections.emptyList(); - } - - Object facetsObj = responseList.get(FACETS_KEY); - if (!(facetsObj instanceof NamedList)) { - return Collections.emptyList(); - } - - NamedList facets = (NamedList) facetsObj; - Object perHourObj = facets.get("per_hour"); - List buckets = extractBuckets(perHourObj); - - if (buckets == null || buckets.isEmpty()) { - return Collections.emptyList(); - } - - List metrics = new ArrayList<>(); - String serviceType = filter != null ? filter.getParam(SearchFilter.SERVICE_TYPE) : null; - String serviceName = filter != null ? filter.getParam(SearchFilter.SERVICE_NAME) : null; - String appId = filter != null ? filter.getParam(SearchFilter.APP_ID) : null; - String clusterName = filter != null ? filter.getParam(SearchFilter.CLUSTER_NAME) : null; - String clientIP = filter != null ? filter.getParam(SearchFilter.CLIENT_IP) : null; - - for (Object bucketObj : buckets) { - Object dateValue = getBucketValue(bucketObj, BUCKET_VAL); - Long auditDate = parseAuditDate(dateValue); - if (auditDate == null) { - continue; - } - - Object countObj = getBucketValue(bucketObj, "count"); - long count = countObj instanceof Number ? ((Number) countObj).longValue() : 0L; - - ZoneId zoneId = resolveZoneId(timezone); - ZonedDateTime dateTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(auditDate), zoneId); - int hour = dateTime.getHour(); - - RangerAuditMetricsByHours metric = new RangerAuditMetricsByHours(serviceType, serviceName, appId, clusterName, clientIP, hour, count); - metrics.add(metric); - } - - return metrics; - } - - private ZoneId resolveZoneId(String timezone) { - if (StringUtils.isBlank(timezone)) { - return ZoneOffset.UTC; - } - - try { - return ZoneId.of(timezone.trim()); - } catch (DateTimeException e) { - LOGGER.warn("Invalid timezone '{}', using UTC", timezone, e); - return ZoneOffset.UTC; - } - } - - private List extractBuckets(Object perDayObj) { - if (perDayObj instanceof NamedList) { - Object bucketsObj = ((NamedList) perDayObj).get("buckets"); - if (bucketsObj instanceof List) { - return (List) bucketsObj; - } - } else if (perDayObj instanceof Map) { - Object bucketsObj = ((Map) perDayObj).get("buckets"); - if (bucketsObj instanceof List) { - return (List) bucketsObj; - } - } - - return Collections.emptyList(); - } - - private Object getBucketValue(Object bucketObj, String key) { - if (bucketObj instanceof NamedList) { - return ((NamedList) bucketObj).get(key); - } - - if (bucketObj instanceof Map) { - return ((Map) bucketObj).get(key); - } - - return null; - } - - private Long parseAuditDate(Object value) { - if (value instanceof Date) { - return ((Date) value).getTime(); - } - - if (value instanceof Number) { - return ((Number) value).longValue(); - } - - if (value instanceof String) { - try { - return Instant.parse((String) value).toEpochMilli(); - } catch (DateTimeParseException e) { - LOGGER.warn("Unable to parse audit metric date value: {}", value, e); - } - } - - return null; + return auditMetricsHelper.getAuditMetricsByHours(filter, timezone); } /** diff --git a/security-admin/src/main/java/org/apache/ranger/solr/SolrAuditMetricsHelper.java b/security-admin/src/main/java/org/apache/ranger/solr/SolrAuditMetricsHelper.java new file mode 100644 index 00000000000..f926b6d2665 --- /dev/null +++ b/security-admin/src/main/java/org/apache/ranger/solr/SolrAuditMetricsHelper.java @@ -0,0 +1,570 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.solr; + +import org.apache.commons.lang3.StringUtils; +import org.apache.ranger.common.MessageEnums; +import org.apache.ranger.common.RESTErrorUtil; +import org.apache.ranger.db.RangerDaoManager; +import org.apache.ranger.entity.XXService; +import org.apache.ranger.entity.XXServiceDef; +import org.apache.ranger.plugin.model.RangerAuditMetrics; +import org.apache.ranger.plugin.model.RangerAuditMetricsByDays; +import org.apache.ranger.plugin.model.RangerAuditMetricsByHours; +import org.apache.ranger.plugin.util.SearchFilter; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.client.solrj.util.ClientUtils; +import org.apache.solr.common.util.NamedList; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.time.DateTimeException; +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.time.format.DateTimeParseException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +@Component +public class SolrAuditMetricsHelper { + private static final Logger LOGGER = LoggerFactory.getLogger(SolrAuditMetricsHelper.class); + private static final long MISSING_REPO_TYPE_SENTINEL = -1L; + + private static final String FACET_REPO = "per_repo"; + private static final String FACET_AGENT = "per_agent"; + private static final String FACET_CLUSTER = "per_cluster"; + private static final String FACET_CLIIP = "per_cliip"; + private static final String BUCKET_VAL = "val"; + private static final String FACETS_KEY = "facets"; + + private final Map repoTypeByServiceType = new ConcurrentHashMap<>(); + + @Autowired + SolrMgr solrMgr; + + @Autowired + SolrUtil solrUtil; + + @Autowired + RESTErrorUtil restErrorUtil; + + @Autowired + RangerDaoManager daoManager; + + public RangerAuditMetrics getLatestAuditMetrics(String serviceType, String serviceName, String timezone) { + SearchFilter filter = buildSearchFilter(serviceType, serviceName, null, null, null); + + SolrQuery query = buildMetricsQuery(); + applyAuditMetricsFilters(query, filter); + addLatestMetricsRangeFilter(query); + applyTimezone(query, timezone); + + QueryResponse response = runMetricsQuery(query, "latest audit metrics"); + long count = response.getResults() != null ? response.getResults().getNumFound() : 0L; + return buildAuditMetrics(serviceType, serviceName, null, null, null, count); + } + + public List getLatestAuditMetricsList(SearchFilter filter, String timezone) { + SolrQuery query = buildMetricsQuery(); + query.set("json.facet", buildAuditMetricsListFacet()); + + applyAuditMetricsFilters(query, filter); + addLatestMetricsRangeFilter(query); + applyTimezone(query, timezone); + + return extractAuditMetricsList(runMetricsQuery(query, "audit metrics list"), filter); + } + + public List getAuditMetricsByDays(int olderThanInDays, SearchFilter filter, String timezone) { + SolrQuery query = buildMetricsQuery(); + query.set("json.facet", buildAuditAccessMetricsFacet(olderThanInDays)); + + applyAuditMetricsFilters(query, filter); + applyTimezone(query, timezone); + + return extractAuditMetricsByDays(runMetricsQuery(query, "audit metrics by days"), filter); + } + + public List getAuditMetricsByHours(SearchFilter filter, String timezone) { + SolrQuery query = buildMetricsQuery(); + query.set("json.facet", buildAuditMetricsByHourFacet()); + + applyAuditMetricsFilters(query, filter); + addTodayMetricsRangeFilter(query); + applyTimezone(query, timezone); + + return extractAuditMetricsByHours(runMetricsQuery(query, "audit metrics by hours"), filter, timezone); + } + + public String resolveServiceType(XXService service) { + String ret = null; + + if (service != null && daoManager != null && daoManager.getXXServiceDef() != null) { + XXServiceDef serviceDef = daoManager.getXXServiceDef().getById(service.getType()); + if (serviceDef != null) { + ret = serviceDef.getName(); + } + } + + return ret; + } + + private SolrQuery buildMetricsQuery() { + SolrQuery query = new SolrQuery(); + query.setQuery("*:*"); + query.setRows(0); + return query; + } + + private QueryResponse runMetricsQuery(SolrQuery query, String context) { + SolrClient solrClient = solrMgr.getSolrClient(); + if (solrClient == null) { + LOGGER.warn("Solr client is null, so not running the query."); + throw restErrorUtil.createRESTException("Error connecting to search engine", MessageEnums.ERROR_SYSTEM); + } + + QueryResponse response; + try { + response = solrUtil.runQuery(solrClient, query); + } catch (Throwable e) { + LOGGER.error("Error running Solr query for {}.", context, e); + throw restErrorUtil.createRESTException("Error running Solr query, please check solr configs. " + e.getMessage(), MessageEnums.ERROR_SYSTEM); + } + + if (response == null || response.getStatus() != 0) { + LOGGER.error("Error running Solr query for {}. Query = {}, response = {}", context, query, response); + throw restErrorUtil.createRESTException("Unable to connect to Audit store !!", MessageEnums.ERROR_SYSTEM); + } + + return response; + } + + private String buildAuditAccessMetricsFacet(int olderThanInDays) { + // If olderThanInDays is 7, we go back 6 days from today to include today + int daysBack = olderThanInDays - 1; + + return String.format("{per_day:{type:range,field:evtTime,start:\"NOW-%dDAYS/DAY\",end:\"NOW\",gap:\"+1DAY\",mincount:1}}", daysBack); + } + + private String buildAuditMetricsListFacet() { + return "{per_repo:{type:terms,field:repo,limit:-1,sort:\"count desc\"," + + "facet:{per_agent:{type:terms,field:agent,limit:-1,missing:true," + + "facet:{per_cliip:{type:terms,field:cliIP,limit:-1,missing:true," + + "facet:{per_cluster:{type:terms,field:cluster,limit:-1,missing:true" + + "}}}}}}}}"; + } + + private String buildAuditMetricsByHourFacet() { + return "{per_hour:{type:range,field:evtTime,start:\"NOW/DAY\",end:\"NOW\",gap:\"+1HOUR\"}}"; + } + + private void addLatestMetricsRangeFilter(SolrQuery query) { + if (query == null) { + return; + } + + query.addFilterQuery("evtTime:[NOW-1DAY TO NOW]"); + } + + private void addTodayMetricsRangeFilter(SolrQuery query) { + if (query == null) { + return; + } + + query.addFilterQuery("evtTime:[NOW/DAY TO NOW]"); + } + + private SearchFilter buildSearchFilter(String serviceType, String serviceName, String appId, String clusterName, String clientIP) { + SearchFilter filter = new SearchFilter(); + filter.setParam(SearchFilter.SERVICE_TYPE, serviceType); + filter.setParam(SearchFilter.SERVICE_NAME, serviceName); + filter.setParam(SearchFilter.PLUGIN_APP_ID, appId); + filter.setParam(SearchFilter.CLUSTER_NAME, clusterName); + filter.setParam(SearchFilter.CLIENT_IP, clientIP); + return filter; + } + + private void applyAuditMetricsFilters(SolrQuery query, SearchFilter filter) { + if (query == null || filter == null) { + return; + } + + String serviceName = filter.getParam(SearchFilter.SERVICE_NAME); + addFilterQuery(query, "repo", serviceName); + + String serviceType = filter.getParam(SearchFilter.SERVICE_TYPE); + if (StringUtils.isNotBlank(serviceType)) { + long repoType = resolveRepoType(serviceType); + if (repoType == MISSING_REPO_TYPE_SENTINEL) { + query.addFilterQuery("repoType:-1"); + } else { + query.addFilterQuery("repoType:" + repoType); + } + } + + addFilterQuery(query, "cluster", filter.getParam(SearchFilter.CLUSTER_NAME)); + addFilterQuery(query, "cliIP", filter.getParam(SearchFilter.CLIENT_IP)); + addFilterQuery(query, "agent", filter.getParam(SearchFilter.PLUGIN_APP_ID)); + } + + private void addFilterQuery(SolrQuery query, String field, String value) { + if (query == null || StringUtils.isBlank(value)) { + return; + } + + String escapedValue = ClientUtils.escapeQueryChars(value.trim().toLowerCase()); + query.addFilterQuery(field + ":" + escapedValue); + } + + private void applyTimezone(SolrQuery query, String timezone) { + if (query == null || StringUtils.isBlank(timezone)) { + return; + } + + String tz = timezone.trim(); + try { + ZoneId.of(tz); + query.set("TZ", tz); + } catch (DateTimeException e) { + query.set("TZ", "UTC"); + } + } + + private long resolveRepoType(String serviceType) { + if (StringUtils.isBlank(serviceType)) { + return MISSING_REPO_TYPE_SENTINEL; + } + + String cacheKey = serviceType.trim().toLowerCase(); + Long cached = repoTypeByServiceType.get(cacheKey); + if (cached != null) { + return cached; + } + + long resolved = MISSING_REPO_TYPE_SENTINEL; + if (daoManager != null && daoManager.getXXServiceDef() != null) { + XXServiceDef serviceDef = daoManager.getXXServiceDef().findByName(serviceType); + if (serviceDef != null && serviceDef.getId() != null) { + resolved = serviceDef.getId(); + } + } + + repoTypeByServiceType.put(cacheKey, resolved); + return resolved; + } + + private List extractAuditMetricsList(QueryResponse response, SearchFilter filter) { + NamedList responseList = response.getResponse(); + if (responseList == null || !(responseList.get(FACETS_KEY) instanceof NamedList)) { + return Collections.emptyList(); + } + + NamedList facets = (NamedList) responseList.get(FACETS_KEY); + List repoBuckets = extractBuckets(facets.get(FACET_REPO)); + + if (repoBuckets == null || repoBuckets.isEmpty()) { + return Collections.emptyList(); + } + + String serviceTypeFilter = filter != null ? filter.getParam(SearchFilter.SERVICE_TYPE) : null; + List metrics = new ArrayList<>(); + + processRepoBuckets(repoBuckets, metrics, serviceTypeFilter); + + return metrics; + } + + private void processRepoBuckets(List repoBuckets, List metrics, String serviceTypeFilter) { + for (Object repoBucket : repoBuckets) { + Object repoVal = getBucketValue(repoBucket, BUCKET_VAL); + if (repoVal == null) { + continue; + } + + String serviceName = repoVal.toString(); + List agentBuckets = extractBuckets(getBucketValue(repoBucket, FACET_AGENT)); + + if (agentBuckets == null || agentBuckets.isEmpty()) { + metrics.add(buildAuditMetrics(serviceTypeFilter, serviceName, null, null, null, bucketCount(repoBucket))); + } else { + processAgentBuckets(agentBuckets, metrics, serviceTypeFilter, serviceName); + } + } + } + + private void processAgentBuckets(List agentBuckets, List metrics, String serviceTypeFilter, String serviceName) { + for (Object agentBucket : agentBuckets) { + String appId = bucketValToString(getBucketValue(agentBucket, BUCKET_VAL)); + + List cliIpBuckets = extractBuckets(getBucketValue(agentBucket, FACET_CLIIP)); + + if (cliIpBuckets == null || cliIpBuckets.isEmpty()) { + metrics.add(buildAuditMetrics(serviceTypeFilter, serviceName, appId, null, null, bucketCount(agentBucket))); + } else { + processCliIpBuckets(cliIpBuckets, metrics, serviceTypeFilter, serviceName, appId); + } + } + } + + private void processCliIpBuckets(List cliIpBuckets, List metrics, String serviceTypeFilter, String serviceName, String appId) { + for (Object cliIpBucket : cliIpBuckets) { + String clientIP = bucketValToString(getBucketValue(cliIpBucket, BUCKET_VAL)); + List clusterBuckets = extractBuckets(getBucketValue(cliIpBucket, FACET_CLUSTER)); + + if (clusterBuckets == null || clusterBuckets.isEmpty()) { + metrics.add(buildAuditMetrics(serviceTypeFilter, serviceName, appId, null, clientIP, bucketCount(cliIpBucket))); + } else { + processClusterBuckets(clusterBuckets, metrics, serviceTypeFilter, serviceName, appId, clientIP); + } + } + } + + private void processClusterBuckets(List clusterBuckets, List metrics, String serviceTypeFilter, String serviceName, String appId, String clientIP) { + for (Object clusterBucket : clusterBuckets) { + String clusterName = bucketValToString(getBucketValue(clusterBucket, BUCKET_VAL)); + metrics.add(buildAuditMetrics(serviceTypeFilter, serviceName, appId, clusterName, clientIP, bucketCount(clusterBucket))); + } + } + + private long bucketCount(Object bucketObj) { + Object countObj = getBucketValue(bucketObj, "count"); + return countObj instanceof Number ? ((Number) countObj).longValue() : 0L; + } + + private String bucketValToString(Object value) { + String ret = null; + + if (value != null) { + ret = value.toString(); + if (StringUtils.isBlank(ret)) { + ret = null; + } + } + + return ret; + } + + private RangerAuditMetrics buildAuditMetrics(String serviceType, String serviceName, String appId, String clusterName, String clientIP, long count) { + RangerAuditMetrics metric = new RangerAuditMetrics(); + metric.setServiceName(serviceName); + metric.setServiceType(resolveServiceType(serviceType, serviceName)); + metric.setAppId(appId); + metric.setClusterName(clusterName); + metric.setClientIP(clientIP); + metric.setNumberOfAudits(count); + + Long serviceId = resolveServiceId(serviceName); + if (serviceId != null) { + metric.setId(serviceId); + } + + return metric; + } + + private Long resolveServiceId(String serviceName) { + Long ret = null; + + if (StringUtils.isNotBlank(serviceName) && daoManager != null && daoManager.getXXService() != null) { + XXService service = daoManager.getXXService().findByName(serviceName); + if (service != null) { + ret = service.getId(); + } + } + + return ret; + } + + private String resolveServiceType(String serviceType, String serviceName) { + String ret = serviceType; + + if (StringUtils.isBlank(ret)) { + if (StringUtils.isNotBlank(serviceName) && daoManager != null && daoManager.getXXService() != null) { + XXService service = daoManager.getXXService().findByName(serviceName); + ret = resolveServiceType(service); + } else { + ret = null; + } + } + + return ret; + } + + private List extractAuditMetricsByDays(QueryResponse response, SearchFilter filter) { + NamedList responseList = response.getResponse(); + if (responseList == null) { + return Collections.emptyList(); + } + + Object facetsObj = responseList.get(FACETS_KEY); + if (!(facetsObj instanceof NamedList)) { + return Collections.emptyList(); + } + + NamedList facets = (NamedList) facetsObj; + Object perDayObj = facets.get("per_day"); + List buckets = extractBuckets(perDayObj); + + if (buckets == null || buckets.isEmpty()) { + return Collections.emptyList(); + } + + List metrics = new ArrayList<>(); + String serviceType = filter != null ? filter.getParam(SearchFilter.SERVICE_TYPE) : null; + String serviceName = filter != null ? filter.getParam(SearchFilter.SERVICE_NAME) : null; + String appId = filter != null ? filter.getParam(SearchFilter.PLUGIN_APP_ID) : null; + String clusterName = filter != null ? filter.getParam(SearchFilter.CLUSTER_NAME) : null; + String clientIP = filter != null ? filter.getParam(SearchFilter.CLIENT_IP) : null; + + for (Object bucketObj : buckets) { + Object dateValue = getBucketValue(bucketObj, BUCKET_VAL); + Long auditDate = parseAuditDate(dateValue); + if (auditDate == null) { + continue; + } + + Object countObj = getBucketValue(bucketObj, "count"); + long count = countObj instanceof Number ? ((Number) countObj).longValue() : 0L; + + RangerAuditMetricsByDays metric = new RangerAuditMetricsByDays(serviceType, serviceName, appId, clusterName, clientIP, auditDate, count); + metrics.add(metric); + } + + return metrics; + } + + private List extractAuditMetricsByHours(QueryResponse response, SearchFilter filter, String timezone) { + NamedList responseList = response.getResponse(); + if (responseList == null) { + return Collections.emptyList(); + } + + Object facetsObj = responseList.get(FACETS_KEY); + if (!(facetsObj instanceof NamedList)) { + return Collections.emptyList(); + } + + NamedList facets = (NamedList) facetsObj; + Object perHourObj = facets.get("per_hour"); + List buckets = extractBuckets(perHourObj); + + if (buckets == null || buckets.isEmpty()) { + return Collections.emptyList(); + } + + List metrics = new ArrayList<>(); + String serviceType = filter != null ? filter.getParam(SearchFilter.SERVICE_TYPE) : null; + String serviceName = filter != null ? filter.getParam(SearchFilter.SERVICE_NAME) : null; + String appId = filter != null ? filter.getParam(SearchFilter.PLUGIN_APP_ID) : null; + String clusterName = filter != null ? filter.getParam(SearchFilter.CLUSTER_NAME) : null; + String clientIP = filter != null ? filter.getParam(SearchFilter.CLIENT_IP) : null; + + for (Object bucketObj : buckets) { + Object dateValue = getBucketValue(bucketObj, BUCKET_VAL); + Long auditDate = parseAuditDate(dateValue); + if (auditDate == null) { + continue; + } + + Object countObj = getBucketValue(bucketObj, "count"); + long count = countObj instanceof Number ? ((Number) countObj).longValue() : 0L; + + ZoneId zoneId = resolveZoneId(timezone); + ZonedDateTime dateTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(auditDate), zoneId); + int hour = dateTime.getHour(); + + RangerAuditMetricsByHours metric = new RangerAuditMetricsByHours(serviceType, serviceName, appId, clusterName, clientIP, hour, count); + metrics.add(metric); + } + + return metrics; + } + + private ZoneId resolveZoneId(String timezone) { + ZoneId ret = ZoneOffset.UTC; + + if (StringUtils.isNotBlank(timezone)) { + try { + ret = ZoneId.of(timezone.trim()); + } catch (DateTimeException e) { + LOGGER.warn("Invalid timezone '{}', using UTC", timezone, e); + } + } + + return ret; + } + + private List extractBuckets(Object perDayObj) { + List ret = Collections.emptyList(); + + if (perDayObj instanceof NamedList) { + Object bucketsObj = ((NamedList) perDayObj).get("buckets"); + if (bucketsObj instanceof List) { + ret = (List) bucketsObj; + } + } else if (perDayObj instanceof Map) { + Object bucketsObj = ((Map) perDayObj).get("buckets"); + if (bucketsObj instanceof List) { + ret = (List) bucketsObj; + } + } + + return ret; + } + + private Object getBucketValue(Object bucketObj, String key) { + Object ret = null; + + if (bucketObj instanceof NamedList) { + ret = ((NamedList) bucketObj).get(key); + } else if (bucketObj instanceof Map) { + ret = ((Map) bucketObj).get(key); + } + + return ret; + } + + private Long parseAuditDate(Object value) { + Long ret = null; + + if (value instanceof Date) { + ret = ((Date) value).getTime(); + } else if (value instanceof Number) { + ret = ((Number) value).longValue(); + } else if (value instanceof String) { + try { + ret = Instant.parse((String) value).toEpochMilli(); + } catch (DateTimeParseException e) { + LOGGER.warn("Unable to parse audit metric date value: {}", value, e); + } + } + + return ret; + } +} diff --git a/security-admin/src/main/java/org/apache/ranger/view/RangerAuditMetricsListByDays.java b/security-admin/src/main/java/org/apache/ranger/view/RangerAuditMetricsListByDays.java index 1bf3b8b71c5..6cf92632c0a 100644 --- a/security-admin/src/main/java/org/apache/ranger/view/RangerAuditMetricsListByDays.java +++ b/security-admin/src/main/java/org/apache/ranger/view/RangerAuditMetricsListByDays.java @@ -42,7 +42,7 @@ public RangerAuditMetricsListByDays(List objList) { this.auditMetricsByDays = objList; } - public List getAuditMetricsListByUnit() { + public List getAuditMetricsListByDays() { return auditMetricsByDays; } diff --git a/security-admin/src/test/java/org/apache/ranger/solr/SolrAccessAuditsServiceTest.java b/security-admin/src/test/java/org/apache/ranger/solr/SolrAccessAuditsServiceTest.java index 01d165432f6..a8f340540cc 100644 --- a/security-admin/src/test/java/org/apache/ranger/solr/SolrAccessAuditsServiceTest.java +++ b/security-admin/src/test/java/org/apache/ranger/solr/SolrAccessAuditsServiceTest.java @@ -26,16 +26,12 @@ import org.apache.ranger.entity.XXService; import org.apache.ranger.entity.XXServiceDef; import org.apache.ranger.plugin.model.RangerAuditMetrics; -import org.apache.ranger.plugin.model.RangerAuditMetricsByDays; -import org.apache.ranger.plugin.model.RangerAuditMetricsByHours; -import org.apache.ranger.plugin.util.SearchFilter; import org.apache.ranger.view.VXAccessAuditList; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrDocumentList; -import org.apache.solr.common.util.NamedList; import org.junit.jupiter.api.MethodOrderer; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestMethodOrder; @@ -45,18 +41,11 @@ import javax.ws.rs.WebApplicationException; import java.lang.reflect.Field; -import java.time.Instant; -import java.time.ZoneId; -import java.time.ZonedDateTime; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; import java.util.Map; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; @@ -207,45 +196,16 @@ public void getXAccessAuditSearchCount_returnsConstant() { assertEquals(100L, service.getXAccessAuditSearchCount(new SearchCriteria()).getValue()); } - @Test - public void getLatestAuditMetrics_returnsSolrCount() throws Throwable { - SolrAccessAuditsService service = createMetricsService(); - SolrClient client = mock(SolrClient.class); - when(service.solrMgr.getSolrClient()).thenReturn(client); - - QueryResponse response = mock(QueryResponse.class); - SolrDocumentList results = new SolrDocumentList(); - results.setNumFound(42); - when(response.getResults()).thenReturn(results); - when(response.getStatus()).thenReturn(0); - when(service.solrUtil.runQuery(eq(client), any(SolrQuery.class))).thenReturn(response); - - RangerAuditMetrics metrics = service.getLatestAuditMetrics("hdfs", "dev_hdfs", null); - - assertEquals("dev_hdfs", metrics.getServiceName()); - assertEquals("hdfs", metrics.getServiceType()); - assertEquals(42L, metrics.getNumberOfAudits()); - assertEquals("", metrics.getClientIP()); - } - - @Test - public void getLatestAuditMetrics_whenSolrClientNull_throws() { - SolrAccessAuditsService service = createMetricsService(); - when(service.solrMgr.getSolrClient()).thenReturn(null); - - assertThrows(WebApplicationException.class, () -> service.getLatestAuditMetrics("hdfs", "dev_hdfs", null)); - } - @Test public void getAuditMetrics_whenIdNull_throws() { - SolrAccessAuditsService service = createMetricsService(); + SolrAccessAuditsService service = createServiceWithHelper(); assertThrows(WebApplicationException.class, () -> service.getAuditMetrics(null, null)); } @Test public void getAuditMetrics_whenServiceNotFound_throws() { - SolrAccessAuditsService service = createMetricsService(); + SolrAccessAuditsService service = createServiceWithHelper(); RangerDaoManager daoManager = mock(RangerDaoManager.class); XXServiceDao svcDao = mock(XXServiceDao.class); when(daoManager.getXXService()).thenReturn(svcDao); @@ -257,16 +217,16 @@ public void getAuditMetrics_whenServiceNotFound_throws() { @Test public void getAuditMetrics_success_setsServiceId() throws Throwable { - SolrAccessAuditsService service = createMetricsService(); + SolrAccessAuditsService service = createServiceWithHelper(); SolrClient client = mock(SolrClient.class); - when(service.solrMgr.getSolrClient()).thenReturn(client); + when(service.auditMetricsHelper.solrMgr.getSolrClient()).thenReturn(client); QueryResponse response = mock(QueryResponse.class); SolrDocumentList results = new SolrDocumentList(); results.setNumFound(10); when(response.getResults()).thenReturn(results); when(response.getStatus()).thenReturn(0); - when(service.solrUtil.runQuery(eq(client), any(SolrQuery.class))).thenReturn(response); + when(service.auditMetricsHelper.solrUtil.runQuery(eq(client), any(SolrQuery.class))).thenReturn(response); RangerDaoManager daoManager = mock(RangerDaoManager.class); XXServiceDao svcDao = mock(XXServiceDao.class); @@ -281,6 +241,7 @@ public void getAuditMetrics_success_setsServiceId() throws Throwable { when(sdDao.getById(1L)).thenReturn(sd); when(sd.getName()).thenReturn("hdfs"); injectField(service, AccessAuditsService.class, "daoManager", daoManager); + service.auditMetricsHelper.daoManager = daoManager; RangerAuditMetrics metrics = service.getAuditMetrics(5L, null); @@ -289,208 +250,16 @@ public void getAuditMetrics_success_setsServiceId() throws Throwable { assertEquals(10L, metrics.getNumberOfAudits()); } - @Test - public void getLatestAuditMetricsList_includesAgentCliIpAndCluster() throws Throwable { - SolrAccessAuditsService service = createMetricsServiceWithDao("dev_hdfs", 1L, "hdfs", 10L); - QueryResponse response = buildFacetQueryResponse("per_repo", buildFullMetricsFacetResponse("dev_hdfs", "hdfs", "172.18.0.16", "ABC cluster", 7L)); - stubSolrQuery(service, response); - - List metrics = service.getLatestAuditMetricsList(new SearchFilter(), null); - - assertEquals(1, metrics.size()); - RangerAuditMetrics metric = metrics.get(0); - assertEquals("dev_hdfs", metric.getServiceName()); - assertEquals("hdfs", metric.getAppId()); - assertEquals("172.18.0.16", metric.getClientIP()); - assertEquals("ABC cluster", metric.getClusterName()); - assertEquals(7L, metric.getNumberOfAudits()); - assertEquals(1L, metric.getId()); - } - - @Test - public void getLatestAuditMetricsList_missingCliIp_usesEmptyClientIP() throws Throwable { - SolrAccessAuditsService service = createMetricsServiceWithDao("dev_hdfs", 1L, "hdfs", 10L); - QueryResponse response = buildFacetQueryResponse("per_repo", buildFullMetricsFacetResponse("dev_hdfs", "hdfs", null, "ABC cluster", 3L)); - stubSolrQuery(service, response); - - List metrics = service.getLatestAuditMetricsList(new SearchFilter(), null); - - assertEquals(1, metrics.size()); - assertEquals("", metrics.get(0).getClientIP()); - assertEquals("ABC cluster", metrics.get(0).getClusterName()); - assertEquals("hdfs", metrics.get(0).getAppId()); - } - - @Test - public void getLatestAuditMetricsList_missingCluster_omitsClusterName() throws Throwable { - SolrAccessAuditsService service = createMetricsServiceWithDao("dev_hdfs", 1L, "hdfs", 10L); - QueryResponse response = buildFacetQueryResponse("per_repo", buildFullMetricsFacetResponse("dev_hdfs", "hdfs", "172.18.0.16", null, 4L)); - stubSolrQuery(service, response); - - List metrics = service.getLatestAuditMetricsList(new SearchFilter(), null); - - assertEquals(1, metrics.size()); - assertEquals("172.18.0.16", metrics.get(0).getClientIP()); - assertNull(metrics.get(0).getClusterName()); - } - - @Test - public void getLatestAuditMetricsList_emptyFacets_returnsEmptyList() throws Throwable { - SolrAccessAuditsService service = createMetricsService(); - QueryResponse response = buildFacetQueryResponse("per_repo", facetBuckets(Collections.emptyList())); - stubSolrQuery(service, response); - - List metrics = service.getLatestAuditMetricsList(new SearchFilter(), null); - - assertTrue(metrics.isEmpty()); - } - - @Test - public void getAuditMetricsByHours_usesTimezoneForHour() throws Throwable { - SolrAccessAuditsService service = createMetricsService(); - long epochMillis = Instant.parse("2026-08-11T12:27:43.634Z").toEpochMilli(); - QueryResponse response = buildFacetQueryResponse("per_hour", buildHourlyFacetResponse(epochMillis, 15L)); - stubSolrQuery(service, response); - - List metricsUtc = service.getAuditMetricsByHours(new SearchFilter(), null); - List metricsIst = service.getAuditMetricsByHours(new SearchFilter(), "Asia/Kolkata"); - - assertEquals(1, metricsUtc.size()); - assertEquals(1, metricsIst.size()); - assertEquals(12, metricsUtc.get(0).getHours()); - assertEquals(17, metricsIst.get(0).getHours()); - assertEquals(15L, metricsUtc.get(0).getNumberOfAudits()); - } - - @Test - public void getAuditMetricsByHours_invalidTimezone_fallsBackToUtcHour() throws Throwable { - SolrAccessAuditsService service = createMetricsService(); - long epochMillis = Instant.parse("2026-08-11T12:27:43.634Z").toEpochMilli(); - QueryResponse response = buildFacetQueryResponse("per_hour", buildHourlyFacetResponse(epochMillis, 8L)); - stubSolrQuery(service, response); - - List metrics = service.getAuditMetricsByHours(new SearchFilter(), "Invalid/Zone"); - - assertEquals(1, metrics.size()); - assertEquals(12, metrics.get(0).getHours()); - assertEquals(8L, metrics.get(0).getNumberOfAudits()); - } - - @Test - public void getAuditMetricsByDays_returnsDailyBuckets() throws Throwable { - SolrAccessAuditsService service = createMetricsService(); - long auditDate = ZonedDateTime.of(2026, 8, 11, 0, 0, 0, 0, ZoneId.of("UTC")).toInstant().toEpochMilli(); - QueryResponse response = buildFacetQueryResponse("per_day", buildDailyFacetResponse(auditDate, 25L)); - stubSolrQuery(service, response); - - SearchFilter filter = new SearchFilter(); - filter.setParam(SearchFilter.SERVICE_NAME, "dev_hdfs"); - filter.setParam(SearchFilter.SERVICE_TYPE, "hdfs"); - - List metrics = service.getAuditMetricsByDays(7, filter, "UTC"); - - assertEquals(1, metrics.size()); - assertEquals(auditDate, metrics.get(0).getAuditDate()); - assertEquals(25L, metrics.get(0).getNumberOfAudits()); - assertEquals("dev_hdfs", metrics.get(0).getServiceName()); - assertEquals("hdfs", metrics.get(0).getServiceType()); - } - - @Test - public void runMetricsQuery_whenSolrThrows_wrapsException() throws Throwable { - SolrAccessAuditsService service = createMetricsService(); - SolrClient client = mock(SolrClient.class); - when(service.solrMgr.getSolrClient()).thenReturn(client); - when(service.solrUtil.runQuery(eq(client), any(SolrQuery.class))).thenThrow(new RuntimeException("Solr down")); - - assertThrows(WebApplicationException.class, () -> service.getLatestAuditMetrics("hdfs", "dev_hdfs", null)); - } - - private SolrAccessAuditsService createMetricsService() { + private SolrAccessAuditsService createServiceWithHelper() { SolrAccessAuditsService service = new SolrAccessAuditsService(); - service.solrMgr = mock(SolrMgr.class); - service.solrUtil = mock(SolrUtil.class); + service.auditMetricsHelper = new SolrAuditMetricsHelper(); + service.auditMetricsHelper.solrMgr = mock(SolrMgr.class); + service.auditMetricsHelper.solrUtil = mock(SolrUtil.class); + service.auditMetricsHelper.restErrorUtil = new RESTErrorUtil(); setRestErrorUtil(service); return service; } - private SolrAccessAuditsService createMetricsServiceWithDao(String serviceName, Long serviceId, String serviceType, Long serviceDefId) { - SolrAccessAuditsService service = createMetricsService(); - RangerDaoManager daoManager = mock(RangerDaoManager.class); - XXServiceDao svcDao = mock(XXServiceDao.class); - XXServiceDefDao sdDao = mock(XXServiceDefDao.class); - XXService xxService = mock(XXService.class); - XXServiceDef sd = mock(XXServiceDef.class); - - when(daoManager.getXXService()).thenReturn(svcDao); - when(daoManager.getXXServiceDef()).thenReturn(sdDao); - when(svcDao.findByName(serviceName)).thenReturn(xxService); - when(xxService.getId()).thenReturn(serviceId); - when(xxService.getType()).thenReturn(serviceDefId); - when(sdDao.getById(serviceDefId)).thenReturn(sd); - when(sd.getName()).thenReturn(serviceType); - injectField(service, AccessAuditsService.class, "daoManager", daoManager); - return service; - } - - private void stubSolrQuery(SolrAccessAuditsService service, QueryResponse response) throws Throwable { - SolrClient client = mock(SolrClient.class); - when(service.solrMgr.getSolrClient()).thenReturn(client); - when(response.getStatus()).thenReturn(0); - when(service.solrUtil.runQuery(eq(client), any(SolrQuery.class))).thenReturn(response); - } - - private QueryResponse buildFacetQueryResponse(String facetKey, Map facetValue) { - QueryResponse response = mock(QueryResponse.class); - NamedList responseList = new NamedList<>(); - NamedList facets = new NamedList<>(); - facets.add(facetKey, facetValue); - responseList.add("facets", facets); - when(response.getResponse()).thenReturn(responseList); - return response; - } - - private Map buildFullMetricsFacetResponse(String serviceName, String appId, String clientIP, - String clusterName, long count) { - Map clusterBucket = facetBucket(clusterName, count, null); - Map cliIpBucket = facetBucket(clientIP, count, nestedFacet("per_cluster", clusterBucket)); - Map agentBucket = facetBucket(appId, count, nestedFacet("per_cliip", cliIpBucket)); - Map repoBucket = facetBucket(serviceName, count, nestedFacet("per_agent", agentBucket)); - return facetBuckets(Collections.singletonList(repoBucket)); - } - - private Map buildHourlyFacetResponse(long epochMillis, long count) { - Map hourBucket = facetBucket(epochMillis, count, null); - return facetBuckets(Collections.singletonList(hourBucket)); - } - - private Map buildDailyFacetResponse(long auditDate, long count) { - Map dayBucket = facetBucket(auditDate, count, null); - return facetBuckets(Collections.singletonList(dayBucket)); - } - - private Map facetBucket(Object val, long count, Map nestedFacet) { - Map bucket = new HashMap<>(); - bucket.put("val", val); - bucket.put("count", count); - if (nestedFacet != null) { - bucket.putAll(nestedFacet); - } - return bucket; - } - - private Map nestedFacet(String facetName, Map childBucket) { - Map nested = new HashMap<>(); - nested.put(facetName, facetBuckets(Collections.singletonList(childBucket))); - return nested; - } - - private Map facetBuckets(List> buckets) { - Map facet = new HashMap<>(); - facet.put("buckets", buckets); - return facet; - } - private static void setRestErrorUtil(AccessAuditsService svc) { try { Field f = AccessAuditsService.class.getDeclaredField("restErrorUtil"); diff --git a/security-admin/src/test/java/org/apache/ranger/solr/SolrAuditMetricsHelperTest.java b/security-admin/src/test/java/org/apache/ranger/solr/SolrAuditMetricsHelperTest.java new file mode 100644 index 00000000000..8ab33963340 --- /dev/null +++ b/security-admin/src/test/java/org/apache/ranger/solr/SolrAuditMetricsHelperTest.java @@ -0,0 +1,312 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ranger.solr; + +import org.apache.ranger.common.RESTErrorUtil; +import org.apache.ranger.db.RangerDaoManager; +import org.apache.ranger.db.XXServiceDao; +import org.apache.ranger.db.XXServiceDefDao; +import org.apache.ranger.entity.XXService; +import org.apache.ranger.entity.XXServiceDef; +import org.apache.ranger.plugin.model.RangerAuditMetrics; +import org.apache.ranger.plugin.model.RangerAuditMetricsByDays; +import org.apache.ranger.plugin.model.RangerAuditMetricsByHours; +import org.apache.ranger.plugin.util.SearchFilter; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.common.SolrDocumentList; +import org.apache.solr.common.util.NamedList; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestMethodOrder; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.junit.jupiter.MockitoExtension; + +import javax.ws.rs.WebApplicationException; + +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * @generated by Cursor + * @description : Unit Test cases for SolrAuditMetricsHelper + */ + +@ExtendWith(MockitoExtension.class) +@TestMethodOrder(MethodOrderer.MethodName.class) +public class SolrAuditMetricsHelperTest { + @Test + public void getLatestAuditMetrics_returnsSolrCount() throws Throwable { + SolrAuditMetricsHelper helper = createHelper(null); + SolrClient client = mock(SolrClient.class); + when(helper.solrMgr.getSolrClient()).thenReturn(client); + + QueryResponse response = mock(QueryResponse.class); + SolrDocumentList results = new SolrDocumentList(); + results.setNumFound(42); + when(response.getResults()).thenReturn(results); + when(response.getStatus()).thenReturn(0); + when(helper.solrUtil.runQuery(eq(client), any(SolrQuery.class))).thenReturn(response); + + RangerAuditMetrics metrics = helper.getLatestAuditMetrics("hdfs", "dev_hdfs", null); + + assertEquals("dev_hdfs", metrics.getServiceName()); + assertEquals("hdfs", metrics.getServiceType()); + assertEquals(42L, metrics.getNumberOfAudits()); + assertNull(metrics.getClientIP()); + } + + @Test + public void getLatestAuditMetrics_whenSolrClientNull_throws() { + SolrAuditMetricsHelper helper = createHelper(null); + when(helper.solrMgr.getSolrClient()).thenReturn(null); + + assertThrows(WebApplicationException.class, () -> helper.getLatestAuditMetrics("hdfs", "dev_hdfs", null)); + } + + @Test + public void getLatestAuditMetricsList_includesAgentCliIpAndCluster() throws Throwable { + SolrAuditMetricsHelper helper = createHelperWithDao("dev_hdfs", 1L, "hdfs", 10L); + QueryResponse response = buildFacetQueryResponse("per_repo", buildFullMetricsFacetResponse("dev_hdfs", "hdfs", "172.18.0.16", "ABC cluster", 7L)); + stubSolrQuery(helper, response); + + List metrics = helper.getLatestAuditMetricsList(new SearchFilter(), null); + + assertEquals(1, metrics.size()); + RangerAuditMetrics metric = metrics.get(0); + assertEquals("dev_hdfs", metric.getServiceName()); + assertEquals("hdfs", metric.getAppId()); + assertEquals("172.18.0.16", metric.getClientIP()); + assertEquals("ABC cluster", metric.getClusterName()); + assertEquals(7L, metric.getNumberOfAudits()); + assertEquals(1L, metric.getId()); + } + + @Test + public void getLatestAuditMetricsList_missingAgent_omitsAppId() throws Throwable { + SolrAuditMetricsHelper helper = createHelperWithDao("dev_hdfs", 1L, "hdfs", 10L); + QueryResponse response = buildFacetQueryResponse("per_repo", buildFullMetricsFacetResponse("dev_hdfs", null, "172.18.0.16", "ABC cluster", 5L)); + stubSolrQuery(helper, response); + + List metrics = helper.getLatestAuditMetricsList(new SearchFilter(), null); + + assertEquals(1, metrics.size()); + assertNull(metrics.get(0).getAppId()); + assertEquals("172.18.0.16", metrics.get(0).getClientIP()); + assertEquals("ABC cluster", metrics.get(0).getClusterName()); + assertEquals(5L, metrics.get(0).getNumberOfAudits()); + } + + @Test + public void getLatestAuditMetricsList_missingCliIp_omitsClientIP() throws Throwable { + SolrAuditMetricsHelper helper = createHelperWithDao("dev_hdfs", 1L, "hdfs", 10L); + QueryResponse response = buildFacetQueryResponse("per_repo", buildFullMetricsFacetResponse("dev_hdfs", "hdfs", null, "ABC cluster", 3L)); + stubSolrQuery(helper, response); + + List metrics = helper.getLatestAuditMetricsList(new SearchFilter(), null); + + assertEquals(1, metrics.size()); + assertNull(metrics.get(0).getClientIP()); + assertEquals("ABC cluster", metrics.get(0).getClusterName()); + assertEquals("hdfs", metrics.get(0).getAppId()); + } + + @Test + public void getLatestAuditMetricsList_missingCluster_omitsClusterName() throws Throwable { + SolrAuditMetricsHelper helper = createHelperWithDao("dev_hdfs", 1L, "hdfs", 10L); + QueryResponse response = buildFacetQueryResponse("per_repo", buildFullMetricsFacetResponse("dev_hdfs", "hdfs", "172.18.0.16", null, 4L)); + stubSolrQuery(helper, response); + + List metrics = helper.getLatestAuditMetricsList(new SearchFilter(), null); + + assertEquals(1, metrics.size()); + assertEquals("172.18.0.16", metrics.get(0).getClientIP()); + assertNull(metrics.get(0).getClusterName()); + } + + @Test + public void getLatestAuditMetricsList_emptyFacets_returnsEmptyList() throws Throwable { + SolrAuditMetricsHelper helper = createHelper(null); + QueryResponse response = buildFacetQueryResponse("per_repo", facetBuckets(Collections.emptyList())); + stubSolrQuery(helper, response); + + List metrics = helper.getLatestAuditMetricsList(new SearchFilter(), null); + + assertTrue(metrics.isEmpty()); + } + + @Test + public void getAuditMetricsByHours_usesTimezoneForHour() throws Throwable { + SolrAuditMetricsHelper helper = createHelper(null); + long epochMillis = Instant.parse("2026-08-11T12:27:43.634Z").toEpochMilli(); + QueryResponse response = buildFacetQueryResponse("per_hour", buildHourlyFacetResponse(epochMillis, 15L)); + stubSolrQuery(helper, response); + + List metricsUtc = helper.getAuditMetricsByHours(new SearchFilter(), null); + List metricsIst = helper.getAuditMetricsByHours(new SearchFilter(), "Asia/Kolkata"); + + assertEquals(1, metricsUtc.size()); + assertEquals(1, metricsIst.size()); + assertEquals(12, metricsUtc.get(0).getHours()); + assertEquals(17, metricsIst.get(0).getHours()); + assertEquals(15L, metricsUtc.get(0).getNumberOfAudits()); + } + + @Test + public void getAuditMetricsByHours_invalidTimezone_fallsBackToUtcHour() throws Throwable { + SolrAuditMetricsHelper helper = createHelper(null); + long epochMillis = Instant.parse("2026-08-11T12:27:43.634Z").toEpochMilli(); + QueryResponse response = buildFacetQueryResponse("per_hour", buildHourlyFacetResponse(epochMillis, 8L)); + stubSolrQuery(helper, response); + + List metrics = helper.getAuditMetricsByHours(new SearchFilter(), "Invalid/Zone"); + + assertEquals(1, metrics.size()); + assertEquals(12, metrics.get(0).getHours()); + assertEquals(8L, metrics.get(0).getNumberOfAudits()); + } + + @Test + public void getAuditMetricsByDays_returnsDailyBuckets() throws Throwable { + SolrAuditMetricsHelper helper = createHelper(null); + long auditDate = ZonedDateTime.of(2026, 8, 11, 0, 0, 0, 0, ZoneId.of("UTC")).toInstant().toEpochMilli(); + QueryResponse response = buildFacetQueryResponse("per_day", buildDailyFacetResponse(auditDate, 25L)); + stubSolrQuery(helper, response); + + SearchFilter filter = new SearchFilter(); + filter.setParam(SearchFilter.SERVICE_NAME, "dev_hdfs"); + filter.setParam(SearchFilter.SERVICE_TYPE, "hdfs"); + + List metrics = helper.getAuditMetricsByDays(7, filter, "UTC"); + + assertEquals(1, metrics.size()); + assertEquals(auditDate, metrics.get(0).getAuditDate()); + assertEquals(25L, metrics.get(0).getNumberOfAudits()); + assertEquals("dev_hdfs", metrics.get(0).getServiceName()); + assertEquals("hdfs", metrics.get(0).getServiceType()); + } + + @Test + public void runMetricsQuery_whenSolrThrows_wrapsException() throws Throwable { + SolrAuditMetricsHelper helper = createHelper(null); + SolrClient client = mock(SolrClient.class); + when(helper.solrMgr.getSolrClient()).thenReturn(client); + when(helper.solrUtil.runQuery(eq(client), any(SolrQuery.class))).thenThrow(new RuntimeException("Solr down")); + + assertThrows(WebApplicationException.class, () -> helper.getLatestAuditMetrics("hdfs", "dev_hdfs", null)); + } + + private SolrAuditMetricsHelper createHelper(RangerDaoManager daoManager) { + SolrAuditMetricsHelper helper = new SolrAuditMetricsHelper(); + helper.solrMgr = mock(SolrMgr.class); + helper.solrUtil = mock(SolrUtil.class); + helper.restErrorUtil = new RESTErrorUtil(); + helper.daoManager = daoManager; + return helper; + } + + private SolrAuditMetricsHelper createHelperWithDao(String serviceName, Long serviceId, String serviceType, Long serviceDefId) { + RangerDaoManager daoManager = mock(RangerDaoManager.class); + XXServiceDao svcDao = mock(XXServiceDao.class); + XXServiceDefDao sdDao = mock(XXServiceDefDao.class); + XXService xxService = mock(XXService.class); + XXServiceDef sd = mock(XXServiceDef.class); + + when(daoManager.getXXService()).thenReturn(svcDao); + when(daoManager.getXXServiceDef()).thenReturn(sdDao); + when(svcDao.findByName(serviceName)).thenReturn(xxService); + when(xxService.getId()).thenReturn(serviceId); + when(xxService.getType()).thenReturn(serviceDefId); + when(sdDao.getById(serviceDefId)).thenReturn(sd); + when(sd.getName()).thenReturn(serviceType); + + return createHelper(daoManager); + } + + private void stubSolrQuery(SolrAuditMetricsHelper helper, QueryResponse response) throws Throwable { + SolrClient client = mock(SolrClient.class); + when(helper.solrMgr.getSolrClient()).thenReturn(client); + when(response.getStatus()).thenReturn(0); + when(helper.solrUtil.runQuery(eq(client), any(SolrQuery.class))).thenReturn(response); + } + + private QueryResponse buildFacetQueryResponse(String facetKey, Map facetValue) { + QueryResponse response = mock(QueryResponse.class); + NamedList responseList = new NamedList<>(); + NamedList facets = new NamedList<>(); + facets.add(facetKey, facetValue); + responseList.add("facets", facets); + when(response.getResponse()).thenReturn(responseList); + return response; + } + + private Map buildFullMetricsFacetResponse(String serviceName, String appId, String clientIP, + String clusterName, long count) { + Map clusterBucket = facetBucket(clusterName, count, null); + Map cliIpBucket = facetBucket(clientIP, count, nestedFacet("per_cluster", clusterBucket)); + Map agentBucket = facetBucket(appId, count, nestedFacet("per_cliip", cliIpBucket)); + Map repoBucket = facetBucket(serviceName, count, nestedFacet("per_agent", agentBucket)); + return facetBuckets(Collections.singletonList(repoBucket)); + } + + private Map buildHourlyFacetResponse(long epochMillis, long count) { + Map hourBucket = facetBucket(epochMillis, count, null); + return facetBuckets(Collections.singletonList(hourBucket)); + } + + private Map buildDailyFacetResponse(long auditDate, long count) { + Map dayBucket = facetBucket(auditDate, count, null); + return facetBuckets(Collections.singletonList(dayBucket)); + } + + private Map facetBucket(Object val, long count, Map nestedFacet) { + Map bucket = new HashMap<>(); + bucket.put("val", val); + bucket.put("count", count); + if (nestedFacet != null) { + bucket.putAll(nestedFacet); + } + return bucket; + } + + private Map nestedFacet(String facetName, Map childBucket) { + Map nested = new HashMap<>(); + nested.put(facetName, facetBuckets(Collections.singletonList(childBucket))); + return nested; + } + + private Map facetBuckets(List> buckets) { + Map facet = new HashMap<>(); + facet.put("buckets", buckets); + return facet; + } +}