adoroszlai commented on code in PR #8251: URL: https://github.com/apache/ozone/pull/8251#discussion_r2034639687
########## hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java: ########## @@ -215,10 +215,14 @@ public void stop() { try { LOG.info("Stopping the RPC server for Client Protocol"); getClientRpcServer().stop(); + } catch (Exception ex) { - LOG.error("Client Protocol RPC stop failed.", ex); + AUDIT.logWriteFailure(buildAuditMessageForFailure( + SCMAction.STOP_SERVER, null, ex)); } IOUtils.cleanupWithLogger(LOG, scm.getScmNodeManager()); + AUDIT.logWriteSuccess(buildAuditMessageForSuccess( + SCMAction.STOP_SERVER, null)); Review Comment: Stopping the server is not a client operation, does not need to be audited. ########## hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java: ########## @@ -1414,6 +1582,7 @@ public void close() throws IOException { @Override public DecommissionScmResponseProto decommissionScm( String scmId) { + Review Comment: nit: let's not add empty lines ########## hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java: ########## @@ -1270,60 +1409,89 @@ public List<HddsProtos.DatanodeUsageInfoProto> getDatanodeUsageInfo( boolean mostUsed, int count, int clientVersion) throws IOException, IllegalArgumentException { - // check admin authorisation + final Map<String, String> auditMap = Maps.newHashMap(); + auditMap.put("mostUsed", String.valueOf(mostUsed)); + auditMap.put("count", String.valueOf(count)); + auditMap.put("clientVersion", String.valueOf(clientVersion)); + try { + // check admin authorisation getScm().checkAdminAccess(getRemoteUser(), true); - } catch (IOException e) { - LOG.error("Authorization failed", e); - throw e; - } + if (count < 1) { + throw new IllegalArgumentException("The specified parameter count must " + + "be an integer greater than zero."); + } + List<DatanodeUsageInfo> datanodeUsageInfoList = + scm.getScmNodeManager().getMostOrLeastUsedDatanodes(mostUsed); - if (count < 1) { - throw new IllegalArgumentException("The specified parameter count must " + - "be an integer greater than zero."); - } + // if count is greater than the size of list containing healthy, + // in-service nodes, just set count to that size + if (count > datanodeUsageInfoList.size()) { + count = datanodeUsageInfoList.size(); + } - List<DatanodeUsageInfo> datanodeUsageInfoList = - scm.getScmNodeManager().getMostOrLeastUsedDatanodes(mostUsed); + // return count number of DatanodeUsageInfoProto + List<HddsProtos.DatanodeUsageInfoProto> result = datanodeUsageInfoList.stream() + .map(each -> each.toProto(clientVersion)) + .limit(count) + .collect(Collectors.toList()); - // if count is greater than the size of list containing healthy, - // in-service nodes, just set count to that size - if (count > datanodeUsageInfoList.size()) { - count = datanodeUsageInfoList.size(); + AUDIT.logReadSuccess(buildAuditMessageForSuccess( + SCMAction.GET_DATANODE_USAGE_INFO, auditMap)); + return result; + } catch (Exception ex) { + AUDIT.logReadFailure(buildAuditMessageForFailure( + SCMAction.GET_DATANODE_USAGE_INFO, auditMap, ex)); + throw ex; } - - // return count number of DatanodeUsageInfoProto - return datanodeUsageInfoList.stream() - .map(each -> each.toProto(clientVersion)) - .limit(count) - .collect(Collectors.toList()); } @Override public Token<?> getContainerToken(ContainerID containerID) throws IOException { - UserGroupInformation remoteUser = getRemoteUser(); - getScm().checkAdminAccess(remoteUser, true); + try { + UserGroupInformation remoteUser = getRemoteUser(); + getScm().checkAdminAccess(getRemoteUser(), true); - return scm.getContainerTokenGenerator() - .generateToken(remoteUser.getUserName(), containerID); + Token<?> token = scm.getContainerTokenGenerator() + .generateToken(remoteUser.getUserName(), containerID); + AUDIT.logReadSuccess(buildAuditMessageForSuccess( + SCMAction.GET_CONTAINER_TOKEN, Collections.singletonMap("containerID", String.valueOf(containerID)))); + return token; + } catch (Exception ex) { + AUDIT.logReadFailure(buildAuditMessageForFailure( + SCMAction.GET_CONTAINER_TOKEN, Collections.singletonMap("containerID", String.valueOf(containerID)), ex)); + throw ex; + } } @Override public long getContainerCount() throws IOException { + AUDIT.logReadSuccess(buildAuditMessageForSuccess( + SCMAction.GET_CONTAINER_COUNT, null)); return scm.getContainerManager().getContainers().size(); } @Override public long getContainerCount(HddsProtos.LifeCycleState state) throws IOException { + AUDIT.logReadSuccess(buildAuditMessageForSuccess( + SCMAction.GET_CONTAINER_COUNT, Collections.singletonMap("state", String.valueOf(state)))); return scm.getContainerManager().getContainers(state).size(); Review Comment: 1. This could log success and then throw exception. 2. Failure is not logged. ########## hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java: ########## @@ -1270,60 +1409,89 @@ public List<HddsProtos.DatanodeUsageInfoProto> getDatanodeUsageInfo( boolean mostUsed, int count, int clientVersion) throws IOException, IllegalArgumentException { - // check admin authorisation + final Map<String, String> auditMap = Maps.newHashMap(); + auditMap.put("mostUsed", String.valueOf(mostUsed)); + auditMap.put("count", String.valueOf(count)); + auditMap.put("clientVersion", String.valueOf(clientVersion)); + try { + // check admin authorisation getScm().checkAdminAccess(getRemoteUser(), true); - } catch (IOException e) { - LOG.error("Authorization failed", e); - throw e; - } + if (count < 1) { + throw new IllegalArgumentException("The specified parameter count must " + + "be an integer greater than zero."); + } + List<DatanodeUsageInfo> datanodeUsageInfoList = + scm.getScmNodeManager().getMostOrLeastUsedDatanodes(mostUsed); - if (count < 1) { - throw new IllegalArgumentException("The specified parameter count must " + - "be an integer greater than zero."); - } + // if count is greater than the size of list containing healthy, + // in-service nodes, just set count to that size + if (count > datanodeUsageInfoList.size()) { + count = datanodeUsageInfoList.size(); + } - List<DatanodeUsageInfo> datanodeUsageInfoList = - scm.getScmNodeManager().getMostOrLeastUsedDatanodes(mostUsed); + // return count number of DatanodeUsageInfoProto + List<HddsProtos.DatanodeUsageInfoProto> result = datanodeUsageInfoList.stream() + .map(each -> each.toProto(clientVersion)) + .limit(count) + .collect(Collectors.toList()); - // if count is greater than the size of list containing healthy, - // in-service nodes, just set count to that size - if (count > datanodeUsageInfoList.size()) { - count = datanodeUsageInfoList.size(); + AUDIT.logReadSuccess(buildAuditMessageForSuccess( + SCMAction.GET_DATANODE_USAGE_INFO, auditMap)); + return result; + } catch (Exception ex) { + AUDIT.logReadFailure(buildAuditMessageForFailure( + SCMAction.GET_DATANODE_USAGE_INFO, auditMap, ex)); + throw ex; } - - // return count number of DatanodeUsageInfoProto - return datanodeUsageInfoList.stream() - .map(each -> each.toProto(clientVersion)) - .limit(count) - .collect(Collectors.toList()); } @Override public Token<?> getContainerToken(ContainerID containerID) throws IOException { - UserGroupInformation remoteUser = getRemoteUser(); - getScm().checkAdminAccess(remoteUser, true); + try { + UserGroupInformation remoteUser = getRemoteUser(); + getScm().checkAdminAccess(getRemoteUser(), true); - return scm.getContainerTokenGenerator() - .generateToken(remoteUser.getUserName(), containerID); + Token<?> token = scm.getContainerTokenGenerator() + .generateToken(remoteUser.getUserName(), containerID); + AUDIT.logReadSuccess(buildAuditMessageForSuccess( + SCMAction.GET_CONTAINER_TOKEN, Collections.singletonMap("containerID", String.valueOf(containerID)))); + return token; + } catch (Exception ex) { + AUDIT.logReadFailure(buildAuditMessageForFailure( + SCMAction.GET_CONTAINER_TOKEN, Collections.singletonMap("containerID", String.valueOf(containerID)), ex)); + throw ex; + } } @Override public long getContainerCount() throws IOException { + AUDIT.logReadSuccess(buildAuditMessageForSuccess( + SCMAction.GET_CONTAINER_COUNT, null)); return scm.getContainerManager().getContainers().size(); } @Override public long getContainerCount(HddsProtos.LifeCycleState state) throws IOException { + AUDIT.logReadSuccess(buildAuditMessageForSuccess( + SCMAction.GET_CONTAINER_COUNT, Collections.singletonMap("state", String.valueOf(state)))); return scm.getContainerManager().getContainers(state).size(); } @Override public List<ContainerInfo> getListOfContainers( long startContainerID, int count, HddsProtos.LifeCycleState state) throws IOException { + + final Map<String, String> auditMap = Maps.newHashMap(); + auditMap.put("startContainerID", String.valueOf(startContainerID)); + auditMap.put("count", String.valueOf(count)); + auditMap.put("state", String.valueOf(state)); + + AUDIT.logReadSuccess(buildAuditMessageForSuccess( + SCMAction.LIST_CONTAINER, auditMap)); return scm.getContainerManager().getContainers( ContainerID.valueOf(startContainerID), count, state); Review Comment: 1. This could log success and then throw exception. 2. Failure is not logged. Please wrap in `try-catch`, and make sure no processing is performed after `logReadSuccess`. ########## hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java: ########## @@ -1425,13 +1594,26 @@ public DecommissionScmResponseProto decommissionScm( decommissionScmResponseBuilder .setSuccess(false) .setErrorMsg(ex.getMessage()); + AUDIT.logWriteSuccess(buildAuditMessageForFailure( Review Comment: Both should be failure. ########## hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java: ########## @@ -1425,13 +1594,26 @@ public DecommissionScmResponseProto decommissionScm( decommissionScmResponseBuilder .setSuccess(false) .setErrorMsg(ex.getMessage()); + AUDIT.logWriteSuccess(buildAuditMessageForFailure( + SCMAction.DECOMMISSION_SCM, Collections.singletonMap("scmId", scmId), ex)); } + AUDIT.logWriteSuccess(buildAuditMessageForSuccess( + SCMAction.DECOMMISSION_SCM, Collections.singletonMap("scmId", scmId))); Review Comment: Please move inside `try`. Instead of throwing exception, this method returns response for both success and failure. Thus, two audit log entries would be created for failure case. ########## hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java: ########## @@ -605,10 +653,14 @@ public List<HddsProtos.Node> queryNode( .addNodeOperationalStates(ns.getOperationalState()) .build()); } catch (NodeNotFoundException e) { - throw new IOException( - "An unexpected error occurred querying the NodeStatus", e); + IOException ex = new IOException("An unexpected error occurred querying the NodeStatus", e); + AUDIT.logReadFailure(buildAuditMessageForFailure( + SCMAction.QUERY_NODE, auditMap, ex)); Review Comment: Pass the original exception (`NodeNotFoundException e`) to audit instead of the constructed one (`IOException ex`). nit: let's avoid unnecessary change (i.e. renaming `e -> ex`) ########## hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java: ########## @@ -1270,60 +1409,89 @@ public List<HddsProtos.DatanodeUsageInfoProto> getDatanodeUsageInfo( boolean mostUsed, int count, int clientVersion) throws IOException, IllegalArgumentException { - // check admin authorisation + final Map<String, String> auditMap = Maps.newHashMap(); + auditMap.put("mostUsed", String.valueOf(mostUsed)); + auditMap.put("count", String.valueOf(count)); + auditMap.put("clientVersion", String.valueOf(clientVersion)); + try { + // check admin authorisation getScm().checkAdminAccess(getRemoteUser(), true); - } catch (IOException e) { - LOG.error("Authorization failed", e); - throw e; - } + if (count < 1) { + throw new IllegalArgumentException("The specified parameter count must " + + "be an integer greater than zero."); + } + List<DatanodeUsageInfo> datanodeUsageInfoList = + scm.getScmNodeManager().getMostOrLeastUsedDatanodes(mostUsed); - if (count < 1) { - throw new IllegalArgumentException("The specified parameter count must " + - "be an integer greater than zero."); - } + // if count is greater than the size of list containing healthy, + // in-service nodes, just set count to that size + if (count > datanodeUsageInfoList.size()) { + count = datanodeUsageInfoList.size(); + } - List<DatanodeUsageInfo> datanodeUsageInfoList = - scm.getScmNodeManager().getMostOrLeastUsedDatanodes(mostUsed); + // return count number of DatanodeUsageInfoProto + List<HddsProtos.DatanodeUsageInfoProto> result = datanodeUsageInfoList.stream() + .map(each -> each.toProto(clientVersion)) + .limit(count) + .collect(Collectors.toList()); - // if count is greater than the size of list containing healthy, - // in-service nodes, just set count to that size - if (count > datanodeUsageInfoList.size()) { - count = datanodeUsageInfoList.size(); + AUDIT.logReadSuccess(buildAuditMessageForSuccess( + SCMAction.GET_DATANODE_USAGE_INFO, auditMap)); + return result; + } catch (Exception ex) { + AUDIT.logReadFailure(buildAuditMessageForFailure( + SCMAction.GET_DATANODE_USAGE_INFO, auditMap, ex)); + throw ex; } - - // return count number of DatanodeUsageInfoProto - return datanodeUsageInfoList.stream() - .map(each -> each.toProto(clientVersion)) - .limit(count) - .collect(Collectors.toList()); } @Override public Token<?> getContainerToken(ContainerID containerID) throws IOException { - UserGroupInformation remoteUser = getRemoteUser(); - getScm().checkAdminAccess(remoteUser, true); + try { + UserGroupInformation remoteUser = getRemoteUser(); + getScm().checkAdminAccess(getRemoteUser(), true); - return scm.getContainerTokenGenerator() - .generateToken(remoteUser.getUserName(), containerID); + Token<?> token = scm.getContainerTokenGenerator() + .generateToken(remoteUser.getUserName(), containerID); + AUDIT.logReadSuccess(buildAuditMessageForSuccess( + SCMAction.GET_CONTAINER_TOKEN, Collections.singletonMap("containerID", String.valueOf(containerID)))); + return token; + } catch (Exception ex) { + AUDIT.logReadFailure(buildAuditMessageForFailure( + SCMAction.GET_CONTAINER_TOKEN, Collections.singletonMap("containerID", String.valueOf(containerID)), ex)); + throw ex; + } } @Override public long getContainerCount() throws IOException { + AUDIT.logReadSuccess(buildAuditMessageForSuccess( + SCMAction.GET_CONTAINER_COUNT, null)); return scm.getContainerManager().getContainers().size(); Review Comment: 1. This could log success and then throw exception. 2. Failure is not logged. ########## hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java: ########## @@ -382,10 +411,14 @@ public List<ContainerWithPipeline> getExistContainerWithPipelinesInBatch( ContainerWithPipeline cp = getContainerWithPipelineCommon(containerID); cpList.add(cp); } catch (IOException ex) { - //not found , just go ahead - LOG.error("Container with common pipeline not found: {}", ex); + AUDIT.logReadFailure(buildAuditMessageForFailure( + SCMAction.GET_EXIST_CONTAINER_WITH_PIPELINE_BATCH, + Collections.singletonMap("containerID", String.valueOf(containerID)), ex)); } } + AUDIT.logReadSuccess(buildAuditMessageForSuccess( + SCMAction.GET_EXIST_CONTAINER_WITH_PIPELINE_BATCH, + Collections.singletonMap("containerIDs", containerIDs.toString()))); Review Comment: Please move inside `try`, otherwise two audit entries will be created in case of failure. ########## hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java: ########## @@ -627,45 +679,72 @@ public HddsProtos.Node queryNode(UUID uuid) .build(); } } catch (NodeNotFoundException e) { - throw new IOException( + IOException ex = new IOException( "An unexpected error occurred querying the NodeStatus", e); + AUDIT.logReadFailure(buildAuditMessageForFailure(SCMAction.QUERY_NODE_BY_UUID, Review Comment: `QUERY_NODE` action can be used, instead of introducing another one (`QUERY_NODE_BY_UUID`). Parameters will indicate the specific method invoked. ########## hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java: ########## @@ -382,10 +411,14 @@ public List<ContainerWithPipeline> getExistContainerWithPipelinesInBatch( ContainerWithPipeline cp = getContainerWithPipelineCommon(containerID); cpList.add(cp); } catch (IOException ex) { - //not found , just go ahead - LOG.error("Container with common pipeline not found: {}", ex); + AUDIT.logReadFailure(buildAuditMessageForFailure( + SCMAction.GET_EXIST_CONTAINER_WITH_PIPELINE_BATCH, + Collections.singletonMap("containerID", String.valueOf(containerID)), ex)); Review Comment: nit: Please store map in variable to reduce duplication. ########## hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java: ########## @@ -986,35 +1106,47 @@ public ReplicationManagerReport getReplicationManagerReport() { @Override public StatusAndMessages finalizeScmUpgrade(String upgradeClientID) throws IOException { - // check admin authorization try { + // check admin authorization getScm().checkAdminAccess(getRemoteUser(), false); - } catch (IOException e) { - LOG.error("Authorization failed for finalize scm upgrade", e); - throw e; + // TODO HDDS-6762: Return to the client once the FINALIZATION_STARTED + // checkpoint has been crossed and continue finalizing asynchronously. + StatusAndMessages result = scm.getFinalizationManager().finalizeUpgrade(upgradeClientID); + AUDIT.logWriteSuccess(buildAuditMessageForSuccess( + SCMAction.FINALIZE_SCM_UPGRADE, Collections.singletonMap("upgradeClientID", upgradeClientID))); Review Comment: nit: please store audit map in variable to avoid duplication. ########## hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java: ########## @@ -1270,60 +1409,89 @@ public List<HddsProtos.DatanodeUsageInfoProto> getDatanodeUsageInfo( boolean mostUsed, int count, int clientVersion) throws IOException, IllegalArgumentException { - // check admin authorisation + final Map<String, String> auditMap = Maps.newHashMap(); + auditMap.put("mostUsed", String.valueOf(mostUsed)); + auditMap.put("count", String.valueOf(count)); + auditMap.put("clientVersion", String.valueOf(clientVersion)); + try { + // check admin authorisation getScm().checkAdminAccess(getRemoteUser(), true); - } catch (IOException e) { - LOG.error("Authorization failed", e); - throw e; - } + if (count < 1) { + throw new IllegalArgumentException("The specified parameter count must " + + "be an integer greater than zero."); + } + List<DatanodeUsageInfo> datanodeUsageInfoList = + scm.getScmNodeManager().getMostOrLeastUsedDatanodes(mostUsed); - if (count < 1) { - throw new IllegalArgumentException("The specified parameter count must " + - "be an integer greater than zero."); - } + // if count is greater than the size of list containing healthy, + // in-service nodes, just set count to that size + if (count > datanodeUsageInfoList.size()) { + count = datanodeUsageInfoList.size(); + } - List<DatanodeUsageInfo> datanodeUsageInfoList = - scm.getScmNodeManager().getMostOrLeastUsedDatanodes(mostUsed); + // return count number of DatanodeUsageInfoProto + List<HddsProtos.DatanodeUsageInfoProto> result = datanodeUsageInfoList.stream() + .map(each -> each.toProto(clientVersion)) + .limit(count) + .collect(Collectors.toList()); - // if count is greater than the size of list containing healthy, - // in-service nodes, just set count to that size - if (count > datanodeUsageInfoList.size()) { - count = datanodeUsageInfoList.size(); + AUDIT.logReadSuccess(buildAuditMessageForSuccess( + SCMAction.GET_DATANODE_USAGE_INFO, auditMap)); + return result; + } catch (Exception ex) { + AUDIT.logReadFailure(buildAuditMessageForFailure( + SCMAction.GET_DATANODE_USAGE_INFO, auditMap, ex)); + throw ex; } - - // return count number of DatanodeUsageInfoProto - return datanodeUsageInfoList.stream() - .map(each -> each.toProto(clientVersion)) - .limit(count) - .collect(Collectors.toList()); } @Override public Token<?> getContainerToken(ContainerID containerID) throws IOException { - UserGroupInformation remoteUser = getRemoteUser(); - getScm().checkAdminAccess(remoteUser, true); + try { + UserGroupInformation remoteUser = getRemoteUser(); + getScm().checkAdminAccess(getRemoteUser(), true); - return scm.getContainerTokenGenerator() - .generateToken(remoteUser.getUserName(), containerID); + Token<?> token = scm.getContainerTokenGenerator() + .generateToken(remoteUser.getUserName(), containerID); + AUDIT.logReadSuccess(buildAuditMessageForSuccess( + SCMAction.GET_CONTAINER_TOKEN, Collections.singletonMap("containerID", String.valueOf(containerID)))); + return token; + } catch (Exception ex) { + AUDIT.logReadFailure(buildAuditMessageForFailure( + SCMAction.GET_CONTAINER_TOKEN, Collections.singletonMap("containerID", String.valueOf(containerID)), ex)); Review Comment: Please store audit map in local variable to reduce duplication. ########## hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java: ########## @@ -579,10 +612,15 @@ public void deleteContainer(long containerID) throws IOException { @Override public Map<String, List<ContainerID>> getContainersOnDecomNode(DatanodeDetails dn) throws IOException { + Map<String, String> auditMap = Maps.newHashMap(); + auditMap.put("datanodeDetails", String.valueOf(dn)); try { - return scm.getScmDecommissionManager().getContainersPendingReplication(dn); - } catch (NodeNotFoundException e) { - throw new IOException("Failed to get containers list. Unable to find required node", e); + Map<String, List<ContainerID>> result = scm.getScmDecommissionManager().getContainersPendingReplication(dn); + AUDIT.logReadSuccess(buildAuditMessageForSuccess(SCMAction.GET_CONTAINERS_ON_DECOM_NODE, auditMap)); + return result; + } catch (NodeNotFoundException ex) { + AUDIT.logReadFailure(buildAuditMessageForFailure(SCMAction.GET_CONTAINERS_ON_DECOM_NODE, auditMap, ex)); + throw new IOException("Failed to get containers list. Unable to find required node", ex); Review Comment: nit: let's avoid unnecessary change (i.e. renaming `e -> ex`) -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: issues-unsubscr...@ozone.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org --------------------------------------------------------------------- To unsubscribe, e-mail: issues-unsubscr...@ozone.apache.org For additional commands, e-mail: issues-h...@ozone.apache.org