This is an automated email from the ASF dual-hosted git repository. aboda pushed a commit to branch NIFIREG-371-RC1 in repository https://gitbox.apache.org/repos/asf/nifi-registry.git
commit 23afadfcb91cc7253b6a7d3caa8db2e671132a94 Author: Bryan Bende <[email protected]> AuthorDate: Thu Jan 23 13:18:51 2020 -0500 NIFIREG-353 Add ShellUserGroupProvider and relax checks to allow a user to have same identity as a group This closes #255. --- .../security/authorization/AuthorizerFactory.java | 18 +- .../authorization/shell/NssShellCommands.java | 89 +++ .../authorization/shell/OsxShellCommands.java | 81 +++ .../authorization/shell/RemoteShellCommands.java | 73 +++ .../authorization/shell/ShellCommandsProvider.java | 100 +++ .../security/authorization/shell/ShellRunner.java | 81 +++ .../shell/ShellUserGroupProvider.java | 678 +++++++++++++++++++++ ...gistry.security.authorization.UserGroupProvider | 3 +- .../src/main/resources/conf/authorizers.xml | 19 + .../org/apache/nifi/registry/util/FormatUtils.java | 232 ++++++- 10 files changed, 1334 insertions(+), 40 deletions(-) diff --git a/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authorization/AuthorizerFactory.java b/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authorization/AuthorizerFactory.java index 916892b..a819e97 100644 --- a/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authorization/AuthorizerFactory.java +++ b/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authorization/AuthorizerFactory.java @@ -611,7 +611,7 @@ public class AuthorizerFactory implements UserGroupProviderLookup, AccessPolicyP @Override public User addUser(User user) throws AuthorizationAccessException { - if (tenantExists(baseConfigurableUserGroupProvider, user.getIdentifier(), user.getIdentity())) { + if (userExists(baseConfigurableUserGroupProvider, user.getIdentifier(), user.getIdentity())) { throw new IllegalStateException(String.format("User/user group already exists with the identity '%s'.", user.getIdentity())); } return baseConfigurableUserGroupProvider.addUser(user); @@ -624,7 +624,7 @@ public class AuthorizerFactory implements UserGroupProviderLookup, AccessPolicyP @Override public User updateUser(User user) throws AuthorizationAccessException { - if (tenantExists(baseConfigurableUserGroupProvider, user.getIdentifier(), user.getIdentity())) { + if (userExists(baseConfigurableUserGroupProvider, user.getIdentifier(), user.getIdentity())) { throw new IllegalStateException(String.format("User/user group already exists with the identity '%s'.", user.getIdentity())); } if (!baseConfigurableUserGroupProvider.isConfigurable(user)) { @@ -651,7 +651,7 @@ public class AuthorizerFactory implements UserGroupProviderLookup, AccessPolicyP @Override public Group addGroup(Group group) throws AuthorizationAccessException { - if (tenantExists(baseConfigurableUserGroupProvider, group.getIdentifier(), group.getName())) { + if (groupExists(baseConfigurableUserGroupProvider, group.getIdentifier(), group.getName())) { throw new IllegalStateException(String.format("User/user group already exists with the identity '%s'.", group.getName())); } if (!allGroupUsersExist(baseUserGroupProvider, group)) { @@ -667,7 +667,7 @@ public class AuthorizerFactory implements UserGroupProviderLookup, AccessPolicyP @Override public Group updateGroup(Group group) throws AuthorizationAccessException { - if (tenantExists(baseConfigurableUserGroupProvider, group.getIdentifier(), group.getName())) { + if (groupExists(baseConfigurableUserGroupProvider, group.getIdentifier(), group.getName())) { throw new IllegalStateException(String.format("User/user group already exists with the identity '%s'.", group.getName())); } if (!baseConfigurableUserGroupProvider.isConfigurable(group)) { @@ -796,14 +796,14 @@ public class AuthorizerFactory implements UserGroupProviderLookup, AccessPolicyP // ensure that only one group exists per identity for (User user : userGroupProvider.getUsers()) { - if (tenantExists(userGroupProvider, user.getIdentifier(), user.getIdentity())) { + if (userExists(userGroupProvider, user.getIdentifier(), user.getIdentity())) { throw new SecurityProviderCreationException(String.format("Found multiple users/user groups with identity '%s'.", user.getIdentity())); } } // ensure that only one group exists per identity for (Group group : userGroupProvider.getGroups()) { - if (tenantExists(userGroupProvider, group.getIdentifier(), group.getName())) { + if (groupExists(userGroupProvider, group.getIdentifier(), group.getName())) { throw new SecurityProviderCreationException(String.format("Found multiple users/user groups with name '%s'.", group.getName())); } } @@ -896,7 +896,7 @@ public class AuthorizerFactory implements UserGroupProviderLookup, AccessPolicyP * @param identity identity of the tenant * @return true if another user exists with the same identity, false otherwise */ - private static boolean tenantExists(final UserGroupProvider userGroupProvider, final String identifier, final String identity) { + private static boolean userExists(final UserGroupProvider userGroupProvider, final String identifier, final String identity) { for (User user : userGroupProvider.getUsers()) { if (!user.getIdentifier().equals(identifier) && user.getIdentity().equals(identity)) { @@ -904,6 +904,10 @@ public class AuthorizerFactory implements UserGroupProviderLookup, AccessPolicyP } } + return false; + } + + private static boolean groupExists(final UserGroupProvider userGroupProvider, final String identifier, final String identity) { for (Group group : userGroupProvider.getGroups()) { if (!group.getIdentifier().equals(identifier) && group.getName().equals(identity)) { diff --git a/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authorization/shell/NssShellCommands.java b/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authorization/shell/NssShellCommands.java new file mode 100644 index 0000000..eef58b0 --- /dev/null +++ b/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authorization/shell/NssShellCommands.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.registry.security.authorization.shell; + +/** + * Provides shell commands to read users and groups on NSS-enabled systems. + * + * See `man 5 nsswitch.conf` for more info. + */ +class NssShellCommands implements ShellCommandsProvider { + /** + * @return Shell command string that will return a list of users. + */ + public String getUsersList() { + return "getent passwd | cut -f 1,3,4 -d ':'"; + } + + /** + * @return Shell command string that will return a list of groups. + */ + public String getGroupsList() { + return "getent group | cut -f 1,3 -d ':'"; + } + + /** + * @param groupName name of group. + * @return Shell command string that will return a list of users for a group. + */ + public String getGroupMembers(String groupName) { + return String.format("getent group %s | cut -f 4 -d ':'", groupName); + } + + /** + * Gets the command for reading a single user by id. + * + * When executed, this command should output a single line, in the format used by `getUsersList`. + * + * @param userId name of user. + * @return Shell command string that will read a single user. + */ + @Override + public String getUserById(String userId) { + return String.format("getent passwd %s | cut -f 1,3,4 -d ':'", userId); + } + + /** + * This method reuses `getUserById` because the getent command is the same for + * both uid and username. + * + * @param userName name of user. + * @return Shell command string that will read a single user. + */ + public String getUserByName(String userName) { + return getUserById(userName); + } + + /** + * This method supports gid or group name because getent does. + * + * @param groupId name of group. + * @return Shell command string that will read a single group. + */ + public String getGroupById(String groupId) { + return String.format("getent group %s | cut -f 1,3,4 -d ':'", groupId); + } + + /** + * This gives exit code 0 on all tested distributions. + * + * @return Shell command string that will exit normally (0) on a suitable system. + */ + public String getSystemCheck() { + return "getent --version"; + } +} \ No newline at end of file diff --git a/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authorization/shell/OsxShellCommands.java b/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authorization/shell/OsxShellCommands.java new file mode 100644 index 0000000..0591662 --- /dev/null +++ b/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authorization/shell/OsxShellCommands.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.registry.security.authorization.shell; + +/** + * Provides shell commands to read users and groups on Mac OSX systems. + * + * See `man dscl` for more info. + */ +class OsxShellCommands implements ShellCommandsProvider { + /** + * @return Shell command string that will return a list of users. + */ + public String getUsersList() { + return "dscl . -readall /Users UniqueID PrimaryGroupID | awk 'BEGIN { OFS = \":\"; ORS=\"\\n\"; i=0;} /RecordName: / {name = $2;i = 0;}" + + "/PrimaryGroupID: / {gid = $2;} /^ / {if (i == 0) { i++; name = $1;}} /UniqueID: / {uid = $2;print name, uid, gid;}' | grep -v ^_"; + } + + /** + * @return Shell command string that will return a list of groups. + */ + public String getGroupsList() { + return "dscl . -list /Groups PrimaryGroupID | grep -v '^_' | sed 's/ \\{1,\\}/:/g'"; + } + + /** + * + * @param groupName name of group. + * @return Shell command string that will return a list of users for a group. + */ + public String getGroupMembers(String groupName) { + return String.format("dscl . -read /Groups/%s GroupMembership | cut -f 2- -d ' ' | sed 's/\\ /,/g'", groupName); + } + + /** + * @param userId name of user. + * @return Shell command string that will read a single user. + */ + @Override + public String getUserById(String userId) { + return String.format("id -P %s | cut -f 1,3,4 -d ':'", userId); + } + + /** + * @param userName name of user. + * @return Shell command string that will read a single user. + */ + public String getUserByName(String userName) { + return getUserById(userName); // 'id' command works for both uid/username + } + + /** + * @param groupId name of group. + * @return Shell command string that will read a single group. + */ + public String getGroupById(String groupId) { + return String.format(" dscl . -read /Groups/`dscl . -search /Groups gid %s | head -n 1 | cut -f 1` RecordName PrimaryGroupID | awk 'BEGIN { OFS = \":\"; ORS=\"\\n\"; i=0;} " + + "/RecordName: / {name = $2;i = 1;}/PrimaryGroupID: / {gid = $2;}; {if (i==1) {print name,gid,\"\"}}'", groupId); + } + + /** + * @return Shell command string that will exit normally (0) on a suitable system. + */ + public String getSystemCheck() { + return "which dscl"; + } +} diff --git a/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authorization/shell/RemoteShellCommands.java b/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authorization/shell/RemoteShellCommands.java new file mode 100644 index 0000000..f622409 --- /dev/null +++ b/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authorization/shell/RemoteShellCommands.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.registry.security.authorization.shell; + +class RemoteShellCommands implements ShellCommandsProvider { + // Carefully crafted command replacement string: + private final static String remoteCommand = "ssh " + + "-o 'StrictHostKeyChecking no' " + + "-o 'PasswordAuthentication no' " + + "-o \"RemoteCommand %s\" " + + "-i %s -p %s -l root %s"; + + private ShellCommandsProvider innerProvider; + private String privateKeyPath; + private String remoteHost; + private Integer remotePort; + + private RemoteShellCommands() { + } + + public static ShellCommandsProvider wrapOtherProvider(ShellCommandsProvider otherProvider, String keyPath, String host, Integer port) { + RemoteShellCommands remote = new RemoteShellCommands(); + + remote.innerProvider = otherProvider; + remote.privateKeyPath = keyPath; + remote.remoteHost = host; + remote.remotePort = port; + + return remote; + } + + public String getUsersList() { + return String.format(remoteCommand, innerProvider.getUsersList(), privateKeyPath, remotePort, remoteHost); + } + + public String getGroupsList() { + return String.format(remoteCommand, innerProvider.getGroupsList(), privateKeyPath, remotePort, remoteHost); + } + + public String getGroupMembers(String groupName) { + return String.format(remoteCommand, innerProvider.getGroupMembers(groupName), privateKeyPath, remotePort, remoteHost); + } + + public String getUserById(String userId) { + return String.format(remoteCommand, innerProvider.getUserById(userId), privateKeyPath, remotePort, remoteHost); + } + + public String getUserByName(String userName) { + return String.format(remoteCommand, innerProvider.getUserByName(userName), privateKeyPath, remotePort, remoteHost); + } + + public String getGroupById(String groupId) { + return String.format(remoteCommand, innerProvider.getGroupById(groupId), privateKeyPath, remotePort, remoteHost); + } + + public String getSystemCheck() { + return String.format(remoteCommand, innerProvider.getSystemCheck(), privateKeyPath, remotePort, remoteHost); + } +} diff --git a/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authorization/shell/ShellCommandsProvider.java b/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authorization/shell/ShellCommandsProvider.java new file mode 100644 index 0000000..ce3e6a4 --- /dev/null +++ b/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authorization/shell/ShellCommandsProvider.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.registry.security.authorization.shell; + +/** + * Common interface for shell command strings to read users and groups. + * + */ +interface ShellCommandsProvider { + /** + * Gets the command for listing users. + * + * When executed, this command should output one record per line in this format: + * + * `username:user-id:primary-group-id` + * + * @return Shell command string that will return a list of users. + */ + String getUsersList(); + + /** + * Gets the command for listing groups. + * + * When executed, this command should output one record per line in this format: + * + * `group-name:group-id` + * + * @return Shell command string that will return a list of groups. + */ + String getGroupsList(); + + /** + * Gets the command for listing the members of a group. + * + * When executed, this command should output one line in this format: + * + * `user-name-1,user-name-2,user-name-n` + * + * @param groupName name of group. + * @return Shell command string that will return a list of users for a group. + */ + String getGroupMembers(String groupName); + + /** + * Gets the command for reading a single user by id. Implementations may return null if reading a single + * user by id is not supported. + * + * When executed, this command should output a single line, in the format used by `getUsersList`. + * + * @param userId name of user. + * @return Shell command string that will read a single user. + */ + String getUserById(String userId); + + /** + * Gets the command for reading a single user. Implementations may return null if reading a single user by + * username is not supported. + * + * When executed, this command should output a single line, in the format used by `getUsersList`. + * + * @param userName name of user. + * @return Shell command string that will read a single user. + */ + String getUserByName(String userName); + + /** + * Gets the command for reading a single group. Implementations may return null if reading a single group + * by name is not supported. + * + * When executed, this command should output a single line, in the format used by `getGroupsList`. + * + * @param groupId name of group. + * @return Shell command string that will read a single group. + */ + String getGroupById(String groupId); + + /** + * Gets the command for checking the suitability of the host system. + * + * The command is expected to exit with status 0 (zero) to indicate success, and any other status + * to indicate failure. + * + * @return Shell command string that will exit normally (0) on a suitable system. + */ + String getSystemCheck(); +} diff --git a/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authorization/shell/ShellRunner.java b/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authorization/shell/ShellRunner.java new file mode 100644 index 0000000..ee7ef41 --- /dev/null +++ b/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authorization/shell/ShellRunner.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.registry.security.authorization.shell; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.Reader; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; + +public class ShellRunner { + private final static Logger logger = LoggerFactory.getLogger(ShellRunner.class); + + static String SHELL = "sh"; + static String OPTS = "-c"; + static Integer TIMEOUT = 60; + + public static List<String> runShell(String command) throws IOException { + return runShell(command, "<unknown>"); + } + + public static List<String> runShell(String command, String description) throws IOException { + final ProcessBuilder builder = new ProcessBuilder(SHELL, OPTS, command); + final List<String> builderCommand = builder.command(); + + logger.debug("Run Command '" + description + "': " + builderCommand); + final Process proc = builder.start(); + + boolean completed; + try { + completed = proc.waitFor(TIMEOUT, TimeUnit.SECONDS); + } catch (InterruptedException irexc) { + throw new IOException(irexc.getMessage(), irexc.getCause()); + } + + if (!completed) { + throw new IllegalStateException("Shell command '" + command + "' did not complete during the allotted time period"); + } + + if (proc.exitValue() != 0) { + try (final Reader stderr = new InputStreamReader(proc.getErrorStream()); + final BufferedReader reader = new BufferedReader(stderr)) { + String line; + while ((line = reader.readLine()) != null) { + logger.warn(line.trim()); + } + } + throw new IOException("Command exit non-zero: " + proc.exitValue()); + } + + final List<String> lines = new ArrayList<>(); + try (final Reader stdin = new InputStreamReader(proc.getInputStream()); + final BufferedReader reader = new BufferedReader(stdin)) { + String line; + while ((line = reader.readLine()) != null) { + lines.add(line.trim()); + } + } + + return lines; + } +} diff --git a/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authorization/shell/ShellUserGroupProvider.java b/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authorization/shell/ShellUserGroupProvider.java new file mode 100644 index 0000000..1d709ac --- /dev/null +++ b/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authorization/shell/ShellUserGroupProvider.java @@ -0,0 +1,678 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.registry.security.authorization.shell; + +import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.registry.security.authorization.AuthorizerConfigurationContext; +import org.apache.nifi.registry.security.authorization.Group; +import org.apache.nifi.registry.security.authorization.User; +import org.apache.nifi.registry.security.authorization.UserAndGroups; +import org.apache.nifi.registry.security.authorization.UserGroupProvider; +import org.apache.nifi.registry.security.authorization.UserGroupProviderInitializationContext; +import org.apache.nifi.registry.security.authorization.exception.AuthorizationAccessException; +import org.apache.nifi.registry.security.exception.SecurityProviderCreationException; +import org.apache.nifi.registry.security.exception.SecurityProviderDestructionException; +import org.apache.nifi.registry.util.FormatUtils; +import org.apache.nifi.registry.util.PropertyValue; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; + +/* + * ShellUserGroupProvider implements UserGroupProvider by way of shell commands. + */ +public class ShellUserGroupProvider implements UserGroupProvider { + private final static Logger logger = LoggerFactory.getLogger(ShellUserGroupProvider.class); + + private final static String OS_TYPE_ERROR = "Unsupported operating system."; + private final static String SYS_CHECK_ERROR = "System check failed - cannot provide users and groups."; + private final static Map<String, User> usersById = new HashMap<>(); // id == identifier + private final static Map<String, User> usersByName = new HashMap<>(); // name == identity + private final static Map<String, Group> groupsById = new HashMap<>(); + + public static final String REFRESH_DELAY_PROPERTY = "Refresh Delay"; + private static final long MINIMUM_SYNC_INTERVAL_MILLISECONDS = 10_000; + + public static final String EXCLUDE_USER_PROPERTY = "Exclude Users"; + public static final String EXCLUDE_GROUP_PROPERTY = "Exclude Groups"; + + private long fixedDelay; + private Pattern excludeUsers; + private Pattern excludeGroups; + + // Our scheduler has one thread for users, one for groups: + private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(2); + + // Our shell timeout, in seconds: + @SuppressWarnings("FieldCanBeLocal") + private final Integer shellTimeout = 10; + + // Commands selected during initialization: + private ShellCommandsProvider selectedShellCommands; + + // Start of the UserGroupProvider implementation. Javadoc strings + // copied from the interface definition for reference. + + /** + * Retrieves all users. Must be non null + * + * @return a list of users + * @throws AuthorizationAccessException if there was an unexpected error performing the operation + */ + @Override + public Set<User> getUsers() throws AuthorizationAccessException { + synchronized (usersById) { + logger.debug("getUsers has user set of size: " + usersById.size()); + return new HashSet<>(usersById.values()); + } + } + + /** + * Retrieves the user with the given identifier. + * + * @param identifier the id of the user to retrieve + * @return the user with the given id, or null if no matching user was found + * @throws AuthorizationAccessException if there was an unexpected error performing the operation + */ + @Override + public User getUser(String identifier) throws AuthorizationAccessException { + User user; + + synchronized (usersById) { + user = usersById.get(identifier); + } + + if (user == null) { + logger.debug("getUser (by id) user not found: " + identifier); + } else { + logger.debug("getUser (by id) found user: " + user + " for id: " + identifier); + } + return user; + } + + /** + * Retrieves the user with the given identity. + * + * @param identity the identity of the user to retrieve + * @return the user with the given identity, or null if no matching user was found + * @throws AuthorizationAccessException if there was an unexpected error performing the operation + */ + @Override + public User getUserByIdentity(String identity) throws AuthorizationAccessException { + User user; + + synchronized (usersByName) { + user = usersByName.get(identity); + } + + if (user == null) { + refreshOneUser(selectedShellCommands.getUserByName(identity), "Get Single User by Name"); + user = usersByName.get(identity); + } + + if (user == null) { + logger.debug("getUser (by name) user not found: " + identity); + } else { + logger.debug("getUser (by name) found user: " + user.getIdentity() + " for name: " + identity); + } + return user; + } + + /** + * Retrieves all groups. Must be non null + * + * @return a list of groups + * @throws AuthorizationAccessException if there was an unexpected error performing the operation + */ + @Override + public Set<Group> getGroups() throws AuthorizationAccessException { + synchronized (groupsById) { + logger.debug("getGroups has group set of size: " + groupsById.size()); + return new HashSet<>(groupsById.values()); + } + } + + /** + * Retrieves a Group by Id. + * + * @param identifier the identifier of the Group to retrieve + * @return the Group with the given identifier, or null if no matching group was found + * @throws AuthorizationAccessException if there was an unexpected error performing the operation + */ + @Override + public Group getGroup(String identifier) throws AuthorizationAccessException { + Group group; + + synchronized (groupsById) { + group = groupsById.get(identifier); + } + + if (group == null) { + refreshOneGroup(selectedShellCommands.getGroupById(identifier), "Get Single Group by Id"); + group = groupsById.get(identifier); + } + + if (group == null) { + logger.debug("getGroup (by id) group not found: " + identifier); + } else { + logger.debug("getGroup (by id) found group: " + group.getName() + " for id: " + identifier); + } + return group; + + } + + /** + * Gets a user and their groups. + * + * @return the UserAndGroups for the specified identity + * @throws AuthorizationAccessException if there was an unexpected error performing the operation + */ + @Override + public UserAndGroups getUserAndGroups(String identity) throws AuthorizationAccessException { + User user = getUserByIdentity(identity); + logger.debug("Retrieved user {} for identity {}", new Object[]{user, identity}); + + Set<Group> groups = new HashSet<>(); + if (user != null) { + for (Group g : getGroups()) { + if (g.getUsers().contains(user.getIdentifier())) { + logger.debug("User {} belongs to group {}", new Object[]{user.getIdentity(), g.getName()}); + groups.add(g); + } + } + } + + if (groups.isEmpty()) { + logger.debug("User {} belongs to no groups", user); + } + + return new UserAndGroups() { + @Override + public User getUser() { + return user; + } + + @Override + public Set<Group> getGroups() { + return groups; + } + }; + } + + /** + * Called immediately after instance creation for implementers to perform additional setup + * + * @param initializationContext in which to initialize + */ + @Override + public void initialize(UserGroupProviderInitializationContext initializationContext) throws SecurityProviderCreationException { + } + + /** + * Called to configure the Authorizer. + * + * @param configurationContext at the time of configuration + * @throws SecurityProviderCreationException for any issues configuring the provider + */ + @Override + public void onConfigured(AuthorizerConfigurationContext configurationContext) throws SecurityProviderCreationException { + fixedDelay = getDelayProperty(configurationContext, REFRESH_DELAY_PROPERTY, "5 mins"); + + // Our next init step is to select the command set based on the operating system name: + ShellCommandsProvider commands = getCommandsProvider(); + + if (commands == null) { + commands = getCommandsProviderFromName(null); + setCommandsProvider(commands); + } + + // Our next init step is to run the system check from that command set to determine if the other commands + // will work on this host or not. + try { + ShellRunner.runShell(commands.getSystemCheck()); + } catch (final Exception e) { + logger.error("initialize exception: " + e + " system check command: " + commands.getSystemCheck()); + throw new SecurityProviderCreationException(SYS_CHECK_ERROR, e); + } + + // The next step is to add the user and group exclude regexes: + try { + excludeGroups = Pattern.compile(getProperty(configurationContext, EXCLUDE_GROUP_PROPERTY, "")); + excludeUsers = Pattern.compile(getProperty(configurationContext, EXCLUDE_USER_PROPERTY, "")); + } catch (final PatternSyntaxException e) { + throw new SecurityProviderCreationException(e); + } + + // With our command set selected, and our system check passed, we can pull in the users and groups: + refreshUsersAndGroups(); + + // And finally, our last init step is to fire off the refresh thread: + scheduler.scheduleWithFixedDelay(() -> { + try { + refreshUsersAndGroups(); + }catch (final Throwable t) { + logger.error("", t); + } + }, fixedDelay, fixedDelay, TimeUnit.MILLISECONDS); + + } + + private static ShellCommandsProvider getCommandsProviderFromName(String osName) { + if (osName == null) { + osName = System.getProperty("os.name"); + } + + ShellCommandsProvider commands; + if (osName.startsWith("Linux")) { + logger.debug("Selected Linux command set."); + commands = new NssShellCommands(); + } else if (osName.startsWith("Mac OS X")) { + logger.debug("Selected OSX command set."); + commands = new OsxShellCommands(); + } else { + throw new SecurityProviderCreationException(OS_TYPE_ERROR); + } + return commands; + } + + private String getProperty(AuthorizerConfigurationContext authContext, String propertyName, String defaultValue) { + final PropertyValue property = authContext.getProperty(propertyName); + final String value; + + if (property != null && property.isSet()) { + value = property.getValue(); + } else { + value = defaultValue; + } + return value; + + } + + private long getDelayProperty(AuthorizerConfigurationContext authContext, String propertyName, String defaultValue) { + final PropertyValue intervalProperty = authContext.getProperty(propertyName); + final String propertyValue; + final long syncInterval; + + if (intervalProperty.isSet()) { + propertyValue = intervalProperty.getValue(); + } else { + propertyValue = defaultValue; + } + + try { + syncInterval = Math.round(FormatUtils.getPreciseTimeDuration(propertyValue, TimeUnit.MILLISECONDS)); + } catch (final IllegalArgumentException ignored) { + throw new SecurityProviderCreationException(String.format("The %s '%s' is not a valid time interval.", propertyName, propertyValue)); + } + + if (syncInterval < MINIMUM_SYNC_INTERVAL_MILLISECONDS) { + throw new SecurityProviderCreationException(String.format("The %s '%s' is below the minimum value of '%d ms'", propertyName, propertyValue, MINIMUM_SYNC_INTERVAL_MILLISECONDS)); + } + return syncInterval; + } + + /** + * Called immediately before instance destruction for implementers to release resources. + * + * @throws SecurityProviderDestructionException If pre-destruction fails. + */ + @Override + public void preDestruction() throws SecurityProviderDestructionException { + try { + scheduler.shutdownNow(); + } catch (final Exception ignored) { + } + } + + public ShellCommandsProvider getCommandsProvider() { + return selectedShellCommands; + } + + public void setCommandsProvider(ShellCommandsProvider commandsProvider) { + selectedShellCommands = commandsProvider; + } + + /** + * Refresh a single user. + * + * @param command Shell command to read a single user. Pre-formatted by caller. + * @param description Shell command description. + */ + private void refreshOneUser(String command, String description) { + if (command != null) { + Map<String, User> idToUser = new HashMap<>(); + Map<String, User> usernameToUser = new HashMap<>(); + Map<String, User> gidToUser = new HashMap<>(); + List<String> userLines; + + try { + userLines = ShellRunner.runShell(command, description); + rebuildUsers(userLines, idToUser, usernameToUser, gidToUser); + } catch (final IOException ioexc) { + logger.error("refreshOneUser shell exception: " + ioexc); + } + + if (idToUser.size() > 0) { + synchronized (usersById) { + usersById.putAll(idToUser); + } + } + + if (usernameToUser.size() > 0) { + synchronized (usersByName) { + usersByName.putAll(usernameToUser); + } + } + } else { + logger.info("Get Single User not supported on this system."); + } + } + + /** + * Refresh a single group. + * + * @param command Shell command to read a single group. Pre-formatted by caller. + * @param description Shell command description. + */ + private void refreshOneGroup(String command, String description) { + if (command != null) { + Map<String, Group> gidToGroup = new HashMap<>(); + List<String> groupLines; + + try { + groupLines = ShellRunner.runShell(command, description); + rebuildGroups(groupLines, gidToGroup); + } catch (final IOException ioexc) { + logger.error("refreshOneGroup shell exception: " + ioexc); + } + + if (gidToGroup.size() > 0) { + synchronized (groupsById) { + groupsById.putAll(gidToGroup); + } + } + } else { + logger.info("Get Single Group not supported on this system."); + } + } + + /** + * This is our entry point for user and group refresh. This method runs the top-level + * `getUserList()` and `getGroupsList()` shell commands, then passes those results to the + * other methods for record parse, extract, and object construction. + */ + private void refreshUsersAndGroups() { + Map<String, User> uidToUser = new HashMap<>(); + Map<String, User> usernameToUser = new HashMap<>(); + Map<String, User> gidToUser = new HashMap<>(); + Map<String, Group> gidToGroup = new HashMap<>(); + + List<String> userLines; + List<String> groupLines; + + try { + userLines = ShellRunner.runShell(selectedShellCommands.getUsersList(), "Get Users List"); + groupLines = ShellRunner.runShell(selectedShellCommands.getGroupsList(), "Get Groups List"); + } catch (final IOException ioexc) { + logger.error("refreshUsersAndGroups shell exception: " + ioexc); + return; + } + + rebuildUsers(userLines, uidToUser, usernameToUser, gidToUser); + rebuildGroups(groupLines, gidToGroup); + reconcilePrimaryGroups(gidToUser, gidToGroup); + + synchronized (usersById) { + usersById.clear(); + usersById.putAll(uidToUser); + + if (logger.isTraceEnabled()) { + logger.trace("=== Users by id..."); + Set<User> sortedUsers = new TreeSet<>(Comparator.comparing(User::getIdentity)); + sortedUsers.addAll(usersById.values()); + sortedUsers.forEach(u -> logger.trace("=== " + u.toString())); + } + } + + synchronized (usersByName) { + usersByName.clear(); + usersByName.putAll(usernameToUser); + logger.debug("users now size: " + usersByName.size()); + } + + synchronized (groupsById) { + groupsById.clear(); + groupsById.putAll(gidToGroup); + logger.debug("groups now size: " + groupsById.size()); + + if (logger.isTraceEnabled()) { + logger.trace("=== Groups by id..."); + Set<Group> sortedGroups = new TreeSet<>(Comparator.comparing(Group::getName)); + sortedGroups.addAll(groupsById.values()); + sortedGroups.forEach(g -> logger.trace("=== " + g.toString())); + } + } + } + + /** + * This method parses the output of the `getUsersList()` shell command, where we expect the output + * to look like `user-name:user-id:primary-group-id`. + * <p> + * This method splits each output line on the ":" and attempts to build a User object + * from the resulting name, uid, and primary gid. Unusable records are logged. + */ + private void rebuildUsers(List<String> userLines, Map<String, User> idToUser, Map<String, User> usernameToUser, Map<String, User> gidToUser) { + userLines.forEach(line -> { + logger.trace("Processing user: {}", new Object[]{line}); + + String[] record = line.split(":"); + if (record.length > 2) { + String userIdentity = record[0], userIdentifier = record[1], primaryGroupIdentifier = record[2]; + + if (!StringUtils.isBlank(userIdentifier) && !StringUtils.isBlank(userIdentity) && !excludeUsers.matcher(userIdentity).matches()) { + User user = new User.Builder() + .identity(userIdentity) + .identifierGenerateFromSeed(getUserIdentifierSeed(userIdentity)) + .build(); + idToUser.put(user.getIdentifier(), user); + usernameToUser.put(userIdentity, user); + logger.debug("Refreshed user {}", new Object[]{user}); + + if (!StringUtils.isBlank(primaryGroupIdentifier)) { + // create a temporary group to deterministically generate the group id and associate this user + Group group = new Group.Builder() + .name(primaryGroupIdentifier) + .identifierGenerateFromSeed(getGroupIdentifierSeed(primaryGroupIdentifier)) + .build(); + gidToUser.put(group.getIdentifier(), user); + logger.debug("Associated primary group {} with user {}", new Object[]{group.getIdentifier(), userIdentity}); + } else { + logger.warn("Null or empty primary group id for: " + userIdentity); + } + + } else { + logger.warn("Null, empty, or skipped user name: " + userIdentity + " or id: " + userIdentifier); + } + } else { + logger.warn("Unexpected record format. Expected 3 or more colon separated values per line."); + } + }); + } + + /** + * This method parses the output of the `getGroupsList()` shell command, where we expect the output + * to look like `group-name:group-id`. + * <p> + * This method splits each output line on the ":" and attempts to build a Group object + * from the resulting name and gid. Unusable records are logged. + * <p> + * This command also runs the `getGroupMembers(username)` command once per group. The expected output + * of that command should look like `group-name-1,group-name-2`. + */ + private void rebuildGroups(List<String> groupLines, Map<String, Group> groupsById) { + groupLines.forEach(line -> { + logger.trace("Processing group: {}", new Object[]{line}); + + String[] record = line.split(":"); + if (record.length > 1) { + Set<String> users = new HashSet<>(); + String groupName = record[0], groupIdentifier = record[1]; + + try { + String groupMembersCommand = selectedShellCommands.getGroupMembers(groupName); + List<String> memberLines = ShellRunner.runShell(groupMembersCommand); + // Use the first line only, and log if the line count isn't exactly one: + if (!memberLines.isEmpty()) { + String memberLine = memberLines.get(0); + if (!StringUtils.isBlank(memberLine)) { + String[] members = memberLine.split(","); + for (String userIdentity : members) { + if (!StringUtils.isBlank(userIdentity)) { + User tempUser = new User.Builder() + .identity(userIdentity) + .identifierGenerateFromSeed(getUserIdentifierSeed(userIdentity)) + .build(); + users.add(tempUser.getIdentifier()); + logger.debug("Added temp user {} for group {}", new Object[]{tempUser, groupName}); + } + } + } else { + logger.debug("list membership returned no members"); + } + } else { + logger.debug("list membership returned zero lines."); + } + if (memberLines.size() > 1) { + logger.error("list membership returned too many lines, only used the first."); + } + + } catch (final IOException ioexc) { + logger.error("list membership shell exception: " + ioexc); + } + + if (!StringUtils.isBlank(groupIdentifier) && !StringUtils.isBlank(groupName) && !excludeGroups.matcher(groupName).matches()) { + Group group = new Group.Builder() + .name(groupName) + .identifierGenerateFromSeed(getGroupIdentifierSeed(groupIdentifier)) + .addUsers(users) + .build(); + groupsById.put(group.getIdentifier(), group); + logger.debug("Refreshed group {}", new Object[] {group}); + } else { + logger.warn("Null, empty, or skipped group name: " + groupName + " or id: " + groupIdentifier); + } + } else { + logger.warn("Unexpected record format. Expected 1 or more comma separated values."); + } + }); + } + + /** + * This method parses the output of the `getGroupsList()` shell command, where we expect the output + * to look like `group-name:group-id`. + * <p> + * This method splits each output line on the ":" and attempts to build a Group object + * from the resulting name and gid. + */ + private void reconcilePrimaryGroups(Map<String, User> uidToUser, Map<String, Group> gidToGroup) { + uidToUser.forEach((primaryGid, primaryUser) -> { + Group primaryGroup = gidToGroup.get(primaryGid); + + if (primaryGroup == null) { + logger.warn("Primary group {} not found for {}", new Object[]{primaryGid, primaryUser.getIdentity()}); + } else if (!excludeGroups.matcher(primaryGroup.getName()).matches()) { + Set<String> groupUsers = primaryGroup.getUsers(); + if (!groupUsers.contains(primaryUser.getIdentifier())) { + Set<String> updatedUserIdentifiers = new HashSet<>(groupUsers); + updatedUserIdentifiers.add(primaryUser.getIdentifier()); + + Group updatedGroup = new Group.Builder() + .identifier(primaryGroup.getIdentifier()) + .name(primaryGroup.getName()) + .addUsers(updatedUserIdentifiers) + .build(); + gidToGroup.put(updatedGroup.getIdentifier(), updatedGroup); + logger.debug("Added user {} to primary group {}", new Object[]{primaryUser, updatedGroup}); + } else { + logger.debug("Primary group {} already contains user {}", new Object[]{primaryGroup, primaryUser}); + } + } else { + logger.debug("Primary group {} excluded from matcher for {}", new Object[]{primaryGroup.getName(), primaryUser.getIdentity()}); + } + }); + } + + private String getUserIdentifierSeed(final String userIdentifier) { + return userIdentifier + "-user"; + } + + private String getGroupIdentifierSeed(final String groupIdentifier) { + return groupIdentifier + "-group"; + } + + + /** + * @return The fixed refresh delay. + */ + public long getRefreshDelay() { + return fixedDelay; + } + + /** + * Testing concession for clearing the internal caches. + */ + void clearCaches() { + synchronized (usersById) { + usersById.clear(); + } + + synchronized (usersByName) { + usersByName.clear(); + } + + synchronized (groupsById) { + groupsById.clear(); + } + } + + /** + * @return The size of the internal user cache. + */ + public int userCacheSize() { + return usersById.size(); + } + + /** + * @return The size of the internal group cache. + */ + public int groupCacheSize() { + return groupsById.size(); + } +} diff --git a/nifi-registry-core/nifi-registry-framework/src/main/resources/META-INF/services/org.apache.nifi.registry.security.authorization.UserGroupProvider b/nifi-registry-core/nifi-registry-framework/src/main/resources/META-INF/services/org.apache.nifi.registry.security.authorization.UserGroupProvider index ee28c07..a4ac129 100644 --- a/nifi-registry-core/nifi-registry-framework/src/main/resources/META-INF/services/org.apache.nifi.registry.security.authorization.UserGroupProvider +++ b/nifi-registry-core/nifi-registry-framework/src/main/resources/META-INF/services/org.apache.nifi.registry.security.authorization.UserGroupProvider @@ -15,4 +15,5 @@ org.apache.nifi.registry.security.authorization.CompositeUserGroupProvider org.apache.nifi.registry.security.authorization.CompositeConfigurableUserGroupProvider org.apache.nifi.registry.security.authorization.file.FileUserGroupProvider -org.apache.nifi.registry.security.ldap.tenants.LdapUserGroupProvider \ No newline at end of file +org.apache.nifi.registry.security.ldap.tenants.LdapUserGroupProvider +org.apache.nifi.registry.security.authorization.shell.ShellUserGroupProvider \ No newline at end of file diff --git a/nifi-registry-core/nifi-registry-resources/src/main/resources/conf/authorizers.xml b/nifi-registry-core/nifi-registry-resources/src/main/resources/conf/authorizers.xml index 9f63754..38a6ee8 100644 --- a/nifi-registry-core/nifi-registry-resources/src/main/resources/conf/authorizers.xml +++ b/nifi-registry-core/nifi-registry-resources/src/main/resources/conf/authorizers.xml @@ -166,6 +166,25 @@ To enable the ldap-user-group-provider remove 2 lines. This is 2 of 2. --> <!-- + The ShellUserGroupProvider provides support for retrieving users and groups by way of shell commands + on systems that support `sh`. Implementations available for Linux and Mac OS, and are selected by the + provider based on the system property `os.name`. + + 'Refresh Delay' - duration to wait between subsequent refreshes. Default is '5 mins'. + 'Exclude Groups' - regular expression used to exclude groups. Default is '', which means no groups are excluded. + 'Exclude Users' - regular expression used to exclude users. Default is '', which means no users are excluded. + --> + <!-- To enable the shell-user-group-provider remove 2 lines. This is 1 of 2. + <userGroupProvider> + <identifier>shell-user-group-provider</identifier> + <class>org.apache.nifi.registry.security.authorization.shell.ShellUserGroupProvider</class> + <property name="Refresh Delay">5 mins</property> + <property name="Exclude Groups"></property> + <property name="Exclude Users"></property> + </userGroupProvider> + To enable the shell-user-group-provider remove 2 lines. This is 2 of 2. --> + + <!-- The CompositeUserGroupProvider will provide support for retrieving users and groups from multiple sources. - User Group Provider [unique key] - The identifier of user group providers to load from. The name of diff --git a/nifi-registry-core/nifi-registry-utils/src/main/java/org/apache/nifi/registry/util/FormatUtils.java b/nifi-registry-core/nifi-registry-utils/src/main/java/org/apache/nifi/registry/util/FormatUtils.java index c1e353d..aa207c5 100644 --- a/nifi-registry-core/nifi-registry-utils/src/main/java/org/apache/nifi/registry/util/FormatUtils.java +++ b/nifi-registry-core/nifi-registry-utils/src/main/java/org/apache/nifi/registry/util/FormatUtils.java @@ -17,12 +17,13 @@ package org.apache.nifi.registry.util; import java.text.NumberFormat; +import java.util.Arrays; +import java.util.List; import java.util.concurrent.TimeUnit; import java.util.regex.Matcher; import java.util.regex.Pattern; public class FormatUtils { - private static final String UNION = "|"; // for Data Sizes @@ -41,8 +42,9 @@ public class FormatUtils { private static final String WEEKS = join(UNION, "w", "wk", "wks", "week", "weeks"); private static final String VALID_TIME_UNITS = join(UNION, NANOS, MILLIS, SECS, MINS, HOURS, DAYS, WEEKS); - public static final String TIME_DURATION_REGEX = "(\\d+)\\s*(" + VALID_TIME_UNITS + ")"; + public static final String TIME_DURATION_REGEX = "([\\d.]+)\\s*(" + VALID_TIME_UNITS + ")"; public static final Pattern TIME_DURATION_PATTERN = Pattern.compile(TIME_DURATION_REGEX); + private static final List<Long> TIME_UNIT_MULTIPLIERS = Arrays.asList(1000L, 1000L, 1000L, 60L, 60L, 24L); /** * Formats the specified count by adding commas. @@ -58,7 +60,7 @@ public class FormatUtils { * Formats the specified duration in 'mm:ss.SSS' format. * * @param sourceDuration the duration to format - * @param sourceUnit the unit to interpret the duration + * @param sourceUnit the unit to interpret the duration * @return representation of the given time data in minutes/seconds */ public static String formatMinutesSeconds(final long sourceDuration, final TimeUnit sourceUnit) { @@ -79,7 +81,7 @@ public class FormatUtils { * Formats the specified duration in 'HH:mm:ss.SSS' format. * * @param sourceDuration the duration to format - * @param sourceUnit the unit to interpret the duration + * @param sourceUnit the unit to interpret the duration * @return representation of the given time data in hours/minutes/seconds */ public static String formatHoursMinutesSeconds(final long sourceDuration, final TimeUnit sourceUnit) { @@ -139,65 +141,230 @@ public class FormatUtils { return format.format(dataSize) + " bytes"; } + /** + * Returns a time duration in the requested {@link TimeUnit} after parsing the {@code String} + * input. If the resulting value is a decimal (i.e. + * {@code 25 hours -> TimeUnit.DAYS = 1.04}), the value is rounded. + * + * @param value the raw String input (i.e. "28 minutes") + * @param desiredUnit the requested output {@link TimeUnit} + * @return the whole number value of this duration in the requested units + * @deprecated As of Apache NiFi 1.9.0, because this method only returns whole numbers, use {@link #getPreciseTimeDuration(String, TimeUnit)} when possible. + */ + @Deprecated public static long getTimeDuration(final String value, final TimeUnit desiredUnit) { + return Math.round(getPreciseTimeDuration(value, desiredUnit)); + } + + /** + * Returns the parsed and converted input in the requested units. + * <p> + * If the value is {@code 0 <= x < 1} in the provided units, the units will first be converted to a smaller unit to get a value >= 1 (i.e. 0.5 seconds -> 500 milliseconds). + * This is because the underlying unit conversion cannot handle decimal values. + * <p> + * If the value is {@code x >= 1} but x is not a whole number, the units will first be converted to a smaller unit to attempt to get a whole number value (i.e. 1.5 seconds -> 1500 milliseconds). + * <p> + * If the value is {@code x < 1000} and the units are {@code TimeUnit.NANOSECONDS}, the result will be a whole number of nanoseconds, rounded (i.e. 123.4 ns -> 123 ns). + * <p> + * This method handles decimal values over {@code 1 ns}, but {@code < 1 ns} will return {@code 0} in any other unit. + * <p> + * Examples: + * <p> + * "10 seconds", {@code TimeUnit.MILLISECONDS} -> 10_000.0 + * "0.010 s", {@code TimeUnit.MILLISECONDS} -> 10.0 + * "0.010 s", {@code TimeUnit.SECONDS} -> 0.010 + * "0.010 ns", {@code TimeUnit.NANOSECONDS} -> 1 + * "0.010 ns", {@code TimeUnit.MICROSECONDS} -> 0 + * + * @param value the {@code String} input + * @param desiredUnit the desired output {@link TimeUnit} + * @return the parsed and converted amount (without a unit) + */ + public static double getPreciseTimeDuration(final String value, final TimeUnit desiredUnit) { final Matcher matcher = TIME_DURATION_PATTERN.matcher(value.toLowerCase()); if (!matcher.matches()) { - throw new IllegalArgumentException("Value '" + value + "' is not a valid Time Duration"); + throw new IllegalArgumentException("Value '" + value + "' is not a valid time duration"); } final String duration = matcher.group(1); final String units = matcher.group(2); - TimeUnit specifiedTimeUnit = null; - switch (units.toLowerCase()) { + + double durationVal = Double.parseDouble(duration); + TimeUnit specifiedTimeUnit; + + // The TimeUnit enum doesn't have a value for WEEKS, so handle this case independently + if (isWeek(units)) { + specifiedTimeUnit = TimeUnit.DAYS; + durationVal *= 7; + } else { + specifiedTimeUnit = determineTimeUnit(units); + } + + // The units are now guaranteed to be in DAYS or smaller + long durationLong; + if (durationVal == Math.rint(durationVal)) { + durationLong = Math.round(durationVal); + } else { + // Try reducing the size of the units to make the input a long + List wholeResults = makeWholeNumberTime(durationVal, specifiedTimeUnit); + durationLong = (long) wholeResults.get(0); + specifiedTimeUnit = (TimeUnit) wholeResults.get(1); + } + + return desiredUnit.convert(durationLong, specifiedTimeUnit); + } + + /** + * Converts the provided time duration value to one that can be represented as a whole number. + * Returns a {@code List} containing the new value as a {@code long} at index 0 and the + * {@link TimeUnit} at index 1. If the incoming value is already whole, it is returned as is. + * If the incoming value cannot be made whole, a whole approximation is returned. For values + * {@code >= 1 TimeUnit.NANOSECONDS}, the value is rounded (i.e. 123.4 ns -> 123 ns). + * For values {@code < 1 TimeUnit.NANOSECONDS}, the constant [1L, {@code TimeUnit.NANOSECONDS}] is returned as the smallest measurable unit of time. + * <p> + * Examples: + * <p> + * 1, {@code TimeUnit.SECONDS} -> [1, {@code TimeUnit.SECONDS}] + * 1.1, {@code TimeUnit.SECONDS} -> [1100, {@code TimeUnit.MILLISECONDS}] + * 0.1, {@code TimeUnit.SECONDS} -> [100, {@code TimeUnit.MILLISECONDS}] + * 0.1, {@code TimeUnit.NANOSECONDS} -> [1, {@code TimeUnit.NANOSECONDS}] + * + * @param decimal the time duration as a decimal + * @param timeUnit the current time unit + * @return the time duration as a whole number ({@code long}) and the smaller time unit used + */ + protected static List<Object> makeWholeNumberTime(double decimal, TimeUnit timeUnit) { + // If the value is already a whole number, return it and the current time unit + if (decimal == Math.rint(decimal)) { + return Arrays.asList(new Object[]{(long) decimal, timeUnit}); + } else if (TimeUnit.NANOSECONDS == timeUnit) { + // The time unit is as small as possible + if (decimal < 1.0) { + decimal = 1; + } else { + decimal = Math.rint(decimal); + } + return Arrays.asList(new Object[]{(long) decimal, timeUnit}); + } else { + // Determine the next time unit and the respective multiplier + TimeUnit smallerTimeUnit = getSmallerTimeUnit(timeUnit); + long multiplier = calculateMultiplier(timeUnit, smallerTimeUnit); + + // Recurse with the original number converted to the smaller unit + return makeWholeNumberTime(decimal * multiplier, smallerTimeUnit); + } + } + + /** + * Returns the numerical multiplier to convert a value from {@code originalTimeUnit} to + * {@code newTimeUnit} (i.e. for {@code TimeUnit.DAYS -> TimeUnit.MINUTES} would return + * 24 * 60 = 1440). If the original and new units are the same, returns 1. If the new unit + * is larger than the original (i.e. the result would be less than 1), throws an + * {@link IllegalArgumentException}. + * + * @param originalTimeUnit the source time unit + * @param newTimeUnit the destination time unit + * @return the numerical multiplier between the units + */ + protected static long calculateMultiplier(TimeUnit originalTimeUnit, TimeUnit newTimeUnit) { + if (originalTimeUnit == newTimeUnit) { + return 1; + } else if (originalTimeUnit.ordinal() < newTimeUnit.ordinal()) { + throw new IllegalArgumentException("The original time unit '" + originalTimeUnit + "' must be larger than the new time unit '" + newTimeUnit + "'"); + } else { + int originalOrd = originalTimeUnit.ordinal(); + int newOrd = newTimeUnit.ordinal(); + + List<Long> unitMultipliers = TIME_UNIT_MULTIPLIERS.subList(newOrd, originalOrd); + return unitMultipliers.stream().reduce(1L, (a, b) -> (long) a * b); + } + } + + /** + * Returns the next smallest {@link TimeUnit} (i.e. {@code TimeUnit.DAYS -> TimeUnit.HOURS}). + * If the parameter is {@code null} or {@code TimeUnit.NANOSECONDS}, an + * {@link IllegalArgumentException} is thrown because there is no valid smaller TimeUnit. + * + * @param originalUnit the TimeUnit + * @return the next smaller TimeUnit + */ + protected static TimeUnit getSmallerTimeUnit(TimeUnit originalUnit) { + if (originalUnit == null || TimeUnit.NANOSECONDS == originalUnit) { + throw new IllegalArgumentException("Cannot determine a smaller time unit than '" + originalUnit + "'"); + } else { + return TimeUnit.values()[originalUnit.ordinal() - 1]; + } + } + + /** + * Returns {@code true} if this raw unit {@code String} is parsed as representing "weeks", which does not have a value in the {@link TimeUnit} enum. + * + * @param rawUnit the String containing the desired unit + * @return true if the unit is "weeks"; false otherwise + */ + protected static boolean isWeek(final String rawUnit) { + switch (rawUnit) { + case "w": + case "wk": + case "wks": + case "week": + case "weeks": + return true; + default: + return false; + } + } + + /** + * Returns the {@link TimeUnit} enum that maps to the provided raw {@code String} input. The + * highest time unit is {@code TimeUnit.DAYS}. Any input that cannot be parsed will result in + * an {@link IllegalArgumentException}. + * + * @param rawUnit the String to parse + * @return the TimeUnit + */ + protected static TimeUnit determineTimeUnit(String rawUnit) { + switch (rawUnit.toLowerCase()) { case "ns": case "nano": case "nanos": case "nanoseconds": - specifiedTimeUnit = TimeUnit.NANOSECONDS; - break; + return TimeUnit.NANOSECONDS; + case "µs": + case "micro": + case "micros": + case "microseconds": + return TimeUnit.MICROSECONDS; case "ms": case "milli": case "millis": case "milliseconds": - specifiedTimeUnit = TimeUnit.MILLISECONDS; - break; + return TimeUnit.MILLISECONDS; case "s": case "sec": case "secs": case "second": case "seconds": - specifiedTimeUnit = TimeUnit.SECONDS; - break; + return TimeUnit.SECONDS; case "m": case "min": case "mins": case "minute": case "minutes": - specifiedTimeUnit = TimeUnit.MINUTES; - break; + return TimeUnit.MINUTES; case "h": case "hr": case "hrs": case "hour": case "hours": - specifiedTimeUnit = TimeUnit.HOURS; - break; + return TimeUnit.HOURS; case "d": case "day": case "days": - specifiedTimeUnit = TimeUnit.DAYS; - break; - case "w": - case "wk": - case "wks": - case "week": - case "weeks": - final long durationVal = Long.parseLong(duration); - return desiredUnit.convert(durationVal, TimeUnit.DAYS)*7; + return TimeUnit.DAYS; + default: + throw new IllegalArgumentException("Could not parse '" + rawUnit + "' to TimeUnit"); } - - final long durationVal = Long.parseLong(duration); - return desiredUnit.convert(durationVal, specifiedTimeUnit); } public static String formatUtilization(final double utilization) { @@ -225,15 +392,15 @@ public class FormatUtils { * 3 seconds, 8 millis, 3 nanos - if includeTotalNanos = false, * 3 seconds, 8 millis, 3 nanos (3008000003 nanos) - if includeTotalNanos = true * - * @param nanos the number of nanoseconds to format + * @param nanos the number of nanoseconds to format * @param includeTotalNanos whether or not to include the total number of nanoseconds in parentheses in the returned value * @return a human-readable String that is a formatted representation of the given number of nanoseconds. */ public static String formatNanos(final long nanos, final boolean includeTotalNanos) { final StringBuilder sb = new StringBuilder(); - final long seconds = nanos > 1000000000L ? nanos / 1000000000L : 0L; - long millis = nanos > 1000000L ? nanos / 1000000L : 0L; + final long seconds = nanos >= 1000000000L ? nanos / 1000000000L : 0L; + long millis = nanos >= 1000000L ? nanos / 1000000L : 0L; final long nanosLeft = nanos % 1000000L; if (seconds > 0) { @@ -258,4 +425,5 @@ public class FormatUtils { return sb.toString(); } + }
