fapifta commented on code in PR #4194: URL: https://github.com/apache/ozone/pull/4194#discussion_r1088139365
########## hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/LocalSecretKeyStore.java: ########## @@ -0,0 +1,227 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.security.symmetric; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.MappingIterator; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectReader; +import com.fasterxml.jackson.databind.SequenceWriter; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; +import com.fasterxml.jackson.databind.ser.std.StdSerializer; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.crypto.SecretKey; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.PosixFilePermission; +import java.time.Instant; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.UUID; + +import static com.google.common.collect.Sets.newHashSet; +import static java.nio.file.attribute.PosixFilePermission.OWNER_READ; +import static java.nio.file.attribute.PosixFilePermission.OWNER_WRITE; +import static java.util.stream.Collectors.toList; + +/** + * A {@link SecretKeyStore} that saves and loads SecretKeys from/to a + * JSON file on local file system. + */ +public class LocalSecretKeyStore implements SecretKeyStore { + private static final Logger LOG = + LoggerFactory.getLogger(LocalSecretKeyStore.class); + + private final Path secretKeysFile; + private final ObjectMapper mapper; + + public LocalSecretKeyStore(Path secretKeysFile) { + this.secretKeysFile = secretKeysFile; + this.mapper = new ObjectMapper() + .registerModule(new JavaTimeModule()) + .configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false); + } + + @Override + public synchronized List<ManagedSecretKey> load() { + if (!secretKeysFile.toFile().exists()) { + return Collections.emptyList(); + } + + ObjectReader reader = mapper.readerFor(ManagedSecretKeyDto.class); + try (MappingIterator<ManagedSecretKeyDto> iterator = + reader.readValues(secretKeysFile.toFile())) { + List<ManagedSecretKeyDto> dtos = iterator.readAll(); + List<ManagedSecretKey> result = dtos.stream() + .map(ManagedSecretKeyDto::toObject) + .collect(toList()); + LOG.info("Loaded {} from {}", result, secretKeysFile); + return result; + } catch (IOException e) { + throw new IllegalStateException("Error reading SecretKeys from " + + secretKeysFile, e); + } + } + + @Override + public synchronized void save(Collection<ManagedSecretKey> secretKeys) { + setFileOwnerPermissions(secretKeysFile); + + List<ManagedSecretKeyDto> dtos = secretKeys.stream() + .map(ManagedSecretKeyDto::new) + .collect(toList()); + + try (SequenceWriter writer = + mapper.writer().writeValues(secretKeysFile.toFile())) { + writer.init(true); + for (ManagedSecretKeyDto dto : dtos) { Review Comment: I know it is nitpicking, but we can use dtos.foreach here, to harmonize with the rest of the code that uses functionals. ########## hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/SecretKeyManager.java: ########## @@ -0,0 +1,164 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.security.symmetric; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.crypto.KeyGenerator; +import java.security.NoSuchAlgorithmException; +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.TimeoutException; + +import static java.time.Duration.between; +import static java.util.Comparator.comparing; +import static java.util.Objects.requireNonNull; +import static java.util.stream.Collectors.toList; + +/** + * This component manages symmetric SecretKey life-cycle, including generation, + * rotation and destruction. + */ +public class SecretKeyManager { + private static final Logger LOG = + LoggerFactory.getLogger(SecretKeyManager.class); + + private final SecretKeyState state; + private boolean pendingInititializedState = false; + private final Duration rotationDuration; + private final Duration validityDuration; + private final SecretKeyStore keyStore; + + private final KeyGenerator keyGenerator; + + public SecretKeyManager(SecretKeyState state, + SecretKeyStore keyStore, + Duration rotationDuration, + Duration validityDuration, + String algorithm) { + this.state = requireNonNull(state); + this.rotationDuration = requireNonNull(rotationDuration); + this.validityDuration = requireNonNull(validityDuration); + this.keyStore = requireNonNull(keyStore); + this.keyGenerator = createKeyGenerator(algorithm); + } + + public SecretKeyManager(SecretKeyState state, + SecretKeyStore keyStore, + SecretKeyConfig config) { + this(state, keyStore, config.getRotateDuration(), + config.getExpiryDuration(), config.getAlgorithm()); + } + + /** + * Initialize the state from by loading SecretKeys from local file, or + * generate new keys if the file doesn't exist. + * + * @throws TimeoutException can possibly occur when replicating the state. + */ + public synchronized boolean initialize() { + if (state.getCurrentKey() != null) { + return false; + } + + List<ManagedSecretKey> sortedKeys = keyStore.load() + .stream() + .filter(x -> !x.isExpired()) + .sorted(comparing(ManagedSecretKey::getCreationTime)) + .collect(toList()); + + ManagedSecretKey currentKey; + if (sortedKeys.isEmpty()) { + // First start, generate new key as the current key. + currentKey = generateSecretKey(); + sortedKeys.add(currentKey); + LOG.info("No keys is loaded, generated new key: {}", currentKey); Review Comment: nit: are ########## hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/package-info.java: ########## @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Encapsulate classes dealing with symmetric key algorithms. + */ +package org.apache.hadoop.hdds.security.symmetric; Review Comment: At the end of the day, can we add a more comprehensive doc on the package, to summarize what we have here in more detail? Best would be to give at least some detail why the package exists, what is the fundamental logic behind storing things, what are the config options that are affecting how the code here behaves, and such? I also would love to have a readme or some kind of a doc linked that summarizes the design. As a first step the design doc might work, but as after implementation we know more about what we ended up with and why, we might write down the original options we discussed, and then the final architecture and the decisions that led to the result. ########## hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/symmetric/LocalSecretKeyStore.java: ########## @@ -0,0 +1,227 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.security.symmetric; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.MappingIterator; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectReader; +import com.fasterxml.jackson.databind.SequenceWriter; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; +import com.fasterxml.jackson.databind.ser.std.StdSerializer; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.crypto.SecretKey; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.PosixFilePermission; +import java.time.Instant; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.UUID; + +import static com.google.common.collect.Sets.newHashSet; +import static java.nio.file.attribute.PosixFilePermission.OWNER_READ; +import static java.nio.file.attribute.PosixFilePermission.OWNER_WRITE; +import static java.util.stream.Collectors.toList; + +/** + * A {@link SecretKeyStore} that saves and loads SecretKeys from/to a + * JSON file on local file system. + */ +public class LocalSecretKeyStore implements SecretKeyStore { + private static final Logger LOG = + LoggerFactory.getLogger(LocalSecretKeyStore.class); + + private final Path secretKeysFile; + private final ObjectMapper mapper; + + public LocalSecretKeyStore(Path secretKeysFile) { + this.secretKeysFile = secretKeysFile; + this.mapper = new ObjectMapper() + .registerModule(new JavaTimeModule()) + .configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false); + } + + @Override + public synchronized List<ManagedSecretKey> load() { + if (!secretKeysFile.toFile().exists()) { + return Collections.emptyList(); + } + + ObjectReader reader = mapper.readerFor(ManagedSecretKeyDto.class); + try (MappingIterator<ManagedSecretKeyDto> iterator = + reader.readValues(secretKeysFile.toFile())) { + List<ManagedSecretKeyDto> dtos = iterator.readAll(); + List<ManagedSecretKey> result = dtos.stream() + .map(ManagedSecretKeyDto::toObject) + .collect(toList()); + LOG.info("Loaded {} from {}", result, secretKeysFile); + return result; + } catch (IOException e) { + throw new IllegalStateException("Error reading SecretKeys from " + + secretKeysFile, e); + } + } + + @Override + public synchronized void save(Collection<ManagedSecretKey> secretKeys) { + setFileOwnerPermissions(secretKeysFile); + + List<ManagedSecretKeyDto> dtos = secretKeys.stream() + .map(ManagedSecretKeyDto::new) + .collect(toList()); + + try (SequenceWriter writer = + mapper.writer().writeValues(secretKeysFile.toFile())) { + writer.init(true); + for (ManagedSecretKeyDto dto : dtos) { + writer.write(dto); + } + } catch (IOException e) { + throw new IllegalStateException("Error saving SecretKeys to file " + + secretKeysFile, e); + } + LOG.info("Saved {} to file {}", secretKeys, secretKeysFile); + } + + private void setFileOwnerPermissions(Path path) { Review Comment: Again a little bit of nitpicking, but will we use this method with any other path then the secretKeysFile? If not, we can omit the parameter. ########## hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/security/SerializableCodec.java: ########## @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.scm.security; Review Comment: Shouldn't this be in the o.a.h.hdds.scm.ha.io.codec package as the other codec implementations? ########## hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/symmetric/LocalSecretKeyStoreTest.java: ########## @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.security.symmetric; + +import com.google.common.collect.ImmutableList; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import javax.crypto.KeyGenerator; +import javax.crypto.SecretKey; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.PosixFilePermission; +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.Set; +import java.util.UUID; +import java.util.stream.Stream; + +import static com.google.common.collect.Lists.newArrayList; +import static com.google.common.collect.Sets.newHashSet; +import static java.nio.file.attribute.PosixFilePermission.OWNER_READ; +import static java.nio.file.attribute.PosixFilePermission.OWNER_WRITE; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Test cases for {@link LocalSecretKeyStore}. + */ +public class LocalSecretKeyStoreTest { + private SecretKeyStore secretKeyStore; + private Path testSecretFile; + + @BeforeEach + private void setup() throws Exception { + testSecretFile = Files.createTempFile("key-strore-test", ".json"); + secretKeyStore = new LocalSecretKeyStore(testSecretFile); + } + + public static Stream<Arguments> saveAndLoadTestCases() throws Exception { + return Stream.of( + // empty + Arguments.of(ImmutableList.of()), + // single secret keys. + Arguments.of(newArrayList( + generateKey("HmacSHA256") + )), + // multiple secret keys. + Arguments.of(newArrayList( + generateKey("HmacSHA1"), + generateKey("HmacSHA256") + )) + ); + } + + @ParameterizedTest + @MethodSource("saveAndLoadTestCases") + public void testSaveAndLoad(List<ManagedSecretKey> keys) throws IOException { Review Comment: Here we are testing whether what we save we can load, and the two will be identical. It would be nice to have a file saved with the patch, that serves as an etalon, so we have a test case that will check if we can load the same data with the current code that we saved with a previous version of the code, with that we can catch cases where we make an incompatible change in the serialization. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
