dimas-b commented on code in PR #2728: URL: https://github.com/apache/polaris/pull/2728#discussion_r2395190247
########## persistence/nosql/nodes/impl/src/main/java/org/apache/polaris/nodes/impl/NodeManagementImpl.java: ########## @@ -0,0 +1,630 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.nodes.impl; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; +import static java.lang.Integer.bitCount; + +import com.google.common.annotations.VisibleForTesting; +import jakarta.annotation.Nonnull; +import jakarta.annotation.PreDestroy; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; +import java.net.InterfaceAddress; +import java.net.NetworkInterface; +import java.net.SocketException; +import java.time.Duration; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.BitSet; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; +import java.util.function.IntSupplier; +import org.apache.polaris.ids.api.IdGenerator; +import org.apache.polaris.ids.api.IdGeneratorSpec; +import org.apache.polaris.ids.api.ImmutableIdGeneratorSpec; +import org.apache.polaris.ids.api.MonotonicClock; +import org.apache.polaris.ids.spi.IdGeneratorFactory; +import org.apache.polaris.ids.spi.IdGeneratorSource; +import org.apache.polaris.nodes.api.ImmutableNode; +import org.apache.polaris.nodes.api.Node; +import org.apache.polaris.nodes.api.NodeLease; +import org.apache.polaris.nodes.api.NodeManagement; +import org.apache.polaris.nodes.api.NodeManagementConfig; +import org.apache.polaris.nodes.spi.ImmutableBuildableNodeManagementState; +import org.apache.polaris.nodes.spi.ImmutableNodeState; +import org.apache.polaris.nodes.spi.NodeState; +import org.apache.polaris.nodes.spi.NodeStore; +import org.apache.polaris.nodes.spi.NodeStoreFactory; +import org.apache.polaris.nosql.async.AsyncExec; +import org.apache.polaris.nosql.async.Cancelable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@ApplicationScoped +class NodeManagementImpl implements NodeManagement { + private static final Logger LOGGER = LoggerFactory.getLogger(NodeManagementImpl.class); + + // should be a power of 2 + private static final int CHECK_BATCH_SIZE = 16; + static final Duration RESCHEDULE_AFTER_FAILURE = Duration.ofSeconds(10); + static final Duration RESCHEDULE_UNTIL_EXPIRATION = Duration.ofMinutes(1); + static final Duration RENEWAL_MIN_LEFT_FOR_RENEWAL = Duration.ofSeconds(30); + private final NodeStore nodeStore; + private final NodeManagementConfig config; + private final MonotonicClock clock; + private final int numNodeIds; + private final IdGeneratorFactory<?> idGenFactory; + private final IdGeneratorSpec idGenSpec; + + private final Set<NodeLeaseImpl> registeredLeases = ConcurrentHashMap.newKeySet(); + private final IdGenerator systemIdGen; + private final AsyncExec scheduler; + + private volatile boolean closed; + + @SuppressWarnings("CdiInjectionPointsInspection") + @Inject + NodeManagementImpl( + NodeManagementConfig config, + MonotonicClock clock, + NodeStoreFactory nodeStoreFactory, + AsyncExec scheduler) { + var activePeriod = config.leaseDuration().minus(config.renewalPeriod()); + this.numNodeIds = config.numNodes(); + checkArgs( + () -> + checkArgument( + config.leaseDuration().compareTo(Duration.ofMinutes(15)) > 0, + "leaseDuration must be at least 15 minutes"), + () -> + checkArgument( + activePeriod.isPositive(), "leaseDuration must be greater than renewalPeriod"), + () -> + checkArgument( + activePeriod.compareTo(Duration.ofMinutes(15)) > 0, + "active period (leaseDuration - renewalPeriod) must be at least 15 minutes"), + () -> + checkArgument( + numNodeIds >= CHECK_BATCH_SIZE && bitCount(numNodeIds) == 1, + "numNodeIds %s must not be smaller than %s and a power of 2", + numNodeIds, + CHECK_BATCH_SIZE)); + this.config = config; + this.clock = clock; + this.scheduler = scheduler; + + var idGenSpec = + (IdGeneratorSpec) ImmutableIdGeneratorSpec.builder().from(config.idGeneratorSpec()).build(); + var validationIdGeneratorSource = + new IdGeneratorSource() { + @Override + public long currentTimeMillis() { + return clock.currentTimeMillis(); + } + + @Override + public int nodeId() { + return 0; + } + }; + while (true) { + var existingNodeManagementState = nodeStoreFactory.fetchManagementState(); + if (existingNodeManagementState.isPresent()) { + var spec = Util.idgenSpecFromManagementState(existingNodeManagementState); + if (!idGenSpec.equals(spec)) { + warnOnIncompatibleIdGeneratorSpec(spec); + idGenSpec = spec; + } + var factory = IdGeneratorFactory.lookupFactory(idGenSpec.type()); + // try to build an ID generator instance to validate the spec + factory.validateParameters(idGenSpec.params(), validationIdGeneratorSource); + idGenFactory = factory; + break; + } else { + var factory = IdGeneratorFactory.lookupFactory(idGenSpec.type()); + // try to build an ID generator instance to validate the spec + factory.validateParameters(idGenSpec.params(), validationIdGeneratorSource); + + if (nodeStoreFactory.storeManagementState( + ImmutableBuildableNodeManagementState.builder().idGeneratorSpec(idGenSpec).build())) { + LOGGER.info("Persisted node management configuration."); + idGenFactory = factory; + break; + } + } + } + + this.idGenSpec = idGenSpec; + + var nodeIdGen = idGenFactory.buildSystemIdGenerator(idGenSpec.params()); + this.systemIdGen = nodeIdGen; + this.nodeStore = nodeStoreFactory.createNodeStore(nodeIdGen); + } + + static void checkArgs(Runnable... checks) { + var violations = new ArrayList<String>(); + for (Runnable check : checks) { + try { + check.run(); + } catch (IllegalArgumentException iae) { + violations.add(iae.getMessage()); + } + } + if (!violations.isEmpty()) { + throw new IllegalArgumentException(String.join(", ", violations)); + } + } + + @Override + public long systemIdForNode(int nodeId) { + return systemIdGen.systemIdForNode(nodeId); + } + + @Override + public IdGenerator buildIdGenerator(@Nonnull NodeLease leasedNode) { + var idGenSource = + new IdGeneratorSource() { + @Override + public long currentTimeMillis() { + return clock.currentTimeMillis(); + } + + @Override + public int nodeId() { + return leasedNode.nodeIdIfValid(); + } + }; + return idGenFactory.buildIdGenerator(idGenSpec.params(), idGenSource); + } + + @VisibleForTesting + void warnOnIncompatibleIdGeneratorSpec(IdGeneratorSpec spec) { + LOGGER.warn( + "Provided ID generator specification does not match the persisted, unmodifiable one: provided: {} - persisted: {}", + idGenSpec, + spec); + } + + @Override + public int maxNumberOfNodes() { + return numNodeIds; + } + + @Override + public Optional<Node> getNodeInfo(int nodeId) { + checkArgument(nodeId >= 0 && nodeId < numNodeIds, "Illegal node ID " + nodeId); + var leased = + registeredLeases.stream() + .map(NodeLeaseImpl::node) + .filter(Objects::nonNull) + .filter(n -> n.id() == nodeId) + .findFirst(); + if (leased.isPresent()) { + return leased; + } + + return nodeStore + .fetch(nodeId) + .map( + nodeObj -> + ImmutableNode.builder() + .id(nodeId) + .leaseTimestamp(nodeObj.leaseTimestamp()) + .expirationTimestamp(nodeObj.expirationTimestamp()) + .build()); + } + + @Override + @Nonnull + public NodeLease lease() { + var leaseParams = leaseInternal(); + var lease = new NodeLeaseImpl(leaseParams); + registeredLeases.add(lease); + return lease; + } + + private LeaseParams leaseInternal() { + LOGGER.debug("Leasing a node ID..."); + + checkState(!closed, "NodeManagement instance has been closed"); + + var now = clock.currentInstant(); + + var checkedIds = new BitSet(numNodeIds); // 128 bytes for 1024 nodes - not too much + var rand = ThreadLocalRandom.current(); + + var generateNodeId = + (IntSupplier) + () -> { + if (checkedIds.cardinality() == numNodeIds) { + return -1; + } + while (true) { Review Comment: Re: step 2: my reading of line 321 below is that this function can be called up to `3 * numNodeIds` number of times. The more it is called, the longer is the expected loop run time... before it returns `-1`... Finding the last "whole" with a randomized index can take an indefinite amount of time... WDYT? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
