[
https://issues.apache.org/jira/browse/APEXMALHAR-1897?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15203378#comment-15203378
]
ASF GitHub Bot commented on APEXMALHAR-1897:
--------------------------------------------
Github user amberarrow commented on a diff in the pull request:
https://github.com/apache/incubator-apex-malhar/pull/145#discussion_r56768167
--- Diff:
library/src/main/java/com/datatorrent/lib/state/managed/AbstractManagedStateImpl.java
---
@@ -0,0 +1,499 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package com.datatorrent.lib.state.managed;
+
+import java.io.IOException;
+import java.util.Comparator;
+import java.util.Map;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicReference;
+
+import javax.validation.constraints.Min;
+import javax.validation.constraints.NotNull;
+
+import org.joda.time.Duration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.esotericsoftware.kryo.serializers.FieldSerializer;
+import com.esotericsoftware.kryo.serializers.JavaSerializer;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Throwables;
+import com.google.common.collect.Maps;
+
+import com.datatorrent.api.Component;
+import com.datatorrent.api.Context.DAGContext;
+import com.datatorrent.api.Context.OperatorContext;
+import com.datatorrent.api.Operator;
+import com.datatorrent.api.annotation.Stateless;
+import com.datatorrent.common.util.NameableThreadFactory;
+import com.datatorrent.lib.fileaccess.FileAccess;
+import com.datatorrent.lib.fileaccess.TFileImpl;
+import com.datatorrent.lib.util.comparator.SliceComparator;
+import com.datatorrent.netlet.util.Slice;
+
+/**
+ * An abstract implementation of managed state.<br/>
+ *
+ * The important sub-components here are:
+ * <ol>
+ * <li>
+ * {@link #checkpointManager}: writes incremental checkpoints in
window files and transfers data from window
+ * files to bucket files.
+ * </li>
+ * <li>
+ * {@link #bucketsFileSystem}: a bucket on disk is sub-divided into
time-buckets. This manages meta-bucket
+ * information (list of {@link BucketsFileSystem.TimeBucketMeta}) per
bucket.
+ * </li>
+ * <li>
+ * {@link #timeBucketAssigner}: assigns time-buckets to keys and
manages the time boundaries.
+ * </li>
+ * <li>
+ * {@link #stateTracker}: tracks the size of data in memory and
requests buckets to free memory when enough memory
+ * is not available.
+ * </li>
+ * <li>
+ * {@link #fileAccess}: plug-able file system abstraction.
+ * </li>
+ * </ol>
+ *
+ * The implementations of put, getSync and getAsync here use windowId as
the time field to derive timeBucket of a key.
+ */
+public abstract class AbstractManagedStateImpl
+ implements ManagedState, Component<OperatorContext>,
Operator.CheckpointNotificationListener, ManagedStateContext,
+ TimeBucketAssigner.PurgeListener
+{
+ private long maxMemorySize;
+
+ protected int numBuckets;
+
+ @NotNull
+ private FileAccess fileAccess = new TFileImpl.DTFileImpl();
+ @NotNull
+ protected TimeBucketAssigner timeBucketAssigner = new
TimeBucketAssigner();
+
+ protected Bucket[] buckets;
+
+ @Min(1)
+ private int numReaders = 1;
+ @NotNull
+ protected transient ExecutorService readerService;
+
+ @NotNull
+ protected IncrementalCheckpointManager checkpointManager = new
IncrementalCheckpointManager();
+
+ @NotNull
+ protected BucketsFileSystem bucketsFileSystem = new BucketsFileSystem();
+
+ protected transient OperatorContext operatorContext;
+ protected transient long windowId;
+
+ @NotNull
+ protected Comparator<Slice> keyComparator = new SliceComparator();
+
+ protected final transient AtomicReference<Throwable> throwable = new
AtomicReference<>();
+
+ @NotNull
+ @FieldSerializer.Bind(JavaSerializer.class)
+ private Duration checkStateSizeInterval = Duration.millis(
+ DAGContext.STREAMING_WINDOW_SIZE_MILLIS.defaultValue *
OperatorContext.APPLICATION_WINDOW_COUNT.defaultValue);
+
+ @FieldSerializer.Bind(JavaSerializer.class)
+ private Duration durationPreventingFreeingSpace;
+
+ private transient StateTracker stateTracker = new StateTracker();
+
+ //accessible to StateTracker
+ final transient Object commitLock = new Object();
+
+ @Override
+ public void setup(OperatorContext context)
+ {
+ operatorContext = context;
+ fileAccess.init();
+
+ timeBucketAssigner.setPurgeListener(this);
+
+ //setup all the managed state components
+ timeBucketAssigner.setup(this);
+ checkpointManager.setup(this);
+ bucketsFileSystem.setup(this);
+
+ if (buckets == null) {
+ //create buckets array only once at start when it is not created.
+ numBuckets = getNumBuckets();
+ buckets = new Bucket[numBuckets];
+ }
+ for (Bucket bucket : buckets) {
+ if (bucket != null) {
--- End diff --
Is it possible for bucket to be null here ?
> Create ManagedState
> -------------------
>
> Key: APEXMALHAR-1897
> URL: https://issues.apache.org/jira/browse/APEXMALHAR-1897
> Project: Apache Apex Malhar
> Issue Type: Sub-task
> Reporter: Chandni Singh
> Assignee: Chandni Singh
> Fix For: 3.4.0
>
>
> ManagedState is described in the document below:
> https://docs.google.com/document/d/1gRWN9ufKSZSZD0N-pthlhpC9TZ8KwJ6hJlAX6nxl5f8/edit#heading=h.z87ti1fwyt0t
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
