This is an automated email from the ASF dual-hosted git repository.
szetszwo pushed a change to branch HDDS-4454
in repository https://gitbox.apache.org/repos/asf/ozone.git.
from f826d12 HDDS-6431. Fix usedBytes for FSO bucket (#3178)
add 22f977a HDDS-5366. [Ozone-Streaming] Implement stream method to
ContainerStateMachine. (#2358). Contributed by mingchao zhao
add 0aabbc0 HDDS-5452. Add link method to ContainerStateMachine for Ratis
streaming (#2422)
add 82af9d7 HDDS-5481. Fix stream() and link() method in
ContainerStateMachine. (#2451)
add decaa57 HDDS-5480. [Ozone-Streaming] Client and server should support
stream setup. (#2452)
add 0634daf HDDS-5488. [Ozone-Streaming] Add a new
BlockOutputStream/KeyOutputStream to support streaming api (#2495)
add c49edfe HDDS-5599. [Ozone-Streaming]drop BufferPool and ChunkBuffer
to avoid buffer copying (#2557)
add 50e96dd HDDS-5705. [Ozone-Streaming] Change ByteBufStreamOutput to
ByteBufferStreamOutput (#2603)
add 7f503b9 HDDS-5742. Avoid unnecessary Bytebuffer conversions (#2673)
add aba2b3b HDDS-5486. [Ozone-Streaming] Streaming supports writing in
Pipline mode (#2682)
add 0ee0c63 HDDS-5849. [Ozone-Streaming]Write exceptions occur after
checksum is enabled (#2729)
add eede797 HDDS-5674.[Ozone-Streaming] Handle client retries on
exception (#2701)
add 786d09a HDDS-5895. [Ozone-Streaming] Make
raft.server.data-stream.client.pool.size configurable (#2766)
add 253ee38 HDDS-5763. Provide an Executor for each LocalStream in
ContainerStateMachine (#2782)
add d86e5fe HDDS-5987. [Ozone-Streaming] Add XceiverClientRatis stream
config (#2841)
add 06dbec5 HDDS-5961. [Ozone-Streaming] update the usage space of
Containers in the stream write (#2833)
add b6f5244 HDDS-5879. [Ozone-Streaming] OzoneBucket add the
createMultipartStreamKey method (#2760)
add 9491324 HDDS-5743. [Ozone-Streaming] Add option to write files via
streaming api in ofs and o3fs. (#2770)
add d75ea44 HDDS-5851. [Ozone-Streaming] Define a PutBlock/maxBuffer
fixed boundary for streaming writes. (#2866)
add 4e21bdf HDDS-6039. Define a minimum packet size during streaming
writes. (#2883)
add 6ab6d0d HDDS-6130. [Ozone-Streaming] When releaseBuffers will get
“Couldn 't find the required future” (#2939)
add fbeda1c HDDS-6139. [Ozone-Streaming] Fix incorrect computation of
totalAckDataLength. (#2978)
add 1792363 HDDS-6178. [Ozone-Streaming] Fix NPE in HDDS-6139. (#2984)
add e8257f9 HDDS-6281. Update ratis version to 2.3.0-94db58b-SNAPSHOT
version (#3059)
add 4f9265e HDDS-6138.[Ozone-Streaming] Define a limit on the size of the
retry bufferList. (#2946)
add 5dfdbcb HDDS-6298. Add XceiverServerRatis stream config (#3070)
add 9134b73 HDDS-5487. [Ozone-Streaming] BlockDataStreamOutput support
FlushDelay. (#3002)
add 45d8f51 HDDS-6282. Fix BlockDataStreamOutput#doFlushIfNeeded NPE
(#3060)
add 4dcc108 HDDS-6229. [Ozone-Streaming] Data Channel abstraction on
datanode (#3023)
add bb39d10 HDDS-6355. [Ozone-Streaming] Fix CheckStyle problem (#3119)
add b6f1921 HDDS-6388. [Ozone-Streaming] Streaming write support both
pipeline model and star model (#3145)
No new revisions were added by this update.
Summary of changes:
.../apache/hadoop/hdds/scm/OzoneClientConfig.java | 63 +++
.../apache/hadoop/hdds/scm/XceiverClientRatis.java | 7 +-
...utputStream.java => BlockDataStreamOutput.java} | 621 ++++++++++-----------
.../hdds/scm/storage/ByteBufferStreamOutput.java | 57 ++
.../hadoop/hdds/scm/storage/StreamBuffer.java} | 47 +-
...CommitWatcher.java => StreamCommitWatcher.java} | 184 +++---
.../hadoop/hdds/protocol/DatanodeDetails.java | 9 +-
.../org/apache/hadoop/hdds/ratis/RatisHelper.java | 32 +-
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 25 +
.../org/apache/hadoop/ozone/audit/DNAction.java | 3 +-
.../org/apache/hadoop/ozone/common/Checksum.java | 5 +
.../helpers/ContainerCommandRequestPBHelper.java | 1 +
.../common/src/main/resources/ozone-default.xml | 27 +
.../org/apache/hadoop/hdds/conf/ConfigTag.java | 3 +-
.../container/common/impl/HddsDispatcher.java | 21 +-
.../common/interfaces/ContainerDispatcher.java | 10 +
.../ozone/container/common/interfaces/Handler.java | 5 +
.../server/ratis/ContainerStateMachine.java | 54 ++
.../common/transport/server/ratis/LocalStream.java | 58 ++
.../transport/server/ratis/XceiverServerRatis.java | 46 +-
.../ozone/container/keyvalue/KeyValueHandler.java | 45 ++
.../keyvalue/impl/ChunkManagerDispatcher.java | 16 +
.../keyvalue/impl/FilePerBlockStrategy.java | 20 +
.../impl/KeyValueStreamDataChannel.java} | 24 +-
.../keyvalue/impl/StreamDataChannelBase.java | 97 ++++
.../keyvalue/interfaces/ChunkManager.java | 13 +
.../container/common/TestDatanodeStateMachine.java | 6 +-
.../TestCreatePipelineCommandHandler.java | 3 +
.../hdds/conf/DatanodeRatisServerConfig.java | 35 ++
.../src/main/proto/DatanodeClientProtocol.proto | 5 +-
.../ozone/container/common/TestEndPoint.java | 4 +
.../apache/hadoop/ozone/client/OzoneBucket.java | 42 ++
.../client/io/BlockDataStreamOutputEntry.java | 287 ++++++++++
...ol.java => BlockDataStreamOutputEntryPool.java} | 183 ++----
...yOutputStream.java => KeyDataStreamOutput.java} | 169 +++---
...utputStream.java => OzoneDataStreamOutput.java} | 49 +-
.../ozone/client/protocol/ClientProtocol.java | 38 ++
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 146 +++++
.../intellij/runConfigurations/Datanode2.xml | 2 +-
.../intellij/runConfigurations/Datanode3.xml | 2 +-
.../dist/src/main/license/update-jar-report.sh | 2 +-
.../org/apache/hadoop/ozone/MiniOzoneCluster.java | 19 +
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 19 +
.../apache/hadoop/ozone/TestMiniOzoneCluster.java | 2 +
.../client/rpc/TestBlockDataStreamOutput.java | 277 +++++++++
.../rpc/TestContainerStateMachineStream.java | 184 ++++++
.../client/rpc/TestOzoneRpcClientAbstract.java | 1 +
.../client/rpc/TestOzoneRpcClientWithRatis.java | 53 ++
.../apache/hadoop/ozone/container/TestHelper.java | 44 +-
.../container/server/TestContainerServer.java | 1 +
.../server/TestSecureContainerServer.java | 2 +
.../fs/ozone/BasicOzoneClientAdapterImpl.java | 33 ++
.../hadoop/fs/ozone/BasicOzoneFileSystem.java | 8 +
.../ozone/BasicRootedOzoneClientAdapterImpl.java | 39 ++
.../fs/ozone/BasicRootedOzoneFileSystem.java | 8 +
.../apache/hadoop/fs/ozone/OzoneClientAdapter.java | 3 +
.../hadoop/fs/ozone/OzoneFSDataStreamOutput.java | 103 ++++
.../hadoop/ozone/shell/keys/PutKeyHandler.java | 36 +-
pom.xml | 2 +-
59 files changed, 2557 insertions(+), 743 deletions(-)
copy
hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/{BlockOutputStream.java
=> BlockDataStreamOutput.java} (55%)
create mode 100644
hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ByteBufferStreamOutput.java
copy
hadoop-hdds/client/src/{test/java/org/apache/hadoop/hdds/scm/storage/TestBufferPool.java
=> main/java/org/apache/hadoop/hdds/scm/storage/StreamBuffer.java} (54%)
copy
hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/{CommitWatcher.java
=> StreamCommitWatcher.java} (54%)
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/LocalStream.java
copy
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/{common/report/IncrementalReportSender.java
=> keyvalue/impl/KeyValueStreamDataChannel.java} (55%)
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/StreamDataChannelBase.java
create mode 100644
hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockDataStreamOutputEntry.java
copy
hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/{BlockOutputStreamEntryPool.java
=> BlockDataStreamOutputEntryPool.java} (66%)
copy
hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/{KeyOutputStream.java
=> KeyDataStreamOutput.java} (82%)
copy
hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/{OzoneOutputStream.java
=> OzoneDataStreamOutput.java} (52%)
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineStream.java
create mode 100644
hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneFSDataStreamOutput.java
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
