This is an automated email from the ASF dual-hosted git repository. duhengforever pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/rocketmq-streams.git
commit d8216f0a8ee72e34a2a6ecd1d67c36c4bf5696dd Merge: b013eb2 607418a Author: xstorm1 <[email protected]> AuthorDate: Mon Aug 2 15:43:16 2021 +0800 modify README.md .gitignore | 25 ++ README.md | 55 ++- pom.xml | 410 +++++++++++++++++ rocketmq-streams-clients/pom.xml | 46 ++ .../rocketmq/streams/client/DataStreamAction.java | 101 +++++ .../rocketmq/streams/client/StreamBuilder.java | 28 ++ .../streams/client/source/DataStreamSource.java | 76 ++++ .../client/strategy/CheckpointStrategy.java | 69 +++ .../streams/client/strategy/StateStrategy.java | 37 ++ .../rocketmq/streams/client/strategy/Strategy.java | 25 ++ .../streams/client/transform/DataStream.java | 437 ++++++++++++++++++ .../streams/client/transform/JoinStream.java | 212 +++++++++ .../streams/client/transform/SplitStream.java | 61 +++ .../streams/client/transform/WindowStream.java | 210 +++++++++ .../client/transform/window/HoppingWindow.java | 32 ++ .../client/transform/window/SessionWindow.java | 32 ++ .../streams/client/transform/window/Time.java | 45 ++ .../client/transform/window/TumblingWindow.java | 33 ++ .../client/transform/window/WindowInfo.java | 83 ++++ .../rocketmq/streams/client/DBDriverTest.java | 75 ++++ .../rocketmq/streams/client/DataStreamTest.java | 107 +++++ .../apache/rocketmq/streams/client/FilterTest.java | 49 +++ .../apache/rocketmq/streams/client/JoinTest.java | 89 ++++ .../apache/rocketmq/streams/client/LeaseTest.java | 98 +++++ .../rocketmq/streams/client/ORMUtilTest.java | 172 ++++++++ .../apache/rocketmq/streams/client/SplitTest.java | 86 ++++ .../apache/rocketmq/streams/client/UnionTest.java | 82 ++++ .../apache/rocketmq/streams/client/WindowTest.java | 86 ++++ .../client/windows/AbstractWindowFireModeTest.java | 189 ++++++++ .../streams/client/windows/WindowFromFileTest.java | 158 +++++++ .../streams/client/windows/WindowFromMetaq.java | 47 ++ .../client/windows/WindowHighAvailabilityTest.java | 131 ++++++ .../src/test/resources/log4j.xml | 36 ++ rocketmq-streams-db-operator/pom.xml | 34 ++ .../rocketmq-streams-db-operator.iml | 16 + .../streams/db/configuable/DBConfigureService.java | 282 ++++++++++++ .../DBSupportParentConfigureService.java | 37 ++ .../rocketmq/streams/db/driver/DriverBuilder.java | 111 +++++ .../rocketmq/streams/db/driver/IDriverBudiler.java | 36 ++ .../rocketmq/streams/db/driver/JDBCDriver.java | 277 ++++++++++++ .../db/driver/batchloader/BatchRowLoader.java | 179 ++++++++ .../db/driver/batchloader/IRowOperator.java | 33 ++ .../rocketmq/streams/db/driver/orm/ORMUtil.java | 490 +++++++++++++++++++++ .../rocketmq/streams/db/operator/SQLOperator.java | 178 ++++++++ .../org/apache/rocketmq/streams/db/Person.java | 110 +++++ .../DBSupportParentConfigureServiceTest.java | 74 ++++ .../streams/db/driver/orm/ORMUtilTest.java | 86 ++++ rocketmq-streams-dim/pom.xml | 47 ++ .../apache/rocketmq/streams/dim/DimComponent.java | 63 +++ .../rocketmq/streams/dim/builder/DimBuilder.java | 94 ++++ .../function/expression/InExpressionResource.java | 80 ++++ .../expression/NotInExpressionResource.java | 45 ++ .../dim/function/script/IntelligenceFunction.java | 81 ++++ .../script/IntelligenceNameListFunction.java | 24 + .../dim/function/script/NameListFunction.java | 203 +++++++++ .../rocketmq/streams/dim/index/DimIndex.java | 319 ++++++++++++++ .../rocketmq/streams/dim/index/IndexExecutor.java | 258 +++++++++++ .../intelligence/AbstractIntelligenceCache.java | 395 +++++++++++++++++ .../dim/intelligence/AccountIntelligenceCache.java | 77 ++++ .../dim/intelligence/DomainIntelligenceCache.java | 83 ++++ .../dim/intelligence/IPIntelligenceCache.java | 108 +++++ .../dim/intelligence/URLIntelligenceCache.java | 80 ++++ .../rocketmq/streams/dim/model/AbstractDim.java | 312 +++++++++++++ .../streams/dim/model/BooleanFieldDBDim.java | 55 +++ .../apache/rocketmq/streams/dim/model/DBDim.java | 140 ++++++ .../rocketmq/streams/dim/service/IDimService.java | 65 +++ .../streams/dim/service/impl/DimServiceImpl.java | 92 ++++ .../com/aliyun/service/ConfigureLoaderTest.java | 37 ++ .../com/aliyun/service/ExpressionExecutorTest.java | 80 ++++ .../java/com/aliyun/service/JsonParserTest.java | 40 ++ .../com/aliyun/service/NameListFunctionTest.java | 90 ++++ .../java/com/aliyun/service/TableCompressTest.java | 26 ++ rocketmq-streams-lease/pom.xml | 25 ++ .../rocketmq/streams/lease/LeaseComponent.java | 103 +++++ .../rocketmq/streams/lease/model/LeaseInfo.java | 127 ++++++ .../streams/lease/service/ILeaseGetCallback.java | 30 ++ .../streams/lease/service/ILeaseService.java | 136 ++++++ .../streams/lease/service/ILeaseStorage.java | 73 +++ .../streams/lease/service/ILeaseStorasge.java | 63 +++ .../lease/service/impl/BasedLesaseImpl.java | 404 +++++++++++++++++ .../lease/service/impl/LeaseServiceImpl.java | 275 ++++++++++++ .../streams/lease/service/impl/MockLeaseImpl.java | 95 ++++ .../lease/service/storages/DBLeaseStorage.java | 229 ++++++++++ .../rocketmq/streams/lease/LeaseComponentTest.java | 119 +++++ .../src/test/resources/log4j.xml | 20 + rocketmq-streams-transport-minio/pom.xml | 25 ++ .../rocketmq-streams-transport-minio.iml | 17 + .../transport/minio/MinioFileTransport.java | 141 ++++++ .../yundun/dipper/configurable/DataTpyeTest.java | 70 +++ .../streams/configuable/model/DataTpyeTest.java | 68 +++ .../rocketmq/streams/configuable/model/Person.java | 97 ++++ .../streams/configurable/model/Person.java | 97 ++++ .../component/ConfigurableComponent.properties | 7 + .../src/test/resources/log4j.xml | 20 + .../src/test/resources/pro-function.txt | 11 + .../src/test/resources/python_script.py | 22 + 96 files changed, 10532 insertions(+), 1 deletion(-) diff --cc README.md index 678bd5a,92a9b2f..d2ff23e --- a/README.md +++ b/README.md @@@ -9,17 -8,15 +8,29 @@@ ## DataStream Example ```java ++<<<<<<< HEAD + import org.apache.rocketmq.streams.client.transform.DataStream; + + + + DataStreamSource source=StreamBuilder.dataStream("namespace","pipeline"); + + source + .fromFile("/Users/junjie.cheng/text.txt",false) + .map(message->message) + .toPrint(1) + .start(); ++======= + import org.apache.rocketmq.streams.client.transform.DataStream; + + DataStreamSource source=StreamBuilder.dataStream("namespace","pipeline"); + + source + .fromFile("/Users/junjie.cheng/text.txt",false) + .map(message->message) + .toPrint(1) + .start(); ++>>>>>>> main ``` ## Maven Repository @@@ -47,6 -44,7 +58,10 @@@ StreamBuilder 用于构建流任务的� ## DataStream API ### Source ++<<<<<<< HEAD ++======= + ++>>>>>>> main DataStreamSource 是分段式编程的源头类，用于对接各种数据源， 从各大消息队列中获取数据； + ```fromFile``` 从文件中读取数据， 该方法包含俩个参数 @@@ -60,13 -58,14 +75,24 @@@ + ```isJson``` 是否json格式，非必填参数 + ```tags``` rocketmq消费的tags值，用于过滤消息，非必填参数 ++<<<<<<< HEAD + ++ ```from``` 自定义的数据源， 通过实现ISource接口实现自己的数据源 + +### transform +transform 允许在流计算过程中对输入源的数据进行修改，进行下一步的操作；DataStream API中包括```DataStream```,```JoinStream```, ```SplitStream```,```WindowStream```等多个transform类； + +#### DataStream ++======= + + ```from``` 自定义的数据源， 通过实现ISource接口实现自己的数据源 + + ### transform + + transform 允许在流计算过程中对输入源的数据进行修改，进行下一步的操作；DataStream API中包括```DataStream```,```JoinStream```, ```SplitStream```,```WindowStream```等多个transform类； + + #### DataStream + ++>>>>>>> main DataStream实现了一系列常见的流计算算子 + ```map``` 通过将源的每个记录传递给函数func来返回一个新的DataStream @@@ -80,30 -79,29 +106,57 @@@ + ```toFile``` 将结果保存为文件，生成一个新的DataStreamAction实例 + ```toDB``` 将结果保存到数据库 + ```toRocketmq``` 将结果输出到rocketmq ++<<<<<<< HEAD ++ ```toSls``` 将结果输出到sls ++ ```to``` 将结果经过自定义的ISink接口输出到指定的存储 ++ ```window``` 在窗口内进行相关的统计分析，一般会与```groupBy```连用， ```window()```用来定义窗口的大小， ```groupBy()```用来定义统计分析的主key，可以指定多个 + + ```count``` 在窗口内计数 + + ```min``` 获取窗口内统计值的最小值 + + ```max``` 获取窗口内统计值得最大值 + + ```avg``` 获取窗口内统计值的平均值 + + ```sum``` 获取窗口内统计值的加和值 + + ```reduce``` 在窗口内进行自定义的汇总运算 ++======= + + ```to``` 将结果经过自定义的ISink接口输出到指定的存储 + + ```window``` 在窗口内进行相关的统计分析，一般会与```groupBy```连用， ```window()```用来定义窗口的大小， ```groupBy()```用来定义统计分析的主key，可以指定多个 + + ```count``` 在窗口内计数 + + ```min``` 获取窗口内统计值的最小值 + + ```max``` 获取窗口内统计值得最大值 + + ```avg``` 获取窗口内统计值的平均值 + + ```sum``` 获取窗口内统计值的加和值 + + ```reduce``` 在窗口内进行自定义的汇总运算 ++>>>>>>> main + ```join``` 根据条件将将俩个流进行关联， 合并为一个大流进行相关的运算 + ```union``` 将俩个流进行合并 + ```split``` 将一个数据流按照标签进行拆分，分为不同的数据流供下游进行分析计算 + ```with``` with算子用来指定计算过程中的相关策略，包括checkpoint的存储策略，state的存储策略等 ++<<<<<<< HEAD + +# Strategy +策略机制主要用来控制计算引擎运行过程中的底层逻辑，如checkpoint，state的存储方式等，后续还会增加对窗口、双流join等的控制；所有的控制策略通过```with```算子传入，可以同时传入多个策略类型； + +```java + //指定checkpoint的存储策略 + source + .fromRocketmq("TSG_META_INFO", "") + .map(message -> message + "--") + .toPrint(1) + .with(CheckpointStrategy.db("jdbc:mysql://XXXXX:3306/XXXXX", "", "", 0L)) + .start(); +``` ++======= + # Strategy + + 策略机制主要用来控制计算引擎运行过程中的底层逻辑，如checkpoint，state的存储方式等，后续还会增加对窗口、双流join等的控制；所有的控制策略通过```with```算子传入，可以同时传入多个策略类型； + + ```java + //指定checkpoint的存储策略 + source + .fromRocketmq("TSG_META_INFO","") + .map(message->message+"--") + .toPrint(1) + .with(CheckpointStrategy.db("jdbc:mysql://XXXXX:3306/XXXXX","","",0L)) + .start(); -``` ++``` ++>>>>>>> main
