[FLINK-1452] Rename 'flink-addons' to 'flink-staging'
Project: http://git-wip-us.apache.org/repos/asf/flink/repo Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/f94112fb Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/f94112fb Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/f94112fb Branch: refs/heads/master Commit: f94112fbbaaf2ecc6a9ecb314a5565203ce779a7 Parents: 563e546 Author: Robert Metzger <rmetz...@apache.org> Authored: Fri Jan 30 15:16:46 2015 +0100 Committer: Robert Metzger <rmetz...@apache.org> Committed: Mon Feb 2 19:40:25 2015 +0100 ---------------------------------------------------------------------- docs/internal_add_operator.md | 2 +- docs/internal_general_arch.md | 3 +- docs/streaming_guide.md | 6 +- flink-addons/flink-avro/pom.xml | 182 --- .../apache/flink/api/avro/DataInputDecoder.java | 213 --- .../flink/api/avro/DataOutputEncoder.java | 183 --- .../api/avro/FSDataInputStreamWrapper.java | 68 - .../api/io/avro/example/AvroTypeExample.java | 111 -- .../apache/flink/api/io/avro/example/User.java | 269 ---- .../flink/api/java/io/AvroInputFormat.java | 124 -- .../flink/api/java/io/AvroOutputFormat.java | 100 -- .../src/test/assembly/test-assembly.xml | 36 - .../api/avro/AvroExternalJarProgramITCase.java | 71 - .../flink/api/avro/AvroOutputFormatTest.java | 173 -- .../flink/api/avro/EncoderDecoderTest.java | 519 ------ .../avro/testjar/AvroExternalJarProgram.java | 219 --- .../api/io/avro/AvroRecordInputFormatTest.java | 167 -- .../flink/api/io/avro/generated/Colors.java | 32 - .../flink/api/io/avro/generated/User.java | 755 --------- .../io/AvroInputFormatTypeExtractionTest.java | 79 - .../src/test/resources/avro/user.avsc | 19 - .../src/test/resources/log4j-test.properties | 27 - .../src/test/resources/logback-test.xml | 29 - .../flink-avro/src/test/resources/testdata.avro | Bin 4572 -> 0 bytes flink-addons/flink-hadoop-compatibility/pom.xml | 80 - .../mapred/HadoopInputFormat.java | 297 ---- .../mapred/HadoopMapFunction.java | 131 -- .../mapred/HadoopOutputFormat.java | 183 --- .../mapred/HadoopReduceCombineFunction.java | 162 -- .../mapred/HadoopReduceFunction.java | 138 -- .../example/HadoopMapredCompatWordCount.java | 133 -- .../mapred/record/HadoopDataSink.java | 109 -- .../mapred/record/HadoopDataSource.java | 82 - .../mapred/record/HadoopRecordInputFormat.java | 174 --- .../mapred/record/HadoopRecordOutputFormat.java | 156 -- .../datatypes/DefaultFlinkTypeConverter.java | 95 -- .../datatypes/DefaultHadoopTypeConverter.java | 83 - .../record/datatypes/FlinkTypeConverter.java | 43 - .../datatypes/HadoopFileOutputCommitter.java | 196 --- .../record/datatypes/HadoopTypeConverter.java | 42 - .../datatypes/WritableComparableWrapper.java | 40 - .../record/datatypes/WritableWrapper.java | 71 - .../datatypes/WritableWrapperConverter.java | 45 - .../mapred/record/example/WordCount.java | 184 --- .../example/WordCountWithOutputFormat.java | 173 -- .../mapred/utils/HadoopUtils.java | 87 -- .../mapred/wrapper/HadoopDummyProgressable.java | 33 - .../mapred/wrapper/HadoopDummyReporter.java | 70 - .../mapred/wrapper/HadoopInputSplit.java | 138 -- .../mapred/wrapper/HadoopOutputCollector.java | 64 - .../wrapper/HadoopTupleUnwrappingIterator.java | 94 -- .../mapreduce/HadoopInputFormat.java | 338 ---- .../mapreduce/HadoopOutputFormat.java | 226 --- .../mapreduce/example/WordCount.java | 120 -- .../mapreduce/utils/HadoopUtils.java | 83 - .../mapreduce/wrapper/HadoopInputSplit.java | 125 -- .../mapred/HadoopIOFormatsITCase.java | 221 --- .../mapred/HadoopInputFormatTest.java | 82 - .../mapred/HadoopMapFunctionITCase.java | 182 --- .../mapred/HadoopMapredITCase.java | 47 - .../HadoopReduceCombineFunctionITCase.java | 265 ---- .../mapred/HadoopReduceFunctionITCase.java | 213 --- .../mapred/HadoopTestData.java | 62 - .../record/HadoopRecordInputOutputITCase.java | 54 - .../HadoopTupleUnwrappingIteratorTest.java | 137 -- .../mapreduce/HadoopInputFormatTest.java | 84 - .../mapreduce/HadoopInputOutputITCase.java | 47 - .../src/test/resources/log4j-test.properties | 27 - .../src/test/resources/logback-test.xml | 29 - flink-addons/flink-hbase/pom.xml | 165 -- .../flink/addons/hbase/TableInputFormat.java | 255 --- .../flink/addons/hbase/TableInputSplit.java | 168 -- .../addons/hbase/example/HBaseReadExample.java | 93 -- .../src/test/resources/hbase-site.xml | 43 - .../src/test/resources/log4j.properties | 23 - flink-addons/flink-jdbc/pom.xml | 64 - .../flink/api/java/io/jdbc/JDBCInputFormat.java | 358 ----- .../api/java/io/jdbc/JDBCOutputFormat.java | 270 ---- .../api/java/io/jdbc/example/JDBCExample.java | 101 -- .../java/record/io/jdbc/JDBCInputFormat.java | 389 ----- .../java/record/io/jdbc/JDBCOutputFormat.java | 359 ----- .../record/io/jdbc/example/JDBCExample.java | 136 -- .../api/java/io/jdbc/JDBCInputFormatTest.java | 195 --- .../api/java/io/jdbc/JDBCOutputFormatTest.java | 240 --- .../java/record/io/jdbc/DevNullLogStream.java | 30 - .../record/io/jdbc/JDBCInputFormatTest.java | 214 --- .../record/io/jdbc/JDBCOutputFormatTest.java | 225 --- .../src/test/resources/log4j-test.properties | 19 - .../src/test/resources/logback-test.xml | 29 - flink-addons/flink-spargel/pom.xml | 60 - .../flink/spargel/java/MessageIterator.java | 58 - .../flink/spargel/java/MessagingFunction.java | 284 ---- .../apache/flink/spargel/java/OutgoingEdge.java | 64 - .../spargel/java/VertexCentricIteration.java | 599 ------- .../spargel/java/VertexUpdateFunction.java | 145 -- .../examples/SpargelConnectedComponents.java | 79 - .../spargel/java/examples/SpargelPageRank.java | 117 -- .../SpargelPageRankCountingVertices.java | 154 -- .../apache/flink/spargel/java/record/Edge.java | 43 - .../spargel/java/record/MessageIterator.java | 59 - .../spargel/java/record/MessagingFunction.java | 163 -- .../spargel/java/record/SpargelIteration.java | 289 ---- .../java/record/VertexUpdateFunction.java | 90 -- .../flink/spargel/java/SpargelCompilerTest.java | 183 --- .../spargel/java/SpargelTranslationTest.java | 211 --- .../SpargelConnectedComponentsITCase.java | 81 - .../src/test/resources/log4j-test.properties | 19 - .../src/test/resources/logback-test.xml | 29 - .../flink-streaming-connectors/pom.xml | 243 --- .../streaming/connectors/ConnectorSource.java | 42 - .../db/CustomSerializationDBState.java | 31 - .../streaming/connectors/db/DBSerializer.java | 36 - .../flink/streaming/connectors/db/DBState.java | 29 - .../connectors/db/DBStateIterator.java | 31 - .../connectors/db/DBStateWithIterator.java | 25 - .../connectors/db/DefaultDBSerializer.java | 35 - .../streaming/connectors/db/LevelDBState.java | 125 -- .../streaming/connectors/db/MemcachedState.java | 73 - .../streaming/connectors/db/RedisState.java | 109 -- .../streaming/connectors/flume/FlumeSink.java | 141 -- .../streaming/connectors/flume/FlumeSource.java | 141 -- .../connectors/flume/FlumeTopology.java | 49 - .../connectors/json/JSONParseFlatMap.java | 144 -- .../streaming/connectors/json/JSONParser.java | 175 --- .../streaming/connectors/kafka/KafkaSink.java | 83 - .../streaming/connectors/kafka/KafkaSource.java | 96 -- .../connectors/kafka/KafkaTopology.java | 57 - .../streaming/connectors/rabbitmq/RMQSink.java | 111 -- .../connectors/rabbitmq/RMQSource.java | 121 -- .../connectors/rabbitmq/RMQTopology.java | 52 - .../connectors/twitter/TwitterSource.java | 283 ---- .../connectors/twitter/TwitterStreaming.java | 99 -- .../connectors/twitter/TwitterTopology.java | 92 -- .../connectors/util/DeserializationSchema.java | 42 - .../streaming/connectors/util/RawSchema.java | 39 - .../connectors/util/SerializationSchema.java | 33 - .../connectors/util/SimpleStringSchema.java | 40 - .../streaming/connectors/db/DBStateTest.java | 105 -- .../connectors/json/JSONParserTest.java | 74 - .../connectors/json/JSONParserTest2.java | 95 -- .../src/test/resources/log4j-test.properties | 19 - .../src/test/resources/logback-test.xml | 30 - .../flink-streaming-core/pom.xml | 75 - .../flink/streaming/api/StreamConfig.java | 406 ----- .../apache/flink/streaming/api/StreamGraph.java | 620 -------- .../api/StreamingJobGraphGenerator.java | 293 ---- .../api/collector/CollectorWrapper.java | 49 - .../api/collector/DirectedCollectorWrapper.java | 131 -- .../streaming/api/collector/OutputSelector.java | 44 - .../streaming/api/collector/StreamOutput.java | 86 - .../api/datastream/ConnectedDataStream.java | 419 ----- .../streaming/api/datastream/DataStream.java | 1271 --------------- .../api/datastream/DataStreamSink.java | 46 - .../api/datastream/DataStreamSource.java | 48 - .../api/datastream/GroupedDataStream.java | 205 --- .../api/datastream/IterativeDataStream.java | 107 -- .../datastream/SingleOutputStreamOperator.java | 174 --- .../api/datastream/SplitDataStream.java | 66 - .../api/datastream/StreamProjection.java | 1471 ------------------ .../api/datastream/WindowedDataStream.java | 658 -------- .../temporaloperator/StreamCrossOperator.java | 101 -- .../temporaloperator/StreamJoinOperator.java | 272 ---- .../temporaloperator/TemporalOperator.java | 124 -- .../temporaloperator/TemporalWindow.java | 45 - .../api/environment/LocalStreamEnvironment.java | 47 - .../environment/RemoteStreamEnvironment.java | 127 -- .../environment/StreamContextEnvironment.java | 76 - .../environment/StreamExecutionEnvironment.java | 618 -------- .../api/environment/StreamPlanEnvironment.java | 64 - .../aggregation/AggregationFunction.java | 35 - .../aggregation/ComparableAggregator.java | 236 --- .../api/function/aggregation/Comparator.java | 104 -- .../api/function/aggregation/SumAggregator.java | 171 -- .../api/function/aggregation/SumFunction.java | 102 -- .../api/function/co/CoFlatMapFunction.java | 42 - .../api/function/co/CoMapFunction.java | 41 - .../api/function/co/CoReduceFunction.java | 107 -- .../api/function/co/CoWindowFunction.java | 30 - .../api/function/co/CrossWindowFunction.java | 44 - .../api/function/co/JoinWindowFunction.java | 55 - .../api/function/co/RichCoFlatMapFunction.java | 40 - .../api/function/co/RichCoMapFunction.java | 40 - .../api/function/co/RichCoReduceFunction.java | 40 - .../api/function/co/RichCoWindowFunction.java | 34 - .../api/function/sink/FileSinkFunction.java | 118 -- .../function/sink/FileSinkFunctionByMillis.java | 59 - .../api/function/sink/PrintSinkFunction.java | 98 -- .../api/function/sink/RichSinkFunction.java | 28 - .../api/function/sink/SinkFunction.java | 28 - .../api/function/sink/WriteFormat.java | 43 - .../api/function/sink/WriteFormatAsCsv.java | 49 - .../api/function/sink/WriteFormatAsText.java | 47 - .../api/function/sink/WriteSinkFunction.java | 92 -- .../sink/WriteSinkFunctionByMillis.java | 50 - .../function/source/FileMonitoringFunction.java | 123 -- .../api/function/source/FileReadFunction.java | 51 - .../api/function/source/FileSourceFunction.java | 142 -- .../function/source/FromElementsFunction.java | 49 - .../function/source/GenSequenceFunction.java | 53 - .../function/source/GenericSourceFunction.java | 25 - .../function/source/ParallelSourceFunction.java | 26 - .../source/RichParallelSourceFunction.java | 27 - .../api/function/source/RichSourceFunction.java | 27 - .../source/SocketTextStreamFunction.java | 86 - .../api/function/source/SourceFunction.java | 29 - .../api/invokable/ChainableInvokable.java | 39 - .../streaming/api/invokable/SinkInvokable.java | 50 - .../api/invokable/SourceInvokable.java | 44 - .../api/invokable/StreamInvokable.java | 186 --- .../api/invokable/operator/BatchIterator.java | 25 - .../invokable/operator/CounterInvokable.java | 43 - .../api/invokable/operator/FilterInvokable.java | 55 - .../invokable/operator/FlatMapInvokable.java | 51 - .../operator/GroupedReduceInvokable.java | 59 - .../operator/GroupedWindowInvokable.java | 506 ------ .../api/invokable/operator/MapInvokable.java | 50 - .../invokable/operator/ProjectInvokable.java | 61 - .../operator/StreamReduceInvokable.java | 69 - .../operator/WindowGroupReduceInvokable.java | 51 - .../api/invokable/operator/WindowInvokable.java | 382 ----- .../operator/WindowReduceInvokable.java | 59 - .../operator/co/CoFlatMapInvokable.java | 54 - .../operator/co/CoGroupedReduceInvokable.java | 88 -- .../api/invokable/operator/co/CoInvokable.java | 133 -- .../invokable/operator/co/CoMapInvokable.java | 54 - .../operator/co/CoReduceInvokable.java | 70 - .../operator/co/CoWindowInvokable.java | 197 --- .../api/streamrecord/StreamRecord.java | 134 -- .../streamrecord/StreamRecordSerializer.java | 114 -- .../flink/streaming/api/streamrecord/UID.java | 122 -- .../api/streamvertex/CoStreamVertex.java | 142 -- .../api/streamvertex/InputHandler.java | 96 -- .../api/streamvertex/OutputHandler.java | 242 --- .../api/streamvertex/StreamIterationHead.java | 101 -- .../api/streamvertex/StreamIterationTail.java | 103 -- .../api/streamvertex/StreamTaskContext.java | 40 - .../api/streamvertex/StreamVertex.java | 156 -- .../api/streamvertex/StreamVertexException.java | 68 - .../streamvertex/StreamingRuntimeContext.java | 88 -- .../windowing/deltafunction/CosineDistance.java | 87 -- .../windowing/deltafunction/DeltaFunction.java | 44 - .../deltafunction/EuclideanDistance.java | 58 - .../ExtractionAwareDeltaFunction.java | 90 -- .../api/windowing/extractor/ArrayFromTuple.java | 74 - .../extractor/ConcatinatedExtract.java | 68 - .../api/windowing/extractor/Extractor.java | 43 - .../api/windowing/extractor/FieldFromArray.java | 59 - .../api/windowing/extractor/FieldFromTuple.java | 58 - .../windowing/extractor/FieldsFromArray.java | 67 - .../windowing/extractor/FieldsFromTuple.java | 53 - .../streaming/api/windowing/helper/Count.java | 101 -- .../streaming/api/windowing/helper/Delta.java | 91 -- .../api/windowing/helper/SystemTimestamp.java | 37 - .../streaming/api/windowing/helper/Time.java | 166 -- .../api/windowing/helper/Timestamp.java | 39 - .../api/windowing/helper/TimestampWrapper.java | 44 - .../api/windowing/helper/WindowingHelper.java | 39 - .../ActiveCloneableEvictionPolicyWrapper.java | 62 - .../windowing/policy/ActiveEvictionPolicy.java | 52 - .../policy/ActiveEvictionPolicyWrapper.java | 64 - .../windowing/policy/ActiveTriggerCallback.java | 45 - .../windowing/policy/ActiveTriggerPolicy.java | 81 - .../policy/CloneableEvictionPolicy.java | 45 - .../policy/CloneableTriggerPolicy.java | 45 - .../windowing/policy/CountEvictionPolicy.java | 119 -- .../windowing/policy/CountTriggerPolicy.java | 87 -- .../api/windowing/policy/DeltaPolicy.java | 112 -- .../api/windowing/policy/EvictionPolicy.java | 47 - .../api/windowing/policy/PunctuationPolicy.java | 116 -- .../windowing/policy/TimeEvictionPolicy.java | 137 -- .../api/windowing/policy/TimeTriggerPolicy.java | 196 --- .../api/windowing/policy/TriggerPolicy.java | 54 - .../policy/TumblingEvictionPolicy.java | 90 -- .../flink/streaming/io/BlockingQueueBroker.java | 41 - .../flink/streaming/io/CoReaderIterator.java | 57 - .../flink/streaming/io/CoRecordReader.java | 221 --- .../flink/streaming/io/StreamRecordWriter.java | 83 - .../partitioner/BroadcastPartitioner.java | 55 - .../partitioner/DistributePartitioner.java | 46 - .../partitioner/FieldsPartitioner.java | 50 - .../partitioner/GlobalPartitioner.java | 38 - .../partitioner/ShufflePartitioner.java | 49 - .../partitioner/StreamPartitioner.java | 44 - .../flink/streaming/state/CircularFifoList.java | 112 -- .../flink/streaming/state/GraphState.java | 61 - .../apache/flink/streaming/state/MapState.java | 201 --- .../streaming/state/NullableCircularBuffer.java | 362 ----- .../flink/streaming/state/OperatorState.java | 105 -- .../streaming/state/PartitionableState.java | 60 - .../flink/streaming/state/SimpleState.java | 53 - .../state/checkpoint/MapCheckpoint.java | 72 - .../state/checkpoint/StateCheckpoint.java | 82 - .../flink/streaming/util/ClusterUtil.java | 76 - .../streaming/util/keys/KeySelectorUtil.java | 126 -- .../streaming/api/AggregationFunctionTest.java | 318 ---- .../apache/flink/streaming/api/IterateTest.java | 95 -- .../flink/streaming/api/OutputSplitterTest.java | 185 --- .../apache/flink/streaming/api/PrintTest.java | 57 - .../apache/flink/streaming/api/SourceTest.java | 70 - .../streaming/api/WindowCrossJoinTest.java | 154 -- .../api/collector/DirectedOutputTest.java | 112 -- .../api/collector/OutputSelectorTest.java | 58 - .../api/collector/StreamCollectorTest.java | 52 - .../api/invokable/operator/CoFlatMapTest.java | 84 - .../invokable/operator/CoGroupedReduceTest.java | 125 -- .../api/invokable/operator/CoMapTest.java | 57 - .../invokable/operator/CoStreamReduceTest.java | 71 - .../api/invokable/operator/CoWindowTest.java | 182 --- .../operator/CounterInvokableTest.java | 39 - .../api/invokable/operator/FilterTest.java | 51 - .../api/invokable/operator/FlatMapTest.java | 54 - .../operator/GroupedReduceInvokableTest.java | 62 - .../operator/GroupedWindowInvokableTest.java | 574 ------- .../api/invokable/operator/MapTest.java | 49 - .../api/invokable/operator/ProjectTest.java | 67 - .../invokable/operator/StreamReduceTest.java | 54 - .../invokable/operator/WindowInvokableTest.java | 261 ---- .../streaming/api/streamrecord/UIDTest.java | 53 - .../api/streamvertex/MockRecordWriter.java | 45 - .../api/streamvertex/StreamVertexTest.java | 177 --- .../deltafunction/CosineDistanceTest.java | 70 - .../deltafunction/EuclideanDistanceTest.java | 71 - .../windowing/extractor/ArrayFromTupleTest.java | 118 -- .../extractor/ConcatinatedExtractTest.java | 83 - .../windowing/extractor/FieldFromArrayTest.java | 55 - .../windowing/extractor/FieldFromTupleTest.java | 84 - .../extractor/FieldsFromArrayTest.java | 108 -- .../extractor/FieldsFromTupleTest.java | 106 -- .../policy/CountEvictionPolicyTest.java | 119 -- .../policy/CountTriggerPolicyTest.java | 94 -- .../api/windowing/policy/DeltaPolicyTest.java | 62 - .../windowing/policy/PunctuationPolicyTest.java | 123 -- .../policy/TimeEvictionPolicyTest.java | 121 -- .../windowing/policy/TimeTriggerPolicyTest.java | 114 -- .../policy/TumblingEvictionPolicyTest.java | 41 - .../partitioner/BroadcastPartitionerTest.java | 55 - .../partitioner/DistributePartitionerTest.java | 56 - .../partitioner/FieldsPartitionerTest.java | 76 - .../partitioner/ForwardPartitionerTest.java | 55 - .../partitioner/GlobalPartitionerTest.java | 50 - .../partitioner/ShufflePartitionerTest.java | 60 - .../flink/streaming/state/MapStateTest.java | 130 -- .../streaming/state/OperatorStateTest.java | 48 - .../flink/streaming/util/MockCoContext.java | 217 --- .../flink/streaming/util/MockCollector.java | 43 - .../flink/streaming/util/MockContext.java | 159 -- .../streaming/util/MockRecordWriterFactory.java | 40 - .../apache/flink/streaming/util/MockSource.java | 36 - .../streaming/util/TestStreamEnvironment.java | 72 - .../src/test/resources/log4j-test.properties | 19 - .../src/test/resources/logback-test.xml | 30 - .../flink-streaming-examples/pom.xml | 518 ------ .../examples/iteration/IterateExample.java | 168 -- .../streaming/examples/join/WindowJoin.java | 190 --- .../ml/IncrementalLearningSkeleton.java | 213 --- .../socket/SocketTextStreamWordCount.java | 108 -- .../examples/twitter/TwitterStream.java | 202 --- .../twitter/util/TwitterStreamData.java | 255 --- .../examples/windowing/DeltaExtractExample.java | 125 -- .../windowing/MultiplePoliciesExample.java | 139 -- .../examples/windowing/SlidingExample.java | 139 -- .../windowing/TimeWindowingExample.java | 152 -- .../windowing/TopSpeedWindowingExample.java | 128 -- .../examples/wordcount/PojoExample.java | 185 --- .../streaming/examples/wordcount/WordCount.java | 148 -- .../socket/SocketTextStreamWordCount.scala | 91 -- .../examples/windowing/TopSpeedWindowing.scala | 94 -- .../scala/examples/windowing/WindowJoin.scala | 72 - .../flink-streaming-scala/pom.xml | 217 --- .../scala/ScalaStreamingAggregator.java | 111 -- .../api/scala/ConnectedDataStream.scala | 380 ----- .../flink/streaming/api/scala/DataStream.scala | 573 ------- .../streaming/api/scala/SplitDataStream.scala | 39 - .../api/scala/StreamCrossOperator.scala | 120 -- .../api/scala/StreamExecutionEnvironment.scala | 289 ---- .../api/scala/StreamJoinOperator.scala | 212 --- .../streaming/api/scala/TemporalOperator.scala | 43 - .../api/scala/WindowedDataStream.scala | 248 --- .../flink/streaming/api/scala/package.scala | 70 - .../streaming/api/scala/windowing/Delta.scala | 47 - .../streaming/api/scala/windowing/Time.scala | 54 - flink-addons/flink-streaming/pom.xml | 75 - flink-addons/flink-tachyon/pom.xml | 117 -- .../java/org/apache/flink/tachyon/HDFSTest.java | 160 -- .../tachyon/TachyonFileSystemWrapperTest.java | 167 -- .../src/test/resources/log4j.properties | 23 - .../src/test/resources/tachyonHadoopConf.xml | 28 - flink-addons/pom.xml | 64 - flink-staging/flink-avro/pom.xml | 182 +++ .../apache/flink/api/avro/DataInputDecoder.java | 213 +++ .../flink/api/avro/DataOutputEncoder.java | 183 +++ .../api/avro/FSDataInputStreamWrapper.java | 68 + .../api/io/avro/example/AvroTypeExample.java | 111 ++ .../apache/flink/api/io/avro/example/User.java | 269 ++++ .../flink/api/java/io/AvroInputFormat.java | 124 ++ .../flink/api/java/io/AvroOutputFormat.java | 100 ++ .../src/test/assembly/test-assembly.xml | 36 + .../api/avro/AvroExternalJarProgramITCase.java | 71 + .../flink/api/avro/AvroOutputFormatTest.java | 173 ++ .../flink/api/avro/EncoderDecoderTest.java | 519 ++++++ .../avro/testjar/AvroExternalJarProgram.java | 219 +++ .../api/io/avro/AvroRecordInputFormatTest.java | 167 ++ .../flink/api/io/avro/generated/Colors.java | 32 + .../flink/api/io/avro/generated/User.java | 755 +++++++++ .../io/AvroInputFormatTypeExtractionTest.java | 79 + .../src/test/resources/avro/user.avsc | 19 + .../src/test/resources/log4j-test.properties | 27 + .../src/test/resources/logback-test.xml | 29 + .../flink-avro/src/test/resources/testdata.avro | Bin 0 -> 4572 bytes .../flink-hadoop-compatibility/pom.xml | 80 + .../mapred/HadoopInputFormat.java | 297 ++++ .../mapred/HadoopMapFunction.java | 131 ++ .../mapred/HadoopOutputFormat.java | 183 +++ .../mapred/HadoopReduceCombineFunction.java | 162 ++ .../mapred/HadoopReduceFunction.java | 138 ++ .../example/HadoopMapredCompatWordCount.java | 133 ++ .../mapred/record/HadoopDataSink.java | 109 ++ .../mapred/record/HadoopDataSource.java | 82 + .../mapred/record/HadoopRecordInputFormat.java | 174 +++ .../mapred/record/HadoopRecordOutputFormat.java | 156 ++ .../datatypes/DefaultFlinkTypeConverter.java | 95 ++ .../datatypes/DefaultHadoopTypeConverter.java | 83 + .../record/datatypes/FlinkTypeConverter.java | 43 + .../datatypes/HadoopFileOutputCommitter.java | 196 +++ .../record/datatypes/HadoopTypeConverter.java | 42 + .../datatypes/WritableComparableWrapper.java | 40 + .../record/datatypes/WritableWrapper.java | 71 + .../datatypes/WritableWrapperConverter.java | 45 + .../mapred/record/example/WordCount.java | 184 +++ .../example/WordCountWithOutputFormat.java | 173 ++ .../mapred/utils/HadoopUtils.java | 87 ++ .../mapred/wrapper/HadoopDummyProgressable.java | 33 + .../mapred/wrapper/HadoopDummyReporter.java | 70 + .../mapred/wrapper/HadoopInputSplit.java | 138 ++ .../mapred/wrapper/HadoopOutputCollector.java | 64 + .../wrapper/HadoopTupleUnwrappingIterator.java | 94 ++ .../mapreduce/HadoopInputFormat.java | 338 ++++ .../mapreduce/HadoopOutputFormat.java | 226 +++ .../mapreduce/example/WordCount.java | 120 ++ .../mapreduce/utils/HadoopUtils.java | 83 + .../mapreduce/wrapper/HadoopInputSplit.java | 125 ++ .../mapred/HadoopIOFormatsITCase.java | 221 +++ .../mapred/HadoopInputFormatTest.java | 82 + .../mapred/HadoopMapFunctionITCase.java | 182 +++ .../mapred/HadoopMapredITCase.java | 47 + .../HadoopReduceCombineFunctionITCase.java | 265 ++++ .../mapred/HadoopReduceFunctionITCase.java | 213 +++ .../mapred/HadoopTestData.java | 62 + .../record/HadoopRecordInputOutputITCase.java | 54 + .../HadoopTupleUnwrappingIteratorTest.java | 137 ++ .../mapreduce/HadoopInputFormatTest.java | 84 + .../mapreduce/HadoopInputOutputITCase.java | 47 + .../src/test/resources/log4j-test.properties | 27 + .../src/test/resources/logback-test.xml | 29 + flink-staging/flink-hbase/pom.xml | 165 ++ .../flink/addons/hbase/TableInputFormat.java | 255 +++ .../flink/addons/hbase/TableInputSplit.java | 168 ++ .../addons/hbase/example/HBaseReadExample.java | 93 ++ .../src/test/resources/hbase-site.xml | 43 + .../src/test/resources/log4j.properties | 23 + flink-staging/flink-jdbc/pom.xml | 64 + .../flink/api/java/io/jdbc/JDBCInputFormat.java | 358 +++++ .../api/java/io/jdbc/JDBCOutputFormat.java | 270 ++++ .../api/java/io/jdbc/example/JDBCExample.java | 101 ++ .../java/record/io/jdbc/JDBCInputFormat.java | 389 +++++ .../java/record/io/jdbc/JDBCOutputFormat.java | 359 +++++ .../record/io/jdbc/example/JDBCExample.java | 136 ++ .../api/java/io/jdbc/JDBCInputFormatTest.java | 195 +++ .../api/java/io/jdbc/JDBCOutputFormatTest.java | 240 +++ .../java/record/io/jdbc/DevNullLogStream.java | 30 + .../record/io/jdbc/JDBCInputFormatTest.java | 214 +++ .../record/io/jdbc/JDBCOutputFormatTest.java | 225 +++ .../src/test/resources/log4j-test.properties | 19 + .../src/test/resources/logback-test.xml | 29 + flink-staging/flink-spargel/pom.xml | 60 + .../flink/spargel/java/MessageIterator.java | 58 + .../flink/spargel/java/MessagingFunction.java | 284 ++++ .../apache/flink/spargel/java/OutgoingEdge.java | 64 + .../spargel/java/VertexCentricIteration.java | 599 +++++++ .../spargel/java/VertexUpdateFunction.java | 145 ++ .../examples/SpargelConnectedComponents.java | 79 + .../spargel/java/examples/SpargelPageRank.java | 117 ++ .../SpargelPageRankCountingVertices.java | 154 ++ .../apache/flink/spargel/java/record/Edge.java | 43 + .../spargel/java/record/MessageIterator.java | 59 + .../spargel/java/record/MessagingFunction.java | 163 ++ .../spargel/java/record/SpargelIteration.java | 289 ++++ .../java/record/VertexUpdateFunction.java | 90 ++ .../flink/spargel/java/SpargelCompilerTest.java | 183 +++ .../spargel/java/SpargelTranslationTest.java | 211 +++ .../SpargelConnectedComponentsITCase.java | 81 + .../src/test/resources/log4j-test.properties | 19 + .../src/test/resources/logback-test.xml | 29 + .../flink-streaming-connectors/pom.xml | 243 +++ .../streaming/connectors/ConnectorSource.java | 42 + .../db/CustomSerializationDBState.java | 31 + .../streaming/connectors/db/DBSerializer.java | 36 + .../flink/streaming/connectors/db/DBState.java | 29 + .../connectors/db/DBStateIterator.java | 31 + .../connectors/db/DBStateWithIterator.java | 25 + .../connectors/db/DefaultDBSerializer.java | 35 + .../streaming/connectors/db/LevelDBState.java | 125 ++ .../streaming/connectors/db/MemcachedState.java | 73 + .../streaming/connectors/db/RedisState.java | 109 ++ .../streaming/connectors/flume/FlumeSink.java | 141 ++ .../streaming/connectors/flume/FlumeSource.java | 141 ++ .../connectors/flume/FlumeTopology.java | 49 + .../connectors/json/JSONParseFlatMap.java | 144 ++ .../streaming/connectors/json/JSONParser.java | 175 +++ .../streaming/connectors/kafka/KafkaSink.java | 83 + .../streaming/connectors/kafka/KafkaSource.java | 96 ++ .../connectors/kafka/KafkaTopology.java | 57 + .../streaming/connectors/rabbitmq/RMQSink.java | 111 ++ .../connectors/rabbitmq/RMQSource.java | 121 ++ .../connectors/rabbitmq/RMQTopology.java | 52 + .../connectors/twitter/TwitterSource.java | 283 ++++ .../connectors/twitter/TwitterStreaming.java | 99 ++ .../connectors/twitter/TwitterTopology.java | 92 ++ .../connectors/util/DeserializationSchema.java | 42 + .../streaming/connectors/util/RawSchema.java | 39 + .../connectors/util/SerializationSchema.java | 33 + .../connectors/util/SimpleStringSchema.java | 40 + .../streaming/connectors/db/DBStateTest.java | 105 ++ .../connectors/json/JSONParserTest.java | 74 + .../connectors/json/JSONParserTest2.java | 95 ++ .../src/test/resources/log4j-test.properties | 19 + .../src/test/resources/logback-test.xml | 30 + .../flink-streaming-core/pom.xml | 75 + .../flink/streaming/api/StreamConfig.java | 406 +++++ .../apache/flink/streaming/api/StreamGraph.java | 620 ++++++++ .../api/StreamingJobGraphGenerator.java | 293 ++++ .../api/collector/CollectorWrapper.java | 49 + .../api/collector/DirectedCollectorWrapper.java | 131 ++ .../streaming/api/collector/OutputSelector.java | 44 + .../streaming/api/collector/StreamOutput.java | 86 + .../api/datastream/ConnectedDataStream.java | 419 +++++ .../streaming/api/datastream/DataStream.java | 1271 +++++++++++++++ .../api/datastream/DataStreamSink.java | 46 + .../api/datastream/DataStreamSource.java | 48 + .../api/datastream/GroupedDataStream.java | 205 +++ .../api/datastream/IterativeDataStream.java | 107 ++ .../datastream/SingleOutputStreamOperator.java | 174 +++ .../api/datastream/SplitDataStream.java | 66 + .../api/datastream/StreamProjection.java | 1471 ++++++++++++++++++ .../api/datastream/WindowedDataStream.java | 658 ++++++++ .../temporaloperator/StreamCrossOperator.java | 101 ++ .../temporaloperator/StreamJoinOperator.java | 272 ++++ .../temporaloperator/TemporalOperator.java | 124 ++ .../temporaloperator/TemporalWindow.java | 45 + .../api/environment/LocalStreamEnvironment.java | 47 + .../environment/RemoteStreamEnvironment.java | 127 ++ .../environment/StreamContextEnvironment.java | 76 + .../environment/StreamExecutionEnvironment.java | 618 ++++++++ .../api/environment/StreamPlanEnvironment.java | 64 + .../aggregation/AggregationFunction.java | 35 + .../aggregation/ComparableAggregator.java | 236 +++ .../api/function/aggregation/Comparator.java | 104 ++ .../api/function/aggregation/SumAggregator.java | 171 ++ .../api/function/aggregation/SumFunction.java | 102 ++ .../api/function/co/CoFlatMapFunction.java | 42 + .../api/function/co/CoMapFunction.java | 41 + .../api/function/co/CoReduceFunction.java | 107 ++ .../api/function/co/CoWindowFunction.java | 30 + .../api/function/co/CrossWindowFunction.java | 44 + .../api/function/co/JoinWindowFunction.java | 55 + .../api/function/co/RichCoFlatMapFunction.java | 40 + .../api/function/co/RichCoMapFunction.java | 40 + .../api/function/co/RichCoReduceFunction.java | 40 + .../api/function/co/RichCoWindowFunction.java | 34 + .../api/function/sink/FileSinkFunction.java | 118 ++ .../function/sink/FileSinkFunctionByMillis.java | 59 + .../api/function/sink/PrintSinkFunction.java | 98 ++ .../api/function/sink/RichSinkFunction.java | 28 + .../api/function/sink/SinkFunction.java | 28 + .../api/function/sink/WriteFormat.java | 43 + .../api/function/sink/WriteFormatAsCsv.java | 49 + .../api/function/sink/WriteFormatAsText.java | 47 + .../api/function/sink/WriteSinkFunction.java | 92 ++ .../sink/WriteSinkFunctionByMillis.java | 50 + .../function/source/FileMonitoringFunction.java | 123 ++ .../api/function/source/FileReadFunction.java | 51 + .../api/function/source/FileSourceFunction.java | 142 ++ .../function/source/FromElementsFunction.java | 49 + .../function/source/GenSequenceFunction.java | 53 + .../function/source/GenericSourceFunction.java | 25 + .../function/source/ParallelSourceFunction.java | 26 + .../source/RichParallelSourceFunction.java | 27 + .../api/function/source/RichSourceFunction.java | 27 + .../source/SocketTextStreamFunction.java | 86 + .../api/function/source/SourceFunction.java | 29 + .../api/invokable/ChainableInvokable.java | 39 + .../streaming/api/invokable/SinkInvokable.java | 50 + .../api/invokable/SourceInvokable.java | 44 + .../api/invokable/StreamInvokable.java | 186 +++ .../api/invokable/operator/BatchIterator.java | 25 + .../invokable/operator/CounterInvokable.java | 43 + .../api/invokable/operator/FilterInvokable.java | 55 + .../invokable/operator/FlatMapInvokable.java | 51 + .../operator/GroupedReduceInvokable.java | 59 + .../operator/GroupedWindowInvokable.java | 506 ++++++ .../api/invokable/operator/MapInvokable.java | 50 + .../invokable/operator/ProjectInvokable.java | 61 + .../operator/StreamReduceInvokable.java | 69 + .../operator/WindowGroupReduceInvokable.java | 51 + .../api/invokable/operator/WindowInvokable.java | 382 +++++ .../operator/WindowReduceInvokable.java | 59 + .../operator/co/CoFlatMapInvokable.java | 54 + .../operator/co/CoGroupedReduceInvokable.java | 88 ++ .../api/invokable/operator/co/CoInvokable.java | 133 ++ .../invokable/operator/co/CoMapInvokable.java | 54 + .../operator/co/CoReduceInvokable.java | 70 + .../operator/co/CoWindowInvokable.java | 197 +++ .../api/streamrecord/StreamRecord.java | 134 ++ .../streamrecord/StreamRecordSerializer.java | 114 ++ .../flink/streaming/api/streamrecord/UID.java | 122 ++ .../api/streamvertex/CoStreamVertex.java | 142 ++ .../api/streamvertex/InputHandler.java | 96 ++ .../api/streamvertex/OutputHandler.java | 242 +++ .../api/streamvertex/StreamIterationHead.java | 101 ++ .../api/streamvertex/StreamIterationTail.java | 103 ++ .../api/streamvertex/StreamTaskContext.java | 40 + .../api/streamvertex/StreamVertex.java | 156 ++ .../api/streamvertex/StreamVertexException.java | 68 + .../streamvertex/StreamingRuntimeContext.java | 88 ++ .../windowing/deltafunction/CosineDistance.java | 87 ++ .../windowing/deltafunction/DeltaFunction.java | 44 + .../deltafunction/EuclideanDistance.java | 58 + .../ExtractionAwareDeltaFunction.java | 90 ++ .../api/windowing/extractor/ArrayFromTuple.java | 74 + .../extractor/ConcatinatedExtract.java | 68 + .../api/windowing/extractor/Extractor.java | 43 + .../api/windowing/extractor/FieldFromArray.java | 59 + .../api/windowing/extractor/FieldFromTuple.java | 58 + .../windowing/extractor/FieldsFromArray.java | 67 + .../windowing/extractor/FieldsFromTuple.java | 53 + .../streaming/api/windowing/helper/Count.java | 101 ++ .../streaming/api/windowing/helper/Delta.java | 91 ++ .../api/windowing/helper/SystemTimestamp.java | 37 + .../streaming/api/windowing/helper/Time.java | 166 ++ .../api/windowing/helper/Timestamp.java | 39 + .../api/windowing/helper/TimestampWrapper.java | 44 + .../api/windowing/helper/WindowingHelper.java | 39 + .../ActiveCloneableEvictionPolicyWrapper.java | 62 + .../windowing/policy/ActiveEvictionPolicy.java | 52 + .../policy/ActiveEvictionPolicyWrapper.java | 64 + .../windowing/policy/ActiveTriggerCallback.java | 45 + .../windowing/policy/ActiveTriggerPolicy.java | 81 + .../policy/CloneableEvictionPolicy.java | 45 + .../policy/CloneableTriggerPolicy.java | 45 + .../windowing/policy/CountEvictionPolicy.java | 119 ++ .../windowing/policy/CountTriggerPolicy.java | 87 ++ .../api/windowing/policy/DeltaPolicy.java | 112 ++ .../api/windowing/policy/EvictionPolicy.java | 47 + .../api/windowing/policy/PunctuationPolicy.java | 116 ++ .../windowing/policy/TimeEvictionPolicy.java | 137 ++ .../api/windowing/policy/TimeTriggerPolicy.java | 196 +++ .../api/windowing/policy/TriggerPolicy.java | 54 + .../policy/TumblingEvictionPolicy.java | 90 ++ .../flink/streaming/io/BlockingQueueBroker.java | 41 + .../flink/streaming/io/CoReaderIterator.java | 57 + .../flink/streaming/io/CoRecordReader.java | 221 +++ .../flink/streaming/io/StreamRecordWriter.java | 83 + .../partitioner/BroadcastPartitioner.java | 55 + .../partitioner/DistributePartitioner.java | 46 + .../partitioner/FieldsPartitioner.java | 50 + .../partitioner/GlobalPartitioner.java | 38 + .../partitioner/ShufflePartitioner.java | 49 + .../partitioner/StreamPartitioner.java | 44 + .../flink/streaming/state/CircularFifoList.java | 112 ++ .../flink/streaming/state/GraphState.java | 61 + .../apache/flink/streaming/state/MapState.java | 201 +++ .../streaming/state/NullableCircularBuffer.java | 362 +++++ .../flink/streaming/state/OperatorState.java | 105 ++ .../streaming/state/PartitionableState.java | 60 + .../flink/streaming/state/SimpleState.java | 53 + .../state/checkpoint/MapCheckpoint.java | 72 + .../state/checkpoint/StateCheckpoint.java | 82 + .../flink/streaming/util/ClusterUtil.java | 76 + .../streaming/util/keys/KeySelectorUtil.java | 126 ++ .../streaming/api/AggregationFunctionTest.java | 318 ++++ .../apache/flink/streaming/api/IterateTest.java | 95 ++ .../flink/streaming/api/OutputSplitterTest.java | 185 +++ .../apache/flink/streaming/api/PrintTest.java | 57 + .../apache/flink/streaming/api/SourceTest.java | 70 + .../streaming/api/WindowCrossJoinTest.java | 154 ++ .../api/collector/DirectedOutputTest.java | 112 ++ .../api/collector/OutputSelectorTest.java | 58 + .../api/collector/StreamCollectorTest.java | 52 + .../api/invokable/operator/CoFlatMapTest.java | 84 + .../invokable/operator/CoGroupedReduceTest.java | 125 ++ .../api/invokable/operator/CoMapTest.java | 57 + .../invokable/operator/CoStreamReduceTest.java | 71 + .../api/invokable/operator/CoWindowTest.java | 182 +++ .../operator/CounterInvokableTest.java | 39 + .../api/invokable/operator/FilterTest.java | 51 + .../api/invokable/operator/FlatMapTest.java | 54 + .../operator/GroupedReduceInvokableTest.java | 62 + .../operator/GroupedWindowInvokableTest.java | 574 +++++++ .../api/invokable/operator/MapTest.java | 49 + .../api/invokable/operator/ProjectTest.java | 67 + .../invokable/operator/StreamReduceTest.java | 54 + .../invokable/operator/WindowInvokableTest.java | 261 ++++ .../streaming/api/streamrecord/UIDTest.java | 53 + .../api/streamvertex/MockRecordWriter.java | 45 + .../api/streamvertex/StreamVertexTest.java | 177 +++ .../deltafunction/CosineDistanceTest.java | 70 + .../deltafunction/EuclideanDistanceTest.java | 71 + .../windowing/extractor/ArrayFromTupleTest.java | 118 ++ .../extractor/ConcatinatedExtractTest.java | 83 + .../windowing/extractor/FieldFromArrayTest.java | 55 + .../windowing/extractor/FieldFromTupleTest.java | 84 + .../extractor/FieldsFromArrayTest.java | 108 ++ .../extractor/FieldsFromTupleTest.java | 106 ++ .../policy/CountEvictionPolicyTest.java | 119 ++ .../policy/CountTriggerPolicyTest.java | 94 ++ .../api/windowing/policy/DeltaPolicyTest.java | 62 + .../windowing/policy/PunctuationPolicyTest.java | 123 ++ .../policy/TimeEvictionPolicyTest.java | 121 ++ .../windowing/policy/TimeTriggerPolicyTest.java | 114 ++ .../policy/TumblingEvictionPolicyTest.java | 41 + .../partitioner/BroadcastPartitionerTest.java | 55 + .../partitioner/DistributePartitionerTest.java | 56 + .../partitioner/FieldsPartitionerTest.java | 76 + .../partitioner/ForwardPartitionerTest.java | 55 + .../partitioner/GlobalPartitionerTest.java | 50 + .../partitioner/ShufflePartitionerTest.java | 60 + .../flink/streaming/state/MapStateTest.java | 130 ++ .../streaming/state/OperatorStateTest.java | 48 + .../flink/streaming/util/MockCoContext.java | 217 +++ .../flink/streaming/util/MockCollector.java | 43 + .../flink/streaming/util/MockContext.java | 159 ++ .../streaming/util/MockRecordWriterFactory.java | 40 + .../apache/flink/streaming/util/MockSource.java | 36 + .../streaming/util/TestStreamEnvironment.java | 72 + .../src/test/resources/log4j-test.properties | 19 + .../src/test/resources/logback-test.xml | 30 + .../flink-streaming-examples/pom.xml | 518 ++++++ .../examples/iteration/IterateExample.java | 168 ++ .../streaming/examples/join/WindowJoin.java | 190 +++ .../ml/IncrementalLearningSkeleton.java | 213 +++ .../socket/SocketTextStreamWordCount.java | 108 ++ .../examples/twitter/TwitterStream.java | 202 +++ .../twitter/util/TwitterStreamData.java | 255 +++ .../examples/windowing/DeltaExtractExample.java | 125 ++ .../windowing/MultiplePoliciesExample.java | 139 ++ .../examples/windowing/SlidingExample.java | 139 ++ .../windowing/TimeWindowingExample.java | 152 ++ .../windowing/TopSpeedWindowingExample.java | 128 ++ .../examples/wordcount/PojoExample.java | 185 +++ .../streaming/examples/wordcount/WordCount.java | 148 ++ .../socket/SocketTextStreamWordCount.scala | 91 ++ .../examples/windowing/TopSpeedWindowing.scala | 94 ++ .../scala/examples/windowing/WindowJoin.scala | 72 + .../flink-streaming-scala/pom.xml | 217 +++ .../scala/ScalaStreamingAggregator.java | 111 ++ .../api/scala/ConnectedDataStream.scala | 380 +++++ .../flink/streaming/api/scala/DataStream.scala | 573 +++++++ .../streaming/api/scala/SplitDataStream.scala | 39 + .../api/scala/StreamCrossOperator.scala | 120 ++ .../api/scala/StreamExecutionEnvironment.scala | 289 ++++ .../api/scala/StreamJoinOperator.scala | 212 +++ .../streaming/api/scala/TemporalOperator.scala | 43 + .../api/scala/WindowedDataStream.scala | 248 +++ .../flink/streaming/api/scala/package.scala | 70 + .../streaming/api/scala/windowing/Delta.scala | 47 + .../streaming/api/scala/windowing/Time.scala | 54 + flink-staging/flink-streaming/pom.xml | 75 + flink-staging/flink-tachyon/pom.xml | 117 ++ .../java/org/apache/flink/tachyon/HDFSTest.java | 160 ++ .../tachyon/TachyonFileSystemWrapperTest.java | 167 ++ .../src/test/resources/log4j.properties | 23 + .../src/test/resources/tachyonHadoopConf.xml | 28 + flink-staging/pom.xml | 64 + pom.xml | 6 +- 774 files changed, 47728 insertions(+), 47727 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/docs/internal_add_operator.md ---------------------------------------------------------------------- diff --git a/docs/internal_add_operator.md b/docs/internal_add_operator.md index ddcb6b5..c564221 100644 --- a/docs/internal_add_operator.md +++ b/docs/internal_add_operator.md @@ -100,7 +100,7 @@ void setInput(DataSet<IN> inputData); DataSet<OUT> createResult(); ~~~ -The {% gh_link /flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/VertexCentricIteration.java "VertexCentricIteration" %} operator is implemented that way. Below is an example how to implement the *count()* operator that way. +The {% gh_link /flink-staging/flink-spargel/src/main/java/org/apache/flink/spargel/java/VertexCentricIteration.java "VertexCentricIteration" %} operator is implemented that way. Below is an example how to implement the *count()* operator that way. ~~~ java public class Counter<T> implements CustomUnaryOperation<T, Long> { http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/docs/internal_general_arch.md ---------------------------------------------------------------------- diff --git a/docs/internal_general_arch.md b/docs/internal_general_arch.md index 81a8cbe..ec07056 100644 --- a/docs/internal_general_arch.md +++ b/docs/internal_general_arch.md @@ -75,12 +75,13 @@ In addition to the projects listed in the figure above, Flink currently contains - `flink-dist`: The *distribution* project. It defines how to assemble the compiled code, scripts, and other resources into the final folder structure that is ready to use. -- `flink-addons`: A series of projects that are in an early version. Currently contains +- `flink-staging`: A series of projects that are in an early version. Currently contains among other things projects for YARN support, JDBC data sources and sinks, hadoop compatibility, graph specific operators, and HBase connectors. - `flink-quickstart`: Scripts, maven archetypes, and example programs for the quickstarts and tutorials. +- `flink-contrib`: Useful tools contributed by users. The code is maintained mainly by external contributors. The requirements for code being accepted into `flink-contrib` are lower compared to the rest of the code. http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/docs/streaming_guide.md ---------------------------------------------------------------------- diff --git a/docs/streaming_guide.md b/docs/streaming_guide.md index d2d54c3..3476390 100644 --- a/docs/streaming_guide.md +++ b/docs/streaming_guide.md @@ -857,7 +857,7 @@ After installing Docker an image can be pulled for each connector. Containers ca For the easiest set up create a jar with all the dependencies of the *flink-streaming-connectors* project. ~~~batch -cd /PATH/TO/GIT/incubator-flink/flink-addons/flink-streaming-connectors +cd /PATH/TO/GIT/flink/flink-staging/flink-streaming-connectors mvn assembly:assembly ~~~batch @@ -974,14 +974,14 @@ Now a terminal started running from the image with all the necessary configurati To have the latest version of Flink type: ~~~batch -cd /git/incubator-flink/ +cd /git/flink/ git pull ~~~ Then build the code with: ~~~batch -cd /git/incubator-flink/flink-addons/flink-streaming/flink-streaming-connectors/ +cd /git/flink/flink-staging/flink-streaming/flink-streaming-connectors/ mvn install -DskipTests ~~~ http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-avro/pom.xml ---------------------------------------------------------------------- diff --git a/flink-addons/flink-avro/pom.xml b/flink-addons/flink-avro/pom.xml deleted file mode 100644 index 8115f9d..0000000 --- a/flink-addons/flink-avro/pom.xml +++ /dev/null @@ -1,182 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. ---> -<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; - - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> - - <modelVersion>4.0.0</modelVersion> - - <parent> - <groupId>org.apache.flink</groupId> - <artifactId>flink-addons</artifactId> - <version>0.9-SNAPSHOT</version> - <relativePath>..</relativePath> - </parent> - - <artifactId>flink-avro</artifactId> - <name>flink-avro</name> - - <packaging>jar</packaging> - - <dependencies> - <dependency> - <groupId>org.apache.flink</groupId> - <artifactId>flink-java</artifactId> - <version>${project.version}</version> - </dependency> - - <dependency> - <groupId>org.apache.flink</groupId> - <artifactId>flink-clients</artifactId> - <version>${project.version}</version> - </dependency> - - <dependency> - <groupId>org.apache.avro</groupId> - <artifactId>avro</artifactId> - <!-- version is derived from base module --> - </dependency> - - <dependency> - <groupId>org.apache.flink</groupId> - <artifactId>flink-test-utils</artifactId> - <version>${project.version}</version> - <scope>test</scope> - </dependency> - - </dependencies> - - <build> - <plugins> - <plugin> - <artifactId>maven-assembly-plugin</artifactId> - <executions> - <execution> - <id>create-test-dependency</id> - <phase>process-test-classes</phase> - <goals> - <goal>single</goal> - </goals> - <configuration> - <archive> - <manifest> - <mainClass>org.apache.flink.api.avro.testjar.AvroExternalJarProgram</mainClass> - </manifest> - </archive> - <finalName>maven</finalName> - <attach>false</attach> - <descriptors> - <descriptor>src/test/assembly/test-assembly.xml</descriptor> - </descriptors> - </configuration> - </execution> - </executions> - </plugin> - <!--Remove the AvroExternalJarProgram code from the test-classes directory since it musn't be in the - classpath when running the tests to actually test whether the user code class loader - is properly used.--> - <plugin> - <artifactId>maven-clean-plugin</artifactId> - <version>2.5</version><!--$NO-MVN-MAN-VER$--> - <executions> - <execution> - <id>remove-avroexternalprogram</id> - <phase>process-test-classes</phase> - <goals> - <goal>clean</goal> - </goals> - <configuration> - <excludeDefaultDirectories>true</excludeDefaultDirectories> - <filesets> - <fileset> - <directory>${project.build.testOutputDirectory}</directory> - <includes> - <include>**/testjar/*.class</include> - </includes> - </fileset> - </filesets> - </configuration> - </execution> - </executions> - </plugin> - - </plugins> - - <pluginManagement> - <plugins> - <!--This plugin's configuration is used to store Eclipse m2e settings only. It has no influence on the Maven build itself.--> - <plugin> - <groupId>org.eclipse.m2e</groupId> - <artifactId>lifecycle-mapping</artifactId> - <version>1.0.0</version> - <configuration> - <lifecycleMappingMetadata> - <pluginExecutions> - <pluginExecution> - <pluginExecutionFilter> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-assembly-plugin</artifactId> - <versionRange>[2.4,)</versionRange> - <goals> - <goal>single</goal> - </goals> - </pluginExecutionFilter> - <action> - <ignore/> - </action> - </pluginExecution> - <pluginExecution> - <pluginExecutionFilter> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-clean-plugin</artifactId> - <versionRange>[1,)</versionRange> - <goals> - <goal>clean</goal> - </goals> - </pluginExecutionFilter> - <action> - <ignore/> - </action> - </pluginExecution> - </pluginExecutions> - </lifecycleMappingMetadata> - </configuration> - </plugin> - </plugins> - </pluginManagement> - </build> - - <profiles> - <profile> - <!-- A bug with java6 is causing the javadoc generation - to fail because the test case contains junit? - See https://github.com/stratosphere/stratosphere/pull/405#issuecomment-32591978 - for further links --> - <id>disable-javadocs-in-java6</id> - <activation> - <jdk>(,1.6]</jdk> <!-- disable javadocs for java6 or lower. --> - </activation> - <properties> - <maven.javadoc.skip>true</maven.javadoc.skip> - </properties> - </profile> - </profiles> - -</project> http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-avro/src/main/java/org/apache/flink/api/avro/DataInputDecoder.java ---------------------------------------------------------------------- diff --git a/flink-addons/flink-avro/src/main/java/org/apache/flink/api/avro/DataInputDecoder.java b/flink-addons/flink-avro/src/main/java/org/apache/flink/api/avro/DataInputDecoder.java deleted file mode 100644 index 59da4cb..0000000 --- a/flink-addons/flink-avro/src/main/java/org/apache/flink/api/avro/DataInputDecoder.java +++ /dev/null @@ -1,213 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.api.avro; - -import java.io.DataInput; -import java.io.IOException; -import java.nio.ByteBuffer; - -import org.apache.avro.io.Decoder; -import org.apache.avro.util.Utf8; - - -public class DataInputDecoder extends Decoder { - - private final Utf8 stringDecoder = new Utf8(); - - private DataInput in; - - public void setIn(DataInput in) { - this.in = in; - } - - // -------------------------------------------------------------------------------------------- - // primitives - // -------------------------------------------------------------------------------------------- - - @Override - public void readNull() {} - - - @Override - public boolean readBoolean() throws IOException { - return in.readBoolean(); - } - - @Override - public int readInt() throws IOException { - return in.readInt(); - } - - @Override - public long readLong() throws IOException { - return in.readLong(); - } - - @Override - public float readFloat() throws IOException { - return in.readFloat(); - } - - @Override - public double readDouble() throws IOException { - return in.readDouble(); - } - - @Override - public int readEnum() throws IOException { - return readInt(); - } - - // -------------------------------------------------------------------------------------------- - // bytes - // -------------------------------------------------------------------------------------------- - - @Override - public void readFixed(byte[] bytes, int start, int length) throws IOException { - in.readFully(bytes, start, length); - } - - @Override - public ByteBuffer readBytes(ByteBuffer old) throws IOException { - int length = readInt(); - ByteBuffer result; - if (old != null && length <= old.capacity() && old.hasArray()) { - result = old; - result.clear(); - } else { - result = ByteBuffer.allocate(length); - } - in.readFully(result.array(), result.arrayOffset() + result.position(), length); - result.limit(length); - return result; - } - - - @Override - public void skipFixed(int length) throws IOException { - skipBytes(length); - } - - @Override - public void skipBytes() throws IOException { - int num = readInt(); - skipBytes(num); - } - - // -------------------------------------------------------------------------------------------- - // strings - // -------------------------------------------------------------------------------------------- - - - @Override - public Utf8 readString(Utf8 old) throws IOException { - int length = readInt(); - Utf8 result = (old != null ? old : new Utf8()); - result.setByteLength(length); - - if (length > 0) { - in.readFully(result.getBytes(), 0, length); - } - - return result; - } - - @Override - public String readString() throws IOException { - return readString(stringDecoder).toString(); - } - - @Override - public void skipString() throws IOException { - int len = readInt(); - skipBytes(len); - } - - // -------------------------------------------------------------------------------------------- - // collection types - // -------------------------------------------------------------------------------------------- - - @Override - public long readArrayStart() throws IOException { - return readVarLongCount(in); - } - - @Override - public long arrayNext() throws IOException { - return readVarLongCount(in); - } - - @Override - public long skipArray() throws IOException { - return readVarLongCount(in); - } - - @Override - public long readMapStart() throws IOException { - return readVarLongCount(in); - } - - @Override - public long mapNext() throws IOException { - return readVarLongCount(in); - } - - @Override - public long skipMap() throws IOException { - return readVarLongCount(in); - } - - // -------------------------------------------------------------------------------------------- - // union - // -------------------------------------------------------------------------------------------- - - @Override - public int readIndex() throws IOException { - return readInt(); - } - - // -------------------------------------------------------------------------------------------- - // utils - // -------------------------------------------------------------------------------------------- - - private void skipBytes(int num) throws IOException { - while (num > 0) { - num -= in.skipBytes(num); - } - } - - public static long readVarLongCount(DataInput in) throws IOException { - long value = in.readUnsignedByte(); - - if ((value & 0x80) == 0) { - return value; - } - else { - long curr; - int shift = 7; - value = value & 0x7f; - while (((curr = in.readUnsignedByte()) & 0x80) != 0){ - value |= (curr & 0x7f) << shift; - shift += 7; - } - value |= curr << shift; - return value; - } - } -} http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-avro/src/main/java/org/apache/flink/api/avro/DataOutputEncoder.java ---------------------------------------------------------------------- diff --git a/flink-addons/flink-avro/src/main/java/org/apache/flink/api/avro/DataOutputEncoder.java b/flink-addons/flink-avro/src/main/java/org/apache/flink/api/avro/DataOutputEncoder.java deleted file mode 100644 index 0102cc1..0000000 --- a/flink-addons/flink-avro/src/main/java/org/apache/flink/api/avro/DataOutputEncoder.java +++ /dev/null @@ -1,183 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.api.avro; - -import java.io.DataOutput; -import java.io.IOException; -import java.nio.ByteBuffer; - -import org.apache.avro.io.Encoder; -import org.apache.avro.util.Utf8; - - -public final class DataOutputEncoder extends Encoder implements java.io.Serializable { - - private static final long serialVersionUID = 1L; - - private DataOutput out; - - - public void setOut(DataOutput out) { - this.out = out; - } - - - @Override - public void flush() throws IOException {} - - // -------------------------------------------------------------------------------------------- - // primitives - // -------------------------------------------------------------------------------------------- - - @Override - public void writeNull() {} - - - @Override - public void writeBoolean(boolean b) throws IOException { - out.writeBoolean(b); - } - - @Override - public void writeInt(int n) throws IOException { - out.writeInt(n); - } - - @Override - public void writeLong(long n) throws IOException { - out.writeLong(n); - } - - @Override - public void writeFloat(float f) throws IOException { - out.writeFloat(f); - } - - @Override - public void writeDouble(double d) throws IOException { - out.writeDouble(d); - } - - @Override - public void writeEnum(int e) throws IOException { - out.writeInt(e); - } - - - // -------------------------------------------------------------------------------------------- - // bytes - // -------------------------------------------------------------------------------------------- - - @Override - public void writeFixed(byte[] bytes, int start, int len) throws IOException { - out.write(bytes, start, len); - } - - @Override - public void writeBytes(byte[] bytes, int start, int len) throws IOException { - out.writeInt(len); - if (len > 0) { - out.write(bytes, start, len); - } - } - - @Override - public void writeBytes(ByteBuffer bytes) throws IOException { - int num = bytes.remaining(); - out.writeInt(num); - - if (num > 0) { - writeFixed(bytes); - } - } - - // -------------------------------------------------------------------------------------------- - // strings - // -------------------------------------------------------------------------------------------- - - @Override - public void writeString(String str) throws IOException { - byte[] bytes = Utf8.getBytesFor(str); - writeBytes(bytes, 0, bytes.length); - } - - @Override - public void writeString(Utf8 utf8) throws IOException { - writeBytes(utf8.getBytes(), 0, utf8.getByteLength()); - - } - - // -------------------------------------------------------------------------------------------- - // collection types - // -------------------------------------------------------------------------------------------- - - @Override - public void writeArrayStart() {} - - @Override - public void setItemCount(long itemCount) throws IOException { - if (itemCount > 0) { - writeVarLongCount(out, itemCount); - } - } - - @Override - public void startItem() {} - - @Override - public void writeArrayEnd() throws IOException { - // write a single byte 0, shortcut for a var-length long of 0 - out.write(0); - } - - @Override - public void writeMapStart() {} - - @Override - public void writeMapEnd() throws IOException { - // write a single byte 0, shortcut for a var-length long of 0 - out.write(0); - } - - // -------------------------------------------------------------------------------------------- - // union - // -------------------------------------------------------------------------------------------- - - @Override - public void writeIndex(int unionIndex) throws IOException { - out.writeInt(unionIndex); - } - - // -------------------------------------------------------------------------------------------- - // utils - // -------------------------------------------------------------------------------------------- - - - public static void writeVarLongCount(DataOutput out, long val) throws IOException { - if (val < 0) { - throw new IOException("Illegal count (must be non-negative): " + val); - } - - while ((val & ~0x7FL) != 0) { - out.write(((int) val) | 0x80); - val >>>= 7; - } - out.write((int) val); - } -} http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-avro/src/main/java/org/apache/flink/api/avro/FSDataInputStreamWrapper.java ---------------------------------------------------------------------- diff --git a/flink-addons/flink-avro/src/main/java/org/apache/flink/api/avro/FSDataInputStreamWrapper.java b/flink-addons/flink-avro/src/main/java/org/apache/flink/api/avro/FSDataInputStreamWrapper.java deleted file mode 100644 index 655b6ff..0000000 --- a/flink-addons/flink-avro/src/main/java/org/apache/flink/api/avro/FSDataInputStreamWrapper.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package org.apache.flink.api.avro; - -import java.io.Closeable; -import java.io.IOException; - -import org.apache.avro.file.SeekableInput; -import org.apache.flink.core.fs.FSDataInputStream; - - -/** - * Code copy pasted from org.apache.avro.mapred.FSInput (which is Apache licensed as well) - * - * The wrapper keeps track of the position in the data stream. - */ -public class FSDataInputStreamWrapper implements Closeable, SeekableInput { - private final FSDataInputStream stream; - private final long len; - private long pos; - - public FSDataInputStreamWrapper(FSDataInputStream stream, long len) { - this.stream = stream; - this.len = len; - this.pos = 0; - } - - public long length() { - return len; - } - - public int read(byte[] b, int off, int len) throws IOException { - int read; - read = stream.read(b, off, len); - pos += read; - return read; - } - - public void seek(long p) throws IOException { - stream.seek(p); - pos = p; - } - - public long tell() throws IOException { - return pos; - } - - public void close() throws IOException { - stream.close(); - } -} http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-avro/src/main/java/org/apache/flink/api/io/avro/example/AvroTypeExample.java ---------------------------------------------------------------------- diff --git a/flink-addons/flink-avro/src/main/java/org/apache/flink/api/io/avro/example/AvroTypeExample.java b/flink-addons/flink-avro/src/main/java/org/apache/flink/api/io/avro/example/AvroTypeExample.java deleted file mode 100644 index 6affeec..0000000 --- a/flink-addons/flink-avro/src/main/java/org/apache/flink/api/io/avro/example/AvroTypeExample.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.api.io.avro.example; - -import java.io.IOException; -import java.util.Random; - -import org.apache.flink.api.common.functions.GroupReduceFunction; -import org.apache.flink.api.common.functions.MapFunction; -import org.apache.flink.api.common.io.GenericInputFormat; -import org.apache.flink.api.java.DataSet; -import org.apache.flink.api.java.ExecutionEnvironment; -import org.apache.flink.api.java.tuple.Tuple2; -import org.apache.flink.util.Collector; - -@SuppressWarnings("serial") -public class AvroTypeExample { - - - public static void main(String[] args) throws Exception { - - ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); - - DataSet<User> users = env.createInput(new UserGeneratingInputFormat()); - - users - .map(new NumberExtractingMapper()) - .groupBy(1) - .reduceGroup(new ConcatenatingReducer()) - .print(); - - env.execute(); - } - - - - public static final class NumberExtractingMapper implements MapFunction<User, Tuple2<User, Integer>> { - - @Override - public Tuple2<User, Integer> map(User user) { - return new Tuple2<User, Integer>(user, user.getFavoriteNumber()); - } - } - - - public static final class ConcatenatingReducer implements GroupReduceFunction<Tuple2<User, Integer>, Tuple2<Integer, String>> { - - @Override - public void reduce(Iterable<Tuple2<User, Integer>> values, Collector<Tuple2<Integer, String>> out) throws Exception { - int number = 0; - StringBuilder colors = new StringBuilder(); - - for (Tuple2<User, Integer> u : values) { - number = u.f1; - colors.append(u.f0.getFavoriteColor()).append(" - "); - } - - colors.setLength(colors.length() - 3); - out.collect(new Tuple2<Integer, String>(number, colors.toString())); - } - } - - - public static final class UserGeneratingInputFormat extends GenericInputFormat<User> { - - private static final long serialVersionUID = 1L; - - private static final int NUM = 100; - - private final Random rnd = new Random(32498562304986L); - - private static final String[] NAMES = { "Peter", "Bob", "Liddy", "Alexander", "Stan" }; - - private static final String[] COLORS = { "mauve", "crimson", "copper", "sky", "grass" }; - - private int count; - - - @Override - public boolean reachedEnd() throws IOException { - return count >= NUM; - } - - @Override - public User nextRecord(User reuse) throws IOException { - count++; - - User u = new User(); - u.setName(NAMES[rnd.nextInt(NAMES.length)]); - u.setFavoriteColor(COLORS[rnd.nextInt(COLORS.length)]); - u.setFavoriteNumber(rnd.nextInt(87)); - return u; - } - } -} http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-avro/src/main/java/org/apache/flink/api/io/avro/example/User.java ---------------------------------------------------------------------- diff --git a/flink-addons/flink-avro/src/main/java/org/apache/flink/api/io/avro/example/User.java b/flink-addons/flink-avro/src/main/java/org/apache/flink/api/io/avro/example/User.java deleted file mode 100644 index 4608f96..0000000 --- a/flink-addons/flink-avro/src/main/java/org/apache/flink/api/io/avro/example/User.java +++ /dev/null @@ -1,269 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -/** - * Autogenerated by Avro - * - * DO NOT EDIT DIRECTLY - */ -package org.apache.flink.api.io.avro.example; -@SuppressWarnings("all") -@org.apache.avro.specific.AvroGenerated -public class User extends org.apache.avro.specific.SpecificRecordBase implements org.apache.avro.specific.SpecificRecord { - public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"record\",\"name\":\"User\",\"namespace\":\"org.apache.flink.api.avro.example\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"int\",\"null\"]},{\"name\":\"favorite_color\",\"type\":[\"string\",\"null\"]}]}"); - public static org.apache.avro.Schema getClassSchema() { return SCHEMA$; } - @Deprecated public java.lang.CharSequence name; - @Deprecated public java.lang.Integer favorite_number; - @Deprecated public java.lang.CharSequence favorite_color; - - /** - * Default constructor. Note that this does not initialize fields - * to their default values from the schema. If that is desired then - * one should use {@link #newBuilder()}. - */ - public User() {} - - /** - * All-args constructor. - */ - public User(java.lang.CharSequence name, java.lang.Integer favorite_number, java.lang.CharSequence favorite_color) { - this.name = name; - this.favorite_number = favorite_number; - this.favorite_color = favorite_color; - } - - public org.apache.avro.Schema getSchema() { return SCHEMA$; } - // Used by DatumWriter. Applications should not call. - public java.lang.Object get(int field$) { - switch (field$) { - case 0: return name; - case 1: return favorite_number; - case 2: return favorite_color; - default: throw new org.apache.avro.AvroRuntimeException("Bad index"); - } - } - // Used by DatumReader. Applications should not call. - @SuppressWarnings(value="unchecked") - public void put(int field$, java.lang.Object value$) { - switch (field$) { - case 0: name = (java.lang.CharSequence)value$; break; - case 1: favorite_number = (java.lang.Integer)value$; break; - case 2: favorite_color = (java.lang.CharSequence)value$; break; - default: throw new org.apache.avro.AvroRuntimeException("Bad index"); - } - } - - /** - * Gets the value of the 'name' field. - */ - public java.lang.CharSequence getName() { - return name; - } - - /** - * Sets the value of the 'name' field. - * @param value the value to set. - */ - public void setName(java.lang.CharSequence value) { - this.name = value; - } - - /** - * Gets the value of the 'favorite_number' field. - */ - public java.lang.Integer getFavoriteNumber() { - return favorite_number; - } - - /** - * Sets the value of the 'favorite_number' field. - * @param value the value to set. - */ - public void setFavoriteNumber(java.lang.Integer value) { - this.favorite_number = value; - } - - /** - * Gets the value of the 'favorite_color' field. - */ - public java.lang.CharSequence getFavoriteColor() { - return favorite_color; - } - - /** - * Sets the value of the 'favorite_color' field. - * @param value the value to set. - */ - public void setFavoriteColor(java.lang.CharSequence value) { - this.favorite_color = value; - } - - /** Creates a new User RecordBuilder */ - public static org.apache.flink.api.io.avro.example.User.Builder newBuilder() { - return new org.apache.flink.api.io.avro.example.User.Builder(); - } - - /** Creates a new User RecordBuilder by copying an existing Builder */ - public static org.apache.flink.api.io.avro.example.User.Builder newBuilder(org.apache.flink.api.io.avro.example.User.Builder other) { - return new org.apache.flink.api.io.avro.example.User.Builder(other); - } - - /** Creates a new User RecordBuilder by copying an existing User instance */ - public static org.apache.flink.api.io.avro.example.User.Builder newBuilder(org.apache.flink.api.io.avro.example.User other) { - return new org.apache.flink.api.io.avro.example.User.Builder(other); - } - - /** - * RecordBuilder for User instances. - */ - public static class Builder extends org.apache.avro.specific.SpecificRecordBuilderBase<User> - implements org.apache.avro.data.RecordBuilder<User> { - - private java.lang.CharSequence name; - private java.lang.Integer favorite_number; - private java.lang.CharSequence favorite_color; - - /** Creates a new Builder */ - private Builder() { - super(org.apache.flink.api.io.avro.example.User.SCHEMA$); - } - - /** Creates a Builder by copying an existing Builder */ - private Builder(org.apache.flink.api.io.avro.example.User.Builder other) { - super(other); - if (isValidValue(fields()[0], other.name)) { - this.name = data().deepCopy(fields()[0].schema(), other.name); - fieldSetFlags()[0] = true; - } - if (isValidValue(fields()[1], other.favorite_number)) { - this.favorite_number = data().deepCopy(fields()[1].schema(), other.favorite_number); - fieldSetFlags()[1] = true; - } - if (isValidValue(fields()[2], other.favorite_color)) { - this.favorite_color = data().deepCopy(fields()[2].schema(), other.favorite_color); - fieldSetFlags()[2] = true; - } - } - - /** Creates a Builder by copying an existing User instance */ - private Builder(org.apache.flink.api.io.avro.example.User other) { - super(org.apache.flink.api.io.avro.example.User.SCHEMA$); - if (isValidValue(fields()[0], other.name)) { - this.name = data().deepCopy(fields()[0].schema(), other.name); - fieldSetFlags()[0] = true; - } - if (isValidValue(fields()[1], other.favorite_number)) { - this.favorite_number = data().deepCopy(fields()[1].schema(), other.favorite_number); - fieldSetFlags()[1] = true; - } - if (isValidValue(fields()[2], other.favorite_color)) { - this.favorite_color = data().deepCopy(fields()[2].schema(), other.favorite_color); - fieldSetFlags()[2] = true; - } - } - - /** Gets the value of the 'name' field */ - public java.lang.CharSequence getName() { - return name; - } - - /** Sets the value of the 'name' field */ - public org.apache.flink.api.io.avro.example.User.Builder setName(java.lang.CharSequence value) { - validate(fields()[0], value); - this.name = value; - fieldSetFlags()[0] = true; - return this; - } - - /** Checks whether the 'name' field has been set */ - public boolean hasName() { - return fieldSetFlags()[0]; - } - - /** Clears the value of the 'name' field */ - public org.apache.flink.api.io.avro.example.User.Builder clearName() { - name = null; - fieldSetFlags()[0] = false; - return this; - } - - /** Gets the value of the 'favorite_number' field */ - public java.lang.Integer getFavoriteNumber() { - return favorite_number; - } - - /** Sets the value of the 'favorite_number' field */ - public org.apache.flink.api.io.avro.example.User.Builder setFavoriteNumber(java.lang.Integer value) { - validate(fields()[1], value); - this.favorite_number = value; - fieldSetFlags()[1] = true; - return this; - } - - /** Checks whether the 'favorite_number' field has been set */ - public boolean hasFavoriteNumber() { - return fieldSetFlags()[1]; - } - - /** Clears the value of the 'favorite_number' field */ - public org.apache.flink.api.io.avro.example.User.Builder clearFavoriteNumber() { - favorite_number = null; - fieldSetFlags()[1] = false; - return this; - } - - /** Gets the value of the 'favorite_color' field */ - public java.lang.CharSequence getFavoriteColor() { - return favorite_color; - } - - /** Sets the value of the 'favorite_color' field */ - public org.apache.flink.api.io.avro.example.User.Builder setFavoriteColor(java.lang.CharSequence value) { - validate(fields()[2], value); - this.favorite_color = value; - fieldSetFlags()[2] = true; - return this; - } - - /** Checks whether the 'favorite_color' field has been set */ - public boolean hasFavoriteColor() { - return fieldSetFlags()[2]; - } - - /** Clears the value of the 'favorite_color' field */ - public org.apache.flink.api.io.avro.example.User.Builder clearFavoriteColor() { - favorite_color = null; - fieldSetFlags()[2] = false; - return this; - } - - @Override - public User build() { - try { - User record = new User(); - record.name = fieldSetFlags()[0] ? this.name : (java.lang.CharSequence) defaultValue(fields()[0]); - record.favorite_number = fieldSetFlags()[1] ? this.favorite_number : (java.lang.Integer) defaultValue(fields()[1]); - record.favorite_color = fieldSetFlags()[2] ? this.favorite_color : (java.lang.CharSequence) defaultValue(fields()[2]); - return record; - } catch (Exception e) { - throw new org.apache.avro.AvroRuntimeException(e); - } - } - } -} http://git-wip-us.apache.org/repos/asf/flink/blob/f94112fb/flink-addons/flink-avro/src/main/java/org/apache/flink/api/java/io/AvroInputFormat.java ---------------------------------------------------------------------- diff --git a/flink-addons/flink-avro/src/main/java/org/apache/flink/api/java/io/AvroInputFormat.java b/flink-addons/flink-avro/src/main/java/org/apache/flink/api/java/io/AvroInputFormat.java deleted file mode 100644 index fbb0f4c..0000000 --- a/flink-addons/flink-avro/src/main/java/org/apache/flink/api/java/io/AvroInputFormat.java +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package org.apache.flink.api.java.io; - -import java.io.IOException; - -import org.apache.avro.file.DataFileReader; -import org.apache.avro.file.FileReader; -import org.apache.avro.file.SeekableInput; -import org.apache.avro.io.DatumReader; -import org.apache.avro.reflect.ReflectDatumReader; -import org.apache.avro.specific.SpecificDatumReader; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.apache.flink.api.avro.FSDataInputStreamWrapper; -import org.apache.flink.api.common.io.FileInputFormat; -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.java.typeutils.ResultTypeQueryable; -import org.apache.flink.api.java.typeutils.TypeExtractor; -import org.apache.flink.core.fs.FileInputSplit; -import org.apache.flink.core.fs.Path; -import org.apache.flink.util.InstantiationUtil; - - -public class AvroInputFormat<E> extends FileInputFormat<E> implements ResultTypeQueryable<E> { - - private static final long serialVersionUID = 1L; - - private static final Logger LOG = LoggerFactory.getLogger(AvroInputFormat.class); - - - private final Class<E> avroValueType; - - private boolean reuseAvroValue = true; - - - private transient FileReader<E> dataFileReader; - - - public AvroInputFormat(Path filePath, Class<E> type) { - super(filePath); - this.avroValueType = type; - this.unsplittable = true; - } - - - /** - * Sets the flag whether to reuse the Avro value instance for all records. - * By default, the input format reuses the Avro value. - * - * @param reuseAvroValue True, if the input format should reuse the Avro value instance, false otherwise. - */ - public void setReuseAvroValue(boolean reuseAvroValue) { - this.reuseAvroValue = reuseAvroValue; - } - - // -------------------------------------------------------------------------------------------- - // Typing - // -------------------------------------------------------------------------------------------- - - @Override - public TypeInformation<E> getProducedType() { - return TypeExtractor.getForClass(this.avroValueType); - } - - // -------------------------------------------------------------------------------------------- - // Input Format Methods - // -------------------------------------------------------------------------------------------- - - @Override - public void open(FileInputSplit split) throws IOException { - super.open(split); - - DatumReader<E> datumReader; - if (org.apache.avro.specific.SpecificRecordBase.class.isAssignableFrom(avroValueType)) { - datumReader = new SpecificDatumReader<E>(avroValueType); - } else { - datumReader = new ReflectDatumReader<E>(avroValueType); - } - - LOG.info("Opening split " + split); - - SeekableInput in = new FSDataInputStreamWrapper(stream, (int) split.getLength()); - - dataFileReader = DataFileReader.openReader(in, datumReader); - dataFileReader.sync(split.getStart()); - } - - @Override - public boolean reachedEnd() throws IOException { - return !dataFileReader.hasNext(); - } - - @Override - public E nextRecord(E reuseValue) throws IOException { - if (!dataFileReader.hasNext()) { - return null; - } - - if (!reuseAvroValue) { - reuseValue = InstantiationUtil.instantiate(avroValueType, Object.class); - } - - reuseValue = dataFileReader.next(reuseValue); - return reuseValue; - } -}
