Repository: incubator-streams-examples Updated Branches: refs/heads/master 6e93a8f7a -> 8fe6860f7
http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/4491cfe1/flink/flink-twitter-collection/src/test/resources/1000twitterids.txt ---------------------------------------------------------------------- diff --git a/flink/flink-twitter-collection/src/test/resources/1000twitterids.txt b/flink/flink-twitter-collection/src/test/resources/1000twitterids.txt new file mode 100644 index 0000000..0590b9d --- /dev/null +++ b/flink/flink-twitter-collection/src/test/resources/1000twitterids.txt @@ -0,0 +1,1000 @@ +twitter:3424266646 +twitter:3277467241 +twitter:3244517214 +twitter:29953647 +twitter:63818319 +twitter:1528436754 +twitter:405580894 +twitter:322778026 +twitter:172382176 +twitter:633076833 +twitter:703735608 +twitter:2347223440 +twitter:2907929487 +twitter:950240089 +twitter:1418546592 +twitter:3318418717 +twitter:2848958704 +twitter:1120797264 +twitter:933623324 +twitter:2977700375 +twitter:328204518 +twitter:585131136 +twitter:2868789793 +twitter:158347647 +twitter:2915413161 +twitter:2217367263 +twitter:2534019247 +twitter:3033565239 +twitter:377379801 +twitter:2525341814 +twitter:3123827524 +twitter:1840932523 +twitter:3307643975 +twitter:3301777832 +twitter:961987748 +twitter:3205632255 +twitter:2799469322 +twitter:17730681 +twitter:1495242662 +twitter:1909516123 +twitter:263933760 +twitter:312651511 +twitter:2479527469 +twitter:2357151036 +twitter:346433828 +twitter:44801893 +twitter:1049697306 +twitter:2779673194 +twitter:18323141 +twitter:2172488902 +twitter:2373431930 +twitter:1038322550 +twitter:2946211549 +twitter:2911057543 +twitter:1186036284 +twitter:2878076317 +twitter:1312950464 +twitter:57323685 +twitter:32929857 +twitter:301933631 +twitter:2852217152 +twitter:330422649 +twitter:98470876 +twitter:933125156 +twitter:3237125761 +twitter:914882005 +twitter:1560239652 +twitter:900444860 +twitter:402918702 +twitter:1820690166 +twitter:3074359086 +twitter:353183684 +twitter:528544881 +twitter:1881638161 +twitter:2751762993 +twitter:3161315692 +twitter:3305680079 +twitter:1721613488 +twitter:513068659 +twitter:627186234 +twitter:3203648416 +twitter:1541163325 +twitter:1882043502 +twitter:29071727 +twitter:610104090 +twitter:2819781014 +twitter:2909115204 +twitter:213886397 +twitter:3249385591 +twitter:3086875073 +twitter:87040031 +twitter:2202487475 +twitter:334896132 +twitter:49163181 +twitter:3433984816 +twitter:543969362 +twitter:489445461 +twitter:855051894 +twitter:2792040175 +twitter:117051455 +twitter:438599410 +twitter:1387329846 +twitter:711595782 +twitter:3230662766 +twitter:2766672269 +twitter:2926781875 +twitter:863203928 +twitter:517199566 +twitter:201645935 +twitter:1555939147 +twitter:2943152669 +twitter:1324775431 +twitter:400234897 +twitter:2347416842 +twitter:1558112510 +twitter:474415350 +twitter:2153710970 +twitter:1408335014 +twitter:3633713483 +twitter:3166021013 +twitter:3530993294 +twitter:332598229 +twitter:308252069 +twitter:3317826986 +twitter:572175644 +twitter:1718271572 +twitter:2869090090 +twitter:23725109 +twitter:1926137280 +twitter:1486830500 +twitter:743080386 +twitter:3250479720 +twitter:2560441544 +twitter:2715649872 +twitter:287089153 +twitter:18761334 +twitter:2305577745 +twitter:724860668 +twitter:193306049 +twitter:2615761979 +twitter:2463299598 +twitter:1436916012 +twitter:919019185 +twitter:90502449 +twitter:50689522 +twitter:1383774679 +twitter:612784850 +twitter:410319975 +twitter:833440153 +twitter:442322844 +twitter:2181167094 +twitter:94012832 +twitter:112748352 +twitter:1474618075 +twitter:158262669 +twitter:2391506308 +twitter:882502026 +twitter:2693660146 +twitter:2971933908 +twitter:55271184 +twitter:2287356556 +twitter:2895756090 +twitter:407147132 +twitter:3262181 +twitter:313317193 +twitter:2729137002 +twitter:2939122360 +twitter:2751601568 +twitter:1215082350 +twitter:124866576 +twitter:274292311 +twitter:3310301042 +twitter:95407473 +twitter:24993769 +twitter:1342908648 +twitter:1805339413 +twitter:3118252036 +twitter:893269387 +twitter:1481149014 +twitter:463288019 +twitter:75008083 +twitter:2895489727 +twitter:965493739 +twitter:278637248 +twitter:1937513246 +twitter:422218268 +twitter:3320995462 +twitter:78682286 +twitter:2777069098 +twitter:2909553730 +twitter:2914338670 +twitter:1251667531 +twitter:2764034755 +twitter:532659717 +twitter:269002510 +twitter:29373713 +twitter:358075450 +twitter:633880614 +twitter:200374379 +twitter:141628294 +twitter:1513028977 +twitter:116798791 +twitter:2937455354 +twitter:246194623 +twitter:793925970 +twitter:115594167 +twitter:82463176 +twitter:324774974 +twitter:185844856 +twitter:2462295999 +twitter:3555105016 +twitter:1029169117 +twitter:2689309484 +twitter:1587145976 +twitter:1607241271 +twitter:3032276402 +twitter:183916933 +twitter:63766245 +twitter:151217255 +twitter:2781098109 +twitter:252081559 +twitter:1608788256 +twitter:41984573 +twitter:1896587353 +twitter:40136999 +twitter:295505814 +twitter:384867933 +twitter:116947371 +twitter:255703939 +twitter:2687800732 +twitter:76543916 +twitter:881649782 +twitter:2765729924 +twitter:1715695669 +twitter:1965383022 +twitter:2888214228 +twitter:21820514 +twitter:1727966414 +twitter:2581992818 +twitter:103999565 +twitter:741018846 +twitter:446792386 +twitter:2568989424 +twitter:2780674777 +twitter:465934916 +twitter:3378294885 +twitter:2885604327 +twitter:3336273419 +twitter:130742941 +twitter:2327629099 +twitter:1103818104 +twitter:3050036073 +twitter:2882456842 +twitter:2702914248 +twitter:2153674818 +twitter:132825659 +twitter:289758699 +twitter:2995946100 +twitter:3027449217 +twitter:2708029160 +twitter:1529367002 +twitter:608170333 +twitter:140446819 +twitter:2790688993 +twitter:1597308192 +twitter:14462028 +twitter:104062608 +twitter:370274893 +twitter:356145607 +twitter:566542629 +twitter:112587243 +twitter:39372070 +twitter:146853060 +twitter:2440984657 +twitter:3074554539 +twitter:204701034 +twitter:887623447 +twitter:1971521630 +twitter:2457208175 +twitter:466113358 +twitter:1574643830 +twitter:1465533884 +twitter:2500404589 +twitter:1633154150 +twitter:1349117870 +twitter:1658071267 +twitter:593022891 +twitter:3094177813 +twitter:1304672510 +twitter:3385525697 +twitter:2916225552 +twitter:2759773715 +twitter:1369215552 +twitter:1058390078 +twitter:2532850321 +twitter:351483656 +twitter:1902796704 +twitter:113000738 +twitter:2241245557 +twitter:2416606754 +twitter:408729540 +twitter:2530294556 +twitter:2936808249 +twitter:3138999692 +twitter:2679987883 +twitter:1448537377 +twitter:2524773906 +twitter:942079406 +twitter:2217584389 +twitter:3059427504 +twitter:3028507725 +twitter:632766658 +twitter:3302663431 +twitter:2914832897 +twitter:93487101 +twitter:2786054379 +twitter:1339647769 +twitter:531402307 +twitter:402066474 +twitter:337936675 +twitter:2760568625 +twitter:1385916396 +twitter:2595560922 +twitter:421910477 +twitter:1713100813 +twitter:352016040 +twitter:415247994 +twitter:1883606209 +twitter:2974994111 +twitter:1118022211 +twitter:3096979637 +twitter:711889867 +twitter:262890561 +twitter:233810062 +twitter:1877177168 +twitter:964106670 +twitter:164985413 +twitter:2920420361 +twitter:318936782 +twitter:3289826764 +twitter:145873735 +twitter:2523059919 +twitter:2409896179 +twitter:2292047201 +twitter:285674825 +twitter:2765549780 +twitter:2359541905 +twitter:2419103894 +twitter:358884588 +twitter:206231205 +twitter:136500778 +twitter:1397885138 +twitter:2625422097 +twitter:2524578002 +twitter:604278657 +twitter:2625634867 +twitter:73168019 +twitter:407448958 +twitter:189276174 +twitter:2507896925 +twitter:80880449 +twitter:520177827 +twitter:418469102 +twitter:2925075456 +twitter:615730636 +twitter:2995998941 +twitter:2697270934 +twitter:497135011 +twitter:2944598402 +twitter:428706893 +twitter:1345291712 +twitter:388751708 +twitter:130092079 +twitter:2984741882 +twitter:1047514436 +twitter:15927135 +twitter:2884357840 +twitter:294362779 +twitter:2870985800 +twitter:1720400449 +twitter:130027314 +twitter:2970518577 +twitter:240923858 +twitter:1613498838 +twitter:708321211 +twitter:1403382426 +twitter:2602186970 +twitter:1596855998 +twitter:280062526 +twitter:2716454552 +twitter:268720451 +twitter:2869044811 +twitter:1911762488 +twitter:392373280 +twitter:2151082712 +twitter:2770919004 +twitter:231541900 +twitter:60122778 +twitter:390006102 +twitter:240167506 +twitter:1558314660 +twitter:221608257 +twitter:852829933 +twitter:461669243 +twitter:239778483 +twitter:502146157 +twitter:1471963970 +twitter:276426707 +twitter:2336546150 +twitter:323595235 +twitter:128670043 +twitter:1308641714 +twitter:1411112756 +twitter:3011727217 +twitter:3082006921 +twitter:450537474 +twitter:2673101407 +twitter:2416030447 +twitter:51952627 +twitter:708057486 +twitter:833620748 +twitter:3024957797 +twitter:2147572362 +twitter:1712467098 +twitter:2899300501 +twitter:1348351772 +twitter:2923114629 +twitter:2779232814 +twitter:21306308 +twitter:1466314507 +twitter:1224588289 +twitter:81307783 +twitter:42717316 +twitter:315972617 +twitter:434649827 +twitter:105839296 +twitter:366063496 +twitter:34045892 +twitter:3076447389 +twitter:92437198 +twitter:3124335006 +twitter:1444393410 +twitter:351737762 +twitter:1919360383 +twitter:2836048345 +twitter:1670939112 +twitter:722140159 +twitter:92939425 +twitter:2932728756 +twitter:2831872033 +twitter:1354255123 +twitter:1689738186 +twitter:463578260 +twitter:2881582438 +twitter:912252510 +twitter:3226221887 +twitter:390827200 +twitter:269169237 +twitter:1450007192 +twitter:2735984326 +twitter:3029836305 +twitter:28291382 +twitter:785668627 +twitter:567287970 +twitter:1480004420 +twitter:131927864 +twitter:2958631308 +twitter:488490020 +twitter:2603422688 +twitter:3186614985 +twitter:177373618 +twitter:2466506329 +twitter:2651294251 +twitter:3367170684 +twitter:2673870882 +twitter:369098635 +twitter:242011326 +twitter:18099277 +twitter:1922210574 +twitter:3093762445 +twitter:470634878 +twitter:1674607392 +twitter:2920526283 +twitter:3261677580 +twitter:2192187078 +twitter:485599960 +twitter:1854850729 +twitter:95198467 +twitter:2228217740 +twitter:2171528344 +twitter:2957461230 +twitter:226615737 +twitter:1624183567 +twitter:158597677 +twitter:2909224690 +twitter:19278114 +twitter:2488284258 +twitter:2777071149 +twitter:1598064697 +twitter:2740691127 +twitter:3100908480 +twitter:1147010126 +twitter:2741161553 +twitter:439971668 +twitter:3247227273 +twitter:2884261062 +twitter:3127250575 +twitter:2942021278 +twitter:539428196 +twitter:409599986 +twitter:3161801331 +twitter:2328613860 +twitter:1903013437 +twitter:313082004 +twitter:2580495721 +twitter:209464435 +twitter:600172085 +twitter:339541217 +twitter:62219810 +twitter:583287316 +twitter:295891933 +twitter:561683767 +twitter:229192352 +twitter:1357869918 +twitter:235438136 +twitter:1599249169 +twitter:583879210 +twitter:507744802 +twitter:1696336261 +twitter:2323537206 +twitter:36882220 +twitter:541528426 +twitter:956202559 +twitter:387936537 +twitter:211658842 +twitter:2685186010 +twitter:2581656488 +twitter:391154378 +twitter:122932105 +twitter:409764153 +twitter:129737967 +twitter:2848806360 +twitter:3054860719 +twitter:372199585 +twitter:2316121597 +twitter:703345746 +twitter:3335505287 +twitter:2466151422 +twitter:380038166 +twitter:420561214 +twitter:2977085351 +twitter:110955327 +twitter:3004295886 +twitter:2362857361 +twitter:3053844460 +twitter:3182081552 +twitter:324208260 +twitter:2571790321 +twitter:1061498868 +twitter:2187395299 +twitter:2187482779 +twitter:3096652530 +twitter:2538239672 +twitter:3809634552 +twitter:2306848839 +twitter:1544061547 +twitter:151075965 +twitter:3250238556 +twitter:16157689 +twitter:1692663644 +twitter:1356000732 +twitter:436774994 +twitter:45503055 +twitter:1086037316 +twitter:2798297775 +twitter:2923485772 +twitter:58731726 +twitter:211816170 +twitter:885013716 +twitter:2608529078 +twitter:2954917057 +twitter:2271021600 +twitter:173743066 +twitter:451543575 +twitter:3219728436 +twitter:399824828 +twitter:2464688153 +twitter:2541069631 +twitter:1522892262 +twitter:3167829845 +twitter:944851321 +twitter:2471474509 +twitter:68073858 +twitter:1496221376 +twitter:13979882 +twitter:2218792189 +twitter:302123873 +twitter:2845915546 +twitter:431402814 +twitter:1364254945 +twitter:2711277666 +twitter:2766696876 +twitter:2495441323 +twitter:2844317433 +twitter:138009079 +twitter:2578631100 +twitter:478167529 +twitter:1222728360 +twitter:1323688411 +twitter:2883066187 +twitter:2443554697 +twitter:411631689 +twitter:68537682 +twitter:1027019269 +twitter:1660752493 +twitter:987324488 +twitter:2764106926 +twitter:2184511674 +twitter:103419315 +twitter:2310456424 +twitter:1572938088 +twitter:2554895281 +twitter:34138105 +twitter:2942100621 +twitter:160517898 +twitter:285075974 +twitter:2260805169 +twitter:19390498 +twitter:301696842 +twitter:2588239985 +twitter:2886588596 +twitter:2962622367 +twitter:1867897483 +twitter:2827053488 +twitter:1447767319 +twitter:2924491293 +twitter:167327096 +twitter:3309592402 +twitter:2795575638 +twitter:578758971 +twitter:2888665561 +twitter:30542348 +twitter:1437049609 +twitter:2242541566 +twitter:74354017 +twitter:58900854 +twitter:2159055031 +twitter:246517688 +twitter:2916873012 +twitter:1110055280 +twitter:562430843 +twitter:761797794 +twitter:1648208552 +twitter:301483343 +twitter:2896842048 +twitter:522103295 +twitter:1578517986 +twitter:2659610776 +twitter:2890560429 +twitter:1427665578 +twitter:268363160 +twitter:563709041 +twitter:2172300002 +twitter:2791262431 +twitter:3039809351 +twitter:2914940301 +twitter:2746560353 +twitter:2892191616 +twitter:71596845 +twitter:233770184 +twitter:1530949130 +twitter:105906110 +twitter:755347622 +twitter:490836906 +twitter:357603454 +twitter:324517203 +twitter:2835402315 +twitter:3285479894 +twitter:86368327 +twitter:238219970 +twitter:3153173945 +twitter:2732361234 +twitter:2357626327 +twitter:346602505 +twitter:13732632 +twitter:44055265 +twitter:2998032219 +twitter:482072312 +twitter:1721073866 +twitter:1386781034 +twitter:168194206 +twitter:1213443144 +twitter:181296114 +twitter:942598400 +twitter:2955577216 +twitter:582056669 +twitter:747540468 +twitter:2371722140 +twitter:360824004 +twitter:3023711736 +twitter:207032580 +twitter:2748107976 +twitter:464428175 +twitter:3150849096 +twitter:85450014 +twitter:2840066340 +twitter:2287819200 +twitter:240931426 +twitter:553606800 +twitter:397876544 +twitter:2195298230 +twitter:2601812005 +twitter:3013344739 +twitter:17599363 +twitter:1572639314 +twitter:3377673407 +twitter:303420278 +twitter:2811879995 +twitter:526860891 +twitter:346333874 +twitter:113568311 +twitter:705488304 +twitter:3238867619 +twitter:333772149 +twitter:373309716 +twitter:300472003 +twitter:3223424681 +twitter:2895699896 +twitter:3241119570 +twitter:1147453440 +twitter:3135402609 +twitter:521763744 +twitter:2702966971 +twitter:2878317616 +twitter:845031697 +twitter:2855454471 +twitter:3051902539 +twitter:482306439 +twitter:129173738 +twitter:306572138 +twitter:2941951538 +twitter:762707233 +twitter:2732608168 +twitter:1228456939 +twitter:246020724 +twitter:1920607602 +twitter:14434245 +twitter:1254943537 +twitter:1520746602 +twitter:150745124 +twitter:1350160351 +twitter:38707222 +twitter:267766858 +twitter:2992121760 +twitter:712666764 +twitter:983036864 +twitter:289490939 +twitter:269797384 +twitter:100215048 +twitter:3099557245 +twitter:2339741570 +twitter:306005146 +twitter:1182227460 +twitter:288235870 +twitter:1412832260 +twitter:455190443 +twitter:489912183 +twitter:448994061 +twitter:2944595072 +twitter:2453094914 +twitter:2899434206 +twitter:59288818 +twitter:2824706688 +twitter:423363992 +twitter:972850482 +twitter:997868714 +twitter:1203750733 +twitter:176147179 +twitter:115110596 +twitter:2978397615 +twitter:2528946267 +twitter:620180433 +twitter:365949935 +twitter:110609853 +twitter:1533494268 +twitter:2723839166 +twitter:34186887 +twitter:2864430424 +twitter:76942977 +twitter:361086733 +twitter:2724200587 +twitter:635206139 +twitter:2757801421 +twitter:19651443 +twitter:3364322949 +twitter:2770576744 +twitter:2168612560 +twitter:764020297 +twitter:2558268513 +twitter:2855384901 +twitter:1881414907 +twitter:2502212139 +twitter:3250037586 +twitter:2525185944 +twitter:591375982 +twitter:707911211 +twitter:3025041666 +twitter:19785599 +twitter:2311172950 +twitter:922817815 +twitter:739363530 +twitter:2812894393 +twitter:2496283986 +twitter:206162815 +twitter:590916342 +twitter:354053245 +twitter:2735195854 +twitter:2788759128 +twitter:3510947235 +twitter:3490740532 +twitter:2920847304 +twitter:2681444558 +twitter:2856805755 +twitter:3103899682 +twitter:145893832 +twitter:3065663910 +twitter:2736009516 +twitter:2835226230 +twitter:1590913771 +twitter:2700889555 +twitter:2221272164 +twitter:109780161 +twitter:700221218 +twitter:541753453 +twitter:126575915 +twitter:274336817 +twitter:2498172455 +twitter:2809515630 +twitter:2588774684 +twitter:296734891 +twitter:2212410182 +twitter:243027454 +twitter:1336526904 +twitter:397062736 +twitter:449331876 +twitter:30619307 +twitter:2310483811 +twitter:2437586509 +twitter:191710730 +twitter:1084185378 +twitter:2831486681 +twitter:1606477879 +twitter:969600636 +twitter:529783214 +twitter:2928131586 +twitter:190041293 +twitter:2967031274 +twitter:2165962781 +twitter:376501355 +twitter:284137985 +twitter:266863824 +twitter:407944074 +twitter:108456036 +twitter:1641294422 +twitter:900733706 +twitter:1063071450 +twitter:1682722328 +twitter:341419520 +twitter:1644293778 +twitter:2245151467 +twitter:511176989 +twitter:241922669 +twitter:3388315624 +twitter:1909431145 +twitter:2223820028 +twitter:600581315 +twitter:1723555076 +twitter:2748445313 +twitter:561211823 +twitter:561022931 +twitter:2751429993 +twitter:2714908343 +twitter:16165257 +twitter:524623359 +twitter:306741266 +twitter:469994381 +twitter:2561892084 +twitter:998802661 +twitter:1492924374 +twitter:789039140 +twitter:210150093 +twitter:817544820 +twitter:35740178 +twitter:326162841 +twitter:1447331628 +twitter:17493441 +twitter:2874693608 +twitter:965027312 +twitter:261936985 +twitter:510564259 +twitter:728031187 +twitter:164696234 +twitter:2204519310 +twitter:1626241164 +twitter:1024940588 +twitter:221486613 +twitter:571084565 +twitter:3029264508 +twitter:221716563 +twitter:2211417135 +twitter:499972359 +twitter:1565989165 +twitter:2436927208 +twitter:381029291 +twitter:2730580620 +twitter:3436438413 +twitter:2466014604 +twitter:538990742 +twitter:2935470687 +twitter:1162845468 +twitter:468108082 +twitter:2383897542 +twitter:2542119658 +twitter:1962281514 +twitter:171235080 +twitter:536915535100125185 +twitter:2841076618 +twitter:3006098500 +twitter:1057158554 +twitter:3245676721 +twitter:251087536 +twitter:3082811549 +twitter:281785349 +twitter:1674871100 +twitter:1898659951 +twitter:1414854156 +twitter:428693618 +twitter:2385953101 +twitter:2281213477 +twitter:2786368894 +twitter:2253203998 +twitter:357277727 +twitter:1358707970 +twitter:545186198 +twitter:3033613587 +twitter:107121821 +twitter:595965259 +twitter:583894637 +twitter:1306698787 +twitter:442262869 +twitter:2868353318 +twitter:1908436844 +twitter:271982042 +twitter:495202171 +twitter:251586884 +twitter:3151032974 +twitter:2213682568 +twitter:1203133039 +twitter:193128957 +twitter:597407120 +twitter:2781102086 +twitter:369254505 +twitter:62831036 +twitter:2328734640 +twitter:2579064082 +twitter:3271313827 +twitter:2880366619 +twitter:2323026113 +twitter:446380518 +twitter:245418139 +twitter:261211664 +twitter:1893329208 +twitter:3406596309 +twitter:584967077 +twitter:1708862304 +twitter:388961426 +twitter:2421535351 +twitter:2194375668 +twitter:2790313673 +twitter:2728894977 +twitter:2829174824 +twitter:784541196 +twitter:959902393 +twitter:249705367 +twitter:1677679309 +twitter:2825975175 +twitter:1305768366 +twitter:373475046 +twitter:785362464 +twitter:419607671 +twitter:61031675 +twitter:3854236343 +twitter:714603248 +twitter:1301447720 +twitter:827660912 +twitter:2383764684 +twitter:3180084906 +twitter:3265558124 +twitter:608536922 +twitter:238943561 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/4491cfe1/flink/flink-twitter-collection/src/test/resources/FlinkTwitterFollowingPipeline.conf ---------------------------------------------------------------------- diff --git a/flink/flink-twitter-collection/src/test/resources/FlinkTwitterFollowingPipeline.conf b/flink/flink-twitter-collection/src/test/resources/FlinkTwitterFollowingPipeline.conf new file mode 100644 index 0000000..e74f00c --- /dev/null +++ b/flink/flink-twitter-collection/src/test/resources/FlinkTwitterFollowingPipeline.conf @@ -0,0 +1,10 @@ +twitter { + endpoint = followers + version = 1.1 + oauth { + consumerKey = "" + consumerSecret = "" + accessToken = "" + accessTokenSecret = "" + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/4491cfe1/flink/flink-twitter-collection/src/test/resources/FlinkTwitterPostsPipeline.conf ---------------------------------------------------------------------- diff --git a/flink/flink-twitter-collection/src/test/resources/FlinkTwitterPostsPipeline.conf b/flink/flink-twitter-collection/src/test/resources/FlinkTwitterPostsPipeline.conf new file mode 100644 index 0000000..63a6481 --- /dev/null +++ b/flink/flink-twitter-collection/src/test/resources/FlinkTwitterPostsPipeline.conf @@ -0,0 +1,10 @@ +twitter { + version = 1.1 + endpoint = statuses + oauth { + consumerKey = "" + consumerSecret = "" + accessToken = "" + accessTokenSecret = "" + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/4491cfe1/flink/flink-twitter-collection/src/test/resources/FlinkTwitterUserInformationPipeline.conf ---------------------------------------------------------------------- diff --git a/flink/flink-twitter-collection/src/test/resources/FlinkTwitterUserInformationPipeline.conf b/flink/flink-twitter-collection/src/test/resources/FlinkTwitterUserInformationPipeline.conf new file mode 100644 index 0000000..6e0a879 --- /dev/null +++ b/flink/flink-twitter-collection/src/test/resources/FlinkTwitterUserInformationPipeline.conf @@ -0,0 +1,10 @@ +twitter { + version = 1.1 + endpoint = users + oauth { + consumerKey = "" + consumerSecret = "" + accessToken = "" + accessTokenSecret = "" + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/4491cfe1/flink/flink-twitter-collection/src/test/resources/asf.txt ---------------------------------------------------------------------- diff --git a/flink/flink-twitter-collection/src/test/resources/asf.txt b/flink/flink-twitter-collection/src/test/resources/asf.txt new file mode 100644 index 0000000..c2b1ea1 --- /dev/null +++ b/flink/flink-twitter-collection/src/test/resources/asf.txt @@ -0,0 +1 @@ +twitter:18055613 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/4491cfe1/flink/flink-twitter-collection/src/test/scala/org/apache/streams/examples/flink/twitter/test/FlinkTwitterFollowingPipelineIT.scala ---------------------------------------------------------------------- diff --git a/flink/flink-twitter-collection/src/test/scala/org/apache/streams/examples/flink/twitter/test/FlinkTwitterFollowingPipelineIT.scala b/flink/flink-twitter-collection/src/test/scala/org/apache/streams/examples/flink/twitter/test/FlinkTwitterFollowingPipelineIT.scala new file mode 100644 index 0000000..aa2b1a9 --- /dev/null +++ b/flink/flink-twitter-collection/src/test/scala/org/apache/streams/examples/flink/twitter/test/FlinkTwitterFollowingPipelineIT.scala @@ -0,0 +1,81 @@ +package com.peoplepattern.streams.twitter.collection + +import java.nio.file.{Files, Paths} + +import com.peoplepattern.streams.pipelines.pdb.{TwitterFollowingPipelineConfiguration, TwitterPostsPipelineConfiguration} +import org.apache.streams.config.{ComponentConfigurator, StreamsConfigurator} +import org.apache.streams.hdfs.{HdfsConfiguration, HdfsReaderConfiguration, HdfsWriterConfiguration} +import org.scalatest.FlatSpec +import org.scalatest.concurrent.Eventually._ +import org.scalatest.time.SpanSugar._ +import org.slf4j.{Logger, LoggerFactory} +import org.testng.annotations.Test + +import scala.io.Source + +/** + * Created by sblackmon on 3/13/16. + */ +class FlinkTwitterFollowingPipelineIT extends FlatSpec { + + private val LOGGER: Logger = LoggerFactory.getLogger(classOf[FlinkTwitterFollowingPipelineIT]) + + @Test + def flinkTwitterFollowersPipelineFriendsIT = { + + val testConfig : TwitterFollowingPipelineConfiguration = + new ComponentConfigurator[TwitterFollowingPipelineConfiguration](classOf[TwitterFollowingPipelineConfiguration]).detectConfiguration(StreamsConfigurator.getConfig) + testConfig.getTwitter.setEndpoint("friends") + val source : HdfsReaderConfiguration = new HdfsReaderConfiguration().withReaderPath("asf.txt").withScheme(HdfsConfiguration.Scheme.FILE).asInstanceOf[HdfsReaderConfiguration] + source.setPath("target/test-classes") + testConfig.setSource(source); + val destination : HdfsWriterConfiguration = new HdfsWriterConfiguration().withWriterPath("pdb-twitter-collect/FlinkTwitterFollowingPipeline/friends").withScheme(HdfsConfiguration.Scheme.FILE).asInstanceOf[HdfsWriterConfiguration] + destination.setPath("target/test-classes") + testConfig.setDestination(destination) + testConfig.setProviderWaitMs(1000l) + testConfig.setTest(true) + + val job = new FlinkTwitterFollowingPipeline(config = testConfig) + val jobThread = new Thread(job) + jobThread.start + jobThread.join + + eventually (timeout(30 seconds), interval(1 seconds)) { + assert(Files.exists(Paths.get("target/test-classes/pdb-twitter-collect/FlinkTwitterFollowingPipeline/friends"))) + assert( + Source.fromFile("target/test-classes/pdb-twitter-collect/FlinkTwitterFollowingPipeline/friends", "UTF-8").getLines.size + > 90) + } + + } + + @Test + def flinkTwitterFollowersPipelineFollowersIT = { + + val testConfig : TwitterFollowingPipelineConfiguration = + new ComponentConfigurator[TwitterFollowingPipelineConfiguration](classOf[TwitterFollowingPipelineConfiguration]).detectConfiguration(StreamsConfigurator.getConfig) + testConfig.getTwitter.setEndpoint("followers") + val source : HdfsReaderConfiguration = new HdfsReaderConfiguration().withReaderPath("asf.txt").withScheme(HdfsConfiguration.Scheme.FILE).asInstanceOf[HdfsReaderConfiguration] + source.setPath("target/test-classes") + testConfig.setSource(source); + val destination : HdfsWriterConfiguration = new HdfsWriterConfiguration().withWriterPath("pdb-twitter-collect/FlinkTwitterFollowingPipeline/followers").withScheme(HdfsConfiguration.Scheme.FILE).asInstanceOf[HdfsWriterConfiguration] + destination.setPath("target/test-classes") + testConfig.setDestination(destination) + testConfig.setProviderWaitMs(1000l) + testConfig.setTest(true) + + val job = new FlinkTwitterFollowingPipeline(config = testConfig) + val jobThread = new Thread(job) + jobThread.start + jobThread.join + + eventually (timeout(30 seconds), interval(1 seconds)) { + assert(Files.exists(Paths.get("target/test-classes/FlinkTwitterFollowingPipeline/followers"))) + assert( + Source.fromFile("target/test-classes/FlinkTwitterFollowingPipeline/followers", "UTF-8").getLines.size + > 500) + } + + } + +} http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/4491cfe1/flink/flink-twitter-collection/src/test/scala/org/apache/streams/examples/flink/twitter/test/FlinkTwitterPostsPipelineIT.scala ---------------------------------------------------------------------- diff --git a/flink/flink-twitter-collection/src/test/scala/org/apache/streams/examples/flink/twitter/test/FlinkTwitterPostsPipelineIT.scala b/flink/flink-twitter-collection/src/test/scala/org/apache/streams/examples/flink/twitter/test/FlinkTwitterPostsPipelineIT.scala new file mode 100644 index 0000000..8a942e5 --- /dev/null +++ b/flink/flink-twitter-collection/src/test/scala/org/apache/streams/examples/flink/twitter/test/FlinkTwitterPostsPipelineIT.scala @@ -0,0 +1,55 @@ +package com.peoplepattern.streams.twitter.collection + +import java.nio.file.{Files, Paths} +import java.util.concurrent.TimeUnit + +import com.google.common.util.concurrent.{Monitor, Uninterruptibles} +import com.peoplepattern.streams.pipelines.pdb.TwitterPostsPipelineConfiguration +import org.apache.flink.api.scala.ExecutionEnvironment +import org.apache.streams.config.{ComponentConfigurator, StreamsConfigurator} +import org.apache.streams.hdfs.{HdfsConfiguration, HdfsReaderConfiguration, HdfsWriterConfiguration} +import org.slf4j.{Logger, LoggerFactory} + +import scala.io.Source +import org.scalatest.FlatSpec +import org.scalatest.concurrent.Eventually._ +import org.scalatest.time.{Seconds, Span} +import org.scalatest.time.SpanSugar._ +import org.testng.annotations.Test + +/** + * Created by sblackmon on 3/13/16. + */ +class FlinkTwitterPostsPipelineIT extends FlatSpec { + + private val LOGGER: Logger = LoggerFactory.getLogger(classOf[FlinkTwitterPostsPipelineIT]) + + @Test + def flinkTwitterPostsPipelineIT = { + + val testConfig : TwitterPostsPipelineConfiguration = + new ComponentConfigurator[TwitterPostsPipelineConfiguration](classOf[TwitterPostsPipelineConfiguration]).detectConfiguration(StreamsConfigurator.getConfig) + val source : HdfsReaderConfiguration = new HdfsReaderConfiguration().withReaderPath("asf.txt").withScheme(HdfsConfiguration.Scheme.FILE).asInstanceOf[HdfsReaderConfiguration] + source.setPath("target/test-classes") + testConfig.setSource(source); + val destination : HdfsWriterConfiguration = new HdfsWriterConfiguration().withWriterPath("pdb-twitter-collect/FlinkTwitterPostsPipeline").withScheme(HdfsConfiguration.Scheme.FILE).asInstanceOf[HdfsWriterConfiguration] + destination.setPath("target/test-classes") + testConfig.setDestination(destination) + testConfig.setProviderWaitMs(1000l) + testConfig.setTest(true) + + val job = new FlinkTwitterPostsPipeline(config = testConfig) + val jobThread = new Thread(job) + jobThread.start + jobThread.join + + eventually (timeout(30 seconds), interval(1 seconds)) { + assert(Files.exists(Paths.get("target/test-classes/FlinkTwitterPostsPipeline"))) + assert( + Source.fromFile("target/test-classes/FlinkTwitterPostsPipeline", "UTF-8").getLines.size + >= 200) + } + + } + +} http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/4491cfe1/flink/flink-twitter-collection/src/test/scala/org/apache/streams/examples/flink/twitter/test/FlinkTwitterUserInformationPipelineIT.scala ---------------------------------------------------------------------- diff --git a/flink/flink-twitter-collection/src/test/scala/org/apache/streams/examples/flink/twitter/test/FlinkTwitterUserInformationPipelineIT.scala b/flink/flink-twitter-collection/src/test/scala/org/apache/streams/examples/flink/twitter/test/FlinkTwitterUserInformationPipelineIT.scala new file mode 100644 index 0000000..3d21244 --- /dev/null +++ b/flink/flink-twitter-collection/src/test/scala/org/apache/streams/examples/flink/twitter/test/FlinkTwitterUserInformationPipelineIT.scala @@ -0,0 +1,56 @@ +package com.peoplepattern.streams.twitter.collection + +import java.nio.file.{Files, Paths} + +import com.peoplepattern.streams.pipelines.pdb.{TwitterPostsPipelineConfiguration, TwitterUserInformationPipelineConfiguration} +import org.apache.flink.api.scala.ExecutionEnvironment +import org.apache.streams.config.{ComponentConfigurator, StreamsConfigurator} +import org.apache.streams.hdfs.{HdfsConfiguration, HdfsReaderConfiguration, HdfsWriterConfiguration} +import org.scalatest.FlatSpec +import org.scalatest._ +import org.scalatest.junit.JUnitRunner +import org.slf4j.{Logger, LoggerFactory} + +import scala.io.Source +import org.scalatest.Ignore +import org.scalatest.concurrent.Eventually._ +import org.scalatest.time.{Seconds, Span} +import org.scalatest.time.SpanSugar._ +import org.testng.annotations.Test + +/** + * Created by sblackmon on 3/13/16. + */ +class FlinkTwitterUserInformationPipelineIT extends FlatSpec { + + private val LOGGER: Logger = LoggerFactory.getLogger(classOf[FlinkTwitterUserInformationPipelineIT]) + + @Test + def flinkTwitterUserInformationPipelineIT = { + + val testConfig : TwitterUserInformationPipelineConfiguration = + new ComponentConfigurator[TwitterUserInformationPipelineConfiguration](classOf[TwitterUserInformationPipelineConfiguration]).detectConfiguration(StreamsConfigurator.getConfig) + val source : HdfsReaderConfiguration = new HdfsReaderConfiguration().withReaderPath("1000twitterids.txt").withScheme(HdfsConfiguration.Scheme.FILE).asInstanceOf[HdfsReaderConfiguration] + source.setPath("target/test-classes") + testConfig.setSource(source); + val destination : HdfsWriterConfiguration = new HdfsWriterConfiguration().withWriterPath("pdb-twitter-collect/TwitterUserInformationPipeline").withScheme(HdfsConfiguration.Scheme.FILE).asInstanceOf[HdfsWriterConfiguration] + destination.setPath("target/test-classes") + testConfig.setDestination(destination) + testConfig.setProviderWaitMs(1000l) + testConfig.setTest(true) + + val job = new FlinkTwitterUserInformationPipeline(config = testConfig) + val jobThread = new Thread(job) + jobThread.start + jobThread.join + + eventually (timeout(30 seconds), interval(1 seconds)) { + assert(Files.exists(Paths.get("target/test-classes/TwitterUserInformationPipeline"))) + assert( + Source.fromFile("target/test-classes/TwitterUserInformationPipeline", "UTF-8").getLines.size + > 500) + } + + } + +} http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/4491cfe1/flink/pom.xml ---------------------------------------------------------------------- diff --git a/flink/pom.xml b/flink/pom.xml new file mode 100644 index 0000000..7054e89 --- /dev/null +++ b/flink/pom.xml @@ -0,0 +1,47 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one + ~ or more contributor license agreements. See the NOTICE file + ~ distributed with this work for additional information + ~ regarding copyright ownership. The ASF licenses this file + ~ to you under the Apache License, Version 2.0 (the + ~ "License"); you may not use this file except in compliance + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, + ~ software distributed under the License is distributed on an + ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + ~ KIND, either express or implied. See the License for the + ~ specific language governing permissions and limitations + ~ under the License. + --> + +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <artifactId>streams-examples</artifactId> + <groupId>org.apache.streams</groupId> + <version>0.4-incubating-SNAPSHOT</version> + <relativePath>..</relativePath> + </parent> + + <artifactId>streams-examples-flink</artifactId> + + <packaging>pom</packaging> + <name>streams-examples-flink</name> + + <description>Contributed examples of use cases for Streams using flink</description> + + <properties> + + </properties> + + <modules> + <module>flink-twitter-collection</module> + </modules> + + <build> + + </build> +</project> http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/4491cfe1/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 5d25974..9984a5b 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ <parent> <artifactId>streams-master</artifactId> <groupId>org.apache.streams</groupId> - <version>0.3-incubating</version> + <version>0.4-incubating-SNAPSHOT</version> </parent> <artifactId>streams-examples</artifactId> @@ -172,6 +172,7 @@ </properties> <modules> + <module>flink</module> <module>local</module> </modules> @@ -186,39 +187,39 @@ <dependency> <groupId>org.apache.streams</groupId> <artifactId>streams-config</artifactId> - <version>0.3-incubating</version> + <version>0.4-incubating-SNAPSHOT</version> </dependency> <dependency> <groupId>org.apache.streams</groupId> <artifactId>streams-core</artifactId> - <version>0.3-incubating</version> + <version>0.4-incubating-SNAPSHOT</version> </dependency> <dependency> <groupId>org.apache.streams</groupId> <artifactId>streams-filters</artifactId> - <version>0.3-incubating</version> + <version>0.4-incubating-SNAPSHOT</version> </dependency> <dependency> <groupId>org.apache.streams</groupId> <artifactId>streams-pojo</artifactId> - <version>0.3-incubating</version> + <version>0.4-incubating-SNAPSHOT</version> </dependency> <dependency> <groupId>org.apache.streams</groupId> <artifactId>streams-pojo</artifactId> - <version>0.3-incubating</version> + <version>0.4-incubating-SNAPSHOT</version> <type>test-jar</type> <scope>test</scope> </dependency> <dependency> <groupId>org.apache.streams</groupId> <artifactId>streams-util</artifactId> - <version>0.3-incubating</version> + <version>0.4-incubating-SNAPSHOT</version> </dependency> <dependency> <groupId>org.apache.streams</groupId> <artifactId>streams-util</artifactId> - <version>0.3-incubating</version> + <version>0.4-incubating-SNAPSHOT</version> <type>test-jar</type> <scope>test</scope> </dependency> @@ -230,32 +231,32 @@ <dependency> <groupId>org.apache.streams</groupId> <artifactId>streams-runtime-local</artifactId> - <version>0.3-incubating</version> + <version>0.4-incubating-SNAPSHOT</version> </dependency> <dependency> <groupId>org.apache.streams</groupId> <artifactId>streams-provider-twitter</artifactId> - <version>0.3-incubating</version> + <version>0.4-incubating-SNAPSHOT</version> </dependency> <dependency> <groupId>org.apache.streams</groupId> <artifactId>streams-persist-elasticsearch</artifactId> - <version>0.3-incubating</version> + <version>0.4-incubating-SNAPSHOT</version> </dependency> <dependency> <groupId>org.apache.streams</groupId> <artifactId>streams-persist-graph</artifactId> - <version>0.3-incubating</version> + <version>0.4-incubating-SNAPSHOT</version> </dependency> <dependency> <groupId>org.apache.streams</groupId> <artifactId>streams-persist-hdfs</artifactId> - <version>0.3-incubating</version> + <version>0.4-incubating-SNAPSHOT</version> </dependency> <dependency> <groupId>org.apache.streams</groupId> <artifactId>streams-persist-mongo</artifactId> - <version>0.3-incubating</version> + <version>0.4-incubating-SNAPSHOT</version> </dependency> </dependencies>
