----------------------------------------------------------- This is an automatically generated e-mail. To reply, visit: https://reviews.apache.org/r/7783/#review13067 -----------------------------------------------------------
Ship it! Ship It! - Bilung Lee On Oct. 31, 2012, 12:26 a.m., Jarek Cecho wrote: > > ----------------------------------------------------------- > This is an automatically generated e-mail. To reply, visit: > https://reviews.apache.org/r/7783/ > ----------------------------------------------------------- > > (Updated Oct. 31, 2012, 12:26 a.m.) > > > Review request for Sqoop. > > > Description > ------- > > This is sort of mega patch that is implementing first (not exactly stable and > finished) version of submission engine. I've got like dozen other notes what > to improve and implement, but the patch is extreme in size already. So I > would like to propose getting it in, to give others chance to see what I'm > working on at the moment and also retrieve their feedback. In meantime I'll > continue my effort to make it very stable and very "nice" :-) My > justification for doing that is that since entire sqoop 2 is alpha, it > doesn't hurt committing alpha submission engine :-) > > Submission engine is asynchronous pluggable interface for submitting > mapreduce jobs. I've provided default mapreduce implementation > (implementation that directly talks to JobTracker). > > Procedure to get it working on dev box: > * Configure your hadoop cluster as usual > * Put config files to /etc/hadoop/conf or reconfigure sqoop to use different > directory > * Put all required hadoop jars to tomcat lib folder (issue with resolving jar > dependency is still open, so this is like temporal hack) > * Use client command submission to launch actual mapreduce jobs > > > This addresses bug SQOOP-656. > https://issues.apache.org/jira/browse/SQOOP-656 > > > Diffs > ----- > > client/src/main/java/org/apache/sqoop/client/core/Environment.java > cb92998d2054c8c7adbbf8fb1764fd5269d474b9 > client/src/main/java/org/apache/sqoop/client/core/RequestCache.java > 03b1eb001f23aa5b3992a479ca9c2fc65136a0bd > client/src/main/java/org/apache/sqoop/client/request/Request.java > b243dfdfe4421236f865c50529a553314ff5b1b3 > client/src/main/java/org/apache/sqoop/client/request/SubmissionRequest.java > PRE-CREATION > > client/src/main/java/org/apache/sqoop/client/shell/CreateConnectionFunction.java > ea1e5d811da9fd817e20363f26498795b29985f8 > client/src/main/java/org/apache/sqoop/client/shell/SqoopShell.java > c9ac848812ec2ae1ed4f1d15daff9eb6a0e19425 > client/src/main/java/org/apache/sqoop/client/shell/SubmissionCommand.java > PRE-CREATION > > client/src/main/java/org/apache/sqoop/client/shell/SubmissionStartFunction.java > PRE-CREATION > > client/src/main/java/org/apache/sqoop/client/shell/SubmissionStatusFunction.java > PRE-CREATION > > client/src/main/java/org/apache/sqoop/client/shell/SubmissionStopFunction.java > PRE-CREATION > client/src/main/java/org/apache/sqoop/client/utils/FormFiller.java.orig > d2fb6cecdf8a6baed02705bf17abf9a788b0c4c0 > client/src/main/java/org/apache/sqoop/client/utils/SubmissionDisplayer.java > PRE-CREATION > client/src/main/java/org/apache/sqoop/client/utils/ThrowableDisplayer.java > 8d6e9b4814a53270e0a1920ce870ad0bef355e47 > common/src/main/java/org/apache/sqoop/common/ImmutableContext.java > PRE-CREATION > common/src/main/java/org/apache/sqoop/common/MapContext.java PRE-CREATION > common/src/main/java/org/apache/sqoop/common/MutableContext.java > PRE-CREATION > common/src/main/java/org/apache/sqoop/common/MutableMapContext.java > PRE-CREATION > common/src/main/java/org/apache/sqoop/json/JobBean.java > 7fd2ce52b7c563ad8570d8ebf88b2143ff5aea29 > common/src/main/java/org/apache/sqoop/json/SubmissionBean.java PRE-CREATION > common/src/main/java/org/apache/sqoop/json/ThrowableBean.java > 91914e84eee4c3e4ee8dc38a1449e51b54e2d2b2 > common/src/main/java/org/apache/sqoop/model/FormUtils.java > 424810c1b36c4b8c5ebca143121a151e499d8231 > common/src/main/java/org/apache/sqoop/model/MSubmission.java PRE-CREATION > common/src/main/java/org/apache/sqoop/submission/SubmissionStatus.java > PRE-CREATION > common/src/main/java/org/apache/sqoop/submission/counter/Counter.java > PRE-CREATION > common/src/main/java/org/apache/sqoop/submission/counter/CounterGroup.java > PRE-CREATION > common/src/main/java/org/apache/sqoop/submission/counter/Counters.java > PRE-CREATION > common/src/main/java/org/apache/sqoop/utils/ClassLoadingUtils.java > 0c7d09f189c1a2fa96e9c5b782f6693e30cdd62a > common/src/main/java/org/apache/sqoop/utils/ClassUtils.java PRE-CREATION > common/src/test/java/org/apache/sqoop/json/TestSubmissionBean.java > PRE-CREATION > common/src/test/java/org/apache/sqoop/json/TestThrowableBean.java > 19a0a27764f957998a7256c98126d2d0d29d1544 > common/src/test/java/org/apache/sqoop/utils/TestClassLoadingUtils.java > e56e017e40bc21e84844ebb20ae72fc81c52eb92 > common/src/test/java/org/apache/sqoop/utils/TestClassUtils.java > PRE-CREATION > connector/connector-generic-jdbc/pom.xml > be4cedd23210d9a5405d4f5ac5f4ee4fbeececf8 > > connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcExecutor.java > 702dd7e3ff50a203d7d5107de7178cc5517a29eb > > connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcExportDestroyer.java > cba8f71f59df7068d6d043509c1cfc8927420aa8 > > connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcExportInitializer.java > a21dc7647e1b405510e050e8006435a293cd3b05 > > connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcExportLoader.java > 0289e29e041527f189a87c44e8e4737a27f83b95 > > connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportDestroyer.java > b0a0f62fc85c6284e346f68e7dfc62a05bb52623 > > connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportExtractor.java > 4499fda0e31ff4d01b30b8e39fefda9e37c2912f > > connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportInitializer.java > 75f3e56a1077cda06c4b3d711c37a84ea7419c15 > > connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportPartition.java > cba313b43075713fb61c3f6661ceeac2c5a94b9e > > connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportPartitioner.java > b741b74186ffdf3047f7a0f1f9ace62f5cb865d6 > > connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/ConnectionConfiguration.java > 212bdf3f41d3ad2ab5f95f0913b1b089f22829ab > > connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/ExportJobConfiguration.java > 91004cf61d954d9e0381573d7543aac0ee10d892 > > connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/ImportJobConfiguration.java > 31ce7777e2f2226787f8a10e96c7a002fbe0413e > > connector/connector-generic-jdbc/src/main/resources/generic-jdbc-connector-resources.properties > 9c8e290bf2dc56b35081ca5501d8a49ef70831b9 > > connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestImportExtractor.java > 70e29e534dffb79dc478876049b26c7ae66beed4 > > connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestImportInitializer.java > 546559348e3b4fa49915565c1742732230252ff9 > > connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestImportPartitioner.java > 0e95a43e59dc74f864f190df8f225afa0076a4a5 > core/src/main/java/org/apache/sqoop/connector/ConnectorHandler.java > 273b4864457ad237c3b5eca9fbb0d98fdbeea993 > core/src/main/java/org/apache/sqoop/connector/ConnectorManager.java > f7228d37e6642e30fce2cba29ee1596fd48a44f5 > core/src/main/java/org/apache/sqoop/core/Context.java > 6eeed137581d3f014b8deeb2d9f9951ec622f0c4 > core/src/main/java/org/apache/sqoop/core/SqoopConfiguration.java > 47340f97251e9b9cbce429ec9286c788067662bd > core/src/main/java/org/apache/sqoop/framework/FrameworkConstants.java > 662a50c77360ae57e9d39a3642dc178585be2cfd > core/src/main/java/org/apache/sqoop/framework/FrameworkError.java > e0d91d4cee29cea8c6da5a67967f17ca8ab84345 > core/src/main/java/org/apache/sqoop/framework/FrameworkManager.java > c243275cc7d9bc5473144f18bd68a807e5b11462 > core/src/main/java/org/apache/sqoop/framework/SubmissionEngine.java > PRE-CREATION > core/src/main/java/org/apache/sqoop/framework/SubmissionRequest.java > PRE-CREATION > > core/src/main/java/org/apache/sqoop/framework/configuration/ImportJobConfiguration.java > 1d1466126eb347f181bef167b8c4ebc51e469a09 > core/src/main/java/org/apache/sqoop/job/JobConstants.java > 2b0ec18b96429e92b3d84595a1c6fe4223d4bdf5 > core/src/main/java/org/apache/sqoop/job/JobEngine.java > fa3e484ad24e116113cbe9af24dc312f3a158efd > core/src/main/java/org/apache/sqoop/job/PrefixContext.java PRE-CREATION > core/src/main/java/org/apache/sqoop/job/etl/EtlContext.java > 09eca58d170492e7621264b35b0cfeec52eb07ac > core/src/main/java/org/apache/sqoop/job/etl/EtlFramework.java > ce7f9884961ddfac77f0973194686a1f7ba35c3c > core/src/main/java/org/apache/sqoop/job/etl/EtlMutableContext.java > e1119567d899f8c234b14eb40efcd531336a9a9e > core/src/main/java/org/apache/sqoop/job/etl/EtlOptions.java > e45c0ff4a2185f04f2631aeeacfff85a48904278 > core/src/main/java/org/apache/sqoop/job/etl/HdfsSequenceImportLoader.java > 854d325ed3336f7f4b69eec19576fdfc2e826621 > core/src/main/java/org/apache/sqoop/job/etl/HdfsTextImportLoader.java > 240265bd8a6a314b826b18b5c930adacd9c4a18c > core/src/main/java/org/apache/sqoop/job/mr/ConfigurationUtils.java > PRE-CREATION > core/src/main/java/org/apache/sqoop/job/mr/MrExecution.java > bd4c108dbc84be65bdc9960a674631168173d9c4 > core/src/main/java/org/apache/sqoop/job/mr/SqoopInputFormat.java > f5ae3df1510c2e4f725ccc5d252a71346a0a93ec > core/src/main/java/org/apache/sqoop/job/mr/SqoopMapper.java > 0a9f46d48754dffe2034c63f3736f20dfb3a9c9e > > core/src/main/java/org/apache/sqoop/job/mr/SqoopOutputFormatLoadExecutor.java > 23fcb6254c2437ce74813dd84601c090341d9669 > core/src/main/java/org/apache/sqoop/job/mr/SqoopSplit.java > f936f6e876785b747a7d3ee600e1d641fb448792 > core/src/main/java/org/apache/sqoop/repository/JdbcRepository.java > 8d7b95cd55ba65b1f597baf7aa3cf96cd830a6cf > core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryContext.java > b2274d335fc9bba919b71e89a7d2661a0b5356a9 > core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryHandler.java > b0c9780bbca33f3fbd3622492576f4074f2ffad7 > core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryProvider.java > e7b6771a715e2b92728bbd70b72f7f30b6c0c68e > core/src/main/java/org/apache/sqoop/repository/Repository.java > ec746d16d5cccd355a917c3b9eb1a0fa45c7cad5 > core/src/main/java/org/apache/sqoop/repository/RepositoryError.java > ff53b1302dce25526ead7c0d15ece2ff2c3b9b5d > core/src/main/java/org/apache/sqoop/repository/RepositoryManager.java > 0fbeeb3d6e5c8b3d5d5e0890f04763425b542970 > core/src/main/java/org/apache/sqoop/repository/RepositoryProvider.java > 1b5d00dc6b8006b845462a5f2ee6e79983622fcf > core/src/main/resources/framework-resources.properties > 4706cf48ce5f0c1cdd1e06d1d2a5a1b6f97a2838 > core/src/test/java/org/apache/sqoop/job/TestHdfsLoad.java > 64c767cf97e5cd5b321fc7e5c5caa55b25bcc5d8 > core/src/test/java/org/apache/sqoop/job/TestJobEngine.java > e653c22e6494d3d3e9449716170c057133384adf > core/src/test/java/org/apache/sqoop/job/TestMapReduce.java > 7646f5732dc45cb18985c1c60c47c35185aaa83d > dist/src/main/server/conf/sqoop.properties > 882191dbbf8ff8ddd68bdf4de61584281150b63c > pom.xml eea0350d5e0d4ed5dd428ebcaee39304c403baef > > repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepoError.java > 94119b1f50cc611b767cb20468dce447e45efb36 > > repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java > 0ce8832f9fb57997955113f6dba8b551ad989b7b > > repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaConstants.java > 95461c9349607ef5ae453172498d43fdc698a300 > > repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaQuery.java > cddace7fd9e5c4fa5a67fdcae845a57b5a9d35e2 > > repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java > ae59933e690a06a346849527c4d90f9f5d20f4da > > repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestSubmissionHandling.java > PRE-CREATION > server/pom.xml 78ad8ee56472bdf5e508c6b641c7fdc92b6aec3a > server/src/main/java/org/apache/sqoop/handler/ConnectionRequestHandler.java > eba334e57bc56becc33d13fd606a11a4158c87fc > server/src/main/java/org/apache/sqoop/handler/ConnectorRequestHandler.java > fda91fd69dc023fede6add6183a311f54dab4d49 > server/src/main/java/org/apache/sqoop/handler/JobRequestHandler.java > 0589e30f9ecd30ee8b0a54fbd5167ac27f9a24d5 > server/src/main/java/org/apache/sqoop/handler/SubmissionRequestHandler.java > PRE-CREATION > server/src/main/java/org/apache/sqoop/server/RequestContext.java > 78950f6901c40e34f521e4d41c2e9b995f5f34d9 > server/src/main/java/org/apache/sqoop/server/ServerInitializer.java > 993c153820891d826e191257fa562d7d12852380 > server/src/main/java/org/apache/sqoop/server/v1/SubmissionServlet.java > PRE-CREATION > server/src/main/webapp/WEB-INF/web.xml > 69229bfdb89ffcbba699bf77e12333fcc4af586c > spi/src/main/java/org/apache/sqoop/job/etl/CallbackBase.java PRE-CREATION > spi/src/main/java/org/apache/sqoop/job/etl/Context.java > fc01c963c01a0e3f786574457e5ccd7775d66d69 > spi/src/main/java/org/apache/sqoop/job/etl/Destroyer.java > af766f3bc4554f864e4da903dc9ab2e2d4fe1eb3 > spi/src/main/java/org/apache/sqoop/job/etl/Exporter.java > ef690bfdc776cba8cab1856e79bb0f93c0e4ba68 > spi/src/main/java/org/apache/sqoop/job/etl/Extractor.java > 20bdeda28dec0dc5a64e5a40469288efa0d6af09 > spi/src/main/java/org/apache/sqoop/job/etl/Importer.java > f0a8d1afdf1d32d77bef44be310f5f84d0097d9b > spi/src/main/java/org/apache/sqoop/job/etl/Initializer.java > 75bd42e75f4050eaf9fb9459fe9b81526d3c604e > spi/src/main/java/org/apache/sqoop/job/etl/Loader.java > 54749274c147bcde7a5e728dd09502a744839a9a > spi/src/main/java/org/apache/sqoop/job/etl/MutableContext.java > 03678c58b4d18062ea458b4cc712928eace33ea4 > spi/src/main/java/org/apache/sqoop/job/etl/Options.java > 2dc467180da9f3d849f45b823aabb49e9f80bf6d > spi/src/main/java/org/apache/sqoop/job/etl/Partition.java > 8834c80e54d8993b6db67999c31b2bf0dbaadc43 > spi/src/main/java/org/apache/sqoop/job/etl/Partitioner.java > 21310be43086631aa943945d0d69ab1ed502b6ff > submission/mapreduce/pom.xml PRE-CREATION > > submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/Constants.java > PRE-CREATION > > submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/MapreduceSubmissionEngine.java > PRE-CREATION > > submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/MapreduceSubmissionError.java > PRE-CREATION > submission/pom.xml PRE-CREATION > > Diff: https://reviews.apache.org/r/7783/diff/ > > > Testing > ------- > > I've tested it on real cluster and I was able to move first rows! Otherwise > my patch is commenting a lot of tests to disable them for the time being. > > > Thanks, > > Jarek Cecho > >
