Repository: sqoop Updated Branches: refs/heads/sqoop2 2fdeeb06d -> 22289d236
SQOOP-1875: Sqoop2: Sync MSubmission with repository and add tests for the max limit handling for submission fields (Veena Basavaraj via Abraham Elmahrek) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/22289d23 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/22289d23 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/22289d23 Branch: refs/heads/sqoop2 Commit: 22289d236aea48ba98d55a7478554d55056e8584 Parents: 2fdeeb0 Author: Abraham Elmahrek <[email protected]> Authored: Mon Dec 22 13:50:26 2014 -0800 Committer: Abraham Elmahrek <[email protected]> Committed: Mon Dec 22 13:50:26 2014 -0800 ---------------------------------------------------------------------- ...RepositoryInsertUpdateDeleteSelectQuery.java | 22 ++--- .../common/CommonRepositorySchemaConstants.java | 4 +- .../derby/DerbyRepositoryHandler.java | 5 +- .../repository/derby/DerbySchemaConstants.java | 7 ++ .../derby/DerbySchemaUpgradeQuery.java | 8 ++ .../sqoop/repository/derby/DerbyTestCase.java | 5 +- .../derby/TestSubmissionHandling.java | 85 ++++++++++++++++++-- 7 files changed, 113 insertions(+), 23 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/22289d23/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryInsertUpdateDeleteSelectQuery.java ---------------------------------------------------------------------- diff --git a/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryInsertUpdateDeleteSelectQuery.java b/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryInsertUpdateDeleteSelectQuery.java index 2102db9..69c55df 100644 --- a/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryInsertUpdateDeleteSelectQuery.java +++ b/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryInsertUpdateDeleteSelectQuery.java @@ -390,8 +390,8 @@ public class CommonRepositoryInsertUpdateDeleteSelectQuery { + COLUMN_SQS_UPDATE_DATE + ", " + COLUMN_SQS_EXTERNAL_ID + ", " + COLUMN_SQS_EXTERNAL_LINK + ", " - + COLUMN_SQS_EXCEPTION + ", " - + COLUMN_SQS_EXCEPTION_TRACE + ") " + + COLUMN_SQS_ERROR_SUMMARY + ", " + + COLUMN_SQS_ERROR_DETAILS + ") " + " VALUES(?, ?, ?, ?, ?, ?, ?, substr(?, 1, 150) , substr(?, 1, 150), substr(?, 1, 750))"; // DML: Update existing submission @@ -400,8 +400,8 @@ public class CommonRepositoryInsertUpdateDeleteSelectQuery { + COLUMN_SQS_STATUS + " = ?, " + COLUMN_SQS_UPDATE_USER + " = ?, " + COLUMN_SQS_UPDATE_DATE + " = ?, " - + COLUMN_SQS_EXCEPTION + " = ?, " - + COLUMN_SQS_EXCEPTION_TRACE + " = ?" + + COLUMN_SQS_ERROR_SUMMARY+ " = substr(?, 1, 150), " + + COLUMN_SQS_ERROR_DETAILS + " = substr(?, 1, 750)" + " WHERE " + COLUMN_SQS_ID + " = ?"; // DML: Check if given submission exists @@ -428,8 +428,8 @@ public class CommonRepositoryInsertUpdateDeleteSelectQuery { + COLUMN_SQS_UPDATE_DATE + ", " + COLUMN_SQS_EXTERNAL_ID + ", " + COLUMN_SQS_EXTERNAL_LINK + ", " - + COLUMN_SQS_EXCEPTION + ", " - + COLUMN_SQS_EXCEPTION_TRACE + + COLUMN_SQS_ERROR_SUMMARY + ", " + + COLUMN_SQS_ERROR_DETAILS + " FROM " + TABLE_SQ_SUBMISSION + " WHERE " + COLUMN_SQS_STATUS + " = ?"; @@ -445,8 +445,8 @@ public class CommonRepositoryInsertUpdateDeleteSelectQuery { + COLUMN_SQS_UPDATE_DATE + ", " + COLUMN_SQS_EXTERNAL_ID + ", " + COLUMN_SQS_EXTERNAL_LINK + ", " - + COLUMN_SQS_EXCEPTION + ", " - + COLUMN_SQS_EXCEPTION_TRACE + + COLUMN_SQS_ERROR_SUMMARY + ", " + + COLUMN_SQS_ERROR_DETAILS + " FROM " + TABLE_SQ_SUBMISSION + " ORDER BY " + COLUMN_SQS_UPDATE_DATE + " DESC"; @@ -462,8 +462,8 @@ public class CommonRepositoryInsertUpdateDeleteSelectQuery { + COLUMN_SQS_UPDATE_DATE + ", " + COLUMN_SQS_EXTERNAL_ID + ", " + COLUMN_SQS_EXTERNAL_LINK + ", " - + COLUMN_SQS_EXCEPTION + ", " - + COLUMN_SQS_EXCEPTION_TRACE + + COLUMN_SQS_ERROR_SUMMARY + ", " + + COLUMN_SQS_ERROR_DETAILS + " FROM " + TABLE_SQ_SUBMISSION + " WHERE " + COLUMN_SQS_JOB + " = ?" + " ORDER BY " + COLUMN_SQS_UPDATE_DATE + " DESC"; @@ -556,4 +556,4 @@ public class CommonRepositoryInsertUpdateDeleteSelectQuery { + " WHERE " + COLUMN_SQ_CFG_DIR_CONFIG + " IN (SELECT " + COLUMN_SQ_CFG_ID + " FROM " + TABLE_SQ_CONFIG + " WHERE " + COLUMN_SQ_CFG_CONFIGURABLE + " = ?)"; -} +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/sqoop/blob/22289d23/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositorySchemaConstants.java ---------------------------------------------------------------------- diff --git a/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositorySchemaConstants.java b/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositorySchemaConstants.java index 4daf203..4ab07b2 100644 --- a/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositorySchemaConstants.java +++ b/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositorySchemaConstants.java @@ -209,9 +209,9 @@ public final class CommonRepositorySchemaConstants { public static final String COLUMN_SQS_EXTERNAL_LINK = "SQS_EXTERNAL_LINK"; - public static final String COLUMN_SQS_EXCEPTION = "SQS_EXCEPTION"; + public static final String COLUMN_SQS_ERROR_SUMMARY = "SQS_ERROR_SUMMARY"; - public static final String COLUMN_SQS_EXCEPTION_TRACE = "SQS_EXCEPTION_TRACE"; + public static final String COLUMN_SQS_ERROR_DETAILS = "SQS_ERROR_DETAILS"; // SQ_COUNTER_GROUP http://git-wip-us.apache.org/repos/asf/sqoop/blob/22289d23/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java ---------------------------------------------------------------------- diff --git a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java index 5bb3a43..6d35143 100644 --- a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java +++ b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java @@ -20,7 +20,6 @@ package org.apache.sqoop.repository.derby; import static org.apache.sqoop.repository.derby.DerbySchemaCreateQuery.*; import static org.apache.sqoop.repository.derby.DerbySchemaInsertUpdateDeleteSelectQuery.*; import static org.apache.sqoop.repository.common.CommonRepositoryInsertUpdateDeleteSelectQuery.*; - import static org.apache.sqoop.repository.derby.DerbySchemaUpgradeQuery.*; import java.net.URL; @@ -294,6 +293,10 @@ public class DerbyRepositoryHandler extends CommonRepositoryHandler { runQuery(QUERY_UPGRADE_TABLE_SQ_CONFIG_ADD_UNIQUE_CONSTRAINT_NAME_TYPE_AND_CONFIGURABLE_ID, conn); runQuery(QUERY_UPGRADE_TABLE_SQ_INPUT_ADD_UNIQUE_CONSTRAINT_NAME_TYPE_AND_CONFIG_ID, conn); + // table column rename + runQuery(QUERY_UPGRADE_RENAME_TABLE_SQ_JOB_SUBMISSION_COLUMN_1, conn); + runQuery(QUERY_UPGRADE_RENAME_TABLE_SQ_JOB_SUBMISSION_COLUMN_2, conn); + } // last step upgrade the repository version to the latest value in the code http://git-wip-us.apache.org/repos/asf/sqoop/blob/22289d23/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaConstants.java ---------------------------------------------------------------------- diff --git a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaConstants.java b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaConstants.java index cdb629d..a551094 100644 --- a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaConstants.java +++ b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaConstants.java @@ -327,6 +327,13 @@ public final class DerbySchemaConstants { public static final String CONSTRAINT_SQS_SQB = SCHEMA_PREFIX + CONSTRAINT_SQS_SQB_NAME; + @Deprecated + // used only for upgrade + public static final String COLUMN_SQS_EXCEPTION = "SQS_EXCEPTION"; + @Deprecated + // used only for upgrade + public static final String COLUMN_SQS_EXCEPTION_TRACE = "SQS_EXCEPTION_TRACE"; + // SQ_COUNTER_SUBMISSION // constraints relevant only in derby public static final String CONSTRAINT_SQRS_SQG_NAME = CONSTRAINT_PREFIX + "SQRS_SQG"; http://git-wip-us.apache.org/repos/asf/sqoop/blob/22289d23/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaUpgradeQuery.java ---------------------------------------------------------------------- diff --git a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaUpgradeQuery.java b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaUpgradeQuery.java index 1cdda1b..d5cabd0 100644 --- a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaUpgradeQuery.java +++ b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaUpgradeQuery.java @@ -500,6 +500,14 @@ public final class DerbySchemaUpgradeQuery { + " UNIQUE (" + COLUMN_SQI_NAME + ", " + COLUMN_SQI_TYPE + ", " + COLUMN_SQI_CONFIG + ")"; + // rename exception to error_summary + public static final String QUERY_UPGRADE_RENAME_TABLE_SQ_JOB_SUBMISSION_COLUMN_1 = "RENAME COLUMN " + + TABLE_SQ_SUBMISSION + "." + COLUMN_SQS_EXCEPTION + " TO " + COLUMN_SQS_ERROR_SUMMARY; + + //rename exception_trace to error_details + public static final String QUERY_UPGRADE_RENAME_TABLE_SQ_JOB_SUBMISSION_COLUMN_2 = "RENAME COLUMN " + + TABLE_SQ_SUBMISSION + "." + COLUMN_SQS_EXCEPTION_TRACE + " TO " + COLUMN_SQS_ERROR_DETAILS; + private DerbySchemaUpgradeQuery() { // Disable explicit object creation } http://git-wip-us.apache.org/repos/asf/sqoop/blob/22289d23/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java ---------------------------------------------------------------------- diff --git a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java index 6d4153c..6cb3eb0 100644 --- a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java +++ b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java @@ -212,6 +212,9 @@ abstract public class DerbyTestCase { runQuery(QUERY_UPGRADE_TABLE_SQ_CONFIG_ADD_UNIQUE_CONSTRAINT_NAME_TYPE_AND_CONFIGURABLE_ID); // add sq_input uniqueness constraint runQuery(QUERY_UPGRADE_TABLE_SQ_INPUT_ADD_UNIQUE_CONSTRAINT_NAME_TYPE_AND_CONFIG_ID); + // add submission table column name renames + runQuery(QUERY_UPGRADE_RENAME_TABLE_SQ_JOB_SUBMISSION_COLUMN_1); + runQuery(QUERY_UPGRADE_RENAME_TABLE_SQ_JOB_SUBMISSION_COLUMN_2); } // deprecated repository version @@ -808,7 +811,7 @@ abstract public class DerbyTestCase { runQuery("INSERT INTO SQOOP.SQ_SUBMISSION" + "(SQS_JOB, SQS_STATUS, SQS_CREATION_DATE, SQS_UPDATE_DATE," - + " SQS_EXTERNAL_ID, SQS_EXTERNAL_LINK, SQS_EXCEPTION," + " SQS_EXCEPTION_TRACE)" + + " SQS_EXTERNAL_ID, SQS_EXTERNAL_LINK, SQS_ERROR_SUMMARY," + " SQS_ERROR_DETAILS)" + "VALUES " + "(1, 'RUNNING', '2012-01-01 01:01:01', '2012-01-01 01:01:01', 'job_1'," + "NULL, NULL, NULL)," + "(2, 'SUCCEEDED', '2012-01-01 01:01:01', '2012-01-02 01:01:01', 'job_2'," http://git-wip-us.apache.org/repos/asf/sqoop/blob/22289d23/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestSubmissionHandling.java ---------------------------------------------------------------------- diff --git a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestSubmissionHandling.java b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestSubmissionHandling.java index dae5e73..b479266 100644 --- a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestSubmissionHandling.java +++ b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestSubmissionHandling.java @@ -122,8 +122,7 @@ public class TestSubmissionHandling extends DerbyTestCase { assertEquals(1, submission.getPersistenceId()); assertCountForTable("SQOOP.SQ_SUBMISSION", 1); - List<MSubmission> submissions = - handler.findUnfinishedSubmissions(getDerbyDatabaseConnection()); + List<MSubmission> submissions = handler.findUnfinishedSubmissions(getDerbyDatabaseConnection()); assertNotNull(submissions); assertEquals(1, submissions.size()); @@ -166,8 +165,7 @@ public class TestSubmissionHandling extends DerbyTestCase { assertEquals(400, counter.getValue()); // Let's create second (simpler) connection - submission = - new MSubmission(1, new Date(), SubmissionStatus.SUCCEEDED, "job-x"); + submission = new MSubmission(1, new Date(), SubmissionStatus.SUCCEEDED, "job-x"); handler.createSubmission(submission, getDerbyDatabaseConnection()); assertEquals(2, submission.getPersistenceId()); @@ -175,11 +173,10 @@ public class TestSubmissionHandling extends DerbyTestCase { } @Test - public void testUpdateConnection() throws Exception { + public void testUpdateSubmission() throws Exception { loadSubmissions(); - List<MSubmission> submissions = - handler.findUnfinishedSubmissions(getDerbyDatabaseConnection()); + List<MSubmission> submissions = handler.findUnfinishedSubmissions(getDerbyDatabaseConnection()); assertNotNull(submissions); assertEquals(2, submissions.size()); @@ -194,6 +191,78 @@ public class TestSubmissionHandling extends DerbyTestCase { } @Test + public void testCreateSubmissionExceptionDetailsMoreThanMaxLimit() throws Exception { + + String externalLink = "http://somewheresomewheresomewheresomewheresomewheresomewheresomewheresomewheresomewheresomewheresomewheresom"; + + "ewheresomewheresomewheresomewheresomewher"; + + String errorSummary = "RuntimeExceptionRuntimeExceptionRuntimeExceptionRuntimeExceptionRuntimeExceptionRuntimeExceptions" + + "RuntimeExceptionRuntimeExceptionRuntimeExceptiontests"; + String errorDetail = "Yeah it happensYeah it happensYeah it happensYeah it happensYeah it happensYeah" + + " it happensYeah it happensYeah it happensYeah it happensYeah it happensYeah it happensYeah it happensYeah it hap" + + "pensYeah it happensYeah it happensYeah it happensYeah it happensYeah it happensYeah it happensYeah it happensYea" + + "h it happensYeah it happensYeah it happensYeah it happensYeah it happensYeah it happensYeah it happensYeah it ha" + + "ppensYeah it happensYeah it happensYeah it happensYeah it happensYeah it happensYeah it happensYeah it happensYeah" + + " it happensYeah it happensYeah it happensYeah it happensYeah it happensYeah it happensYeah it happensYeah it happe" + + "nsYeah it happensYeah it happensYeah it happensYeah it happensYeah it happensYeah it happensYeah it happens"; + MSubmission submission = new MSubmission(); + submission.setJobId(1); + submission.setStatus(SubmissionStatus.RUNNING); + submission.setCreationDate(new Date()); + submission.setLastUpdateDate(new Date()); + submission.setExternalJobId("job-x"); + submission.setExternalLink(externalLink + "more than 150"); + submission.getError().setErrorSummary("RuntimeException"); + submission.getError().setErrorDetails(errorDetail + "morethan750"); + submission.getError().setErrorSummary(errorSummary + "morethan150"); + + handler.createSubmission(submission, getDerbyDatabaseConnection()); + List<MSubmission> submissions = handler.findSubmissionsForJob(1, getDerbyDatabaseConnection()); + assertNotNull(submissions); + + assertEquals(errorDetail, submissions.get(0).getError().getErrorDetails()); + assertEquals(errorSummary, submissions.get(0).getError().getErrorSummary()); + assertEquals(externalLink, submissions.get(0).getExternalLink()); + + } + + @Test + public void testUpdateSubmissionExceptionDetailsMoreThanMaxLimit() throws Exception { + loadSubmissions(); + + List<MSubmission> submissions = handler.findUnfinishedSubmissions(getDerbyDatabaseConnection()); + assertNotNull(submissions); + assertEquals(2, submissions.size()); + + String errorSummary = "RuntimeExceptionRuntimeExceptionRuntimeExceptionRuntimeExceptionRuntimeExceptionRuntimeExceptions" + + "RuntimeExceptionRuntimeExceptionRuntimeExceptiontests"; + + String errorDetail = "Yeah it happensYeah it happensYeah it happensYeah it happensYeah it happensYeah" + + " it happensYeah it happensYeah it happensYeah it happensYeah it happensYeah it happensYeah it happensYeah it hap" + + "pensYeah it happensYeah it happensYeah it happensYeah it happensYeah it happensYeah it happensYeah it happensYea" + + "h it happensYeah it happensYeah it happensYeah it happensYeah it happensYeah it happensYeah it happensYeah it ha" + + "ppensYeah it happensYeah it happensYeah it happensYeah it happensYeah it happensYeah it happensYeah it happensYeah" + + " it happensYeah it happensYeah it happensYeah it happensYeah it happensYeah it happensYeah it happensYeah it happe" + + "nsYeah it happensYeah it happensYeah it happensYeah it happensYeah it happensYeah it happensYeah it happens"; + MSubmission submission = submissions.get(0); + String externalLink = submission.getExternalLink(); + submission.getError().setErrorDetails(errorDetail + "morethan750"); + submission.getError().setErrorSummary(errorSummary + "morethan150"); + submission.setExternalLink("cantupdate"); + + handler.updateSubmission(submission, getDerbyDatabaseConnection()); + + submissions = handler.findUnfinishedSubmissions(getDerbyDatabaseConnection()); + + assertNotNull(submissions); + assertEquals(errorDetail, submissions.get(0).getError().getErrorDetails()); + assertEquals(errorSummary, submissions.get(0).getError().getErrorSummary()); + // note we dont allow external link update + assertEquals(externalLink, submissions.get(0).getExternalLink()); + + } + + @Test public void testPurgeSubmissions() throws Exception { loadSubmissions(); List<MSubmission> submissions; @@ -251,4 +320,4 @@ public class TestSubmissionHandling extends DerbyTestCase { handler.deleteJob(4, getDerbyDatabaseConnection()); assertCountForTable("SQOOP.SQ_SUBMISSION", 0); } -} \ No newline at end of file +}
