[GOBBLIN-409] Set collation to latin1_bin for the MySql state store backing table
Closes #2286 from htran1/mysql_state_store_collation Project: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/commit/c43cf268 Tree: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/tree/c43cf268 Diff: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/diff/c43cf268 Branch: refs/heads/0.12.0 Commit: c43cf268940bed4d98c0ccba6389d15bacdfaa26 Parents: 0791ee9 Author: Hung Tran <[email protected]> Authored: Tue Feb 13 15:48:12 2018 -0800 Committer: Hung Tran <[email protected]> Committed: Tue Feb 13 15:48:12 2018 -0800 ---------------------------------------------------------------------- .../gobblin/metastore/MysqlStateStore.java | 4 +-- .../runtime/MysqlDatasetStateStoreTest.java | 38 +++++++++++++++++++- 2 files changed, 39 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/c43cf268/gobblin-metastore/src/main/java/org/apache/gobblin/metastore/MysqlStateStore.java ---------------------------------------------------------------------- diff --git a/gobblin-metastore/src/main/java/org/apache/gobblin/metastore/MysqlStateStore.java b/gobblin-metastore/src/main/java/org/apache/gobblin/metastore/MysqlStateStore.java index 9756507..b276702 100644 --- a/gobblin-metastore/src/main/java/org/apache/gobblin/metastore/MysqlStateStore.java +++ b/gobblin-metastore/src/main/java/org/apache/gobblin/metastore/MysqlStateStore.java @@ -101,8 +101,8 @@ public class MysqlStateStore<T extends State> implements StateStore<T> { // MySQL key length limit is 767 bytes private static final String CREATE_JOB_STATE_TABLE_TEMPLATE = - "CREATE TABLE IF NOT EXISTS $TABLE$ (store_name varchar(100) CHARACTER SET latin1 not null," - + "table_name varchar(667) CHARACTER SET latin1 not null," + "CREATE TABLE IF NOT EXISTS $TABLE$ (store_name varchar(100) CHARACTER SET latin1 COLLATE latin1_bin not null," + + "table_name varchar(667) CHARACTER SET latin1 COLLATE latin1_bin not null," + " modified_time timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP," + " state longblob, primary key(store_name, table_name))"; http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/c43cf268/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/MysqlDatasetStateStoreTest.java ---------------------------------------------------------------------- diff --git a/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/MysqlDatasetStateStoreTest.java b/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/MysqlDatasetStateStoreTest.java index 86ba8ba..45bb44d 100644 --- a/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/MysqlDatasetStateStoreTest.java +++ b/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/MysqlDatasetStateStoreTest.java @@ -43,9 +43,11 @@ public class MysqlDatasetStateStoreTest { private static final String TEST_STATE_STORE = "TestStateStore"; private static final String TEST_JOB_NAME = "TestJob"; + private static final String TEST_JOB_NAME_LOWER = "testjob"; private static final String TEST_JOB_ID = "TestJob1"; private static final String TEST_TASK_ID_PREFIX = "TestTask-"; private static final String TEST_DATASET_URN = "TestDataset"; + private static final String TEST_DATASET_URN_LOWER = "testdataset"; private static final String TEST_DATASET_URN2 = "TestDataset2"; private StateStore<JobState> dbJobStateStore; @@ -109,6 +111,13 @@ public class MysqlDatasetStateStoreTest { dbJobStateStore.put(TEST_JOB_NAME, MysqlDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + MysqlDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX, jobState); + + // for testing a colliding lowercase job name + jobState.setJobName(TEST_JOB_NAME_LOWER); + jobState.setProp("lower", "case"); + dbJobStateStore.put(TEST_JOB_NAME_LOWER, + MysqlDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + MysqlDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX, + jobState); } @Test(dependsOnMethods = "testPersistJobState") @@ -119,6 +128,8 @@ public class MysqlDatasetStateStoreTest { Assert.assertEquals(jobState.getJobName(), TEST_JOB_NAME); Assert.assertEquals(jobState.getJobId(), TEST_JOB_ID); + Assert.assertEquals(jobState.getProp("foo"), "bar"); + Assert.assertNotEquals(jobState.getProp("lower"), "case"); Assert.assertEquals(jobState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(jobState.getStartTime(), this.startTime); Assert.assertEquals(jobState.getEndTime(), this.startTime + 1000); @@ -132,6 +143,15 @@ public class MysqlDatasetStateStoreTest { Assert.assertEquals(taskState.getId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); } + + jobState = dbJobStateStore.get(TEST_JOB_NAME_LOWER, + dbDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + dbDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX, + TEST_JOB_ID); + + Assert.assertEquals(jobState.getJobName(), TEST_JOB_NAME_LOWER); + Assert.assertEquals(jobState.getJobId(), TEST_JOB_ID); + Assert.assertEquals(jobState.getProp("foo"), "bar"); + Assert.assertEquals(jobState.getProp("lower"), "case"); } @Test(dependsOnMethods = "testGetJobState") @@ -162,6 +182,13 @@ public class MysqlDatasetStateStoreTest { datasetState.setDuration(2000); dbDatasetStateStore.persistDatasetState(TEST_DATASET_URN2, datasetState); + + // persist a colliding lowercase dataset state to test that retrieval is case sensitive + datasetState.setDatasetUrn(TEST_DATASET_URN_LOWER); + datasetState.setId(TEST_DATASET_URN_LOWER ); + datasetState.setDuration(3000); + + dbDatasetStateStore.persistDatasetState(TEST_DATASET_URN_LOWER, datasetState); } @Test(dependsOnMethods = "testPersistDatasetState") @@ -191,7 +218,7 @@ public class MysqlDatasetStateStoreTest { public void testGetPreviousDatasetStatesByUrns() throws IOException { Map<String, JobState.DatasetState> datasetStatesByUrns = dbDatasetStateStore.getLatestDatasetStatesByUrns(TEST_JOB_NAME); - Assert.assertEquals(datasetStatesByUrns.size(), 2); + Assert.assertEquals(datasetStatesByUrns.size(), 3); JobState.DatasetState datasetState = datasetStatesByUrns.get(TEST_DATASET_URN); Assert.assertEquals(datasetState.getDatasetUrn(), TEST_DATASET_URN); @@ -210,6 +237,15 @@ public class MysqlDatasetStateStoreTest { Assert.assertEquals(datasetState.getStartTime(), this.startTime); Assert.assertEquals(datasetState.getEndTime(), this.startTime + 1000); Assert.assertEquals(datasetState.getDuration(), 2000); + + datasetState = datasetStatesByUrns.get(TEST_DATASET_URN_LOWER); + Assert.assertEquals(datasetState.getDatasetUrn(), TEST_DATASET_URN_LOWER); + Assert.assertEquals(datasetState.getJobName(), TEST_JOB_NAME); + Assert.assertEquals(datasetState.getJobId(), TEST_JOB_ID); + Assert.assertEquals(datasetState.getState(), JobState.RunningState.COMMITTED); + Assert.assertEquals(datasetState.getStartTime(), this.startTime); + Assert.assertEquals(datasetState.getEndTime(), this.startTime + 1000); + Assert.assertEquals(datasetState.getDuration(), 3000); } @Test(dependsOnMethods = "testGetPreviousDatasetStatesByUrns")
