Repository: incubator-gobblin
Updated Branches:
  refs/heads/master 0791ee992 -> c43cf2689


[GOBBLIN-409] Set collation to latin1_bin for the MySql state store backing 
table

Closes #2286 from
htran1/mysql_state_store_collation


Project: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/commit/c43cf268
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/tree/c43cf268
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/diff/c43cf268

Branch: refs/heads/master
Commit: c43cf268940bed4d98c0ccba6389d15bacdfaa26
Parents: 0791ee9
Author: Hung Tran <hut...@linkedin.com>
Authored: Tue Feb 13 15:48:12 2018 -0800
Committer: Hung Tran <hut...@linkedin.com>
Committed: Tue Feb 13 15:48:12 2018 -0800

----------------------------------------------------------------------
 .../gobblin/metastore/MysqlStateStore.java      |  4 +--
 .../runtime/MysqlDatasetStateStoreTest.java     | 38 +++++++++++++++++++-
 2 files changed, 39 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/c43cf268/gobblin-metastore/src/main/java/org/apache/gobblin/metastore/MysqlStateStore.java
----------------------------------------------------------------------
diff --git 
a/gobblin-metastore/src/main/java/org/apache/gobblin/metastore/MysqlStateStore.java
 
b/gobblin-metastore/src/main/java/org/apache/gobblin/metastore/MysqlStateStore.java
index 9756507..b276702 100644
--- 
a/gobblin-metastore/src/main/java/org/apache/gobblin/metastore/MysqlStateStore.java
+++ 
b/gobblin-metastore/src/main/java/org/apache/gobblin/metastore/MysqlStateStore.java
@@ -101,8 +101,8 @@ public class MysqlStateStore<T extends State> implements 
StateStore<T> {
 
   // MySQL key length limit is 767 bytes
   private static final String CREATE_JOB_STATE_TABLE_TEMPLATE =
-      "CREATE TABLE IF NOT EXISTS $TABLE$ (store_name varchar(100) CHARACTER 
SET latin1 not null,"
-          + "table_name varchar(667) CHARACTER SET latin1 not null,"
+      "CREATE TABLE IF NOT EXISTS $TABLE$ (store_name varchar(100) CHARACTER 
SET latin1 COLLATE latin1_bin not null,"
+          + "table_name varchar(667) CHARACTER SET latin1 COLLATE latin1_bin 
not null,"
           + " modified_time timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON 
UPDATE CURRENT_TIMESTAMP,"
           + " state longblob, primary key(store_name, table_name))";
 

http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/c43cf268/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/MysqlDatasetStateStoreTest.java
----------------------------------------------------------------------
diff --git 
a/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/MysqlDatasetStateStoreTest.java
 
b/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/MysqlDatasetStateStoreTest.java
index 86ba8ba..45bb44d 100644
--- 
a/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/MysqlDatasetStateStoreTest.java
+++ 
b/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/MysqlDatasetStateStoreTest.java
@@ -43,9 +43,11 @@ public class MysqlDatasetStateStoreTest {
 
   private static final String TEST_STATE_STORE = "TestStateStore";
   private static final String TEST_JOB_NAME = "TestJob";
+  private static final String TEST_JOB_NAME_LOWER = "testjob";
   private static final String TEST_JOB_ID = "TestJob1";
   private static final String TEST_TASK_ID_PREFIX = "TestTask-";
   private static final String TEST_DATASET_URN = "TestDataset";
+  private static final String TEST_DATASET_URN_LOWER = "testdataset";
   private static final String TEST_DATASET_URN2 = "TestDataset2";
 
   private StateStore<JobState> dbJobStateStore;
@@ -109,6 +111,13 @@ public class MysqlDatasetStateStoreTest {
     dbJobStateStore.put(TEST_JOB_NAME,
         MysqlDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + 
MysqlDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX,
         jobState);
+
+    // for testing a colliding lowercase job name
+    jobState.setJobName(TEST_JOB_NAME_LOWER);
+    jobState.setProp("lower", "case");
+    dbJobStateStore.put(TEST_JOB_NAME_LOWER,
+        MysqlDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + 
MysqlDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX,
+        jobState);
   }
 
   @Test(dependsOnMethods = "testPersistJobState")
@@ -119,6 +128,8 @@ public class MysqlDatasetStateStoreTest {
 
     Assert.assertEquals(jobState.getJobName(), TEST_JOB_NAME);
     Assert.assertEquals(jobState.getJobId(), TEST_JOB_ID);
+    Assert.assertEquals(jobState.getProp("foo"), "bar");
+    Assert.assertNotEquals(jobState.getProp("lower"), "case");
     Assert.assertEquals(jobState.getState(), JobState.RunningState.COMMITTED);
     Assert.assertEquals(jobState.getStartTime(), this.startTime);
     Assert.assertEquals(jobState.getEndTime(), this.startTime + 1000);
@@ -132,6 +143,15 @@ public class MysqlDatasetStateStoreTest {
       Assert.assertEquals(taskState.getId(), TEST_TASK_ID_PREFIX + i);
       Assert.assertEquals(taskState.getWorkingState(), 
WorkUnitState.WorkingState.COMMITTED);
     }
+
+    jobState = dbJobStateStore.get(TEST_JOB_NAME_LOWER,
+        dbDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + 
dbDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX,
+        TEST_JOB_ID);
+
+    Assert.assertEquals(jobState.getJobName(), TEST_JOB_NAME_LOWER);
+    Assert.assertEquals(jobState.getJobId(), TEST_JOB_ID);
+    Assert.assertEquals(jobState.getProp("foo"), "bar");
+    Assert.assertEquals(jobState.getProp("lower"), "case");
   }
 
   @Test(dependsOnMethods = "testGetJobState")
@@ -162,6 +182,13 @@ public class MysqlDatasetStateStoreTest {
     datasetState.setDuration(2000);
 
     dbDatasetStateStore.persistDatasetState(TEST_DATASET_URN2, datasetState);
+
+    // persist a colliding lowercase dataset state to test that retrieval is 
case sensitive
+    datasetState.setDatasetUrn(TEST_DATASET_URN_LOWER);
+    datasetState.setId(TEST_DATASET_URN_LOWER );
+    datasetState.setDuration(3000);
+
+    dbDatasetStateStore.persistDatasetState(TEST_DATASET_URN_LOWER, 
datasetState);
   }
 
   @Test(dependsOnMethods = "testPersistDatasetState")
@@ -191,7 +218,7 @@ public class MysqlDatasetStateStoreTest {
   public void testGetPreviousDatasetStatesByUrns() throws IOException {
     Map<String, JobState.DatasetState> datasetStatesByUrns =
         dbDatasetStateStore.getLatestDatasetStatesByUrns(TEST_JOB_NAME);
-    Assert.assertEquals(datasetStatesByUrns.size(), 2);
+    Assert.assertEquals(datasetStatesByUrns.size(), 3);
 
     JobState.DatasetState datasetState = 
datasetStatesByUrns.get(TEST_DATASET_URN);
     Assert.assertEquals(datasetState.getDatasetUrn(), TEST_DATASET_URN);
@@ -210,6 +237,15 @@ public class MysqlDatasetStateStoreTest {
     Assert.assertEquals(datasetState.getStartTime(), this.startTime);
     Assert.assertEquals(datasetState.getEndTime(), this.startTime + 1000);
     Assert.assertEquals(datasetState.getDuration(), 2000);
+
+    datasetState = datasetStatesByUrns.get(TEST_DATASET_URN_LOWER);
+    Assert.assertEquals(datasetState.getDatasetUrn(), TEST_DATASET_URN_LOWER);
+    Assert.assertEquals(datasetState.getJobName(), TEST_JOB_NAME);
+    Assert.assertEquals(datasetState.getJobId(), TEST_JOB_ID);
+    Assert.assertEquals(datasetState.getState(), 
JobState.RunningState.COMMITTED);
+    Assert.assertEquals(datasetState.getStartTime(), this.startTime);
+    Assert.assertEquals(datasetState.getEndTime(), this.startTime + 1000);
+    Assert.assertEquals(datasetState.getDuration(), 3000);
   }
 
   @Test(dependsOnMethods = "testGetPreviousDatasetStatesByUrns")

Reply via email to