Repository: incubator-slider Updated Branches: refs/heads/develop 8dc94a721 -> 963602489
SLIDER-600: better rolehistory r/w Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/fd44d1cf Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/fd44d1cf Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/fd44d1cf Branch: refs/heads/develop Commit: fd44d1cfef0357e4e4abedc2e9dd27cc3807b8cd Parents: f5f837c Author: Steve Loughran <[email protected]> Authored: Thu Mar 26 18:01:42 2015 +0000 Committer: Steve Loughran <[email protected]> Committed: Thu Mar 26 18:01:42 2015 +0000 ---------------------------------------------------------------------- .../slider/server/avro/RoleHistoryRecord.avsc | 4 +++ .../server/appmaster/state/RoleHistory.java | 14 ++++++++++ .../slider/server/avro/RoleHistoryWriter.java | 28 +++++++++++++++----- .../model/history/TestRoleHistoryRW.groovy | 13 +++++++++ .../slider/server/avro/history-v01-4-role.json | 6 +++++ 5 files changed, 58 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fd44d1cf/slider-core/src/main/avro/org/apache/slider/server/avro/RoleHistoryRecord.avsc ---------------------------------------------------------------------- diff --git a/slider-core/src/main/avro/org/apache/slider/server/avro/RoleHistoryRecord.avsc b/slider-core/src/main/avro/org/apache/slider/server/avro/RoleHistoryRecord.avsc index 0380664..0e42e13 100644 --- a/slider-core/src/main/avro/org/apache/slider/server/avro/RoleHistoryRecord.avsc +++ b/slider-core/src/main/avro/org/apache/slider/server/avro/RoleHistoryRecord.avsc @@ -30,6 +30,10 @@ "type": "int" }, { + "name": "rolename", + "type": "string" + }, + { "name": "active", "type": "boolean" }, http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fd44d1cf/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java index c2a741c..c0de492 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java @@ -166,6 +166,20 @@ public class RoleHistory { } /** + * Lookup a role by ID + * @param roleId role Id + * @return role or null if not found + */ + public ProviderRole lookupRole(int roleId) { + for (ProviderRole role : providerRoles) { + if (role.id == roleId) { + return role; + } + } + return null; + } + + /** * Clear the lists of available nodes */ private synchronized void resetAvailableNodeLists() { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fd44d1cf/slider-core/src/main/java/org/apache/slider/server/avro/RoleHistoryWriter.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/avro/RoleHistoryWriter.java b/slider-core/src/main/java/org/apache/slider/server/avro/RoleHistoryWriter.java index 031632b..b938f30 100644 --- a/slider-core/src/main/java/org/apache/slider/server/avro/RoleHistoryWriter.java +++ b/slider-core/src/main/java/org/apache/slider/server/avro/RoleHistoryWriter.java @@ -39,6 +39,7 @@ import org.apache.hadoop.fs.PathFilter; import org.apache.slider.common.SliderKeys; import org.apache.slider.common.tools.SliderUtils; import org.apache.slider.core.exceptions.BadConfigException; +import org.apache.slider.providers.ProviderRole; import org.apache.slider.server.appmaster.state.NodeEntry; import org.apache.slider.server.appmaster.state.NodeInstance; import org.apache.slider.server.appmaster.state.RoleHistory; @@ -106,10 +107,12 @@ public class RoleHistoryWriter { Collection<NodeInstance> instances = history.cloneNodemap().values(); for (NodeInstance instance : instances) { for (int role = 0; role < roles; role++) { + ProviderRole providerRole = history.lookupRole(role); + String rolename = providerRole != null ? providerRole.name : ""; NodeEntry nodeEntry = instance.get(role); if (nodeEntry != null) { - NodeEntryRecord ner = build(nodeEntry, role, instance.hostname); + NodeEntryRecord ner = build(nodeEntry, role, "", instance.hostname); record = new RoleHistoryRecord(ner); writer.write(record, encoder); count++; @@ -121,7 +124,6 @@ public class RoleHistoryWriter { footer.setCount(count); writer.write(new RoleHistoryRecord(footer), encoder); encoder.flush(); - out.close(); return count; } finally { out.close(); @@ -140,8 +142,12 @@ public class RoleHistoryWriter { * @return no of records written * @throws IOException IO failures */ - public long write(FileSystem fs, Path path, boolean overwrite, - RoleHistory history, long savetime) throws IOException { + public long write(FileSystem fs, + Path path, + boolean overwrite, + RoleHistory history, + long savetime) + throws IOException { FSDataOutputStream out = fs.create(path, overwrite); return write(out, history, savetime); } @@ -159,10 +165,18 @@ public class RoleHistoryWriter { Path path = new Path(historyPath, filename); return path; } - - private NodeEntryRecord build(NodeEntry entry, int role, String hostname) { + + /** + * Build a {@link NodeEntryRecord} from a node entry; include whether + * the node is in use and when it was last used. + * @param entry entry count + * @param role role index + * @param rolename + *@param hostname name @return + */ + private NodeEntryRecord build(NodeEntry entry, int role, String rolename, String hostname) { NodeEntryRecord record = new NodeEntryRecord( - hostname, role, entry.getLive() > 0, entry.getLastUsed() + hostname, role, rolename, entry.getLive() > 0, entry.getLastUsed() ); return record; } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fd44d1cf/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy index 5575076..83cf2c6 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy @@ -255,4 +255,17 @@ class TestRoleHistoryRW extends BaseMockAppStateTest { assert fs.exists(badfile ) } + /** + * Test that a v1 JSON file can be read. This is the one without any name + * @throws Throwable + */ + @Test + public void testReloadDataV1() throws Throwable { + String source = "org/apache/slider/server/avro/history-v01-4-role.json" + RoleHistoryWriter writer = new RoleHistoryWriter() + RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES) + assert 4 == writer.read(source, roleHistory) + + } + } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fd44d1cf/slider-core/src/test/resources/org/apache/slider/server/avro/history-v01-4-role.json ---------------------------------------------------------------------- diff --git a/slider-core/src/test/resources/org/apache/slider/server/avro/history-v01-4-role.json b/slider-core/src/test/resources/org/apache/slider/server/avro/history-v01-4-role.json new file mode 100644 index 0000000..565e998 --- /dev/null +++ b/slider-core/src/test/resources/org/apache/slider/server/avro/history-v01-4-role.json @@ -0,0 +1,6 @@ +{"entry":{"org.apache.slider.server.avro.RoleHistoryHeader":{"version":1,"saved":1415296260647,"savedx":"149863b1a27","savedate":"6 Nov 2014 17:51:00 GMT","roles":6}}} +{"entry":{"org.apache.slider.server.avro.NodeEntryRecord":{"host":"192.168.1.85","role":1,"active":false,"last_used":0}}} +{"entry":{"org.apache.slider.server.avro.NodeEntryRecord":{"host":"192.168.1.85","role":2,"active":false,"last_used":0}}} +{"entry":{"org.apache.slider.server.avro.NodeEntryRecord":{"host":"192.168.1.85","role":3,"active":false,"last_used":0}}} +{"entry":{"org.apache.slider.server.avro.NodeEntryRecord":{"host":"192.168.1.85","role":4,"active":true,"last_used":0}}} +{"entry":{"org.apache.slider.server.avro.RoleHistoryFooter":{"count":4}}} \ No newline at end of file
