Repository: incubator-crail Updated Branches: refs/heads/master 0db64fdda -> 46925ad19
Make default configuration easier to run Better default values for cachepath and tcp storage path. Throw an exception if paths are invalid. Throw an exception if interface is invalid. Trim down crail-site.conf.template to only include the necessary parameters. https://issues.apache.org/jira/browse/CRAIL-15 Signed-off-by: Patrick Stuedi <pstu...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/incubator-crail/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-crail/commit/84f608e5 Tree: http://git-wip-us.apache.org/repos/asf/incubator-crail/tree/84f608e5 Diff: http://git-wip-us.apache.org/repos/asf/incubator-crail/diff/84f608e5 Branch: refs/heads/master Commit: 84f608e542d6eed044bdf12ef45616ec645f7a4d Parents: 0db64fd Author: Patrick Stuedi <pstu...@apache.org> Authored: Tue Jun 19 15:21:07 2018 +0200 Committer: Patrick Stuedi <pstu...@apache.org> Committed: Tue Jun 19 15:21:07 2018 +0200 ---------------------------------------------------------------------- .../main/java/org/apache/crail/CrailStore.java | 2 +- .../org/apache/crail/conf/CrailConstants.java | 10 +++++----- .../apache/crail/memory/MappedBufferCache.java | 4 +++- conf/core-site.xml.template | 2 +- conf/crail-site.conf.template | 19 ++++++------------- .../crail/storage/tcp/TcpStorageConstants.java | 2 +- .../crail/storage/tcp/TcpStorageServer.java | 16 +++++++++++----- .../apache/crail/storage/rdma/RdmaConstants.java | 2 +- 8 files changed, 29 insertions(+), 28 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-crail/blob/84f608e5/client/src/main/java/org/apache/crail/CrailStore.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/crail/CrailStore.java b/client/src/main/java/org/apache/crail/CrailStore.java index 5f9a229..cb91b7e 100644 --- a/client/src/main/java/org/apache/crail/CrailStore.java +++ b/client/src/main/java/org/apache/crail/CrailStore.java @@ -69,7 +69,7 @@ public abstract class CrailStore { public static CrailStore newInstance(CrailConfiguration conf) throws Exception { synchronized(referenceCounter){ - boolean isSingleton = conf.getBoolean(CrailConstants.SINGLETON_KEY, false); + boolean isSingleton = conf.getBoolean(CrailConstants.SINGLETON_KEY, CrailConstants.SINGLETON); if (isSingleton) { referenceCounter.incrementAndGet(); if (instance == null) { http://git-wip-us.apache.org/repos/asf/incubator-crail/blob/84f608e5/client/src/main/java/org/apache/crail/conf/CrailConstants.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/crail/conf/CrailConstants.java b/client/src/main/java/org/apache/crail/conf/CrailConstants.java index 56d99db..733d622 100644 --- a/client/src/main/java/org/apache/crail/conf/CrailConstants.java +++ b/client/src/main/java/org/apache/crail/conf/CrailConstants.java @@ -36,16 +36,16 @@ public class CrailConstants { public static long TOKEN_EXPIRATION = 10; public static final String BLOCK_SIZE_KEY = "crail.blocksize"; - public static long BLOCK_SIZE = 67108864; + public static long BLOCK_SIZE = 1048576; public static final String CACHE_LIMIT_KEY = "crail.cachelimit"; public static long CACHE_LIMIT = 1073741824; public static final String CACHE_PATH_KEY = "crail.cachepath"; - public static String CACHE_PATH = "/home/stu/craildata/cache"; + public static String CACHE_PATH = "/dev/hugepages/cache"; public static final String USER_KEY = "crail.user"; - public static String USER = "stu"; + public static String USER = "crail"; public static final String SHADOW_REPLICATION_KEY = "crail.shadowreplication"; public static int SHADOW_REPLICATION = 1; @@ -69,7 +69,7 @@ public class CrailConstants { public static int SLICE_SIZE = 524288; public static final String SINGLETON_KEY = "crail.singleton"; - public static boolean SINGLETON = false; + public static boolean SINGLETON = true; public static final String REGION_SIZE_KEY = "crail.regionsize"; public static long REGION_SIZE = 1073741824; @@ -88,7 +88,7 @@ public class CrailConstants { //namenode interface public static final String NAMENODE_ADDRESS_KEY = "crail.namenode.address"; - public static String NAMENODE_ADDRESS = ""; + public static String NAMENODE_ADDRESS = "crail://localhost:9060"; public static final String NAMENODE_FILEBLOCKS_KEY = "crail.namenode.fileblocks"; public static int NAMENODE_FILEBLOCKS = 16; http://git-wip-us.apache.org/repos/asf/incubator-crail/blob/84f608e5/client/src/main/java/org/apache/crail/memory/MappedBufferCache.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/crail/memory/MappedBufferCache.java b/client/src/main/java/org/apache/crail/memory/MappedBufferCache.java index bc78afe..429bbe0 100644 --- a/client/src/main/java/org/apache/crail/memory/MappedBufferCache.java +++ b/client/src/main/java/org/apache/crail/memory/MappedBufferCache.java @@ -54,7 +54,9 @@ public class MappedBufferCache extends BufferCache { directory = CrailUtils.getCacheDirectory(id); dir = new File(directory); if (!dir.exists()){ - dir.mkdirs(); + if (!dir.mkdirs()) { + throw new IOException("Cannot create cache directory [crail.cachepath] set to path " + directory + ", check if crail.cachepath exists and has write permissions"); + } } for (File child : dir.listFiles()) { child.delete(); http://git-wip-us.apache.org/repos/asf/incubator-crail/blob/84f608e5/conf/core-site.xml.template ---------------------------------------------------------------------- diff --git a/conf/core-site.xml.template b/conf/core-site.xml.template index 1126e57..ec7e64f 100644 --- a/conf/core-site.xml.template +++ b/conf/core-site.xml.template @@ -23,7 +23,7 @@ </property> <property> <name>fs.defaultFS</name> - <value>crail://hostname:9060</value> + <value>crail://localhost:9060</value> </property> <property> <name>fs.AbstractFileSystem.crail.impl</name> http://git-wip-us.apache.org/repos/asf/incubator-crail/blob/84f608e5/conf/crail-site.conf.template ---------------------------------------------------------------------- diff --git a/conf/crail-site.conf.template b/conf/crail-site.conf.template index 1159dc0..a375dc8 100644 --- a/conf/crail-site.conf.template +++ b/conf/crail-site.conf.template @@ -1,14 +1,7 @@ -crail.blocksize 1048576 -crail.buffersize 1048576 -crail.slicesize 524288 -crail.regionsize 1073741824 -crail.cachelimit 1073741824 -crail.cachepath /dev/hugepages/cache -crail.singleton true -crail.statistics true -crail.namenode.address crail://localhost:9060 -crail.namenode.blockselection roundrobin -crail.namenode.darpc.queuesize 64 -crail.namenode.darpc.polling false -crail.storage.types org.apache.crail.storage.tcp.TcpStorageTier +crail.namenode.address crail://localhost:9060 +crail.cachepath /dev/hugepages/cache +crail.cachelimit 1073741824 +crail.storage.tcp.interface eth0 +crail.storage.tcp.datapath /dev/hugepages/data +crail.storage.tcp.storagelimit 1073741824 http://git-wip-us.apache.org/repos/asf/incubator-crail/blob/84f608e5/storage-narpc/src/main/java/org/apache/crail/storage/tcp/TcpStorageConstants.java ---------------------------------------------------------------------- diff --git a/storage-narpc/src/main/java/org/apache/crail/storage/tcp/TcpStorageConstants.java b/storage-narpc/src/main/java/org/apache/crail/storage/tcp/TcpStorageConstants.java index 92ba236..a883453 100644 --- a/storage-narpc/src/main/java/org/apache/crail/storage/tcp/TcpStorageConstants.java +++ b/storage-narpc/src/main/java/org/apache/crail/storage/tcp/TcpStorageConstants.java @@ -47,7 +47,7 @@ public class TcpStorageConstants { public static long STORAGE_TCP_ALLOCATION_SIZE = CrailConstants.REGION_SIZE; public static final String STORAGE_TCP_DATA_PATH_KEY = "crail.storage.tcp.datapath"; - public static String STORAGE_TCP_DATA_PATH = "/home/stu/craildata/data"; + public static String STORAGE_TCP_DATA_PATH = "/dev/hugepages/data"; public static final String STORAGE_TCP_QUEUE_DEPTH_KEY = "crail.storage.tcp.queuedepth"; public static int STORAGE_TCP_QUEUE_DEPTH = 16; http://git-wip-us.apache.org/repos/asf/incubator-crail/blob/84f608e5/storage-narpc/src/main/java/org/apache/crail/storage/tcp/TcpStorageServer.java ---------------------------------------------------------------------- diff --git a/storage-narpc/src/main/java/org/apache/crail/storage/tcp/TcpStorageServer.java b/storage-narpc/src/main/java/org/apache/crail/storage/tcp/TcpStorageServer.java index 6c8211c..4685607 100644 --- a/storage-narpc/src/main/java/org/apache/crail/storage/tcp/TcpStorageServer.java +++ b/storage-narpc/src/main/java/org/apache/crail/storage/tcp/TcpStorageServer.java @@ -167,7 +167,7 @@ public class TcpStorageServer implements Runnable, StorageServer, NaRPCService<T NetworkInterface netif = NetworkInterface.getByName(ifname); if (netif == null){ - return null; + throw new IOException("Cannot find network interface with name " + TcpStorageConstants.STORAGE_TCP_INTERFACE); } List<InterfaceAddress> addresses = netif.getInterfaceAddresses(); InetAddress addr = null; @@ -176,13 +176,19 @@ public class TcpStorageServer implements Runnable, StorageServer, NaRPCService<T InetAddress _addr = address.getAddress(); addr = _addr; } - } + } + + if (addr == null){ + throw new IOException("Cannot find valid network interface with name " + TcpStorageConstants.STORAGE_TCP_INTERFACE); + } InetSocketAddress inetAddr = new InetSocketAddress(addr, port); return inetAddr; } - public static String getDatanodeDirectory(InetSocketAddress inetAddress){ - String address = inetAddress.getAddress().toString(); - return TcpStorageConstants.STORAGE_TCP_DATA_PATH + address + "-" + inetAddress.getPort(); + public static String getDatanodeDirectory(InetSocketAddress address) throws IllegalArgumentException { + if (address == null) { + throw new IllegalArgumentException("Address paramater cannot be null!"); + } + return TcpStorageConstants.STORAGE_TCP_DATA_PATH + address.getAddress() + "-" + address.getPort(); } } http://git-wip-us.apache.org/repos/asf/incubator-crail/blob/84f608e5/storage-rdma/src/main/java/org/apache/crail/storage/rdma/RdmaConstants.java ---------------------------------------------------------------------- diff --git a/storage-rdma/src/main/java/org/apache/crail/storage/rdma/RdmaConstants.java b/storage-rdma/src/main/java/org/apache/crail/storage/rdma/RdmaConstants.java index 6a54509..af11096 100644 --- a/storage-rdma/src/main/java/org/apache/crail/storage/rdma/RdmaConstants.java +++ b/storage-rdma/src/main/java/org/apache/crail/storage/rdma/RdmaConstants.java @@ -49,7 +49,7 @@ public class RdmaConstants { public static long STORAGE_RDMA_ALLOCATION_SIZE = CrailConstants.REGION_SIZE; public static final String STORAGE_RDMA_DATA_PATH_KEY = "crail.storage.rdma.datapath"; - public static String STORAGE_RDMA_DATA_PATH = "/home/stu/craildata/data"; + public static String STORAGE_RDMA_DATA_PATH = "/dev/hugepages/data"; public static final String STORAGE_RDMA_LOCAL_MAP_KEY = "crail.storage.rdma.localmap"; public static boolean STORAGE_RDMA_LOCAL_MAP = true;
