Repository: incubator-crail Updated Branches: refs/heads/master d58abd828 -> 8f812810f
Applications can now choose on a per node basis (files, directories, tables, key/value pairs, etc.) whether the node should be enumerable or not. If a node is enumerable it will be included in an enumeration operation of the parent node. https://issues.apache.org/jira/browse/CRAIL-10 Close #6 Signed-off-by: Jonas Pfefferle <peppe...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/incubator-crail/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-crail/commit/8f812810 Tree: http://git-wip-us.apache.org/repos/asf/incubator-crail/tree/8f812810 Diff: http://git-wip-us.apache.org/repos/asf/incubator-crail/diff/8f812810 Branch: refs/heads/master Commit: 8f812810f3f23492abd2d46f888890fef5cddb5e Parents: d58abd8 Author: Patrick Stuedi <s...@zurich.ibm.com> Authored: Fri Feb 23 15:44:49 2018 +0100 Committer: Jonas Pfefferle <peppe...@apache.org> Committed: Mon Feb 26 13:55:54 2018 +0100 ---------------------------------------------------------------------- .../main/java/org/apache/crail/CrailStore.java | 2 +- .../org/apache/crail/core/CoreDataStore.java | 4 +- .../org/apache/crail/metadata/FileInfo.java | 13 +++++-- .../org/apache/crail/rpc/RpcConnection.java | 2 +- .../org/apache/crail/rpc/RpcDispatcher.java | 4 +- .../org/apache/crail/tools/CrailBenchmark.java | 36 ++++++++++------- .../java/org/apache/crail/tools/CrailFsck.java | 2 +- .../test/java/org/apache/crail/ClientTest.java | 12 +++--- .../java/org/apache/crail/hdfs/CrailHDFS.java | 6 +-- .../crail/hdfs/CrailHadoopFileSystem.java | 6 +-- .../org/apache/crail/namenode/AbstractNode.java | 4 +- .../apache/crail/namenode/DirectoryBlocks.java | 10 +++-- .../org/apache/crail/namenode/FileBlocks.java | 4 +- .../org/apache/crail/namenode/FileStore.java | 14 +++---- .../apache/crail/namenode/KeyValueBlocks.java | 4 +- .../apache/crail/namenode/MultiFileBlocks.java | 4 +- .../apache/crail/namenode/NameNodeService.java | 3 +- .../org/apache/crail/namenode/TableBlocks.java | 11 ++++-- .../rpc/darpc/DaRPCNameNodeConnection.java | 4 +- .../namenode/rpc/tcp/TcpRpcConnection.java | 4 +- .../org/apache/crail/rpc/RpcRequestMessage.java | 41 +++++++++++--------- 21 files changed, 108 insertions(+), 82 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-crail/blob/8f812810/client/src/main/java/org/apache/crail/CrailStore.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/crail/CrailStore.java b/client/src/main/java/org/apache/crail/CrailStore.java index 1d6c0bf..256c530 100644 --- a/client/src/main/java/org/apache/crail/CrailStore.java +++ b/client/src/main/java/org/apache/crail/CrailStore.java @@ -33,7 +33,7 @@ public abstract class CrailStore { private static AtomicLong referenceCounter = new AtomicLong(0); private static CrailStore instance = null; - public abstract Upcoming<CrailNode> create(String path, CrailNodeType type, CrailStorageClass storageClass, CrailLocationClass locationClass) throws Exception; + public abstract Upcoming<CrailNode> create(String path, CrailNodeType type, CrailStorageClass storageClass, CrailLocationClass locationClass, boolean enumerable) throws Exception; public abstract Upcoming<CrailNode> lookup(String path) throws Exception; public abstract Upcoming<CrailNode> rename(String srcPath, String dstPath) throws Exception; public abstract Upcoming<CrailNode> delete(String path, boolean recursive) throws Exception; http://git-wip-us.apache.org/repos/asf/incubator-crail/blob/8f812810/client/src/main/java/org/apache/crail/core/CoreDataStore.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/crail/core/CoreDataStore.java b/client/src/main/java/org/apache/crail/core/CoreDataStore.java index 4961af8..ce68633 100644 --- a/client/src/main/java/org/apache/crail/core/CoreDataStore.java +++ b/client/src/main/java/org/apache/crail/core/CoreDataStore.java @@ -162,14 +162,14 @@ public class CoreDataStore extends CrailStore { statistics.addProvider(datanodeEndpointCache); } - public Upcoming<CrailNode> create(String path, CrailNodeType type, CrailStorageClass storageClass, CrailLocationClass locationClass) throws Exception { + public Upcoming<CrailNode> create(String path, CrailNodeType type, CrailStorageClass storageClass, CrailLocationClass locationClass, boolean enumerable) throws Exception { FileName name = new FileName(path); if (CrailConstants.DEBUG){ LOG.info("createNode: name " + path + ", type " + type + ", storageAffinity " + storageClass + ", locationAffinity " + locationClass); } - RpcFuture<RpcCreateFile> fileRes = rpcConnection.createFile(name, type, storageClass.value(), locationClass.value()); + RpcFuture<RpcCreateFile> fileRes = rpcConnection.createFile(name, type, storageClass.value(), locationClass.value(), enumerable); return new CreateNodeFuture(this, path, type, fileRes); } http://git-wip-us.apache.org/repos/asf/incubator-crail/blob/8f812810/client/src/main/java/org/apache/crail/metadata/FileInfo.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/crail/metadata/FileInfo.java b/client/src/main/java/org/apache/crail/metadata/FileInfo.java index ca54811..ad0f06a 100644 --- a/client/src/main/java/org/apache/crail/metadata/FileInfo.java +++ b/client/src/main/java/org/apache/crail/metadata/FileInfo.java @@ -30,6 +30,9 @@ import org.apache.crail.conf.CrailConstants; public class FileInfo { public static final int CSIZE = 44; + public static final long ENUMERABLE = -1; + public static final long NOT_ENUMERABLE = -2; + private long fd; protected AtomicLong capacity; private CrailNodeType type; @@ -38,13 +41,13 @@ public class FileInfo { private long modificationTime; public FileInfo(){ - this(-1, CrailNodeType.DATAFILE); + this(-1, CrailNodeType.DATAFILE, true); } - protected FileInfo(long fd, CrailNodeType type){ + protected FileInfo(long fd, CrailNodeType type, boolean enumerable){ this.fd = fd; this.type = type; - this.dirOffset = -1; + this.dirOffset = enumerable ? ENUMERABLE : NOT_ENUMERABLE; this.capacity = new AtomicLong(0); this.token = 0; this.modificationTime = 0; @@ -156,4 +159,8 @@ public class FileInfo { public void setToken(long value) { this.token = value; } + + public boolean isEnumerable() { + return dirOffset > -2; + } } http://git-wip-us.apache.org/repos/asf/incubator-crail/blob/8f812810/client/src/main/java/org/apache/crail/rpc/RpcConnection.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/crail/rpc/RpcConnection.java b/client/src/main/java/org/apache/crail/rpc/RpcConnection.java index 268cab9..cf5d3b0 100644 --- a/client/src/main/java/org/apache/crail/rpc/RpcConnection.java +++ b/client/src/main/java/org/apache/crail/rpc/RpcConnection.java @@ -29,7 +29,7 @@ import org.apache.crail.metadata.FileName; public interface RpcConnection { public abstract RpcFuture<RpcCreateFile> createFile( - FileName filename, CrailNodeType type, int storageClass, int locationClass) throws IOException; + FileName filename, CrailNodeType type, int storageClass, int locationClass, boolean enumerable) throws IOException; public abstract RpcFuture<RpcGetFile> getFile(FileName filename, boolean writeable) throws IOException; http://git-wip-us.apache.org/repos/asf/incubator-crail/blob/8f812810/client/src/main/java/org/apache/crail/rpc/RpcDispatcher.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/crail/rpc/RpcDispatcher.java b/client/src/main/java/org/apache/crail/rpc/RpcDispatcher.java index 48a6fc3..ab9dda0 100644 --- a/client/src/main/java/org/apache/crail/rpc/RpcDispatcher.java +++ b/client/src/main/java/org/apache/crail/rpc/RpcDispatcher.java @@ -48,11 +48,11 @@ public class RpcDispatcher implements RpcConnection { @Override public RpcFuture<RpcCreateFile> createFile(FileName filename, - CrailNodeType type, int storageClass, int locationClass) + CrailNodeType type, int storageClass, int locationClass, boolean enumerable) throws IOException { int index = computeIndex(filename.getComponent(0)); // LOG.info("issuing create file for filename [" + filename.toString() + "], on index " + index); - return connections[index].createFile(filename, type, storageClass, locationClass); + return connections[index].createFile(filename, type, storageClass, locationClass, enumerable); } @Override http://git-wip-us.apache.org/repos/asf/incubator-crail/blob/8f812810/client/src/main/java/org/apache/crail/tools/CrailBenchmark.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/crail/tools/CrailBenchmark.java b/client/src/main/java/org/apache/crail/tools/CrailBenchmark.java index 7d173ff..1493497 100644 --- a/client/src/main/java/org/apache/crail/tools/CrailBenchmark.java +++ b/client/src/main/java/org/apache/crail/tools/CrailBenchmark.java @@ -78,7 +78,7 @@ public class CrailBenchmark { } } - void write(String filename, int size, int loop, int storageClass, int locationClass, boolean buffered) throws Exception { + void write(String filename, int size, int loop, int storageClass, int locationClass, boolean buffered, boolean skipDir) throws Exception { System.out.println("write, filename " + filename + ", size " + size + ", loop " + loop + ", storageClass " + storageClass + ", locationClass " + locationClass + ", buffered " + buffered); CrailBuffer buf = null; @@ -105,7 +105,7 @@ public class CrailBenchmark { long _capacity = _loop*_bufsize; double sumbytes = 0; double ops = 0; - CrailFile file = fs.create(filename, CrailNodeType.DATAFILE, CrailStorageClass.get(storageClass), CrailLocationClass.get(locationClass)).get().asFile(); + CrailFile file = fs.create(filename, CrailNodeType.DATAFILE, CrailStorageClass.get(storageClass), CrailLocationClass.get(locationClass), !skipDir).get().asFile(); CrailBufferedOutputStream bufferedStream = buffered ? file.getBufferedOutputStream(_capacity) : null; CrailOutputStream directStream = !buffered ? file.getDirectOutputStream(_capacity) : null; long start = System.currentTimeMillis(); @@ -141,7 +141,7 @@ public class CrailBenchmark { fs.getStatistics().print("close"); } - void writeAsync(String filename, int size, int loop, int batch, int storageClass, int locationClass) throws Exception { + void writeAsync(String filename, int size, int loop, int batch, int storageClass, int locationClass, boolean skipDir) throws Exception { System.out.println("writeAsync, filename " + filename + ", size " + size + ", loop " + loop + ", batch " + batch + ", storageClass " + storageClass + ", locationClass " + locationClass); ConcurrentLinkedQueue<CrailBuffer> bufferQueue = new ConcurrentLinkedQueue<CrailBuffer>(); @@ -172,7 +172,7 @@ public class CrailBenchmark { long _capacity = _loop*_bufsize; double sumbytes = 0; double ops = 0; - CrailFile file = fs.create(filename, CrailNodeType.DATAFILE, CrailStorageClass.get(storageClass), CrailLocationClass.get(locationClass)).get().asFile(); + CrailFile file = fs.create(filename, CrailNodeType.DATAFILE, CrailStorageClass.get(storageClass), CrailLocationClass.get(locationClass), !skipDir).get().asFile(); CrailOutputStream directStream = file.getDirectOutputStream(_capacity); long start = System.currentTimeMillis(); for (int i = 0; i < batch - 1 && ops < loop; i++){ @@ -552,7 +552,7 @@ public class CrailBenchmark { System.out.println("starting benchmark..."); fs.getStatistics().reset(); LinkedBlockingQueue<String> pathQueue = new LinkedBlockingQueue<String>(); - fs.create(filename, CrailNodeType.DIRECTORY, CrailStorageClass.DEFAULT, CrailLocationClass.DEFAULT).get().syncDir(); + fs.create(filename, CrailNodeType.DIRECTORY, CrailStorageClass.DEFAULT, CrailLocationClass.DEFAULT, true).get().syncDir(); int filecounter = 0; for (int i = 0; i < loop; i++){ String name = "" + filecounter++; @@ -564,7 +564,7 @@ public class CrailBenchmark { long start = System.currentTimeMillis(); while(!pathQueue.isEmpty()){ String path = pathQueue.poll(); - fs.create(path, CrailNodeType.DATAFILE, CrailStorageClass.DEFAULT, CrailLocationClass.DEFAULT).get().syncDir(); + fs.create(path, CrailNodeType.DATAFILE, CrailStorageClass.DEFAULT, CrailLocationClass.DEFAULT, true).get().syncDir(); } long end = System.currentTimeMillis(); double executionTime = ((double) (end - start)) / 1000.0; @@ -596,7 +596,7 @@ public class CrailBenchmark { LinkedBlockingQueue<Future<CrailNode>> futureQueue = new LinkedBlockingQueue<Future<CrailNode>>(); LinkedBlockingQueue<CrailFile> fileQueue = new LinkedBlockingQueue<CrailFile>(); LinkedBlockingQueue<String> pathQueue = new LinkedBlockingQueue<String>(); - fs.create(filename, CrailNodeType.DIRECTORY, CrailStorageClass.DEFAULT, CrailLocationClass.DEFAULT).get().syncDir(); + fs.create(filename, CrailNodeType.DIRECTORY, CrailStorageClass.DEFAULT, CrailLocationClass.DEFAULT, true).get().syncDir(); for (int i = 0; i < loop; i++){ String name = "/" + i; @@ -609,7 +609,7 @@ public class CrailBenchmark { //single operation == loop for (int j = 0; j < batch; j++) { String path = pathQueue.poll(); - Future<CrailNode> future = fs.create(path, CrailNodeType.DATAFILE, CrailStorageClass.DEFAULT, CrailLocationClass.DEFAULT); + Future<CrailNode> future = fs.create(path, CrailNodeType.DATAFILE, CrailStorageClass.DEFAULT, CrailLocationClass.DEFAULT, true); futureQueue.add(future); } for (int j = 0; j < batch; j++){ @@ -636,14 +636,14 @@ public class CrailBenchmark { void createMultiFile(String filename, int storageClass) throws Exception, InterruptedException { System.out.println("createMultiFile, filename " + filename); - fs.create(filename, CrailNodeType.MULTIFILE, CrailStorageClass.get(storageClass), CrailLocationClass.DEFAULT).get().syncDir(); + fs.create(filename, CrailNodeType.MULTIFILE, CrailStorageClass.get(storageClass), CrailLocationClass.DEFAULT, true).get().syncDir(); } void getKey(String filename, int size, int loop) throws Exception { System.out.println("getKey, path " + filename + ", size " + size + ", loop " + loop); CrailBuffer buf = fs.allocateBuffer().clear().limit(size).slice(); - CrailFile file = fs.create(filename, CrailNodeType.DATAFILE, CrailStorageClass.DEFAULT, CrailLocationClass.DEFAULT).get().asFile(); + CrailFile file = fs.create(filename, CrailNodeType.DATAFILE, CrailStorageClass.DEFAULT, CrailLocationClass.DEFAULT, true).get().asFile(); file.syncDir(); CrailOutputStream directOutputStream = file.getDirectOutputStream(0); directOutputStream.write(buf).get(); @@ -785,7 +785,7 @@ public class CrailBenchmark { void early(String filename) throws Exception { ByteBuffer buf = ByteBuffer.allocateDirect(32); - CrailFile file = fs.create(filename, CrailNodeType.DATAFILE, CrailStorageClass.DEFAULT, CrailLocationClass.DEFAULT).early().asFile(); + CrailFile file = fs.create(filename, CrailNodeType.DATAFILE, CrailStorageClass.DEFAULT, CrailLocationClass.DEFAULT, true).early().asFile(); CrailBufferedOutputStream stream = file.getBufferedOutputStream(0); System.out.println("buffered stream initialized"); @@ -811,7 +811,7 @@ public class CrailBenchmark { //benchmark System.out.println("starting benchmark..."); double ops = 0; - CrailFile file = fs.create(filename, CrailNodeType.DATAFILE, CrailStorageClass.DEFAULT, CrailLocationClass.DEFAULT).get().asFile(); + CrailFile file = fs.create(filename, CrailNodeType.DATAFILE, CrailStorageClass.DEFAULT, CrailLocationClass.DEFAULT, true).get().asFile(); CrailBufferedOutputStream outputStream = file.getBufferedOutputStream(loop*4); int intValue = 0; System.out.println("starting write at position " + outputStream.position()); @@ -968,7 +968,7 @@ public class CrailBenchmark { String warmupFilename = filename + random.nextInt(); System.out.println("warmUp, warmupFile " + warmupFilename + ", operations " + operations); if (operations > 0){ - CrailFile warmupFile = fs.create(warmupFilename, CrailNodeType.DATAFILE, CrailStorageClass.DEFAULT, CrailLocationClass.DEFAULT).get().asFile(); + CrailFile warmupFile = fs.create(warmupFilename, CrailNodeType.DATAFILE, CrailStorageClass.DEFAULT, CrailLocationClass.DEFAULT, true).get().asFile(); CrailBufferedOutputStream warmupStream = warmupFile.getBufferedOutputStream(0); for (int i = 0; i < operations; i++){ CrailBuffer buf = bufferList.poll(); @@ -994,6 +994,7 @@ public class CrailBenchmark { int storageClass = 0; int locationClass = 0; boolean useBuffered = true; + boolean skipDir = false; String benchmarkTypes = "write|writeAsync|readSequential|readRandom|readSequentialAsync|readMultiStream|" + "createFile|createFileAsync|createMultiFile|getKey|getFile|getFileAsync|enumerateDir|browseDir|" @@ -1008,6 +1009,7 @@ public class CrailBenchmark { Option warmupOption = Option.builder("w").desc("number of warmup operations [1..n]").hasArg().build(); Option experimentOption = Option.builder("e").desc("number of experiments [1..n]").hasArg().build(); Option openOption = Option.builder("o").desc("whether to keep the file system open [true|false]").hasArg().build(); + Option skipDirOption = Option.builder("d").desc("skip writing the directory record [true|false]").hasArg().build(); Option bufferedOption = Option.builder("m").desc("use buffer streams [true|false]").hasArg().build(); Options options = new Options(); @@ -1022,6 +1024,7 @@ public class CrailBenchmark { options.addOption(experimentOption); options.addOption(openOption); options.addOption(bufferedOption); + options.addOption(skipDirOption); CommandLineParser parser = new DefaultParser(); CommandLine line = parser.parse(options, Arrays.copyOfRange(args, 0, args.length)); @@ -1057,16 +1060,19 @@ public class CrailBenchmark { } if (line.hasOption(bufferedOption.getOpt())) { useBuffered = Boolean.parseBoolean(line.getOptionValue(bufferedOption.getOpt())); + } + if (line.hasOption(skipDirOption.getOpt())) { + skipDir = Boolean.parseBoolean(line.getOptionValue(skipDirOption.getOpt())); } CrailBenchmark benchmark = new CrailBenchmark(warmup); if (type.equals("write")){ benchmark.open(); - benchmark.write(filename, size, loop, storageClass, locationClass, useBuffered); + benchmark.write(filename, size, loop, storageClass, locationClass, useBuffered, skipDir); benchmark.close(); } else if (type.equalsIgnoreCase("writeAsync")) { benchmark.open(); - benchmark.writeAsync(filename, size, loop, batch, storageClass, locationClass); + benchmark.writeAsync(filename, size, loop, batch, storageClass, locationClass, skipDir); benchmark.close(); } else if (type.equalsIgnoreCase("readSequential")){ if (keepOpen) benchmark.open(); http://git-wip-us.apache.org/repos/asf/incubator-crail/blob/8f812810/client/src/main/java/org/apache/crail/tools/CrailFsck.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/crail/tools/CrailFsck.java b/client/src/main/java/org/apache/crail/tools/CrailFsck.java index 0e4bb74..b24fc47 100644 --- a/client/src/main/java/org/apache/crail/tools/CrailFsck.java +++ b/client/src/main/java/org/apache/crail/tools/CrailFsck.java @@ -133,7 +133,7 @@ public class CrailFsck { System.out.println("createDirectory, filename " + filename + ", storageClass " + storageClass + ", locationClass " + locationClass); CrailConfiguration conf = new CrailConfiguration(); CrailStore fs = CrailStore.newInstance(conf); - fs.create(filename, CrailNodeType.DIRECTORY, CrailStorageClass.get(storageClass), CrailLocationClass.get(locationClass)).get().syncDir(); + fs.create(filename, CrailNodeType.DIRECTORY, CrailStorageClass.get(storageClass), CrailLocationClass.get(locationClass), true).get().syncDir(); fs.close(); } http://git-wip-us.apache.org/repos/asf/incubator-crail/blob/8f812810/client/src/test/java/org/apache/crail/ClientTest.java ---------------------------------------------------------------------- diff --git a/client/src/test/java/org/apache/crail/ClientTest.java b/client/src/test/java/org/apache/crail/ClientTest.java index e490985..b4bbf17 100644 --- a/client/src/test/java/org/apache/crail/ClientTest.java +++ b/client/src/test/java/org/apache/crail/ClientTest.java @@ -49,7 +49,7 @@ public class ClientTest { public void init() throws Exception { CrailConfiguration conf = new CrailConfiguration(); fs = CrailStore.newInstance(conf); - fs.create(basePath, CrailNodeType.DIRECTORY, CrailStorageClass.DEFAULT, CrailLocationClass.DEFAULT).get(); + fs.create(basePath, CrailNodeType.DIRECTORY, CrailStorageClass.DEFAULT, CrailLocationClass.DEFAULT, true).get(); } @After @@ -60,14 +60,14 @@ public class ClientTest { @Test public void testCreateFile() throws Exception { String filename = basePath + "/fooCreate"; - fs.create(filename, CrailNodeType.DATAFILE, CrailStorageClass.DEFAULT, CrailLocationClass.DEFAULT).get(); + fs.create(filename, CrailNodeType.DATAFILE, CrailStorageClass.DEFAULT, CrailLocationClass.DEFAULT, true).get(); fs.lookup(filename).get().asFile(); } @Test public void testDeleteFile() throws Exception { String filename = basePath + "/fooDelete"; - fs.create(filename, CrailNodeType.DATAFILE, CrailStorageClass.DEFAULT, CrailLocationClass.DEFAULT).get(); + fs.create(filename, CrailNodeType.DATAFILE, CrailStorageClass.DEFAULT, CrailLocationClass.DEFAULT, true).get(); fs.delete(filename, false).get(); Assert.assertNull(fs.lookup(filename).get()); } @@ -75,7 +75,7 @@ public class ClientTest { @Test public void testRenameFile() throws Exception { String srcname = basePath + "/fooRename"; - fs.create(srcname, CrailNodeType.DATAFILE, CrailStorageClass.DEFAULT, CrailLocationClass.DEFAULT).get(); + fs.create(srcname, CrailNodeType.DATAFILE, CrailStorageClass.DEFAULT, CrailLocationClass.DEFAULT, true).get(); String dstname = basePath + "/barRename"; fs.rename(srcname, dstname).get(); Assert.assertNull(fs.lookup(srcname).get()); @@ -90,7 +90,7 @@ public class ClientTest { @Test public void testCreateDirectory() throws Exception { String filename = basePath + "/fooDir"; - fs.create(filename, CrailNodeType.DIRECTORY, CrailStorageClass.DEFAULT, CrailLocationClass.DEFAULT).get(); + fs.create(filename, CrailNodeType.DIRECTORY, CrailStorageClass.DEFAULT, CrailLocationClass.DEFAULT, true).get(); fs.lookup(filename).get().asDirectory(); } @@ -124,7 +124,7 @@ public class ClientTest { position + ", length = " + length + ", remoteOffset = " + remoteOffset); String filename = basePath + "/fooOutputStream" + length; - CrailFile file = fs.create(filename,CrailNodeType.DATAFILE, CrailStorageClass.DEFAULT, CrailLocationClass.DEFAULT).get().asFile(); + CrailFile file = fs.create(filename,CrailNodeType.DATAFILE, CrailStorageClass.DEFAULT, CrailLocationClass.DEFAULT, true).get().asFile(); CrailOutputStream outputStream = file.getDirectOutputStream(0); CrailInputStream inputStream = file.getDirectInputStream(0); http://git-wip-us.apache.org/repos/asf/incubator-crail/blob/8f812810/hdfs/src/main/java/org/apache/crail/hdfs/CrailHDFS.java ---------------------------------------------------------------------- diff --git a/hdfs/src/main/java/org/apache/crail/hdfs/CrailHDFS.java b/hdfs/src/main/java/org/apache/crail/hdfs/CrailHDFS.java index 17b38ee..84dd52d 100644 --- a/hdfs/src/main/java/org/apache/crail/hdfs/CrailHDFS.java +++ b/hdfs/src/main/java/org/apache/crail/hdfs/CrailHDFS.java @@ -96,7 +96,7 @@ public class CrailHDFS extends AbstractFileSystem { public FSDataOutputStream createInternal(Path path, EnumSet<CreateFlag> flag, FsPermission absolutePermission, int bufferSize, short replication, long blockSize, Progressable progress, ChecksumOpt checksumOpt, boolean createParent) throws AccessControlException, FileAlreadyExistsException, FileNotFoundException, ParentNotDirectoryException, UnsupportedFileSystemException, UnresolvedLinkException, IOException { CrailFile fileInfo = null; try { - fileInfo = dfs.create(path.toUri().getRawPath(), CrailNodeType.DATAFILE, CrailStorageClass.PARENT, CrailLocationClass.PARENT).get().asFile(); + fileInfo = dfs.create(path.toUri().getRawPath(), CrailNodeType.DATAFILE, CrailStorageClass.PARENT, CrailLocationClass.PARENT, true).get().asFile(); } catch(Exception e){ if (e.getMessage().contains(RpcErrors.messages[RpcErrors.ERR_PARENT_MISSING])){ fileInfo = null; @@ -109,7 +109,7 @@ public class CrailHDFS extends AbstractFileSystem { Path parent = path.getParent(); this.mkdir(parent, FsPermission.getDirDefault(), true); try { - fileInfo = dfs.create(path.toUri().getRawPath(), CrailNodeType.DATAFILE, CrailStorageClass.PARENT, CrailLocationClass.PARENT).get().asFile(); + fileInfo = dfs.create(path.toUri().getRawPath(), CrailNodeType.DATAFILE, CrailStorageClass.PARENT, CrailLocationClass.PARENT, true).get().asFile(); } catch(Exception e){ throw new IOException(e); } @@ -137,7 +137,7 @@ public class CrailHDFS extends AbstractFileSystem { @Override public void mkdir(Path path, FsPermission permission, boolean createParent) throws AccessControlException, FileAlreadyExistsException, FileNotFoundException, UnresolvedLinkException, IOException { try { - CrailDirectory file = dfs.create(path.toUri().getRawPath(), CrailNodeType.DIRECTORY, CrailStorageClass.PARENT, CrailLocationClass.DEFAULT).get().asDirectory(); + CrailDirectory file = dfs.create(path.toUri().getRawPath(), CrailNodeType.DIRECTORY, CrailStorageClass.PARENT, CrailLocationClass.DEFAULT, true).get().asDirectory(); file.syncDir(); } catch(Exception e){ if (e.getMessage().contains(RpcErrors.messages[RpcErrors.ERR_PARENT_MISSING])){ http://git-wip-us.apache.org/repos/asf/incubator-crail/blob/8f812810/hdfs/src/main/java/org/apache/crail/hdfs/CrailHadoopFileSystem.java ---------------------------------------------------------------------- diff --git a/hdfs/src/main/java/org/apache/crail/hdfs/CrailHadoopFileSystem.java b/hdfs/src/main/java/org/apache/crail/hdfs/CrailHadoopFileSystem.java index e98310a..efcbfa5 100644 --- a/hdfs/src/main/java/org/apache/crail/hdfs/CrailHadoopFileSystem.java +++ b/hdfs/src/main/java/org/apache/crail/hdfs/CrailHadoopFileSystem.java @@ -104,7 +104,7 @@ public class CrailHadoopFileSystem extends FileSystem { long blockSize, Progressable progress) throws IOException { CrailFile fileInfo = null; try { - fileInfo = dfs.create(path.toUri().getRawPath(), CrailNodeType.DATAFILE, CrailStorageClass.PARENT, CrailLocationClass.PARENT).get().asFile(); + fileInfo = dfs.create(path.toUri().getRawPath(), CrailNodeType.DATAFILE, CrailStorageClass.PARENT, CrailLocationClass.PARENT, true).get().asFile(); } catch (Exception e) { if (e.getMessage().contains(RpcErrors.messages[RpcErrors.ERR_PARENT_MISSING])) { fileInfo = null; @@ -117,7 +117,7 @@ public class CrailHadoopFileSystem extends FileSystem { Path parent = path.getParent(); this.mkdirs(parent, FsPermission.getDirDefault()); try { - fileInfo = dfs.create(path.toUri().getRawPath(), CrailNodeType.DATAFILE, CrailStorageClass.PARENT, CrailLocationClass.PARENT).get().asFile(); + fileInfo = dfs.create(path.toUri().getRawPath(), CrailNodeType.DATAFILE, CrailStorageClass.PARENT, CrailLocationClass.PARENT, true).get().asFile(); } catch (Exception e) { throw new IOException(e); } @@ -210,7 +210,7 @@ public class CrailHadoopFileSystem extends FileSystem { @Override public boolean mkdirs(Path path, FsPermission permission) throws IOException { try { - CrailDirectory file = dfs.create(path.toUri().getRawPath(), CrailNodeType.DIRECTORY, CrailStorageClass.PARENT, CrailLocationClass.DEFAULT).get().asDirectory(); + CrailDirectory file = dfs.create(path.toUri().getRawPath(), CrailNodeType.DIRECTORY, CrailStorageClass.PARENT, CrailLocationClass.DEFAULT, true).get().asDirectory(); file.syncDir(); return true; } catch(Exception e){ http://git-wip-us.apache.org/repos/asf/incubator-crail/blob/8f812810/namenode/src/main/java/org/apache/crail/namenode/AbstractNode.java ---------------------------------------------------------------------- diff --git a/namenode/src/main/java/org/apache/crail/namenode/AbstractNode.java b/namenode/src/main/java/org/apache/crail/namenode/AbstractNode.java index 7ed6ff1..f8488a5 100644 --- a/namenode/src/main/java/org/apache/crail/namenode/AbstractNode.java +++ b/namenode/src/main/java/org/apache/crail/namenode/AbstractNode.java @@ -51,8 +51,8 @@ public abstract class AbstractNode extends FileInfo implements Delayed { //clear all the blocks (used by GC) public abstract void freeBlocks(BlockStore blockStore) throws Exception; - public AbstractNode(long fd, int fileComponent, CrailNodeType type, int storageClass, int locationAffinity){ - super(fd, type); + public AbstractNode(long fd, int fileComponent, CrailNodeType type, int storageClass, int locationAffinity, boolean enumerable){ + super(fd, type, enumerable); this.fileComponent = fileComponent; this.storageClass = storageClass; http://git-wip-us.apache.org/repos/asf/incubator-crail/blob/8f812810/namenode/src/main/java/org/apache/crail/namenode/DirectoryBlocks.java ---------------------------------------------------------------------- diff --git a/namenode/src/main/java/org/apache/crail/namenode/DirectoryBlocks.java b/namenode/src/main/java/org/apache/crail/namenode/DirectoryBlocks.java index 3b8a430..f6f84d8 100644 --- a/namenode/src/main/java/org/apache/crail/namenode/DirectoryBlocks.java +++ b/namenode/src/main/java/org/apache/crail/namenode/DirectoryBlocks.java @@ -29,12 +29,12 @@ import org.apache.crail.conf.CrailConstants; import org.apache.crail.metadata.BlockInfo; public class DirectoryBlocks extends AbstractNode { - private AtomicLong dirOffsetCounter; + protected AtomicLong dirOffsetCounter; protected ConcurrentHashMap<Integer, AbstractNode> children; private ConcurrentHashMap<Integer, NameNodeBlockInfo> blocks; - DirectoryBlocks(long fd, int fileComponent, CrailNodeType type, int storageClass, int locationClass) { - super(fd, fileComponent, type, storageClass, locationClass); + DirectoryBlocks(long fd, int fileComponent, CrailNodeType type, int storageClass, int locationClass, boolean enumerable) { + super(fd, fileComponent, type, storageClass, locationClass, enumerable); this.children = new ConcurrentHashMap<Integer, AbstractNode>(); this.dirOffsetCounter = new AtomicLong(0); this.blocks = new ConcurrentHashMap<Integer, NameNodeBlockInfo>(); @@ -45,7 +45,9 @@ public class DirectoryBlocks extends AbstractNode { if (old != null){ throw new Exception("File exists"); } - child.setDirOffset(dirOffsetCounter.getAndAdd(CrailConstants.DIRECTORY_RECORD)); + if (child.isEnumerable()) { + child.setDirOffset(dirOffsetCounter.getAndAdd(CrailConstants.DIRECTORY_RECORD)); + } return old; } http://git-wip-us.apache.org/repos/asf/incubator-crail/blob/8f812810/namenode/src/main/java/org/apache/crail/namenode/FileBlocks.java ---------------------------------------------------------------------- diff --git a/namenode/src/main/java/org/apache/crail/namenode/FileBlocks.java b/namenode/src/main/java/org/apache/crail/namenode/FileBlocks.java index 76a19ee..f046fa5 100644 --- a/namenode/src/main/java/org/apache/crail/namenode/FileBlocks.java +++ b/namenode/src/main/java/org/apache/crail/namenode/FileBlocks.java @@ -34,8 +34,8 @@ public class FileBlocks extends AbstractNode { private final Lock readLock; private final Lock writeLock; - public FileBlocks(long fd, int fileComponent, CrailNodeType type, int storageClass, int locationClass) { - super(fd, fileComponent, type, storageClass, locationClass); + public FileBlocks(long fd, int fileComponent, CrailNodeType type, int storageClass, int locationClass, boolean enumerable) { + super(fd, fileComponent, type, storageClass, locationClass, enumerable); this.blocks = new ArrayList<NameNodeBlockInfo>(CrailConstants.NAMENODE_FILEBLOCKS); this.lock = new ReentrantReadWriteLock(); this.readLock = lock.readLock(); http://git-wip-us.apache.org/repos/asf/incubator-crail/blob/8f812810/namenode/src/main/java/org/apache/crail/namenode/FileStore.java ---------------------------------------------------------------------- diff --git a/namenode/src/main/java/org/apache/crail/namenode/FileStore.java b/namenode/src/main/java/org/apache/crail/namenode/FileStore.java index 672a550..66c4953 100644 --- a/namenode/src/main/java/org/apache/crail/namenode/FileStore.java +++ b/namenode/src/main/java/org/apache/crail/namenode/FileStore.java @@ -33,20 +33,20 @@ public class FileStore { public FileStore(Sequencer sequencer) throws IOException { this.sequencer = sequencer; - this.root = createNode(new FileName("/").getFileComponent(), CrailNodeType.DIRECTORY, CrailConstants.STORAGE_ROOTCLASS, 0); + this.root = createNode(new FileName("/").getFileComponent(), CrailNodeType.DIRECTORY, CrailConstants.STORAGE_ROOTCLASS, 0, false); } - public AbstractNode createNode(int fileComponent, CrailNodeType type, int storageClass, int locationClass) throws IOException { + public AbstractNode createNode(int fileComponent, CrailNodeType type, int storageClass, int locationClass, boolean enumerable) throws IOException { if (type == CrailNodeType.DIRECTORY){ - return new DirectoryBlocks(sequencer.getNextId(), fileComponent, type, storageClass, locationClass); + return new DirectoryBlocks(sequencer.getNextId(), fileComponent, type, storageClass, locationClass, enumerable); } else if (type == CrailNodeType.MULTIFILE){ - return new MultiFileBlocks(sequencer.getNextId(), fileComponent, type, storageClass, locationClass); + return new MultiFileBlocks(sequencer.getNextId(), fileComponent, type, storageClass, locationClass, enumerable); } else if (type == CrailNodeType.TABLE){ - return new TableBlocks(sequencer.getNextId(), fileComponent, type, storageClass, locationClass); + return new TableBlocks(sequencer.getNextId(), fileComponent, type, storageClass, locationClass, enumerable); } else if (type == CrailNodeType.KEYVALUE){ - return new KeyValueBlocks(sequencer.getNextId(), fileComponent, type, storageClass, locationClass); + return new KeyValueBlocks(sequencer.getNextId(), fileComponent, type, storageClass, locationClass, enumerable); } else if (type == CrailNodeType.DATAFILE){ - return new FileBlocks(sequencer.getNextId(), fileComponent, type, storageClass, locationClass); + return new FileBlocks(sequencer.getNextId(), fileComponent, type, storageClass, locationClass, enumerable); } else { throw new IOException("File type unkown: " + type); } http://git-wip-us.apache.org/repos/asf/incubator-crail/blob/8f812810/namenode/src/main/java/org/apache/crail/namenode/KeyValueBlocks.java ---------------------------------------------------------------------- diff --git a/namenode/src/main/java/org/apache/crail/namenode/KeyValueBlocks.java b/namenode/src/main/java/org/apache/crail/namenode/KeyValueBlocks.java index 2ad79bf..2967079 100644 --- a/namenode/src/main/java/org/apache/crail/namenode/KeyValueBlocks.java +++ b/namenode/src/main/java/org/apache/crail/namenode/KeyValueBlocks.java @@ -23,8 +23,8 @@ import org.apache.crail.CrailNodeType; public class KeyValueBlocks extends FileBlocks { public KeyValueBlocks(long fd, int fileComponent, CrailNodeType type, - int storageClass, int locationClass) { - super(fd, fileComponent, type, storageClass, locationClass); + int storageClass, int locationClass, boolean enumerable) { + super(fd, fileComponent, type, storageClass, locationClass, enumerable); } } http://git-wip-us.apache.org/repos/asf/incubator-crail/blob/8f812810/namenode/src/main/java/org/apache/crail/namenode/MultiFileBlocks.java ---------------------------------------------------------------------- diff --git a/namenode/src/main/java/org/apache/crail/namenode/MultiFileBlocks.java b/namenode/src/main/java/org/apache/crail/namenode/MultiFileBlocks.java index 153a25e..b118c34 100644 --- a/namenode/src/main/java/org/apache/crail/namenode/MultiFileBlocks.java +++ b/namenode/src/main/java/org/apache/crail/namenode/MultiFileBlocks.java @@ -24,8 +24,8 @@ import org.apache.crail.CrailNodeType; public class MultiFileBlocks extends DirectoryBlocks { MultiFileBlocks(long fd, int fileComponent, CrailNodeType type, - int storageClass, int locationClass) { - super(fd, fileComponent, type, storageClass, locationClass); + int storageClass, int locationClass, boolean enumerable) { + super(fd, fileComponent, type, storageClass, locationClass, enumerable); } } http://git-wip-us.apache.org/repos/asf/incubator-crail/blob/8f812810/namenode/src/main/java/org/apache/crail/namenode/NameNodeService.java ---------------------------------------------------------------------- diff --git a/namenode/src/main/java/org/apache/crail/namenode/NameNodeService.java b/namenode/src/main/java/org/apache/crail/namenode/NameNodeService.java index 195a060..0157bdd 100644 --- a/namenode/src/main/java/org/apache/crail/namenode/NameNodeService.java +++ b/namenode/src/main/java/org/apache/crail/namenode/NameNodeService.java @@ -90,6 +90,7 @@ public class NameNodeService implements RpcNameNodeService, Sequencer { boolean writeable = type.isDirectory() ? false : true; int storageClass = request.getStorageClass(); int locationClass = request.getLocationClass(); + boolean enumerable = request.isEnumerable(); //check params if (type.isContainer() && locationClass > 0){ @@ -115,7 +116,7 @@ public class NameNodeService implements RpcNameNodeService, Sequencer { locationClass = parentInfo.getLocationClass(); } - AbstractNode fileInfo = fileTree.createNode(fileHash.getFileComponent(), type, storageClass, locationClass); + AbstractNode fileInfo = fileTree.createNode(fileHash.getFileComponent(), type, storageClass, locationClass, enumerable); try { AbstractNode oldNode = parentInfo.putChild(fileInfo); if (oldNode != null && oldNode.getFd() != fileInfo.getFd()){ http://git-wip-us.apache.org/repos/asf/incubator-crail/blob/8f812810/namenode/src/main/java/org/apache/crail/namenode/TableBlocks.java ---------------------------------------------------------------------- diff --git a/namenode/src/main/java/org/apache/crail/namenode/TableBlocks.java b/namenode/src/main/java/org/apache/crail/namenode/TableBlocks.java index d8e73ce..6bb831d 100644 --- a/namenode/src/main/java/org/apache/crail/namenode/TableBlocks.java +++ b/namenode/src/main/java/org/apache/crail/namenode/TableBlocks.java @@ -20,12 +20,13 @@ package org.apache.crail.namenode; import org.apache.crail.CrailNodeType; +import org.apache.crail.conf.CrailConstants; public class TableBlocks extends DirectoryBlocks { TableBlocks(long fd, int fileComponent, CrailNodeType type, - int storageClass, int locationClass) { - super(fd, fileComponent, type, storageClass, locationClass); + int storageClass, int locationClass, boolean enumerable) { + super(fd, fileComponent, type, storageClass, locationClass, enumerable); } @Override @@ -34,6 +35,10 @@ public class TableBlocks extends DirectoryBlocks { throw new Exception("Attempt to create key/value pair in container other than a table"); } - return children.put(child.getComponent(), child); + AbstractNode oldNode = children.put(child.getComponent(), child); + if (child.isEnumerable()) { + child.setDirOffset(dirOffsetCounter.getAndAdd(CrailConstants.DIRECTORY_RECORD)); + } + return oldNode; } } http://git-wip-us.apache.org/repos/asf/incubator-crail/blob/8f812810/rpc-darpc/src/main/java/org/apache/crail/namenode/rpc/darpc/DaRPCNameNodeConnection.java ---------------------------------------------------------------------- diff --git a/rpc-darpc/src/main/java/org/apache/crail/namenode/rpc/darpc/DaRPCNameNodeConnection.java b/rpc-darpc/src/main/java/org/apache/crail/namenode/rpc/darpc/DaRPCNameNodeConnection.java index df09210..d5f8ece 100644 --- a/rpc-darpc/src/main/java/org/apache/crail/namenode/rpc/darpc/DaRPCNameNodeConnection.java +++ b/rpc-darpc/src/main/java/org/apache/crail/namenode/rpc/darpc/DaRPCNameNodeConnection.java @@ -60,12 +60,12 @@ public class DaRPCNameNodeConnection implements RpcConnection { } @Override - public RpcFuture<RpcCreateFile> createFile(FileName filename, CrailNodeType type, int storageClass, int locationClass) throws IOException { + public RpcFuture<RpcCreateFile> createFile(FileName filename, CrailNodeType type, int storageClass, int locationClass, boolean enumerable) throws IOException { if (CrailConstants.DEBUG){ LOG.debug("RPC: createFile, fileType " + type + ", storageClass " + storageClass + ", locationClass " + locationClass); } - RpcRequestMessage.CreateFileReq createFileReq = new RpcRequestMessage.CreateFileReq(filename, type, storageClass, locationClass); + RpcRequestMessage.CreateFileReq createFileReq = new RpcRequestMessage.CreateFileReq(filename, type, storageClass, locationClass, enumerable); DaRPCNameNodeRequest request = new DaRPCNameNodeRequest(createFileReq); request.setCommand(RpcProtocol.CMD_CREATE_FILE); http://git-wip-us.apache.org/repos/asf/incubator-crail/blob/8f812810/rpc-narpc/src/main/java/org/apache/crail/namenode/rpc/tcp/TcpRpcConnection.java ---------------------------------------------------------------------- diff --git a/rpc-narpc/src/main/java/org/apache/crail/namenode/rpc/tcp/TcpRpcConnection.java b/rpc-narpc/src/main/java/org/apache/crail/namenode/rpc/tcp/TcpRpcConnection.java index 5cfe6a2..c955301 100644 --- a/rpc-narpc/src/main/java/org/apache/crail/namenode/rpc/tcp/TcpRpcConnection.java +++ b/rpc-narpc/src/main/java/org/apache/crail/namenode/rpc/tcp/TcpRpcConnection.java @@ -56,9 +56,9 @@ public class TcpRpcConnection implements RpcConnection { } public RpcFuture<RpcCreateFile> createFile(FileName fileName, - CrailNodeType type, int storageAffinity, int locationAffinity) + CrailNodeType type, int storageAffinity, int locationAffinity, boolean enumerable) throws IOException { - RpcRequestMessage.CreateFileReq req = new RpcRequestMessage.CreateFileReq(fileName, type, storageAffinity, locationAffinity); + RpcRequestMessage.CreateFileReq req = new RpcRequestMessage.CreateFileReq(fileName, type, storageAffinity, locationAffinity, enumerable); RpcResponseMessage.CreateFileRes resp = new RpcResponseMessage.CreateFileRes(); TcpNameNodeRequest request = new TcpNameNodeRequest(req); http://git-wip-us.apache.org/repos/asf/incubator-crail/blob/8f812810/rpc/src/main/java/org/apache/crail/rpc/RpcRequestMessage.java ---------------------------------------------------------------------- diff --git a/rpc/src/main/java/org/apache/crail/rpc/RpcRequestMessage.java b/rpc/src/main/java/org/apache/crail/rpc/RpcRequestMessage.java index 52830d9..3e05d45 100644 --- a/rpc/src/main/java/org/apache/crail/rpc/RpcRequestMessage.java +++ b/rpc/src/main/java/org/apache/crail/rpc/RpcRequestMessage.java @@ -30,25 +30,28 @@ import org.apache.crail.metadata.FileName; public class RpcRequestMessage { public static class CreateFileReq implements RpcProtocol.NameNodeRpcMessage { - public static int CSIZE = FileName.CSIZE + 12; + public static int CSIZE = FileName.CSIZE + 16; protected FileName filename; protected CrailNodeType type; protected int storageClass; protected int locationClass; + protected boolean enumerable; public CreateFileReq(){ this.filename = new FileName(); this.type = CrailNodeType.DATAFILE; this.storageClass = 0; this.locationClass = 0; + this.enumerable = true; } - public CreateFileReq(FileName filename, CrailNodeType type, int storageClass, int locationClass) { + public CreateFileReq(FileName filename, CrailNodeType type, int storageClass, int locationClass, boolean enumerable) { this.filename = filename; this.type = type; this.storageClass = storageClass; this.locationClass = locationClass; + this.enumerable = enumerable; } public FileName getFileName() { @@ -67,7 +70,10 @@ public class RpcRequestMessage { return locationClass; } - + public boolean isEnumerable() { + return enumerable; + } + public int size() { return CSIZE; } @@ -77,28 +83,30 @@ public class RpcRequestMessage { } public int write(ByteBuffer buffer) { - int written = filename.write(buffer); + filename.write(buffer); buffer.putInt(type.getLabel()); buffer.putInt(storageClass); buffer.putInt(locationClass); - written += 12; + buffer.putInt(enumerable ? 1 : 0); - return written; + return CSIZE; } public void update(ByteBuffer buffer) { filename.update(buffer); - int tmp = buffer.getInt(); - type = CrailNodeType.parse(tmp); + int _type = buffer.getInt(); + type = CrailNodeType.parse(_type); storageClass = buffer.getInt(); locationClass = buffer.getInt(); + int _enumerable = buffer.getInt(); + enumerable = (_enumerable == 1) ? true : false; } @Override public String toString() { return "CreateFileReq [filename=" + filename + ", type=" + type + ", storageClass=" + storageClass + ", locationClass=" - + locationClass + "]"; + + locationClass + ", enumerable=" + enumerable + "]"; } } @@ -136,10 +144,9 @@ public class RpcRequestMessage { } public int write(ByteBuffer buffer) { - int written = filename.write(buffer); + filename.write(buffer); buffer.putInt(writeable ? 1 : 0); - written += 4; - return written; + return CSIZE; } public void update(ByteBuffer buffer) { @@ -182,10 +189,9 @@ public class RpcRequestMessage { } public int write(ByteBuffer buffer) { - int written = fileInfo.write(buffer, true); + fileInfo.write(buffer, true); buffer.putInt(close ? 1 : 0); - written += 4; - return written; + return CSIZE; } public void update(ByteBuffer buffer) { @@ -239,10 +245,9 @@ public class RpcRequestMessage { } public int write(ByteBuffer buffer) { - int written = filename.write(buffer); + filename.write(buffer); buffer.putInt(recursive ? 1 : 0); - written += 4; - return written; + return CSIZE; } public void update(ByteBuffer buffer) {
