paul-rogers commented on a change in pull request #1500: DRILL-6820: Msgpack
format reader
URL: https://github.com/apache/drill/pull/1500#discussion_r230999435
##########
File path:
contrib/format-msgpack/src/main/java/org/apache/drill/exec/store/msgpack/MsgpackSchema.java
##########
@@ -0,0 +1,114 @@
+package org.apache.drill.exec.store.msgpack;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.drill.common.exceptions.DrillRuntimeException;
+import org.apache.drill.common.types.TypeProtos.MinorType;
+import org.apache.drill.exec.exception.SchemaChangeRuntimeException;
+import org.apache.drill.exec.proto.UserBitShared.SerializedField;
+import org.apache.drill.exec.proto.UserBitShared.SerializedField.Builder;
+import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.store.dfs.DrillFileSystem;
+import org.apache.drill.shaded.guava.com.google.common.base.Preconditions;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.security.AccessControlException;
+
+import com.google.protobuf.TextFormat;
+import com.google.protobuf.TextFormat.ParseException;
+
+public class MsgpackSchema {
+ public static final String SCHEMA_FILE_NAME = ".schema.proto";
+
+ @SuppressWarnings("unused")
+ private static final org.slf4j.Logger logger =
org.slf4j.LoggerFactory.getLogger(MsgpackSchema.class);
+
+ private DrillFileSystem fileSystem;
+
+ public MsgpackSchema(DrillFileSystem fileSystem) {
+ this.fileSystem = fileSystem;
+ }
+
+ public MaterializedField load(Path schemaLocation) throws
AccessControlException, FileNotFoundException, IOException {
+ MaterializedField previousMapField = null;
+ if (schemaLocation != null && fileSystem.exists(schemaLocation)) {
+ try (FSDataInputStream in = fileSystem.open(schemaLocation)) {
+ String schemaData = IOUtils.toString(in);
+ Builder newBuilder = SerializedField.newBuilder();
+ try {
+ TextFormat.merge(schemaData, newBuilder);
+ } catch (ParseException e) {
+ throw new DrillRuntimeException("Failed to read schema file: " +
schemaLocation, e);
+ }
+ SerializedField read = newBuilder.build();
+ previousMapField = MaterializedField.create(read);
+ }
+ }
+ return previousMapField;
+ }
+
+ public void save(MaterializedField mapField, Path schemaLocation) throws
IOException {
Review comment:
Will this work in a highly concurrent, distributed system? If we have, say,
10 scan operators running concurrently on each of 100 nodes? Or, does this work
only for the single machine, single thread case?
In a multi-thread case, how will we merge the, say 1000 files produced? This
is not a trivial question: it is one that Drill has wrested with and failed to
solve for many years. I suspect that there is no good, general solution. But,
maybe there are special cases in MsgPack that can be exploited?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
