This is an automated email from the ASF dual-hosted git repository. dlmarion pushed a commit to branch 1451-external-compactions-feature in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/1451-external-compactions-feature by this push: new 6c37802 More work on compactor and coordinator 6c37802 is described below commit 6c378021b12906ccb33675d80d4ad8686df9d06b Author: Dave Marion <dlmar...@apache.org> AuthorDate: Wed Mar 3 16:33:45 2021 +0000 More work on compactor and coordinator --- .../compaction/thrift/CompactionCoordinator.java | 68 +- .../core/tabletserver/thrift/CompactionJob.java | 258 +++++++- .../core/tabletserver/thrift/CompactionKind.java | 66 ++ .../core/tabletserver/thrift/CompactionStats.java | 582 +++++++++++++++++ .../core/tabletserver/thrift/InputFile.java | 687 +++++++++++++++++++++ core/src/main/thrift/compaction-coordinator.thrift | 33 +- core/src/main/thrift/data.thrift | 6 - core/src/main/thrift/tabletserver.thrift | 24 +- .../server/compaction/ExternalCompactionUtil.java} | 17 +- .../server/compaction}/RetryableThriftCall.java | 3 +- .../compaction}/RetryableThriftFunction.java | 2 +- .../coordinator/CompactionCoordinator.java | 82 ++- .../org/apache/accumulo/compactor/Compactor.java | 271 ++++---- 13 files changed, 1846 insertions(+), 253 deletions(-) diff --git a/core/src/main/java/org/apache/accumulo/core/compaction/thrift/CompactionCoordinator.java b/core/src/main/java/org/apache/accumulo/core/compaction/thrift/CompactionCoordinator.java index ffd00f9..9397c22 100644 --- a/core/src/main/java/org/apache/accumulo/core/compaction/thrift/CompactionCoordinator.java +++ b/core/src/main/java/org/apache/accumulo/core/compaction/thrift/CompactionCoordinator.java @@ -35,7 +35,7 @@ public class CompactionCoordinator { public org.apache.accumulo.core.tabletserver.thrift.CompactionJob getCompactionJob(java.lang.String queueName, java.lang.String compactor) throws org.apache.thrift.TException; - public void compactionCompleted(org.apache.accumulo.core.tabletserver.thrift.CompactionJob job, org.apache.accumulo.core.dataImpl.thrift.CompactionStats stats) throws org.apache.thrift.TException; + public void compactionCompleted(org.apache.accumulo.core.tabletserver.thrift.CompactionJob job, org.apache.accumulo.core.tabletserver.thrift.CompactionStats stats) throws org.apache.thrift.TException; public void updateCompactionStatus(org.apache.accumulo.core.tabletserver.thrift.CompactionJob compaction, CompactionState state, java.lang.String message, long timestamp) throws org.apache.thrift.TException; @@ -49,7 +49,7 @@ public class CompactionCoordinator { public void getCompactionJob(java.lang.String queueName, java.lang.String compactor, org.apache.thrift.async.AsyncMethodCallback<org.apache.accumulo.core.tabletserver.thrift.CompactionJob> resultHandler) throws org.apache.thrift.TException; - public void compactionCompleted(org.apache.accumulo.core.tabletserver.thrift.CompactionJob job, org.apache.accumulo.core.dataImpl.thrift.CompactionStats stats, org.apache.thrift.async.AsyncMethodCallback<Void> resultHandler) throws org.apache.thrift.TException; + public void compactionCompleted(org.apache.accumulo.core.tabletserver.thrift.CompactionJob job, org.apache.accumulo.core.tabletserver.thrift.CompactionStats stats, org.apache.thrift.async.AsyncMethodCallback<Void> resultHandler) throws org.apache.thrift.TException; public void updateCompactionStatus(org.apache.accumulo.core.tabletserver.thrift.CompactionJob compaction, CompactionState state, java.lang.String message, long timestamp, org.apache.thrift.async.AsyncMethodCallback<Void> resultHandler) throws org.apache.thrift.TException; @@ -146,13 +146,13 @@ public class CompactionCoordinator { throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "getCompactionJob failed: unknown result"); } - public void compactionCompleted(org.apache.accumulo.core.tabletserver.thrift.CompactionJob job, org.apache.accumulo.core.dataImpl.thrift.CompactionStats stats) throws org.apache.thrift.TException + public void compactionCompleted(org.apache.accumulo.core.tabletserver.thrift.CompactionJob job, org.apache.accumulo.core.tabletserver.thrift.CompactionStats stats) throws org.apache.thrift.TException { send_compactionCompleted(job, stats); recv_compactionCompleted(); } - public void send_compactionCompleted(org.apache.accumulo.core.tabletserver.thrift.CompactionJob job, org.apache.accumulo.core.dataImpl.thrift.CompactionStats stats) throws org.apache.thrift.TException + public void send_compactionCompleted(org.apache.accumulo.core.tabletserver.thrift.CompactionJob job, org.apache.accumulo.core.tabletserver.thrift.CompactionStats stats) throws org.apache.thrift.TException { compactionCompleted_args args = new compactionCompleted_args(); args.setJob(job); @@ -319,7 +319,7 @@ public class CompactionCoordinator { } } - public void compactionCompleted(org.apache.accumulo.core.tabletserver.thrift.CompactionJob job, org.apache.accumulo.core.dataImpl.thrift.CompactionStats stats, org.apache.thrift.async.AsyncMethodCallback<Void> resultHandler) throws org.apache.thrift.TException { + public void compactionCompleted(org.apache.accumulo.core.tabletserver.thrift.CompactionJob job, org.apache.accumulo.core.tabletserver.thrift.CompactionStats stats, org.apache.thrift.async.AsyncMethodCallback<Void> resultHandler) throws org.apache.thrift.TException { checkReady(); compactionCompleted_call method_call = new compactionCompleted_call(job, stats, resultHandler, this, ___protocolFactory, ___transport); this.___currentMethod = method_call; @@ -328,8 +328,8 @@ public class CompactionCoordinator { public static class compactionCompleted_call extends org.apache.thrift.async.TAsyncMethodCall<Void> { private org.apache.accumulo.core.tabletserver.thrift.CompactionJob job; - private org.apache.accumulo.core.dataImpl.thrift.CompactionStats stats; - public compactionCompleted_call(org.apache.accumulo.core.tabletserver.thrift.CompactionJob job, org.apache.accumulo.core.dataImpl.thrift.CompactionStats stats, org.apache.thrift.async.AsyncMethodCallback<Void> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + private org.apache.accumulo.core.tabletserver.thrift.CompactionStats stats; + public compactionCompleted_call(org.apache.accumulo.core.tabletserver.thrift.CompactionJob job, org.apache.accumulo.core.tabletserver.thrift.CompactionStats stats, org.apache.thrift.async.AsyncMethodCallback<Void> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { super(client, protocolFactory, transport, resultHandler, false); this.job = job; this.stats = stats; @@ -2594,14 +2594,14 @@ public class CompactionCoordinator { case 0: // SUCCESS if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { { - org.apache.thrift.protocol.TList _list8 = iprot.readListBegin(); - struct.success = new java.util.ArrayList<Status>(_list8.size); - @org.apache.thrift.annotation.Nullable Status _elem9; - for (int _i10 = 0; _i10 < _list8.size; ++_i10) + org.apache.thrift.protocol.TList _list0 = iprot.readListBegin(); + struct.success = new java.util.ArrayList<Status>(_list0.size); + @org.apache.thrift.annotation.Nullable Status _elem1; + for (int _i2 = 0; _i2 < _list0.size; ++_i2) { - _elem9 = new Status(); - _elem9.read(iprot); - struct.success.add(_elem9); + _elem1 = new Status(); + _elem1.read(iprot); + struct.success.add(_elem1); } iprot.readListEnd(); } @@ -2629,9 +2629,9 @@ public class CompactionCoordinator { oprot.writeFieldBegin(SUCCESS_FIELD_DESC); { oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.success.size())); - for (Status _iter11 : struct.success) + for (Status _iter3 : struct.success) { - _iter11.write(oprot); + _iter3.write(oprot); } oprot.writeListEnd(); } @@ -2662,9 +2662,9 @@ public class CompactionCoordinator { if (struct.isSetSuccess()) { { oprot.writeI32(struct.success.size()); - for (Status _iter12 : struct.success) + for (Status _iter4 : struct.success) { - _iter12.write(oprot); + _iter4.write(oprot); } } } @@ -2676,14 +2676,14 @@ public class CompactionCoordinator { java.util.BitSet incoming = iprot.readBitSet(1); if (incoming.get(0)) { { - org.apache.thrift.protocol.TList _list13 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32()); - struct.success = new java.util.ArrayList<Status>(_list13.size); - @org.apache.thrift.annotation.Nullable Status _elem14; - for (int _i15 = 0; _i15 < _list13.size; ++_i15) + org.apache.thrift.protocol.TList _list5 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32()); + struct.success = new java.util.ArrayList<Status>(_list5.size); + @org.apache.thrift.annotation.Nullable Status _elem6; + for (int _i7 = 0; _i7 < _list5.size; ++_i7) { - _elem14 = new Status(); - _elem14.read(iprot); - struct.success.add(_elem14); + _elem6 = new Status(); + _elem6.read(iprot); + struct.success.add(_elem6); } } struct.setSuccessIsSet(true); @@ -3550,7 +3550,7 @@ public class CompactionCoordinator { private static final org.apache.thrift.scheme.SchemeFactory TUPLE_SCHEME_FACTORY = new compactionCompleted_argsTupleSchemeFactory(); public @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.tabletserver.thrift.CompactionJob job; // required - public @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.dataImpl.thrift.CompactionStats stats; // required + public @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.tabletserver.thrift.CompactionStats stats; // required /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ public enum _Fields implements org.apache.thrift.TFieldIdEnum { @@ -3622,7 +3622,7 @@ public class CompactionCoordinator { tmpMap.put(_Fields.JOB, new org.apache.thrift.meta_data.FieldMetaData("job", org.apache.thrift.TFieldRequirementType.DEFAULT, new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.tabletserver.thrift.CompactionJob.class))); tmpMap.put(_Fields.STATS, new org.apache.thrift.meta_data.FieldMetaData("stats", org.apache.thrift.TFieldRequirementType.DEFAULT, - new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.dataImpl.thrift.CompactionStats.class))); + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.tabletserver.thrift.CompactionStats.class))); metaDataMap = java.util.Collections.unmodifiableMap(tmpMap); org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(compactionCompleted_args.class, metaDataMap); } @@ -3632,7 +3632,7 @@ public class CompactionCoordinator { public compactionCompleted_args( org.apache.accumulo.core.tabletserver.thrift.CompactionJob job, - org.apache.accumulo.core.dataImpl.thrift.CompactionStats stats) + org.apache.accumulo.core.tabletserver.thrift.CompactionStats stats) { this(); this.job = job; @@ -3647,7 +3647,7 @@ public class CompactionCoordinator { this.job = new org.apache.accumulo.core.tabletserver.thrift.CompactionJob(other.job); } if (other.isSetStats()) { - this.stats = new org.apache.accumulo.core.dataImpl.thrift.CompactionStats(other.stats); + this.stats = new org.apache.accumulo.core.tabletserver.thrift.CompactionStats(other.stats); } } @@ -3687,11 +3687,11 @@ public class CompactionCoordinator { } @org.apache.thrift.annotation.Nullable - public org.apache.accumulo.core.dataImpl.thrift.CompactionStats getStats() { + public org.apache.accumulo.core.tabletserver.thrift.CompactionStats getStats() { return this.stats; } - public compactionCompleted_args setStats(@org.apache.thrift.annotation.Nullable org.apache.accumulo.core.dataImpl.thrift.CompactionStats stats) { + public compactionCompleted_args setStats(@org.apache.thrift.annotation.Nullable org.apache.accumulo.core.tabletserver.thrift.CompactionStats stats) { this.stats = stats; return this; } @@ -3725,7 +3725,7 @@ public class CompactionCoordinator { if (value == null) { unsetStats(); } else { - setStats((org.apache.accumulo.core.dataImpl.thrift.CompactionStats)value); + setStats((org.apache.accumulo.core.tabletserver.thrift.CompactionStats)value); } break; @@ -3935,7 +3935,7 @@ public class CompactionCoordinator { break; case 2: // STATS if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { - struct.stats = new org.apache.accumulo.core.dataImpl.thrift.CompactionStats(); + struct.stats = new org.apache.accumulo.core.tabletserver.thrift.CompactionStats(); struct.stats.read(iprot); struct.setStatsIsSet(true); } else { @@ -4010,7 +4010,7 @@ public class CompactionCoordinator { struct.setJobIsSet(true); } if (incoming.get(1)) { - struct.stats = new org.apache.accumulo.core.dataImpl.thrift.CompactionStats(); + struct.stats = new org.apache.accumulo.core.tabletserver.thrift.CompactionStats(); struct.stats.read(iprot); struct.setStatsIsSet(true); } diff --git a/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/CompactionJob.java b/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/CompactionJob.java index 8ff1214..e74ea50 100644 --- a/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/CompactionJob.java +++ b/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/CompactionJob.java @@ -40,6 +40,8 @@ public class CompactionJob implements org.apache.thrift.TBase<CompactionJob, Com private static final org.apache.thrift.protocol.TField TYPE_FIELD_DESC = new org.apache.thrift.protocol.TField("type", org.apache.thrift.protocol.TType.I32, (short)11); private static final org.apache.thrift.protocol.TField REASON_FIELD_DESC = new org.apache.thrift.protocol.TField("reason", org.apache.thrift.protocol.TType.I32, (short)12); private static final org.apache.thrift.protocol.TField OUTPUT_FILE_FIELD_DESC = new org.apache.thrift.protocol.TField("outputFile", org.apache.thrift.protocol.TType.STRING, (short)13); + private static final org.apache.thrift.protocol.TField PROPAGATE_DELETES_FIELD_DESC = new org.apache.thrift.protocol.TField("propagateDeletes", org.apache.thrift.protocol.TType.BOOL, (short)14); + private static final org.apache.thrift.protocol.TField KIND_FIELD_DESC = new org.apache.thrift.protocol.TField("kind", org.apache.thrift.protocol.TType.I32, (short)15); private static final org.apache.thrift.scheme.SchemeFactory STANDARD_SCHEME_FACTORY = new CompactionJobStandardSchemeFactory(); private static final org.apache.thrift.scheme.SchemeFactory TUPLE_SCHEME_FACTORY = new CompactionJobTupleSchemeFactory(); @@ -48,7 +50,7 @@ public class CompactionJob implements org.apache.thrift.TBase<CompactionJob, Com public @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials; // required public long compactionId; // required public @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent; // required - public @org.apache.thrift.annotation.Nullable java.util.List<java.lang.String> files; // required + public @org.apache.thrift.annotation.Nullable java.util.List<InputFile> files; // required public int priority; // required public int readRate; // required public int writeRate; // required @@ -64,6 +66,8 @@ public class CompactionJob implements org.apache.thrift.TBase<CompactionJob, Com */ public @org.apache.thrift.annotation.Nullable CompactionReason reason; // required public @org.apache.thrift.annotation.Nullable java.lang.String outputFile; // required + public boolean propagateDeletes; // required + public @org.apache.thrift.annotation.Nullable CompactionKind kind; // required /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ public enum _Fields implements org.apache.thrift.TFieldIdEnum { @@ -86,7 +90,9 @@ public class CompactionJob implements org.apache.thrift.TBase<CompactionJob, Com * @see CompactionReason */ REASON((short)12, "reason"), - OUTPUT_FILE((short)13, "outputFile"); + OUTPUT_FILE((short)13, "outputFile"), + PROPAGATE_DELETES((short)14, "propagateDeletes"), + KIND((short)15, "kind"); private static final java.util.Map<java.lang.String, _Fields> byName = new java.util.HashMap<java.lang.String, _Fields>(); @@ -126,6 +132,10 @@ public class CompactionJob implements org.apache.thrift.TBase<CompactionJob, Com return REASON; case 13: // OUTPUT_FILE return OUTPUT_FILE; + case 14: // PROPAGATE_DELETES + return PROPAGATE_DELETES; + case 15: // KIND + return KIND; default: return null; } @@ -171,6 +181,7 @@ public class CompactionJob implements org.apache.thrift.TBase<CompactionJob, Com private static final int __PRIORITY_ISSET_ID = 1; private static final int __READRATE_ISSET_ID = 2; private static final int __WRITERATE_ISSET_ID = 3; + private static final int __PROPAGATEDELETES_ISSET_ID = 4; private byte __isset_bitfield = 0; public static final java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; static { @@ -185,7 +196,7 @@ public class CompactionJob implements org.apache.thrift.TBase<CompactionJob, Com new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent.class))); tmpMap.put(_Fields.FILES, new org.apache.thrift.meta_data.FieldMetaData("files", org.apache.thrift.TFieldRequirementType.DEFAULT, new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, - new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)))); + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, InputFile.class)))); tmpMap.put(_Fields.PRIORITY, new org.apache.thrift.meta_data.FieldMetaData("priority", org.apache.thrift.TFieldRequirementType.DEFAULT, new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32))); tmpMap.put(_Fields.READ_RATE, new org.apache.thrift.meta_data.FieldMetaData("readRate", org.apache.thrift.TFieldRequirementType.DEFAULT, @@ -200,6 +211,10 @@ public class CompactionJob implements org.apache.thrift.TBase<CompactionJob, Com new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, CompactionReason.class))); tmpMap.put(_Fields.OUTPUT_FILE, new org.apache.thrift.meta_data.FieldMetaData("outputFile", org.apache.thrift.TFieldRequirementType.DEFAULT, new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + tmpMap.put(_Fields.PROPAGATE_DELETES, new org.apache.thrift.meta_data.FieldMetaData("propagateDeletes", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BOOL))); + tmpMap.put(_Fields.KIND, new org.apache.thrift.meta_data.FieldMetaData("kind", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.ENUM , "CompactionKind"))); metaDataMap = java.util.Collections.unmodifiableMap(tmpMap); org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(CompactionJob.class, metaDataMap); } @@ -212,14 +227,16 @@ public class CompactionJob implements org.apache.thrift.TBase<CompactionJob, Com org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, long compactionId, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent, - java.util.List<java.lang.String> files, + java.util.List<InputFile> files, int priority, int readRate, int writeRate, IteratorConfig iteratorSettings, CompactionType type, CompactionReason reason, - java.lang.String outputFile) + java.lang.String outputFile, + boolean propagateDeletes, + CompactionKind kind) { this(); this.traceInfo = traceInfo; @@ -238,6 +255,9 @@ public class CompactionJob implements org.apache.thrift.TBase<CompactionJob, Com this.type = type; this.reason = reason; this.outputFile = outputFile; + this.propagateDeletes = propagateDeletes; + setPropagateDeletesIsSet(true); + this.kind = kind; } /** @@ -256,7 +276,10 @@ public class CompactionJob implements org.apache.thrift.TBase<CompactionJob, Com this.extent = new org.apache.accumulo.core.dataImpl.thrift.TKeyExtent(other.extent); } if (other.isSetFiles()) { - java.util.List<java.lang.String> __this__files = new java.util.ArrayList<java.lang.String>(other.files); + java.util.List<InputFile> __this__files = new java.util.ArrayList<InputFile>(other.files.size()); + for (InputFile other_element : other.files) { + __this__files.add(new InputFile(other_element)); + } this.files = __this__files; } this.priority = other.priority; @@ -274,6 +297,10 @@ public class CompactionJob implements org.apache.thrift.TBase<CompactionJob, Com if (other.isSetOutputFile()) { this.outputFile = other.outputFile; } + this.propagateDeletes = other.propagateDeletes; + if (other.isSetKind()) { + this.kind = other.kind; + } } public CompactionJob deepCopy() { @@ -298,6 +325,9 @@ public class CompactionJob implements org.apache.thrift.TBase<CompactionJob, Com this.type = null; this.reason = null; this.outputFile = null; + setPropagateDeletesIsSet(false); + this.propagateDeletes = false; + this.kind = null; } @org.apache.thrift.annotation.Nullable @@ -403,23 +433,23 @@ public class CompactionJob implements org.apache.thrift.TBase<CompactionJob, Com } @org.apache.thrift.annotation.Nullable - public java.util.Iterator<java.lang.String> getFilesIterator() { + public java.util.Iterator<InputFile> getFilesIterator() { return (this.files == null) ? null : this.files.iterator(); } - public void addToFiles(java.lang.String elem) { + public void addToFiles(InputFile elem) { if (this.files == null) { - this.files = new java.util.ArrayList<java.lang.String>(); + this.files = new java.util.ArrayList<InputFile>(); } this.files.add(elem); } @org.apache.thrift.annotation.Nullable - public java.util.List<java.lang.String> getFiles() { + public java.util.List<InputFile> getFiles() { return this.files; } - public CompactionJob setFiles(@org.apache.thrift.annotation.Nullable java.util.List<java.lang.String> files) { + public CompactionJob setFiles(@org.apache.thrift.annotation.Nullable java.util.List<InputFile> files) { this.files = files; return this; } @@ -624,6 +654,54 @@ public class CompactionJob implements org.apache.thrift.TBase<CompactionJob, Com } } + public boolean isPropagateDeletes() { + return this.propagateDeletes; + } + + public CompactionJob setPropagateDeletes(boolean propagateDeletes) { + this.propagateDeletes = propagateDeletes; + setPropagateDeletesIsSet(true); + return this; + } + + public void unsetPropagateDeletes() { + __isset_bitfield = org.apache.thrift.EncodingUtils.clearBit(__isset_bitfield, __PROPAGATEDELETES_ISSET_ID); + } + + /** Returns true if field propagateDeletes is set (has been assigned a value) and false otherwise */ + public boolean isSetPropagateDeletes() { + return org.apache.thrift.EncodingUtils.testBit(__isset_bitfield, __PROPAGATEDELETES_ISSET_ID); + } + + public void setPropagateDeletesIsSet(boolean value) { + __isset_bitfield = org.apache.thrift.EncodingUtils.setBit(__isset_bitfield, __PROPAGATEDELETES_ISSET_ID, value); + } + + @org.apache.thrift.annotation.Nullable + public CompactionKind getKind() { + return this.kind; + } + + public CompactionJob setKind(@org.apache.thrift.annotation.Nullable CompactionKind kind) { + this.kind = kind; + return this; + } + + public void unsetKind() { + this.kind = null; + } + + /** Returns true if field kind is set (has been assigned a value) and false otherwise */ + public boolean isSetKind() { + return this.kind != null; + } + + public void setKindIsSet(boolean value) { + if (!value) { + this.kind = null; + } + } + public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable java.lang.Object value) { switch (field) { case TRACE_INFO: @@ -662,7 +740,7 @@ public class CompactionJob implements org.apache.thrift.TBase<CompactionJob, Com if (value == null) { unsetFiles(); } else { - setFiles((java.util.List<java.lang.String>)value); + setFiles((java.util.List<InputFile>)value); } break; @@ -722,6 +800,22 @@ public class CompactionJob implements org.apache.thrift.TBase<CompactionJob, Com } break; + case PROPAGATE_DELETES: + if (value == null) { + unsetPropagateDeletes(); + } else { + setPropagateDeletes((java.lang.Boolean)value); + } + break; + + case KIND: + if (value == null) { + unsetKind(); + } else { + setKind((CompactionKind)value); + } + break; + } } @@ -764,6 +858,12 @@ public class CompactionJob implements org.apache.thrift.TBase<CompactionJob, Com case OUTPUT_FILE: return getOutputFile(); + case PROPAGATE_DELETES: + return isPropagateDeletes(); + + case KIND: + return getKind(); + } throw new java.lang.IllegalStateException(); } @@ -799,6 +899,10 @@ public class CompactionJob implements org.apache.thrift.TBase<CompactionJob, Com return isSetReason(); case OUTPUT_FILE: return isSetOutputFile(); + case PROPAGATE_DELETES: + return isSetPropagateDeletes(); + case KIND: + return isSetKind(); } throw new java.lang.IllegalStateException(); } @@ -926,6 +1030,24 @@ public class CompactionJob implements org.apache.thrift.TBase<CompactionJob, Com return false; } + boolean this_present_propagateDeletes = true; + boolean that_present_propagateDeletes = true; + if (this_present_propagateDeletes || that_present_propagateDeletes) { + if (!(this_present_propagateDeletes && that_present_propagateDeletes)) + return false; + if (this.propagateDeletes != that.propagateDeletes) + return false; + } + + boolean this_present_kind = true && this.isSetKind(); + boolean that_present_kind = true && that.isSetKind(); + if (this_present_kind || that_present_kind) { + if (!(this_present_kind && that_present_kind)) + return false; + if (!this.kind.equals(that.kind)) + return false; + } + return true; } @@ -973,6 +1095,12 @@ public class CompactionJob implements org.apache.thrift.TBase<CompactionJob, Com if (isSetOutputFile()) hashCode = hashCode * 8191 + outputFile.hashCode(); + hashCode = hashCode * 8191 + ((propagateDeletes) ? 131071 : 524287); + + hashCode = hashCode * 8191 + ((isSetKind()) ? 131071 : 524287); + if (isSetKind()) + hashCode = hashCode * 8191 + kind.getValue(); + return hashCode; } @@ -1104,6 +1232,26 @@ public class CompactionJob implements org.apache.thrift.TBase<CompactionJob, Com return lastComparison; } } + lastComparison = java.lang.Boolean.valueOf(isSetPropagateDeletes()).compareTo(other.isSetPropagateDeletes()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetPropagateDeletes()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.propagateDeletes, other.propagateDeletes); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = java.lang.Boolean.valueOf(isSetKind()).compareTo(other.isSetKind()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetKind()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.kind, other.kind); + if (lastComparison != 0) { + return lastComparison; + } + } return 0; } @@ -1204,6 +1352,18 @@ public class CompactionJob implements org.apache.thrift.TBase<CompactionJob, Com sb.append(this.outputFile); } first = false; + if (!first) sb.append(", "); + sb.append("propagateDeletes:"); + sb.append(this.propagateDeletes); + first = false; + if (!first) sb.append(", "); + sb.append("kind:"); + if (this.kind == null) { + sb.append("null"); + } else { + sb.append(this.kind); + } + first = false; sb.append(")"); return sb.toString(); } @@ -1300,11 +1460,12 @@ public class CompactionJob implements org.apache.thrift.TBase<CompactionJob, Com if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { { org.apache.thrift.protocol.TList _list116 = iprot.readListBegin(); - struct.files = new java.util.ArrayList<java.lang.String>(_list116.size); - @org.apache.thrift.annotation.Nullable java.lang.String _elem117; + struct.files = new java.util.ArrayList<InputFile>(_list116.size); + @org.apache.thrift.annotation.Nullable InputFile _elem117; for (int _i118 = 0; _i118 < _list116.size; ++_i118) { - _elem117 = iprot.readString(); + _elem117 = new InputFile(); + _elem117.read(iprot); struct.files.add(_elem117); } iprot.readListEnd(); @@ -1371,6 +1532,22 @@ public class CompactionJob implements org.apache.thrift.TBase<CompactionJob, Com org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } break; + case 14: // PROPAGATE_DELETES + if (schemeField.type == org.apache.thrift.protocol.TType.BOOL) { + struct.propagateDeletes = iprot.readBool(); + struct.setPropagateDeletesIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 15: // KIND + if (schemeField.type == org.apache.thrift.protocol.TType.I32) { + struct.kind = org.apache.accumulo.core.tabletserver.thrift.CompactionKind.findByValue(iprot.readI32()); + struct.setKindIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; default: org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } @@ -1407,10 +1584,10 @@ public class CompactionJob implements org.apache.thrift.TBase<CompactionJob, Com if (struct.files != null) { oprot.writeFieldBegin(FILES_FIELD_DESC); { - oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, struct.files.size())); - for (java.lang.String _iter119 : struct.files) + oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.files.size())); + for (InputFile _iter119 : struct.files) { - oprot.writeString(_iter119); + _iter119.write(oprot); } oprot.writeListEnd(); } @@ -1445,6 +1622,14 @@ public class CompactionJob implements org.apache.thrift.TBase<CompactionJob, Com oprot.writeString(struct.outputFile); oprot.writeFieldEnd(); } + oprot.writeFieldBegin(PROPAGATE_DELETES_FIELD_DESC); + oprot.writeBool(struct.propagateDeletes); + oprot.writeFieldEnd(); + if (struct.kind != null) { + oprot.writeFieldBegin(KIND_FIELD_DESC); + oprot.writeI32(struct.kind.getValue()); + oprot.writeFieldEnd(); + } oprot.writeFieldStop(); oprot.writeStructEnd(); } @@ -1499,7 +1684,13 @@ public class CompactionJob implements org.apache.thrift.TBase<CompactionJob, Com if (struct.isSetOutputFile()) { optionals.set(11); } - oprot.writeBitSet(optionals, 12); + if (struct.isSetPropagateDeletes()) { + optionals.set(12); + } + if (struct.isSetKind()) { + optionals.set(13); + } + oprot.writeBitSet(optionals, 14); if (struct.isSetTraceInfo()) { struct.traceInfo.write(oprot); } @@ -1515,9 +1706,9 @@ public class CompactionJob implements org.apache.thrift.TBase<CompactionJob, Com if (struct.isSetFiles()) { { oprot.writeI32(struct.files.size()); - for (java.lang.String _iter120 : struct.files) + for (InputFile _iter120 : struct.files) { - oprot.writeString(_iter120); + _iter120.write(oprot); } } } @@ -1542,12 +1733,18 @@ public class CompactionJob implements org.apache.thrift.TBase<CompactionJob, Com if (struct.isSetOutputFile()) { oprot.writeString(struct.outputFile); } + if (struct.isSetPropagateDeletes()) { + oprot.writeBool(struct.propagateDeletes); + } + if (struct.isSetKind()) { + oprot.writeI32(struct.kind.getValue()); + } } @Override public void read(org.apache.thrift.protocol.TProtocol prot, CompactionJob struct) throws org.apache.thrift.TException { org.apache.thrift.protocol.TTupleProtocol iprot = (org.apache.thrift.protocol.TTupleProtocol) prot; - java.util.BitSet incoming = iprot.readBitSet(12); + java.util.BitSet incoming = iprot.readBitSet(14); if (incoming.get(0)) { struct.traceInfo = new org.apache.accumulo.core.trace.thrift.TInfo(); struct.traceInfo.read(iprot); @@ -1569,12 +1766,13 @@ public class CompactionJob implements org.apache.thrift.TBase<CompactionJob, Com } if (incoming.get(4)) { { - org.apache.thrift.protocol.TList _list121 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32()); - struct.files = new java.util.ArrayList<java.lang.String>(_list121.size); - @org.apache.thrift.annotation.Nullable java.lang.String _elem122; + org.apache.thrift.protocol.TList _list121 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32()); + struct.files = new java.util.ArrayList<InputFile>(_list121.size); + @org.apache.thrift.annotation.Nullable InputFile _elem122; for (int _i123 = 0; _i123 < _list121.size; ++_i123) { - _elem122 = iprot.readString(); + _elem122 = new InputFile(); + _elem122.read(iprot); struct.files.add(_elem122); } } @@ -1609,6 +1807,14 @@ public class CompactionJob implements org.apache.thrift.TBase<CompactionJob, Com struct.outputFile = iprot.readString(); struct.setOutputFileIsSet(true); } + if (incoming.get(12)) { + struct.propagateDeletes = iprot.readBool(); + struct.setPropagateDeletesIsSet(true); + } + if (incoming.get(13)) { + struct.kind = org.apache.accumulo.core.tabletserver.thrift.CompactionKind.findByValue(iprot.readI32()); + struct.setKindIsSet(true); + } } } diff --git a/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/CompactionKind.java b/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/CompactionKind.java new file mode 100644 index 0000000..f3599a5 --- /dev/null +++ b/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/CompactionKind.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/** + * Autogenerated by Thrift Compiler (0.12.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +package org.apache.accumulo.core.tabletserver.thrift; + + +public enum CompactionKind implements org.apache.thrift.TEnum { + CHOP(0), + SELECTOR(1), + SYSTEM(2), + USER(3); + + private final int value; + + private CompactionKind(int value) { + this.value = value; + } + + /** + * Get the integer value of this enum value, as defined in the Thrift IDL. + */ + public int getValue() { + return value; + } + + /** + * Find a the enum type by its integer value, as defined in the Thrift IDL. + * @return null if the value is not found. + */ + @org.apache.thrift.annotation.Nullable + public static CompactionKind findByValue(int value) { + switch (value) { + case 0: + return CHOP; + case 1: + return SELECTOR; + case 2: + return SYSTEM; + case 3: + return USER; + default: + return null; + } + } +} diff --git a/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/CompactionStats.java b/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/CompactionStats.java new file mode 100644 index 0000000..4189a6f --- /dev/null +++ b/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/CompactionStats.java @@ -0,0 +1,582 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/** + * Autogenerated by Thrift Compiler (0.12.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +package org.apache.accumulo.core.tabletserver.thrift; + +@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked", "unused"}) +public class CompactionStats implements org.apache.thrift.TBase<CompactionStats, CompactionStats._Fields>, java.io.Serializable, Cloneable, Comparable<CompactionStats> { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("CompactionStats"); + + private static final org.apache.thrift.protocol.TField ENTRIES_READ_FIELD_DESC = new org.apache.thrift.protocol.TField("entriesRead", org.apache.thrift.protocol.TType.I64, (short)1); + private static final org.apache.thrift.protocol.TField ENTRIES_WRITTEN_FIELD_DESC = new org.apache.thrift.protocol.TField("entriesWritten", org.apache.thrift.protocol.TType.I64, (short)2); + private static final org.apache.thrift.protocol.TField FILE_SIZE_FIELD_DESC = new org.apache.thrift.protocol.TField("fileSize", org.apache.thrift.protocol.TType.I64, (short)3); + + private static final org.apache.thrift.scheme.SchemeFactory STANDARD_SCHEME_FACTORY = new CompactionStatsStandardSchemeFactory(); + private static final org.apache.thrift.scheme.SchemeFactory TUPLE_SCHEME_FACTORY = new CompactionStatsTupleSchemeFactory(); + + public long entriesRead; // required + public long entriesWritten; // required + public long fileSize; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + ENTRIES_READ((short)1, "entriesRead"), + ENTRIES_WRITTEN((short)2, "entriesWritten"), + FILE_SIZE((short)3, "fileSize"); + + private static final java.util.Map<java.lang.String, _Fields> byName = new java.util.HashMap<java.lang.String, _Fields>(); + + static { + for (_Fields field : java.util.EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + @org.apache.thrift.annotation.Nullable + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // ENTRIES_READ + return ENTRIES_READ; + case 2: // ENTRIES_WRITTEN + return ENTRIES_WRITTEN; + case 3: // FILE_SIZE + return FILE_SIZE; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new java.lang.IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + @org.apache.thrift.annotation.Nullable + public static _Fields findByName(java.lang.String name) { + return byName.get(name); + } + + private final short _thriftId; + private final java.lang.String _fieldName; + + _Fields(short thriftId, java.lang.String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public java.lang.String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __ENTRIESREAD_ISSET_ID = 0; + private static final int __ENTRIESWRITTEN_ISSET_ID = 1; + private static final int __FILESIZE_ISSET_ID = 2; + private byte __isset_bitfield = 0; + public static final java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new java.util.EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.ENTRIES_READ, new org.apache.thrift.meta_data.FieldMetaData("entriesRead", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))); + tmpMap.put(_Fields.ENTRIES_WRITTEN, new org.apache.thrift.meta_data.FieldMetaData("entriesWritten", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))); + tmpMap.put(_Fields.FILE_SIZE, new org.apache.thrift.meta_data.FieldMetaData("fileSize", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))); + metaDataMap = java.util.Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(CompactionStats.class, metaDataMap); + } + + public CompactionStats() { + } + + public CompactionStats( + long entriesRead, + long entriesWritten, + long fileSize) + { + this(); + this.entriesRead = entriesRead; + setEntriesReadIsSet(true); + this.entriesWritten = entriesWritten; + setEntriesWrittenIsSet(true); + this.fileSize = fileSize; + setFileSizeIsSet(true); + } + + /** + * Performs a deep copy on <i>other</i>. + */ + public CompactionStats(CompactionStats other) { + __isset_bitfield = other.__isset_bitfield; + this.entriesRead = other.entriesRead; + this.entriesWritten = other.entriesWritten; + this.fileSize = other.fileSize; + } + + public CompactionStats deepCopy() { + return new CompactionStats(this); + } + + @Override + public void clear() { + setEntriesReadIsSet(false); + this.entriesRead = 0; + setEntriesWrittenIsSet(false); + this.entriesWritten = 0; + setFileSizeIsSet(false); + this.fileSize = 0; + } + + public long getEntriesRead() { + return this.entriesRead; + } + + public CompactionStats setEntriesRead(long entriesRead) { + this.entriesRead = entriesRead; + setEntriesReadIsSet(true); + return this; + } + + public void unsetEntriesRead() { + __isset_bitfield = org.apache.thrift.EncodingUtils.clearBit(__isset_bitfield, __ENTRIESREAD_ISSET_ID); + } + + /** Returns true if field entriesRead is set (has been assigned a value) and false otherwise */ + public boolean isSetEntriesRead() { + return org.apache.thrift.EncodingUtils.testBit(__isset_bitfield, __ENTRIESREAD_ISSET_ID); + } + + public void setEntriesReadIsSet(boolean value) { + __isset_bitfield = org.apache.thrift.EncodingUtils.setBit(__isset_bitfield, __ENTRIESREAD_ISSET_ID, value); + } + + public long getEntriesWritten() { + return this.entriesWritten; + } + + public CompactionStats setEntriesWritten(long entriesWritten) { + this.entriesWritten = entriesWritten; + setEntriesWrittenIsSet(true); + return this; + } + + public void unsetEntriesWritten() { + __isset_bitfield = org.apache.thrift.EncodingUtils.clearBit(__isset_bitfield, __ENTRIESWRITTEN_ISSET_ID); + } + + /** Returns true if field entriesWritten is set (has been assigned a value) and false otherwise */ + public boolean isSetEntriesWritten() { + return org.apache.thrift.EncodingUtils.testBit(__isset_bitfield, __ENTRIESWRITTEN_ISSET_ID); + } + + public void setEntriesWrittenIsSet(boolean value) { + __isset_bitfield = org.apache.thrift.EncodingUtils.setBit(__isset_bitfield, __ENTRIESWRITTEN_ISSET_ID, value); + } + + public long getFileSize() { + return this.fileSize; + } + + public CompactionStats setFileSize(long fileSize) { + this.fileSize = fileSize; + setFileSizeIsSet(true); + return this; + } + + public void unsetFileSize() { + __isset_bitfield = org.apache.thrift.EncodingUtils.clearBit(__isset_bitfield, __FILESIZE_ISSET_ID); + } + + /** Returns true if field fileSize is set (has been assigned a value) and false otherwise */ + public boolean isSetFileSize() { + return org.apache.thrift.EncodingUtils.testBit(__isset_bitfield, __FILESIZE_ISSET_ID); + } + + public void setFileSizeIsSet(boolean value) { + __isset_bitfield = org.apache.thrift.EncodingUtils.setBit(__isset_bitfield, __FILESIZE_ISSET_ID, value); + } + + public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable java.lang.Object value) { + switch (field) { + case ENTRIES_READ: + if (value == null) { + unsetEntriesRead(); + } else { + setEntriesRead((java.lang.Long)value); + } + break; + + case ENTRIES_WRITTEN: + if (value == null) { + unsetEntriesWritten(); + } else { + setEntriesWritten((java.lang.Long)value); + } + break; + + case FILE_SIZE: + if (value == null) { + unsetFileSize(); + } else { + setFileSize((java.lang.Long)value); + } + break; + + } + } + + @org.apache.thrift.annotation.Nullable + public java.lang.Object getFieldValue(_Fields field) { + switch (field) { + case ENTRIES_READ: + return getEntriesRead(); + + case ENTRIES_WRITTEN: + return getEntriesWritten(); + + case FILE_SIZE: + return getFileSize(); + + } + throw new java.lang.IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new java.lang.IllegalArgumentException(); + } + + switch (field) { + case ENTRIES_READ: + return isSetEntriesRead(); + case ENTRIES_WRITTEN: + return isSetEntriesWritten(); + case FILE_SIZE: + return isSetFileSize(); + } + throw new java.lang.IllegalStateException(); + } + + @Override + public boolean equals(java.lang.Object that) { + if (that == null) + return false; + if (that instanceof CompactionStats) + return this.equals((CompactionStats)that); + return false; + } + + public boolean equals(CompactionStats that) { + if (that == null) + return false; + if (this == that) + return true; + + boolean this_present_entriesRead = true; + boolean that_present_entriesRead = true; + if (this_present_entriesRead || that_present_entriesRead) { + if (!(this_present_entriesRead && that_present_entriesRead)) + return false; + if (this.entriesRead != that.entriesRead) + return false; + } + + boolean this_present_entriesWritten = true; + boolean that_present_entriesWritten = true; + if (this_present_entriesWritten || that_present_entriesWritten) { + if (!(this_present_entriesWritten && that_present_entriesWritten)) + return false; + if (this.entriesWritten != that.entriesWritten) + return false; + } + + boolean this_present_fileSize = true; + boolean that_present_fileSize = true; + if (this_present_fileSize || that_present_fileSize) { + if (!(this_present_fileSize && that_present_fileSize)) + return false; + if (this.fileSize != that.fileSize) + return false; + } + + return true; + } + + @Override + public int hashCode() { + int hashCode = 1; + + hashCode = hashCode * 8191 + org.apache.thrift.TBaseHelper.hashCode(entriesRead); + + hashCode = hashCode * 8191 + org.apache.thrift.TBaseHelper.hashCode(entriesWritten); + + hashCode = hashCode * 8191 + org.apache.thrift.TBaseHelper.hashCode(fileSize); + + return hashCode; + } + + @Override + public int compareTo(CompactionStats other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + + lastComparison = java.lang.Boolean.valueOf(isSetEntriesRead()).compareTo(other.isSetEntriesRead()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetEntriesRead()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.entriesRead, other.entriesRead); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = java.lang.Boolean.valueOf(isSetEntriesWritten()).compareTo(other.isSetEntriesWritten()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetEntriesWritten()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.entriesWritten, other.entriesWritten); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = java.lang.Boolean.valueOf(isSetFileSize()).compareTo(other.isSetFileSize()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetFileSize()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.fileSize, other.fileSize); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + @org.apache.thrift.annotation.Nullable + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + scheme(iprot).read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + scheme(oprot).write(oprot, this); + } + + @Override + public java.lang.String toString() { + java.lang.StringBuilder sb = new java.lang.StringBuilder("CompactionStats("); + boolean first = true; + + sb.append("entriesRead:"); + sb.append(this.entriesRead); + first = false; + if (!first) sb.append(", "); + sb.append("entriesWritten:"); + sb.append(this.entriesWritten); + first = false; + if (!first) sb.append(", "); + sb.append("fileSize:"); + sb.append(this.fileSize); + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + // check for sub-struct validity + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, java.lang.ClassNotFoundException { + try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bitfield = 0; + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class CompactionStatsStandardSchemeFactory implements org.apache.thrift.scheme.SchemeFactory { + public CompactionStatsStandardScheme getScheme() { + return new CompactionStatsStandardScheme(); + } + } + + private static class CompactionStatsStandardScheme extends org.apache.thrift.scheme.StandardScheme<CompactionStats> { + + public void read(org.apache.thrift.protocol.TProtocol iprot, CompactionStats struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // ENTRIES_READ + if (schemeField.type == org.apache.thrift.protocol.TType.I64) { + struct.entriesRead = iprot.readI64(); + struct.setEntriesReadIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // ENTRIES_WRITTEN + if (schemeField.type == org.apache.thrift.protocol.TType.I64) { + struct.entriesWritten = iprot.readI64(); + struct.setEntriesWrittenIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 3: // FILE_SIZE + if (schemeField.type == org.apache.thrift.protocol.TType.I64) { + struct.fileSize = iprot.readI64(); + struct.setFileSizeIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, CompactionStats struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + oprot.writeFieldBegin(ENTRIES_READ_FIELD_DESC); + oprot.writeI64(struct.entriesRead); + oprot.writeFieldEnd(); + oprot.writeFieldBegin(ENTRIES_WRITTEN_FIELD_DESC); + oprot.writeI64(struct.entriesWritten); + oprot.writeFieldEnd(); + oprot.writeFieldBegin(FILE_SIZE_FIELD_DESC); + oprot.writeI64(struct.fileSize); + oprot.writeFieldEnd(); + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class CompactionStatsTupleSchemeFactory implements org.apache.thrift.scheme.SchemeFactory { + public CompactionStatsTupleScheme getScheme() { + return new CompactionStatsTupleScheme(); + } + } + + private static class CompactionStatsTupleScheme extends org.apache.thrift.scheme.TupleScheme<CompactionStats> { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, CompactionStats struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TTupleProtocol oprot = (org.apache.thrift.protocol.TTupleProtocol) prot; + java.util.BitSet optionals = new java.util.BitSet(); + if (struct.isSetEntriesRead()) { + optionals.set(0); + } + if (struct.isSetEntriesWritten()) { + optionals.set(1); + } + if (struct.isSetFileSize()) { + optionals.set(2); + } + oprot.writeBitSet(optionals, 3); + if (struct.isSetEntriesRead()) { + oprot.writeI64(struct.entriesRead); + } + if (struct.isSetEntriesWritten()) { + oprot.writeI64(struct.entriesWritten); + } + if (struct.isSetFileSize()) { + oprot.writeI64(struct.fileSize); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, CompactionStats struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TTupleProtocol iprot = (org.apache.thrift.protocol.TTupleProtocol) prot; + java.util.BitSet incoming = iprot.readBitSet(3); + if (incoming.get(0)) { + struct.entriesRead = iprot.readI64(); + struct.setEntriesReadIsSet(true); + } + if (incoming.get(1)) { + struct.entriesWritten = iprot.readI64(); + struct.setEntriesWrittenIsSet(true); + } + if (incoming.get(2)) { + struct.fileSize = iprot.readI64(); + struct.setFileSizeIsSet(true); + } + } + } + + private static <S extends org.apache.thrift.scheme.IScheme> S scheme(org.apache.thrift.protocol.TProtocol proto) { + return (org.apache.thrift.scheme.StandardScheme.class.equals(proto.getScheme()) ? STANDARD_SCHEME_FACTORY : TUPLE_SCHEME_FACTORY).getScheme(); + } + private static void unusedMethod() {} +} + diff --git a/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/InputFile.java b/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/InputFile.java new file mode 100644 index 0000000..10c0008 --- /dev/null +++ b/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/InputFile.java @@ -0,0 +1,687 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/** + * Autogenerated by Thrift Compiler (0.12.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +package org.apache.accumulo.core.tabletserver.thrift; + +@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked", "unused"}) +public class InputFile implements org.apache.thrift.TBase<InputFile, InputFile._Fields>, java.io.Serializable, Cloneable, Comparable<InputFile> { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("InputFile"); + + private static final org.apache.thrift.protocol.TField METADATA_FILE_ENTRY_FIELD_DESC = new org.apache.thrift.protocol.TField("metadataFileEntry", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField SIZE_FIELD_DESC = new org.apache.thrift.protocol.TField("size", org.apache.thrift.protocol.TType.I64, (short)2); + private static final org.apache.thrift.protocol.TField ENTRIES_FIELD_DESC = new org.apache.thrift.protocol.TField("entries", org.apache.thrift.protocol.TType.I64, (short)3); + private static final org.apache.thrift.protocol.TField TIMESTAMP_FIELD_DESC = new org.apache.thrift.protocol.TField("timestamp", org.apache.thrift.protocol.TType.I64, (short)4); + + private static final org.apache.thrift.scheme.SchemeFactory STANDARD_SCHEME_FACTORY = new InputFileStandardSchemeFactory(); + private static final org.apache.thrift.scheme.SchemeFactory TUPLE_SCHEME_FACTORY = new InputFileTupleSchemeFactory(); + + public @org.apache.thrift.annotation.Nullable java.lang.String metadataFileEntry; // required + public long size; // required + public long entries; // required + public long timestamp; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + METADATA_FILE_ENTRY((short)1, "metadataFileEntry"), + SIZE((short)2, "size"), + ENTRIES((short)3, "entries"), + TIMESTAMP((short)4, "timestamp"); + + private static final java.util.Map<java.lang.String, _Fields> byName = new java.util.HashMap<java.lang.String, _Fields>(); + + static { + for (_Fields field : java.util.EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + @org.apache.thrift.annotation.Nullable + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // METADATA_FILE_ENTRY + return METADATA_FILE_ENTRY; + case 2: // SIZE + return SIZE; + case 3: // ENTRIES + return ENTRIES; + case 4: // TIMESTAMP + return TIMESTAMP; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new java.lang.IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + @org.apache.thrift.annotation.Nullable + public static _Fields findByName(java.lang.String name) { + return byName.get(name); + } + + private final short _thriftId; + private final java.lang.String _fieldName; + + _Fields(short thriftId, java.lang.String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public java.lang.String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __SIZE_ISSET_ID = 0; + private static final int __ENTRIES_ISSET_ID = 1; + private static final int __TIMESTAMP_ISSET_ID = 2; + private byte __isset_bitfield = 0; + public static final java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new java.util.EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.METADATA_FILE_ENTRY, new org.apache.thrift.meta_data.FieldMetaData("metadataFileEntry", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + tmpMap.put(_Fields.SIZE, new org.apache.thrift.meta_data.FieldMetaData("size", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))); + tmpMap.put(_Fields.ENTRIES, new org.apache.thrift.meta_data.FieldMetaData("entries", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))); + tmpMap.put(_Fields.TIMESTAMP, new org.apache.thrift.meta_data.FieldMetaData("timestamp", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))); + metaDataMap = java.util.Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(InputFile.class, metaDataMap); + } + + public InputFile() { + } + + public InputFile( + java.lang.String metadataFileEntry, + long size, + long entries, + long timestamp) + { + this(); + this.metadataFileEntry = metadataFileEntry; + this.size = size; + setSizeIsSet(true); + this.entries = entries; + setEntriesIsSet(true); + this.timestamp = timestamp; + setTimestampIsSet(true); + } + + /** + * Performs a deep copy on <i>other</i>. + */ + public InputFile(InputFile other) { + __isset_bitfield = other.__isset_bitfield; + if (other.isSetMetadataFileEntry()) { + this.metadataFileEntry = other.metadataFileEntry; + } + this.size = other.size; + this.entries = other.entries; + this.timestamp = other.timestamp; + } + + public InputFile deepCopy() { + return new InputFile(this); + } + + @Override + public void clear() { + this.metadataFileEntry = null; + setSizeIsSet(false); + this.size = 0; + setEntriesIsSet(false); + this.entries = 0; + setTimestampIsSet(false); + this.timestamp = 0; + } + + @org.apache.thrift.annotation.Nullable + public java.lang.String getMetadataFileEntry() { + return this.metadataFileEntry; + } + + public InputFile setMetadataFileEntry(@org.apache.thrift.annotation.Nullable java.lang.String metadataFileEntry) { + this.metadataFileEntry = metadataFileEntry; + return this; + } + + public void unsetMetadataFileEntry() { + this.metadataFileEntry = null; + } + + /** Returns true if field metadataFileEntry is set (has been assigned a value) and false otherwise */ + public boolean isSetMetadataFileEntry() { + return this.metadataFileEntry != null; + } + + public void setMetadataFileEntryIsSet(boolean value) { + if (!value) { + this.metadataFileEntry = null; + } + } + + public long getSize() { + return this.size; + } + + public InputFile setSize(long size) { + this.size = size; + setSizeIsSet(true); + return this; + } + + public void unsetSize() { + __isset_bitfield = org.apache.thrift.EncodingUtils.clearBit(__isset_bitfield, __SIZE_ISSET_ID); + } + + /** Returns true if field size is set (has been assigned a value) and false otherwise */ + public boolean isSetSize() { + return org.apache.thrift.EncodingUtils.testBit(__isset_bitfield, __SIZE_ISSET_ID); + } + + public void setSizeIsSet(boolean value) { + __isset_bitfield = org.apache.thrift.EncodingUtils.setBit(__isset_bitfield, __SIZE_ISSET_ID, value); + } + + public long getEntries() { + return this.entries; + } + + public InputFile setEntries(long entries) { + this.entries = entries; + setEntriesIsSet(true); + return this; + } + + public void unsetEntries() { + __isset_bitfield = org.apache.thrift.EncodingUtils.clearBit(__isset_bitfield, __ENTRIES_ISSET_ID); + } + + /** Returns true if field entries is set (has been assigned a value) and false otherwise */ + public boolean isSetEntries() { + return org.apache.thrift.EncodingUtils.testBit(__isset_bitfield, __ENTRIES_ISSET_ID); + } + + public void setEntriesIsSet(boolean value) { + __isset_bitfield = org.apache.thrift.EncodingUtils.setBit(__isset_bitfield, __ENTRIES_ISSET_ID, value); + } + + public long getTimestamp() { + return this.timestamp; + } + + public InputFile setTimestamp(long timestamp) { + this.timestamp = timestamp; + setTimestampIsSet(true); + return this; + } + + public void unsetTimestamp() { + __isset_bitfield = org.apache.thrift.EncodingUtils.clearBit(__isset_bitfield, __TIMESTAMP_ISSET_ID); + } + + /** Returns true if field timestamp is set (has been assigned a value) and false otherwise */ + public boolean isSetTimestamp() { + return org.apache.thrift.EncodingUtils.testBit(__isset_bitfield, __TIMESTAMP_ISSET_ID); + } + + public void setTimestampIsSet(boolean value) { + __isset_bitfield = org.apache.thrift.EncodingUtils.setBit(__isset_bitfield, __TIMESTAMP_ISSET_ID, value); + } + + public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable java.lang.Object value) { + switch (field) { + case METADATA_FILE_ENTRY: + if (value == null) { + unsetMetadataFileEntry(); + } else { + setMetadataFileEntry((java.lang.String)value); + } + break; + + case SIZE: + if (value == null) { + unsetSize(); + } else { + setSize((java.lang.Long)value); + } + break; + + case ENTRIES: + if (value == null) { + unsetEntries(); + } else { + setEntries((java.lang.Long)value); + } + break; + + case TIMESTAMP: + if (value == null) { + unsetTimestamp(); + } else { + setTimestamp((java.lang.Long)value); + } + break; + + } + } + + @org.apache.thrift.annotation.Nullable + public java.lang.Object getFieldValue(_Fields field) { + switch (field) { + case METADATA_FILE_ENTRY: + return getMetadataFileEntry(); + + case SIZE: + return getSize(); + + case ENTRIES: + return getEntries(); + + case TIMESTAMP: + return getTimestamp(); + + } + throw new java.lang.IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new java.lang.IllegalArgumentException(); + } + + switch (field) { + case METADATA_FILE_ENTRY: + return isSetMetadataFileEntry(); + case SIZE: + return isSetSize(); + case ENTRIES: + return isSetEntries(); + case TIMESTAMP: + return isSetTimestamp(); + } + throw new java.lang.IllegalStateException(); + } + + @Override + public boolean equals(java.lang.Object that) { + if (that == null) + return false; + if (that instanceof InputFile) + return this.equals((InputFile)that); + return false; + } + + public boolean equals(InputFile that) { + if (that == null) + return false; + if (this == that) + return true; + + boolean this_present_metadataFileEntry = true && this.isSetMetadataFileEntry(); + boolean that_present_metadataFileEntry = true && that.isSetMetadataFileEntry(); + if (this_present_metadataFileEntry || that_present_metadataFileEntry) { + if (!(this_present_metadataFileEntry && that_present_metadataFileEntry)) + return false; + if (!this.metadataFileEntry.equals(that.metadataFileEntry)) + return false; + } + + boolean this_present_size = true; + boolean that_present_size = true; + if (this_present_size || that_present_size) { + if (!(this_present_size && that_present_size)) + return false; + if (this.size != that.size) + return false; + } + + boolean this_present_entries = true; + boolean that_present_entries = true; + if (this_present_entries || that_present_entries) { + if (!(this_present_entries && that_present_entries)) + return false; + if (this.entries != that.entries) + return false; + } + + boolean this_present_timestamp = true; + boolean that_present_timestamp = true; + if (this_present_timestamp || that_present_timestamp) { + if (!(this_present_timestamp && that_present_timestamp)) + return false; + if (this.timestamp != that.timestamp) + return false; + } + + return true; + } + + @Override + public int hashCode() { + int hashCode = 1; + + hashCode = hashCode * 8191 + ((isSetMetadataFileEntry()) ? 131071 : 524287); + if (isSetMetadataFileEntry()) + hashCode = hashCode * 8191 + metadataFileEntry.hashCode(); + + hashCode = hashCode * 8191 + org.apache.thrift.TBaseHelper.hashCode(size); + + hashCode = hashCode * 8191 + org.apache.thrift.TBaseHelper.hashCode(entries); + + hashCode = hashCode * 8191 + org.apache.thrift.TBaseHelper.hashCode(timestamp); + + return hashCode; + } + + @Override + public int compareTo(InputFile other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + + lastComparison = java.lang.Boolean.valueOf(isSetMetadataFileEntry()).compareTo(other.isSetMetadataFileEntry()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetMetadataFileEntry()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.metadataFileEntry, other.metadataFileEntry); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = java.lang.Boolean.valueOf(isSetSize()).compareTo(other.isSetSize()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetSize()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.size, other.size); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = java.lang.Boolean.valueOf(isSetEntries()).compareTo(other.isSetEntries()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetEntries()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.entries, other.entries); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = java.lang.Boolean.valueOf(isSetTimestamp()).compareTo(other.isSetTimestamp()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTimestamp()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.timestamp, other.timestamp); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + @org.apache.thrift.annotation.Nullable + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + scheme(iprot).read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + scheme(oprot).write(oprot, this); + } + + @Override + public java.lang.String toString() { + java.lang.StringBuilder sb = new java.lang.StringBuilder("InputFile("); + boolean first = true; + + sb.append("metadataFileEntry:"); + if (this.metadataFileEntry == null) { + sb.append("null"); + } else { + sb.append(this.metadataFileEntry); + } + first = false; + if (!first) sb.append(", "); + sb.append("size:"); + sb.append(this.size); + first = false; + if (!first) sb.append(", "); + sb.append("entries:"); + sb.append(this.entries); + first = false; + if (!first) sb.append(", "); + sb.append("timestamp:"); + sb.append(this.timestamp); + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + // check for sub-struct validity + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, java.lang.ClassNotFoundException { + try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bitfield = 0; + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class InputFileStandardSchemeFactory implements org.apache.thrift.scheme.SchemeFactory { + public InputFileStandardScheme getScheme() { + return new InputFileStandardScheme(); + } + } + + private static class InputFileStandardScheme extends org.apache.thrift.scheme.StandardScheme<InputFile> { + + public void read(org.apache.thrift.protocol.TProtocol iprot, InputFile struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // METADATA_FILE_ENTRY + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.metadataFileEntry = iprot.readString(); + struct.setMetadataFileEntryIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // SIZE + if (schemeField.type == org.apache.thrift.protocol.TType.I64) { + struct.size = iprot.readI64(); + struct.setSizeIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 3: // ENTRIES + if (schemeField.type == org.apache.thrift.protocol.TType.I64) { + struct.entries = iprot.readI64(); + struct.setEntriesIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 4: // TIMESTAMP + if (schemeField.type == org.apache.thrift.protocol.TType.I64) { + struct.timestamp = iprot.readI64(); + struct.setTimestampIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, InputFile struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.metadataFileEntry != null) { + oprot.writeFieldBegin(METADATA_FILE_ENTRY_FIELD_DESC); + oprot.writeString(struct.metadataFileEntry); + oprot.writeFieldEnd(); + } + oprot.writeFieldBegin(SIZE_FIELD_DESC); + oprot.writeI64(struct.size); + oprot.writeFieldEnd(); + oprot.writeFieldBegin(ENTRIES_FIELD_DESC); + oprot.writeI64(struct.entries); + oprot.writeFieldEnd(); + oprot.writeFieldBegin(TIMESTAMP_FIELD_DESC); + oprot.writeI64(struct.timestamp); + oprot.writeFieldEnd(); + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class InputFileTupleSchemeFactory implements org.apache.thrift.scheme.SchemeFactory { + public InputFileTupleScheme getScheme() { + return new InputFileTupleScheme(); + } + } + + private static class InputFileTupleScheme extends org.apache.thrift.scheme.TupleScheme<InputFile> { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, InputFile struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TTupleProtocol oprot = (org.apache.thrift.protocol.TTupleProtocol) prot; + java.util.BitSet optionals = new java.util.BitSet(); + if (struct.isSetMetadataFileEntry()) { + optionals.set(0); + } + if (struct.isSetSize()) { + optionals.set(1); + } + if (struct.isSetEntries()) { + optionals.set(2); + } + if (struct.isSetTimestamp()) { + optionals.set(3); + } + oprot.writeBitSet(optionals, 4); + if (struct.isSetMetadataFileEntry()) { + oprot.writeString(struct.metadataFileEntry); + } + if (struct.isSetSize()) { + oprot.writeI64(struct.size); + } + if (struct.isSetEntries()) { + oprot.writeI64(struct.entries); + } + if (struct.isSetTimestamp()) { + oprot.writeI64(struct.timestamp); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, InputFile struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TTupleProtocol iprot = (org.apache.thrift.protocol.TTupleProtocol) prot; + java.util.BitSet incoming = iprot.readBitSet(4); + if (incoming.get(0)) { + struct.metadataFileEntry = iprot.readString(); + struct.setMetadataFileEntryIsSet(true); + } + if (incoming.get(1)) { + struct.size = iprot.readI64(); + struct.setSizeIsSet(true); + } + if (incoming.get(2)) { + struct.entries = iprot.readI64(); + struct.setEntriesIsSet(true); + } + if (incoming.get(3)) { + struct.timestamp = iprot.readI64(); + struct.setTimestampIsSet(true); + } + } + } + + private static <S extends org.apache.thrift.scheme.IScheme> S scheme(org.apache.thrift.protocol.TProtocol proto) { + return (org.apache.thrift.scheme.StandardScheme.class.equals(proto.getScheme()) ? STANDARD_SCHEME_FACTORY : TUPLE_SCHEME_FACTORY).getScheme(); + } + private static void unusedMethod() {} +} + diff --git a/core/src/main/thrift/compaction-coordinator.thrift b/core/src/main/thrift/compaction-coordinator.thrift index 7cc2caf..1b754cf 100644 --- a/core/src/main/thrift/compaction-coordinator.thrift +++ b/core/src/main/thrift/compaction-coordinator.thrift @@ -39,37 +39,6 @@ enum CompactionState { CANCELLED } -struct InputFile { - 1:string metadataFileEntry - 2:i64 size - 3:i64 entries - 4:i64 timestamp -} - -enum CompactionKind { - CHOP - SELECTOR - SYSTEM - USER -} - -struct CompactionJob { - 1:trace.TInfo traceInfo - 2:security.TCredentials credentials - 3:i64 compactionId - 5:data.TKeyExtent extent - 6:list<InputFile> files - 7:i32 priority - 8:i32 readRate - 9:i32 writeRate - 10:tabletserver.IteratorConfig iteratorSettings - 11:tabletserver.CompactionType type - 12:tabletserver.CompactionReason reason - 13:string outputFile - 14:bool propagateDeletes - 15:CompactionKind kind -} - struct Status { 1:i64 timestamp 2:i64 compactionId @@ -111,7 +80,7 @@ service CompactionCoordinator { */ void compactionCompleted( 1:tabletserver.CompactionJob job - 2:data.CompactionStats stats + 2:tabletserver.CompactionStats stats ) /* diff --git a/core/src/main/thrift/data.thrift b/core/src/main/thrift/data.thrift index 7ee0b9d..f0426a4 100644 --- a/core/src/main/thrift/data.thrift +++ b/core/src/main/thrift/data.thrift @@ -191,9 +191,3 @@ typedef map<TKeyExtent, list<TMutation>> UpdateBatch typedef map<TKeyExtent, map<string, MapFileInfo>> TabletFiles -struct CompactionStats{ - 1:i64 entriesRead; - 2:i64 entriesWritten; - 3:i64 fileSize; -} - diff --git a/core/src/main/thrift/tabletserver.thrift b/core/src/main/thrift/tabletserver.thrift index e5f2194..a59abfa 100644 --- a/core/src/main/thrift/tabletserver.thrift +++ b/core/src/main/thrift/tabletserver.thrift @@ -158,12 +158,19 @@ enum TUnloadTabletGoal { DELETED } +struct InputFile { + 1:string metadataFileEntry + 2:i64 size + 3:i64 entries + 4:i64 timestamp +} + struct CompactionJob { 1:trace.TInfo traceInfo 2:security.TCredentials credentials 3:i64 compactionId 5:data.TKeyExtent extent - 6:list<string> files + 6:list<InputFile> files 7:i32 priority 8:i32 readRate 9:i32 writeRate @@ -172,6 +179,15 @@ struct CompactionJob { # Need to add SELECTOR To CompactionReason, delete CompactionKind? 12:CompactionReason reason 13:string outputFile + 14:bool propagateDeletes + 15:CompactionKind kind +} + +enum CompactionKind { + CHOP + SELECTOR + SYSTEM + USER } struct CompactionQueueSummary { @@ -180,6 +196,12 @@ struct CompactionQueueSummary { 3:i32 count } +struct CompactionStats{ + 1:i64 entriesRead; + 2:i64 entriesWritten; + 3:i64 fileSize; +} + service TabletClientService extends client.ClientService { // scan a range of keys diff --git a/server/compactor/src/main/java/org/apache/accumulo/compactor/RetryableThriftFunction.java b/server/base/src/main/java/org/apache/accumulo/server/compaction/ExternalCompactionUtil.java similarity index 59% copy from server/compactor/src/main/java/org/apache/accumulo/compactor/RetryableThriftFunction.java copy to server/base/src/main/java/org/apache/accumulo/server/compaction/ExternalCompactionUtil.java index bfdbb4c..51c743f 100644 --- a/server/compactor/src/main/java/org/apache/accumulo/compactor/RetryableThriftFunction.java +++ b/server/base/src/main/java/org/apache/accumulo/server/compaction/ExternalCompactionUtil.java @@ -16,11 +16,18 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.accumulo.compactor; +package org.apache.accumulo.server.compaction; -import org.apache.thrift.TException; +import org.apache.accumulo.core.Constants; +import org.apache.accumulo.core.util.HostAndPort; +import org.apache.accumulo.server.ServerContext; -@FunctionalInterface -public interface RetryableThriftFunction<T> { - T execute() throws TException; +public class ExternalCompactionUtil { + + public static HostAndPort findCompactionCoordinator(ServerContext context) { + final String lockPath = context.getZooKeeperRoot() + Constants.ZCOORDINATOR_LOCK; + byte[] address = context.getZooCache().get(lockPath); + String coordinatorAddress = new String(address); + return HostAndPort.fromString(coordinatorAddress); + } } diff --git a/server/compactor/src/main/java/org/apache/accumulo/compactor/RetryableThriftCall.java b/server/base/src/main/java/org/apache/accumulo/server/compaction/RetryableThriftCall.java similarity index 96% rename from server/compactor/src/main/java/org/apache/accumulo/compactor/RetryableThriftCall.java rename to server/base/src/main/java/org/apache/accumulo/server/compaction/RetryableThriftCall.java index 5ce599e..6981d96 100644 --- a/server/compactor/src/main/java/org/apache/accumulo/compactor/RetryableThriftCall.java +++ b/server/base/src/main/java/org/apache/accumulo/server/compaction/RetryableThriftCall.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.accumulo.compactor; +package org.apache.accumulo.server.compaction; import org.apache.accumulo.fate.util.UtilWaitThread; import org.apache.thrift.TException; @@ -26,6 +26,7 @@ import org.slf4j.LoggerFactory; public class RetryableThriftCall<T> { private static final Logger LOG = LoggerFactory.getLogger(RetryableThriftCall.class); + public static final long MAX_WAIT_TIME = 60000; private final long start; private final long maxWaitTime; diff --git a/server/compactor/src/main/java/org/apache/accumulo/compactor/RetryableThriftFunction.java b/server/base/src/main/java/org/apache/accumulo/server/compaction/RetryableThriftFunction.java similarity index 95% rename from server/compactor/src/main/java/org/apache/accumulo/compactor/RetryableThriftFunction.java rename to server/base/src/main/java/org/apache/accumulo/server/compaction/RetryableThriftFunction.java index bfdbb4c..e87f020 100644 --- a/server/compactor/src/main/java/org/apache/accumulo/compactor/RetryableThriftFunction.java +++ b/server/base/src/main/java/org/apache/accumulo/server/compaction/RetryableThriftFunction.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.accumulo.compactor; +package org.apache.accumulo.server.compaction; import org.apache.thrift.TException; diff --git a/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java b/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java index 73670c0..6c747ed 100644 --- a/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java +++ b/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java @@ -19,6 +19,7 @@ package org.apache.accumulo.coordinator; import java.net.UnknownHostException; +import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashSet; @@ -35,15 +36,16 @@ import java.util.concurrent.TimeUnit; import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.clientImpl.ThriftTransportPool; import org.apache.accumulo.core.compaction.thrift.CompactionState; +import org.apache.accumulo.core.compaction.thrift.Compactor; import org.apache.accumulo.core.compaction.thrift.Status; import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.Property; -import org.apache.accumulo.core.dataImpl.thrift.CompactionStats; import org.apache.accumulo.core.dataImpl.thrift.TKeyExtent; import org.apache.accumulo.core.metadata.TServerInstance; import org.apache.accumulo.core.rpc.ThriftUtil; import org.apache.accumulo.core.tabletserver.thrift.CompactionJob; import org.apache.accumulo.core.tabletserver.thrift.CompactionQueueSummary; +import org.apache.accumulo.core.tabletserver.thrift.CompactionStats; import org.apache.accumulo.core.tabletserver.thrift.TabletClientService; import org.apache.accumulo.core.trace.TraceUtil; import org.apache.accumulo.core.util.Halt; @@ -57,6 +59,8 @@ import org.apache.accumulo.server.AbstractServer; import org.apache.accumulo.server.GarbageCollectionLogger; import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.ServerOpts; +import org.apache.accumulo.server.compaction.RetryableThriftCall; +import org.apache.accumulo.server.compaction.RetryableThriftFunction; import org.apache.accumulo.server.manager.LiveTServerSet; import org.apache.accumulo.server.manager.LiveTServerSet.TServerConnection; import org.apache.accumulo.server.rpc.ServerAddress; @@ -200,14 +204,14 @@ public class CompactionCoordinator extends AbstractServer implements private static class RunningCompaction { private final CompactionJob job; - private final String compactor; + private final String compactorAddress; private final TServerInstance tserver; private Map<Long, CompactionUpdate> updates = new TreeMap<>(); private CompactionStats stats = null; - public RunningCompaction(CompactionJob job, String compactor, TServerInstance tserver) { + public RunningCompaction(CompactionJob job, String compactorAddress, TServerInstance tserver) { super(); this.job = job; - this.compactor = compactor; + this.compactorAddress = compactorAddress; this.tserver = tserver; } public Map<Long,CompactionUpdate> getUpdates() { @@ -216,9 +220,6 @@ public class CompactionCoordinator extends AbstractServer implements public void addUpdate(Long timestamp, String message, CompactionState state) { this.updates.put(timestamp, new CompactionUpdate(timestamp, message, state)); } - public void setUpdates(Map<Long,CompactionUpdate> updates) { - this.updates = updates; - } public CompactionStats getStats() { return stats; } @@ -228,8 +229,8 @@ public class CompactionCoordinator extends AbstractServer implements public CompactionJob getJob() { return job; } - public String getCompactor() { - return compactor; + public String getCompactorAddress() { + return compactorAddress; } public TServerInstance getTserver() { return tserver; @@ -293,7 +294,7 @@ public class CompactionCoordinator extends AbstractServer implements * @return address of this CompactionCoordinator client service * @throws UnknownHostException */ - private ServerAddress startCompactorClientService() throws UnknownHostException { + private ServerAddress startCoordinatorClientService() throws UnknownHostException { CompactionCoordinator rpcProxy = TraceUtil.wrapService(this); final org.apache.accumulo.core.compaction.thrift.CompactionCoordinator.Processor<CompactionCoordinator> processor; if (getContext().getThriftServerType() == ThriftServerType.SASL) { @@ -319,7 +320,7 @@ public class CompactionCoordinator extends AbstractServer implements ServerAddress coordinatorAddress = null; try { - coordinatorAddress = startCompactorClientService(); + coordinatorAddress = startCoordinatorClientService(); } catch (UnknownHostException e1) { throw new RuntimeException("Failed to start the coordinator service", e1); } @@ -364,7 +365,7 @@ public class CompactionCoordinator extends AbstractServer implements /** - * Callback for the LiveTServerSet object to update us current set of tablet servers, including + * Callback for the LiveTServerSet object to update current set of tablet servers, including * ones that were deleted and added * * @param current current set of live tservers @@ -394,14 +395,14 @@ public class CompactionCoordinator extends AbstractServer implements } /** - * Return the next compaction job for the queue to a Compactor + * Return the next compaction job from the queue to a Compactor * * @param queueName queue - * @param compactor compactor address + * @param compactorAddress compactor address * @return compaction job */ @Override - public CompactionJob getCompactionJob(String queueName, String compactor) throws TException { + public CompactionJob getCompactionJob(String queueName, String compactorAddress) throws TException { String queue = queueName.intern(); TServerInstance tserver = null; Long priority = null; @@ -434,20 +435,17 @@ public class CompactionCoordinator extends AbstractServer implements break; } } - } else { - return (CompactionJob) null; //TODO: or should we thrown an error for no tserver for this queue? } } - - try { - TabletClientService.Client client = getTabletServerConnection(tserver); - CompactionJob job = client.reserveCompactionJob(queue, priority, compactor); - RUNNING.put(job.getCompactionId(), new RunningCompaction(job, compactor, tserver)); - return job; - } catch (TException e) { - // TODO Auto-generated catch block - e.printStackTrace(); + + if (null == tserver) { + return null; } + + TabletClientService.Client client = getTabletServerConnection(tserver); + CompactionJob job = client.reserveCompactionJob(queue, priority, compactorAddress); + RUNNING.put(job.getCompactionId(), new RunningCompaction(job, compactorAddress, tserver)); + return job; } private TabletClientService.Client getTabletServerConnection(TServerInstance tserver) throws TTransportException { @@ -456,19 +454,43 @@ public class CompactionCoordinator extends AbstractServer implements return ThriftUtil.createClient(new TabletClientService.Client.Factory(), transport); } + private Compactor.Client getCompactorConnection(HostAndPort compactorAddress) throws TTransportException { + TTransport transport = ThriftTransportPool.getInstance().getTransport(compactorAddress, 0, getContext()); + return ThriftUtil.createClient(new Compactor.Client.Factory(), transport); + } + /** + * Called by the TabletServer to cancel the running compaction. + */ @Override public void cancelCompaction(TKeyExtent extent, String queueName, long priority) throws TException { - // TODO Auto-generated method stub - + RunningCompaction rc = RUNNING.get(null/* compactionId */); // TODO: Need to change thrift inputs here + HostAndPort compactor = HostAndPort.fromString(rc.getCompactorAddress()); + RetryableThriftCall<Void> cancelThriftCall = new RetryableThriftCall<>(1000, + RetryableThriftCall.MAX_WAIT_TIME, 0, new RetryableThriftFunction<Void>() { + @Override + public Void execute() throws TException { + try { + getCompactorConnection(compactor).cancel(rc.getJob()); + return null; + } catch (TException e) { + throw e; + } + } + }); + cancelThriftCall.run(); } @Override public List<Status> getCompactionStatus(TKeyExtent extent, String queueName, long priority) throws TException { - // TODO Auto-generated method stub - return null; + RunningCompaction rc = RUNNING.get(null/* compactionId */); // TODO: Need to change thrift inputs here + List<Status> status = new ArrayList<>(); + rc.getUpdates().forEach((k,v) -> { + status.add(new Status(v.getTimestamp(), rc.getJob().getCompactionId(), rc.getCompactorAddress(), v.getState(), v.getMessage())); + }); + return status; } /** diff --git a/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java b/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java index 75d81cf..5eb8cab 100644 --- a/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java +++ b/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java @@ -37,7 +37,6 @@ import java.util.concurrent.atomic.LongAdder; import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.client.IteratorSetting; import org.apache.accumulo.core.compaction.thrift.CompactionCoordinator; -import org.apache.accumulo.core.compaction.thrift.CompactionJob; import org.apache.accumulo.core.compaction.thrift.CompactionState; import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.Property; @@ -53,7 +52,9 @@ import org.apache.accumulo.core.metadata.TabletFile; import org.apache.accumulo.core.metadata.schema.DataFileValue; import org.apache.accumulo.core.rpc.ThriftUtil; import org.apache.accumulo.core.spi.compaction.CompactionKind; +import org.apache.accumulo.core.tabletserver.thrift.CompactionJob; import org.apache.accumulo.core.tabletserver.thrift.CompactionReason; +import org.apache.accumulo.core.tabletserver.thrift.CompactionStats; import org.apache.accumulo.core.trace.TraceUtil; import org.apache.accumulo.core.util.Halt; import org.apache.accumulo.core.util.HostAndPort; @@ -72,8 +73,10 @@ import org.apache.accumulo.server.GarbageCollectionLogger; import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.ServerOpts; import org.apache.accumulo.server.compaction.CompactionInfo; -import org.apache.accumulo.server.compaction.CompactionStats; import org.apache.accumulo.server.compaction.Compactor.CompactionEnv; +import org.apache.accumulo.server.compaction.ExternalCompactionUtil; +import org.apache.accumulo.server.compaction.RetryableThriftCall; +import org.apache.accumulo.server.compaction.RetryableThriftFunction; import org.apache.accumulo.server.conf.TableConfiguration; import org.apache.accumulo.server.iterators.SystemIteratorEnvironment; import org.apache.accumulo.server.iterators.TabletIteratorEnvironment; @@ -173,14 +176,15 @@ public class Compactor extends AbstractServer private static final Logger LOG = LoggerFactory.getLogger(Compactor.class); private static final long TIME_BETWEEN_GC_CHECKS = 5000; - private static final long MAX_WAIT_TIME = 60000; + private static final CompactionJobHolder jobHolder = new CompactionJobHolder(); private final GarbageCollectionLogger gcLogger = new GarbageCollectionLogger(); private final UUID compactorId = UUID.randomUUID(); private final AccumuloConfiguration aconf; private final String queueName; - private final CompactionJobHolder jobHolder; + private final AtomicReference<CompactionCoordinator.Client> coordinatorClient = null; private ZooLock compactorLock; + private ServerAddress compactorAddress = null; Compactor(CompactorServerOpts opts, String[] args) { super("compactor", opts, args); @@ -188,7 +192,6 @@ public class Compactor extends AbstractServer ServerContext context = super.getContext(); context.setupCrypto(); - this.jobHolder = new CompactionJobHolder(); aconf = getConfiguration(); ThreadPools.createGeneralScheduledExecutorService(aconf).scheduleWithFixedDelay( () -> gcLogger.logGCInfo(getConfiguration()), 0, TIME_BETWEEN_GC_CHECKS, @@ -265,26 +268,6 @@ public class Compactor extends AbstractServer } /** - * Get the address of the CompactionCoordinator - * - * @return address of Coordinator - */ - private HostAndPort getCoordinatorAddress() { - try { - // TODO: Get the coordinator location from ZooKeeper - List<String> locations = null; - if (locations.isEmpty()) { - return null; - } - return HostAndPort.fromString(locations.get(0)); - } catch (Exception e) { - LOG.warn("Failed to obtain manager host " + e); - } - - return null; - } - - /** * Start this Compactors thrift service to handle incoming client requests * * @return address of this compactor client service @@ -341,15 +324,48 @@ public class Compactor extends AbstractServer * @param message * updated message */ - private void updateCompactionState(CompactionCoordinator.Client coordinatorClient, - CompactionJob job, CompactionState state, String message) { + private void updateCompactionState(CompactionJob job, CompactionState state, String message) { RetryableThriftCall<Void> thriftCall = - new RetryableThriftCall<>(1000, MAX_WAIT_TIME, 25, new RetryableThriftFunction<Void>() { + new RetryableThriftCall<>(1000, RetryableThriftCall.MAX_WAIT_TIME, 25, new RetryableThriftFunction<Void>() { @Override public Void execute() throws TException { - coordinatorClient.updateCompactionState(job, state, message, - System.currentTimeMillis()); - return null; + try { + if (null == coordinatorClient.get()) { + coordinatorClient.set(getCoordinatorClient()); + } + coordinatorClient.get().updateCompactionStatus(job, state, message, + System.currentTimeMillis()); + return null; + } catch (TException e) { + coordinatorClient.set(null); + throw e; + } + } + }); + thriftCall.run(); + } + + /** + * Update the coordinator with the stats from the job + * + * @param job current compaction job + * @param stats compaction stats + */ + private void updateCompactionCompleted(CompactionJob job, CompactionStats stats) { + RetryableThriftCall<Void> thriftCall = + new RetryableThriftCall<>(1000, RetryableThriftCall.MAX_WAIT_TIME, 25, new RetryableThriftFunction<Void>() { + @Override + public Void execute() throws TException { + try { + if (null == coordinatorClient.get()) { + coordinatorClient.set(getCoordinatorClient()); + } + coordinatorClient.get().compactionCompleted(job, stats); + return null; + } catch (TException e) { + coordinatorClient.set(null); + throw e; + } } }); thriftCall.run(); @@ -364,22 +380,103 @@ public class Compactor extends AbstractServer * address of this Compactor * @return CompactionJob */ - private CompactionJob getNextJob(CompactionCoordinator.Client coordinatorClient, - String compactorAddress) { + private CompactionJob getNextJob() { RetryableThriftCall<CompactionJob> nextJobThriftCall = new RetryableThriftCall<>(1000, - MAX_WAIT_TIME, 0, new RetryableThriftFunction<CompactionJob>() { + RetryableThriftCall.MAX_WAIT_TIME, 0, new RetryableThriftFunction<CompactionJob>() { @Override public CompactionJob execute() throws TException { - return coordinatorClient.getCompactionJob(queueName, compactorAddress); + try { + if (null == coordinatorClient.get()) { + coordinatorClient.set(getCoordinatorClient()); + } + return coordinatorClient.get().getCompactionJob(queueName, getHostPortString(compactorAddress.getAddress())); + } catch (TException e) { + coordinatorClient.set(null); + throw e; + } } }); return nextJobThriftCall.run(); } + + /** + * Get the client to the CompactionCoordinator + * + * @return compaction coordinator client + * @throws TTransportException when unable to get client + */ + private CompactionCoordinator.Client getCoordinatorClient() throws TTransportException { + HostAndPort coordinatorHost = ExternalCompactionUtil.findCompactionCoordinator(getContext()); + if (null == coordinatorHost) { + throw new TTransportException("Unable to get CompactionCoordinator address from ZooKeeper"); + } + LOG.info("CompactionCoordinator address is: {}", coordinatorHost); + return ThriftUtil.getClient(new CompactionCoordinator.Client.Factory(), + coordinatorHost, getContext()); + } + /** + * Create and return a new CompactionEnv for the current compaction job + * + * @param job current compaction job + * @return new env + */ + private CompactionEnv getCompactionEnvironment(CompactionJob job) { + return new CompactionEnv() { + @Override + public boolean isCompactionEnabled() { + return !jobHolder.isCancelled(); + } + + @Override + public IteratorScope getIteratorScope() { + return IteratorScope.majc; + } + + @Override + public RateLimiter getReadLimiter() { + return SharedRateLimiterFactory.getInstance(getContext().getConfiguration()) + .create("read_rate_limiter", () -> job.getReadRate()); + } + + @Override + public RateLimiter getWriteLimiter() { + return SharedRateLimiterFactory.getInstance(getContext().getConfiguration()) + .create("write_rate_limiter", () -> job.getWriteRate()); + } + + @Override + public SystemIteratorEnvironment createIteratorEnv(ServerContext context, + AccumuloConfiguration acuTableConf, TableId tableId) { + return new TabletIteratorEnvironment(getContext(), IteratorScope.majc, + !job.isPropagateDeletes(), acuTableConf, tableId, + CompactionKind.valueOf(job.getKind().name())); + } + + @Override + public SortedKeyValueIterator<Key,Value> getMinCIterator() { + throw new UnsupportedOperationException(); + } + + @Override + public CompactionReason getReason() { + switch (job.getKind()) { + case USER: + return CompactionReason.USER; + case CHOP: + return CompactionReason.CHOP; + case SELECTOR: + case SYSTEM: + default: + return CompactionReason.SYSTEM; + } + } + }; + } + @Override public void run() { - ServerAddress compactorAddress = null; try { compactorAddress = startCompactorClientService(); } catch (UnknownHostException e1) { @@ -393,19 +490,6 @@ public class Compactor extends AbstractServer throw new RuntimeException("Erroring registering in ZooKeeper", e); } - HostAndPort coordinatorHost = getCoordinatorAddress(); - if (null == coordinatorHost) { - throw new RuntimeException("Unable to get CompactionCoordinator address from ZooKeeper"); - } - LOG.info("CompactionCoordinator address is: {}", coordinatorHost); - CompactionCoordinator.Client coordinatorClient; - try { - coordinatorClient = ThriftUtil.getClient(new CompactionCoordinator.Client.Factory(), - coordinatorHost, getContext()); - } catch (TTransportException e2) { - throw new RuntimeException("Erroring connecting to CompactionCoordinator", e2); - } - LOG.info("Compactor started, waiting for work"); try { @@ -414,8 +498,8 @@ public class Compactor extends AbstractServer while (true) { err.set(null); jobHolder.reset(); - final CompactionJob job = getNextJob(coordinatorClient, getHostPortString(clientAddress)); - + + final CompactionJob job = getNextJob(); LOG.info("Received next compaction job: {}", job); final LongAdder totalInputSize = new LongAdder(); @@ -425,12 +509,12 @@ public class Compactor extends AbstractServer Thread compactionThread = Threads.createThread( "Compaction job for tablet " + job.getExtent().toString(), new Runnable() { + @Override public void run() { try { LOG.info("Setting up to run compactor"); - updateCompactionState(coordinatorClient, job, CompactionState.STARTED, - "Compaction started"); + updateCompactionState(job, CompactionState.STARTED, "Compaction started"); final TableId tableId = TableId.of(new String(job.getExtent().getTable(), UTF_8)); final TableConfiguration tConfig = getContext().getTableConfiguration(tableId); @@ -445,56 +529,7 @@ public class Compactor extends AbstractServer final TabletFile outputFile = new TabletFile(new Path(job.getOutputFile())); - final CompactionEnv cenv = new CompactionEnv() { - @Override - public boolean isCompactionEnabled() { - return !jobHolder.isCancelled(); - } - - @Override - public IteratorScope getIteratorScope() { - return IteratorScope.majc; - } - - @Override - public RateLimiter getReadLimiter() { - return SharedRateLimiterFactory.getInstance(getContext().getConfiguration()) - .create("read_rate_limiter", () -> job.getReadRate()); - } - - @Override - public RateLimiter getWriteLimiter() { - return SharedRateLimiterFactory.getInstance(getContext().getConfiguration()) - .create("write_rate_limiter", () -> job.getWriteRate()); - } - - @Override - public SystemIteratorEnvironment createIteratorEnv(ServerContext context, - AccumuloConfiguration acuTableConf, TableId tableId) { - return new TabletIteratorEnvironment(getContext(), IteratorScope.majc, - !job.isPropagateDeletes(), acuTableConf, tableId, - CompactionKind.valueOf(job.getKind().name())); - } - - @Override - public SortedKeyValueIterator<Key,Value> getMinCIterator() { - throw new UnsupportedOperationException(); - } - - @Override - public CompactionReason getReason() { - switch (job.getKind()) { - case USER: - return CompactionReason.USER; - case CHOP: - return CompactionReason.CHOP; - case SELECTOR: - case SYSTEM: - default: - return CompactionReason.SYSTEM; - } - } - }; + final CompactionEnv cenv = getCompactionEnvironment(job); final List<IteratorSetting> iters = new ArrayList<>(); job.getIteratorSettings().getIterators() @@ -507,12 +542,16 @@ public class Compactor extends AbstractServer LOG.info("Starting compactor"); started.countDown(); - jobHolder.setStats(compactor.call()); - + + org.apache.accumulo.server.compaction.CompactionStats stat = compactor.call(); + CompactionStats cs = new CompactionStats(); + cs.setEntriesRead(stat.getEntriesRead()); + cs.setEntriesWritten(stat.getEntriesWritten()); + cs.setFileSize(stat.getFileSize()); + jobHolder.setStats(cs); LOG.info("Compaction completed successfully"); // Update state when completed - updateCompactionState(coordinatorClient, job, CompactionState.SUCCEEDED, - "Compaction completed successfully"); + updateCompactionState(job, CompactionState.SUCCEEDED, "Compaction completed successfully"); } catch (Exception e) { LOG.error("Compaction failed", e); err.set(e); @@ -544,16 +583,14 @@ public class Compactor extends AbstractServer info.getEntriesRead(), inputEntries, (info.getEntriesRead() / inputEntries) * 100, info.getEntriesWritten()); LOG.info(message); - updateCompactionState(coordinatorClient, job, CompactionState.IN_PROGRESS, message); + updateCompactionState(job, CompactionState.IN_PROGRESS, message); } } - UtilWaitThread.sleep(MAX_WAIT_TIME); + UtilWaitThread.sleep(60000); } try { compactionThread.join(); - CompactionStats stats = jobHolder.getStats(); - // TODO: Tell coordinator that we are finished, send stats. - + this.updateCompactionCompleted(job, jobHolder.getStats()); } catch (InterruptedException e) { LOG.error( "Compactor thread was interrupted waiting for compaction to finish, cancelling job", @@ -570,20 +607,20 @@ public class Compactor extends AbstractServer if (compactionThread.isInterrupted()) { LOG.warn("Compaction thread was interrupted, sending CANCELLED state"); - updateCompactionState(coordinatorClient, job, CompactionState.CANCELLED, - "Compaction cancelled"); + updateCompactionState(job, CompactionState.CANCELLED, "Compaction cancelled"); } Throwable thrown = err.get(); if (thrown != null) { - updateCompactionState(coordinatorClient, job, CompactionState.FAILED, - "Compaction failed due to: " + thrown.getMessage()); + updateCompactionState(job, CompactionState.FAILED, "Compaction failed due to: " + thrown.getMessage()); } } } finally { // close connection to coordinator - ThriftUtil.returnClient(coordinatorClient); + if (null != coordinatorClient.get()) { + ThriftUtil.returnClient(coordinatorClient.get()); + } // Shutdown local thrift server LOG.debug("Stopping Thrift Servers");