This is an automated email from the ASF dual-hosted git repository. dlmarion pushed a commit to branch 1451-external-compactions-feature in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/1451-external-compactions-feature by this push: new 347e1a9 re #2102 - modified Thrift definition, TServer, and Coordinator to pass extent on compaction completion (success or fail) 347e1a9 is described below commit 347e1a926f7ff555bab68be0ae45a826a4c5d9e0 Author: Dave Marion <dlmar...@apache.org> AuthorDate: Tue Apr 13 12:15:49 2021 +0000 re #2102 - modified Thrift definition, TServer, and Coordinator to pass extent on compaction completion (success or fail) --- .../tabletserver/thrift/TabletClientService.java | 302 ++++++++++++++++++--- core/src/main/thrift/tabletserver.thrift | 6 +- .../accumulo/coordinator/CompactionFinalizer.java | 5 +- .../accumulo/tserver/ThriftClientHandler.java | 8 +- .../tserver/compactions/CompactionManager.java | 10 +- .../accumulo/test/performance/NullTserver.java | 4 +- 6 files changed, 286 insertions(+), 49 deletions(-) diff --git a/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/TabletClientService.java b/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/TabletClientService.java index a2d6eab..767d016 100644 --- a/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/TabletClientService.java +++ b/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/TabletClientService.java @@ -105,9 +105,9 @@ public class TabletClientService { public TExternalCompactionJob reserveCompactionJob(org.apache.accumulo.core.trace.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String queueName, long priority, java.lang.String compactor, java.lang.String externalCompactionId) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.thrift.TException; - public void compactionJobFinished(org.apache.accumulo.core.trace.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, long fileSize, long entries) throws org.apache.thrift.TException; + public void compactionJobFinished(org.apache.accumulo.core.trace.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent, long fileSize, long entries) throws org.apache.thrift.TException; - public void compactionJobFailed(org.apache.accumulo.core.trace.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId) throws org.apache.thrift.TException; + public void compactionJobFailed(org.apache.accumulo.core.trace.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent) throws org.apache.thrift.TException; public boolean isRunningExternalCompaction(org.apache.accumulo.core.trace.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent) throws org.apache.thrift.TException; @@ -191,9 +191,9 @@ public class TabletClientService { public void reserveCompactionJob(org.apache.accumulo.core.trace.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String queueName, long priority, java.lang.String compactor, java.lang.String externalCompactionId, org.apache.thrift.async.AsyncMethodCallback<TExternalCompactionJob> resultHandler) throws org.apache.thrift.TException; - public void compactionJobFinished(org.apache.accumulo.core.trace.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, long fileSize, long entries, org.apache.thrift.async.AsyncMethodCallback<Void> resultHandler) throws org.apache.thrift.TException; + public void compactionJobFinished(org.apache.accumulo.core.trace.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent, long fileSize, long entries, org.apache.thrift.async.AsyncMethodCallback<Void> resultHandler) throws org.apache.thrift.TException; - public void compactionJobFailed(org.apache.accumulo.core.trace.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.thrift.async.AsyncMethodCallback<Void> resultHandler) throws org.apache.thrift.TException; + public void compactionJobFailed(org.apache.accumulo.core.trace.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent, org.apache.thrift.async.AsyncMethodCallback<Void> resultHandler) throws org.apache.thrift.TException; public void isRunningExternalCompaction(org.apache.accumulo.core.trace.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent, org.apache.thrift.async.AsyncMethodCallback<java.lang.Boolean> resultHandler) throws org.apache.thrift.TException; @@ -1169,33 +1169,35 @@ public class TabletClientService { throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "reserveCompactionJob failed: unknown result"); } - public void compactionJobFinished(org.apache.accumulo.core.trace.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, long fileSize, long entries) throws org.apache.thrift.TException + public void compactionJobFinished(org.apache.accumulo.core.trace.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent, long fileSize, long entries) throws org.apache.thrift.TException { - send_compactionJobFinished(tinfo, credentials, externalCompactionId, fileSize, entries); + send_compactionJobFinished(tinfo, credentials, externalCompactionId, extent, fileSize, entries); } - public void send_compactionJobFinished(org.apache.accumulo.core.trace.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, long fileSize, long entries) throws org.apache.thrift.TException + public void send_compactionJobFinished(org.apache.accumulo.core.trace.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent, long fileSize, long entries) throws org.apache.thrift.TException { compactionJobFinished_args args = new compactionJobFinished_args(); args.setTinfo(tinfo); args.setCredentials(credentials); args.setExternalCompactionId(externalCompactionId); + args.setExtent(extent); args.setFileSize(fileSize); args.setEntries(entries); sendBaseOneway("compactionJobFinished", args); } - public void compactionJobFailed(org.apache.accumulo.core.trace.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId) throws org.apache.thrift.TException + public void compactionJobFailed(org.apache.accumulo.core.trace.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent) throws org.apache.thrift.TException { - send_compactionJobFailed(tinfo, credentials, externalCompactionId); + send_compactionJobFailed(tinfo, credentials, externalCompactionId, extent); } - public void send_compactionJobFailed(org.apache.accumulo.core.trace.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId) throws org.apache.thrift.TException + public void send_compactionJobFailed(org.apache.accumulo.core.trace.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent) throws org.apache.thrift.TException { compactionJobFailed_args args = new compactionJobFailed_args(); args.setTinfo(tinfo); args.setCredentials(credentials); args.setExternalCompactionId(externalCompactionId); + args.setExtent(extent); sendBaseOneway("compactionJobFailed", args); } @@ -2804,9 +2806,9 @@ public class TabletClientService { } } - public void compactionJobFinished(org.apache.accumulo.core.trace.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, long fileSize, long entries, org.apache.thrift.async.AsyncMethodCallback<Void> resultHandler) throws org.apache.thrift.TException { + public void compactionJobFinished(org.apache.accumulo.core.trace.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent, long fileSize, long entries, org.apache.thrift.async.AsyncMethodCallback<Void> resultHandler) throws org.apache.thrift.TException { checkReady(); - compactionJobFinished_call method_call = new compactionJobFinished_call(tinfo, credentials, externalCompactionId, fileSize, entries, resultHandler, this, ___protocolFactory, ___transport); + compactionJobFinished_call method_call = new compactionJobFinished_call(tinfo, credentials, externalCompactionId, extent, fileSize, entries, resultHandler, this, ___protocolFactory, ___transport); this.___currentMethod = method_call; ___manager.call(method_call); } @@ -2815,13 +2817,15 @@ public class TabletClientService { private org.apache.accumulo.core.trace.thrift.TInfo tinfo; private org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials; private java.lang.String externalCompactionId; + private org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent; private long fileSize; private long entries; - public compactionJobFinished_call(org.apache.accumulo.core.trace.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, long fileSize, long entries, org.apache.thrift.async.AsyncMethodCallback<Void> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + public compactionJobFinished_call(org.apache.accumulo.core.trace.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent, long fileSize, long entries, org.apache.thrift.async.AsyncMethodCallback<Void> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNo [...] super(client, protocolFactory, transport, resultHandler, true); this.tinfo = tinfo; this.credentials = credentials; this.externalCompactionId = externalCompactionId; + this.extent = extent; this.fileSize = fileSize; this.entries = entries; } @@ -2832,6 +2836,7 @@ public class TabletClientService { args.setTinfo(tinfo); args.setCredentials(credentials); args.setExternalCompactionId(externalCompactionId); + args.setExtent(extent); args.setFileSize(fileSize); args.setEntries(entries); args.write(prot); @@ -2848,9 +2853,9 @@ public class TabletClientService { } } - public void compactionJobFailed(org.apache.accumulo.core.trace.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.thrift.async.AsyncMethodCallback<Void> resultHandler) throws org.apache.thrift.TException { + public void compactionJobFailed(org.apache.accumulo.core.trace.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent, org.apache.thrift.async.AsyncMethodCallback<Void> resultHandler) throws org.apache.thrift.TException { checkReady(); - compactionJobFailed_call method_call = new compactionJobFailed_call(tinfo, credentials, externalCompactionId, resultHandler, this, ___protocolFactory, ___transport); + compactionJobFailed_call method_call = new compactionJobFailed_call(tinfo, credentials, externalCompactionId, extent, resultHandler, this, ___protocolFactory, ___transport); this.___currentMethod = method_call; ___manager.call(method_call); } @@ -2859,11 +2864,13 @@ public class TabletClientService { private org.apache.accumulo.core.trace.thrift.TInfo tinfo; private org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials; private java.lang.String externalCompactionId; - public compactionJobFailed_call(org.apache.accumulo.core.trace.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.thrift.async.AsyncMethodCallback<Void> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + private org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent; + public compactionJobFailed_call(org.apache.accumulo.core.trace.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent, org.apache.thrift.async.AsyncMethodCallback<Void> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) t [...] super(client, protocolFactory, transport, resultHandler, true); this.tinfo = tinfo; this.credentials = credentials; this.externalCompactionId = externalCompactionId; + this.extent = extent; } public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { @@ -2872,6 +2879,7 @@ public class TabletClientService { args.setTinfo(tinfo); args.setCredentials(credentials); args.setExternalCompactionId(externalCompactionId); + args.setExtent(extent); args.write(prot); prot.writeMessageEnd(); } @@ -4062,7 +4070,7 @@ public class TabletClientService { } public org.apache.thrift.TBase getResult(I iface, compactionJobFinished_args args) throws org.apache.thrift.TException { - iface.compactionJobFinished(args.tinfo, args.credentials, args.externalCompactionId, args.fileSize, args.entries); + iface.compactionJobFinished(args.tinfo, args.credentials, args.externalCompactionId, args.extent, args.fileSize, args.entries); return null; } } @@ -4086,7 +4094,7 @@ public class TabletClientService { } public org.apache.thrift.TBase getResult(I iface, compactionJobFailed_args args) throws org.apache.thrift.TException { - iface.compactionJobFailed(args.tinfo, args.credentials, args.externalCompactionId); + iface.compactionJobFailed(args.tinfo, args.credentials, args.externalCompactionId, args.extent); return null; } } @@ -6338,7 +6346,7 @@ public class TabletClientService { } public void start(I iface, compactionJobFinished_args args, org.apache.thrift.async.AsyncMethodCallback<Void> resultHandler) throws org.apache.thrift.TException { - iface.compactionJobFinished(args.tinfo, args.credentials, args.externalCompactionId, args.fileSize, args.entries,resultHandler); + iface.compactionJobFinished(args.tinfo, args.credentials, args.externalCompactionId, args.extent, args.fileSize, args.entries,resultHandler); } } @@ -6372,7 +6380,7 @@ public class TabletClientService { } public void start(I iface, compactionJobFailed_args args, org.apache.thrift.async.AsyncMethodCallback<Void> resultHandler) throws org.apache.thrift.TException { - iface.compactionJobFailed(args.tinfo, args.credentials, args.externalCompactionId,resultHandler); + iface.compactionJobFailed(args.tinfo, args.credentials, args.externalCompactionId, args.extent,resultHandler); } } @@ -47515,8 +47523,9 @@ public class TabletClientService { private static final org.apache.thrift.protocol.TField TINFO_FIELD_DESC = new org.apache.thrift.protocol.TField("tinfo", org.apache.thrift.protocol.TType.STRUCT, (short)1); private static final org.apache.thrift.protocol.TField CREDENTIALS_FIELD_DESC = new org.apache.thrift.protocol.TField("credentials", org.apache.thrift.protocol.TType.STRUCT, (short)2); private static final org.apache.thrift.protocol.TField EXTERNAL_COMPACTION_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("externalCompactionId", org.apache.thrift.protocol.TType.STRING, (short)3); - private static final org.apache.thrift.protocol.TField FILE_SIZE_FIELD_DESC = new org.apache.thrift.protocol.TField("fileSize", org.apache.thrift.protocol.TType.I64, (short)4); - private static final org.apache.thrift.protocol.TField ENTRIES_FIELD_DESC = new org.apache.thrift.protocol.TField("entries", org.apache.thrift.protocol.TType.I64, (short)5); + private static final org.apache.thrift.protocol.TField EXTENT_FIELD_DESC = new org.apache.thrift.protocol.TField("extent", org.apache.thrift.protocol.TType.STRUCT, (short)4); + private static final org.apache.thrift.protocol.TField FILE_SIZE_FIELD_DESC = new org.apache.thrift.protocol.TField("fileSize", org.apache.thrift.protocol.TType.I64, (short)5); + private static final org.apache.thrift.protocol.TField ENTRIES_FIELD_DESC = new org.apache.thrift.protocol.TField("entries", org.apache.thrift.protocol.TType.I64, (short)6); private static final org.apache.thrift.scheme.SchemeFactory STANDARD_SCHEME_FACTORY = new compactionJobFinished_argsStandardSchemeFactory(); private static final org.apache.thrift.scheme.SchemeFactory TUPLE_SCHEME_FACTORY = new compactionJobFinished_argsTupleSchemeFactory(); @@ -47524,6 +47533,7 @@ public class TabletClientService { public @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.trace.thrift.TInfo tinfo; // required public @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials; // required public @org.apache.thrift.annotation.Nullable java.lang.String externalCompactionId; // required + public @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent; // required public long fileSize; // required public long entries; // required @@ -47532,8 +47542,9 @@ public class TabletClientService { TINFO((short)1, "tinfo"), CREDENTIALS((short)2, "credentials"), EXTERNAL_COMPACTION_ID((short)3, "externalCompactionId"), - FILE_SIZE((short)4, "fileSize"), - ENTRIES((short)5, "entries"); + EXTENT((short)4, "extent"), + FILE_SIZE((short)5, "fileSize"), + ENTRIES((short)6, "entries"); private static final java.util.Map<java.lang.String, _Fields> byName = new java.util.HashMap<java.lang.String, _Fields>(); @@ -47555,9 +47566,11 @@ public class TabletClientService { return CREDENTIALS; case 3: // EXTERNAL_COMPACTION_ID return EXTERNAL_COMPACTION_ID; - case 4: // FILE_SIZE + case 4: // EXTENT + return EXTENT; + case 5: // FILE_SIZE return FILE_SIZE; - case 5: // ENTRIES + case 6: // ENTRIES return ENTRIES; default: return null; @@ -47612,6 +47625,8 @@ public class TabletClientService { new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.securityImpl.thrift.TCredentials.class))); tmpMap.put(_Fields.EXTERNAL_COMPACTION_ID, new org.apache.thrift.meta_data.FieldMetaData("externalCompactionId", org.apache.thrift.TFieldRequirementType.DEFAULT, new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + tmpMap.put(_Fields.EXTENT, new org.apache.thrift.meta_data.FieldMetaData("extent", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent.class))); tmpMap.put(_Fields.FILE_SIZE, new org.apache.thrift.meta_data.FieldMetaData("fileSize", org.apache.thrift.TFieldRequirementType.DEFAULT, new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))); tmpMap.put(_Fields.ENTRIES, new org.apache.thrift.meta_data.FieldMetaData("entries", org.apache.thrift.TFieldRequirementType.DEFAULT, @@ -47627,6 +47642,7 @@ public class TabletClientService { org.apache.accumulo.core.trace.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, + org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent, long fileSize, long entries) { @@ -47634,6 +47650,7 @@ public class TabletClientService { this.tinfo = tinfo; this.credentials = credentials; this.externalCompactionId = externalCompactionId; + this.extent = extent; this.fileSize = fileSize; setFileSizeIsSet(true); this.entries = entries; @@ -47654,6 +47671,9 @@ public class TabletClientService { if (other.isSetExternalCompactionId()) { this.externalCompactionId = other.externalCompactionId; } + if (other.isSetExtent()) { + this.extent = new org.apache.accumulo.core.dataImpl.thrift.TKeyExtent(other.extent); + } this.fileSize = other.fileSize; this.entries = other.entries; } @@ -47667,6 +47687,7 @@ public class TabletClientService { this.tinfo = null; this.credentials = null; this.externalCompactionId = null; + this.extent = null; setFileSizeIsSet(false); this.fileSize = 0; setEntriesIsSet(false); @@ -47748,6 +47769,31 @@ public class TabletClientService { } } + @org.apache.thrift.annotation.Nullable + public org.apache.accumulo.core.dataImpl.thrift.TKeyExtent getExtent() { + return this.extent; + } + + public compactionJobFinished_args setExtent(@org.apache.thrift.annotation.Nullable org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent) { + this.extent = extent; + return this; + } + + public void unsetExtent() { + this.extent = null; + } + + /** Returns true if field extent is set (has been assigned a value) and false otherwise */ + public boolean isSetExtent() { + return this.extent != null; + } + + public void setExtentIsSet(boolean value) { + if (!value) { + this.extent = null; + } + } + public long getFileSize() { return this.fileSize; } @@ -47820,6 +47866,14 @@ public class TabletClientService { } break; + case EXTENT: + if (value == null) { + unsetExtent(); + } else { + setExtent((org.apache.accumulo.core.dataImpl.thrift.TKeyExtent)value); + } + break; + case FILE_SIZE: if (value == null) { unsetFileSize(); @@ -47851,6 +47905,9 @@ public class TabletClientService { case EXTERNAL_COMPACTION_ID: return getExternalCompactionId(); + case EXTENT: + return getExtent(); + case FILE_SIZE: return getFileSize(); @@ -47874,6 +47931,8 @@ public class TabletClientService { return isSetCredentials(); case EXTERNAL_COMPACTION_ID: return isSetExternalCompactionId(); + case EXTENT: + return isSetExtent(); case FILE_SIZE: return isSetFileSize(); case ENTRIES: @@ -47924,6 +47983,15 @@ public class TabletClientService { return false; } + boolean this_present_extent = true && this.isSetExtent(); + boolean that_present_extent = true && that.isSetExtent(); + if (this_present_extent || that_present_extent) { + if (!(this_present_extent && that_present_extent)) + return false; + if (!this.extent.equals(that.extent)) + return false; + } + boolean this_present_fileSize = true; boolean that_present_fileSize = true; if (this_present_fileSize || that_present_fileSize) { @@ -47961,6 +48029,10 @@ public class TabletClientService { if (isSetExternalCompactionId()) hashCode = hashCode * 8191 + externalCompactionId.hashCode(); + hashCode = hashCode * 8191 + ((isSetExtent()) ? 131071 : 524287); + if (isSetExtent()) + hashCode = hashCode * 8191 + extent.hashCode(); + hashCode = hashCode * 8191 + org.apache.thrift.TBaseHelper.hashCode(fileSize); hashCode = hashCode * 8191 + org.apache.thrift.TBaseHelper.hashCode(entries); @@ -48006,6 +48078,16 @@ public class TabletClientService { return lastComparison; } } + lastComparison = java.lang.Boolean.valueOf(isSetExtent()).compareTo(other.isSetExtent()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetExtent()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.extent, other.extent); + if (lastComparison != 0) { + return lastComparison; + } + } lastComparison = java.lang.Boolean.valueOf(isSetFileSize()).compareTo(other.isSetFileSize()); if (lastComparison != 0) { return lastComparison; @@ -48071,6 +48153,14 @@ public class TabletClientService { } first = false; if (!first) sb.append(", "); + sb.append("extent:"); + if (this.extent == null) { + sb.append("null"); + } else { + sb.append(this.extent); + } + first = false; + if (!first) sb.append(", "); sb.append("fileSize:"); sb.append(this.fileSize); first = false; @@ -48091,6 +48181,9 @@ public class TabletClientService { if (credentials != null) { credentials.validate(); } + if (extent != null) { + extent.validate(); + } } private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { @@ -48155,7 +48248,16 @@ public class TabletClientService { org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } break; - case 4: // FILE_SIZE + case 4: // EXTENT + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.extent = new org.apache.accumulo.core.dataImpl.thrift.TKeyExtent(); + struct.extent.read(iprot); + struct.setExtentIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 5: // FILE_SIZE if (schemeField.type == org.apache.thrift.protocol.TType.I64) { struct.fileSize = iprot.readI64(); struct.setFileSizeIsSet(true); @@ -48163,7 +48265,7 @@ public class TabletClientService { org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } break; - case 5: // ENTRIES + case 6: // ENTRIES if (schemeField.type == org.apache.thrift.protocol.TType.I64) { struct.entries = iprot.readI64(); struct.setEntriesIsSet(true); @@ -48201,6 +48303,11 @@ public class TabletClientService { oprot.writeString(struct.externalCompactionId); oprot.writeFieldEnd(); } + if (struct.extent != null) { + oprot.writeFieldBegin(EXTENT_FIELD_DESC); + struct.extent.write(oprot); + oprot.writeFieldEnd(); + } oprot.writeFieldBegin(FILE_SIZE_FIELD_DESC); oprot.writeI64(struct.fileSize); oprot.writeFieldEnd(); @@ -48234,13 +48341,16 @@ public class TabletClientService { if (struct.isSetExternalCompactionId()) { optionals.set(2); } - if (struct.isSetFileSize()) { + if (struct.isSetExtent()) { optionals.set(3); } - if (struct.isSetEntries()) { + if (struct.isSetFileSize()) { optionals.set(4); } - oprot.writeBitSet(optionals, 5); + if (struct.isSetEntries()) { + optionals.set(5); + } + oprot.writeBitSet(optionals, 6); if (struct.isSetTinfo()) { struct.tinfo.write(oprot); } @@ -48250,6 +48360,9 @@ public class TabletClientService { if (struct.isSetExternalCompactionId()) { oprot.writeString(struct.externalCompactionId); } + if (struct.isSetExtent()) { + struct.extent.write(oprot); + } if (struct.isSetFileSize()) { oprot.writeI64(struct.fileSize); } @@ -48261,7 +48374,7 @@ public class TabletClientService { @Override public void read(org.apache.thrift.protocol.TProtocol prot, compactionJobFinished_args struct) throws org.apache.thrift.TException { org.apache.thrift.protocol.TTupleProtocol iprot = (org.apache.thrift.protocol.TTupleProtocol) prot; - java.util.BitSet incoming = iprot.readBitSet(5); + java.util.BitSet incoming = iprot.readBitSet(6); if (incoming.get(0)) { struct.tinfo = new org.apache.accumulo.core.trace.thrift.TInfo(); struct.tinfo.read(iprot); @@ -48277,10 +48390,15 @@ public class TabletClientService { struct.setExternalCompactionIdIsSet(true); } if (incoming.get(3)) { + struct.extent = new org.apache.accumulo.core.dataImpl.thrift.TKeyExtent(); + struct.extent.read(iprot); + struct.setExtentIsSet(true); + } + if (incoming.get(4)) { struct.fileSize = iprot.readI64(); struct.setFileSizeIsSet(true); } - if (incoming.get(4)) { + if (incoming.get(5)) { struct.entries = iprot.readI64(); struct.setEntriesIsSet(true); } @@ -48298,6 +48416,7 @@ public class TabletClientService { private static final org.apache.thrift.protocol.TField TINFO_FIELD_DESC = new org.apache.thrift.protocol.TField("tinfo", org.apache.thrift.protocol.TType.STRUCT, (short)1); private static final org.apache.thrift.protocol.TField CREDENTIALS_FIELD_DESC = new org.apache.thrift.protocol.TField("credentials", org.apache.thrift.protocol.TType.STRUCT, (short)2); private static final org.apache.thrift.protocol.TField EXTERNAL_COMPACTION_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("externalCompactionId", org.apache.thrift.protocol.TType.STRING, (short)3); + private static final org.apache.thrift.protocol.TField EXTENT_FIELD_DESC = new org.apache.thrift.protocol.TField("extent", org.apache.thrift.protocol.TType.STRUCT, (short)4); private static final org.apache.thrift.scheme.SchemeFactory STANDARD_SCHEME_FACTORY = new compactionJobFailed_argsStandardSchemeFactory(); private static final org.apache.thrift.scheme.SchemeFactory TUPLE_SCHEME_FACTORY = new compactionJobFailed_argsTupleSchemeFactory(); @@ -48305,12 +48424,14 @@ public class TabletClientService { public @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.trace.thrift.TInfo tinfo; // required public @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials; // required public @org.apache.thrift.annotation.Nullable java.lang.String externalCompactionId; // required + public @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent; // required /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ public enum _Fields implements org.apache.thrift.TFieldIdEnum { TINFO((short)1, "tinfo"), CREDENTIALS((short)2, "credentials"), - EXTERNAL_COMPACTION_ID((short)3, "externalCompactionId"); + EXTERNAL_COMPACTION_ID((short)3, "externalCompactionId"), + EXTENT((short)4, "extent"); private static final java.util.Map<java.lang.String, _Fields> byName = new java.util.HashMap<java.lang.String, _Fields>(); @@ -48332,6 +48453,8 @@ public class TabletClientService { return CREDENTIALS; case 3: // EXTERNAL_COMPACTION_ID return EXTERNAL_COMPACTION_ID; + case 4: // EXTENT + return EXTENT; default: return null; } @@ -48382,6 +48505,8 @@ public class TabletClientService { new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.securityImpl.thrift.TCredentials.class))); tmpMap.put(_Fields.EXTERNAL_COMPACTION_ID, new org.apache.thrift.meta_data.FieldMetaData("externalCompactionId", org.apache.thrift.TFieldRequirementType.DEFAULT, new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + tmpMap.put(_Fields.EXTENT, new org.apache.thrift.meta_data.FieldMetaData("extent", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent.class))); metaDataMap = java.util.Collections.unmodifiableMap(tmpMap); org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(compactionJobFailed_args.class, metaDataMap); } @@ -48392,12 +48517,14 @@ public class TabletClientService { public compactionJobFailed_args( org.apache.accumulo.core.trace.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, - java.lang.String externalCompactionId) + java.lang.String externalCompactionId, + org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent) { this(); this.tinfo = tinfo; this.credentials = credentials; this.externalCompactionId = externalCompactionId; + this.extent = extent; } /** @@ -48413,6 +48540,9 @@ public class TabletClientService { if (other.isSetExternalCompactionId()) { this.externalCompactionId = other.externalCompactionId; } + if (other.isSetExtent()) { + this.extent = new org.apache.accumulo.core.dataImpl.thrift.TKeyExtent(other.extent); + } } public compactionJobFailed_args deepCopy() { @@ -48424,6 +48554,7 @@ public class TabletClientService { this.tinfo = null; this.credentials = null; this.externalCompactionId = null; + this.extent = null; } @org.apache.thrift.annotation.Nullable @@ -48501,6 +48632,31 @@ public class TabletClientService { } } + @org.apache.thrift.annotation.Nullable + public org.apache.accumulo.core.dataImpl.thrift.TKeyExtent getExtent() { + return this.extent; + } + + public compactionJobFailed_args setExtent(@org.apache.thrift.annotation.Nullable org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent) { + this.extent = extent; + return this; + } + + public void unsetExtent() { + this.extent = null; + } + + /** Returns true if field extent is set (has been assigned a value) and false otherwise */ + public boolean isSetExtent() { + return this.extent != null; + } + + public void setExtentIsSet(boolean value) { + if (!value) { + this.extent = null; + } + } + public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable java.lang.Object value) { switch (field) { case TINFO: @@ -48527,6 +48683,14 @@ public class TabletClientService { } break; + case EXTENT: + if (value == null) { + unsetExtent(); + } else { + setExtent((org.apache.accumulo.core.dataImpl.thrift.TKeyExtent)value); + } + break; + } } @@ -48542,6 +48706,9 @@ public class TabletClientService { case EXTERNAL_COMPACTION_ID: return getExternalCompactionId(); + case EXTENT: + return getExtent(); + } throw new java.lang.IllegalStateException(); } @@ -48559,6 +48726,8 @@ public class TabletClientService { return isSetCredentials(); case EXTERNAL_COMPACTION_ID: return isSetExternalCompactionId(); + case EXTENT: + return isSetExtent(); } throw new java.lang.IllegalStateException(); } @@ -48605,6 +48774,15 @@ public class TabletClientService { return false; } + boolean this_present_extent = true && this.isSetExtent(); + boolean that_present_extent = true && that.isSetExtent(); + if (this_present_extent || that_present_extent) { + if (!(this_present_extent && that_present_extent)) + return false; + if (!this.extent.equals(that.extent)) + return false; + } + return true; } @@ -48624,6 +48802,10 @@ public class TabletClientService { if (isSetExternalCompactionId()) hashCode = hashCode * 8191 + externalCompactionId.hashCode(); + hashCode = hashCode * 8191 + ((isSetExtent()) ? 131071 : 524287); + if (isSetExtent()) + hashCode = hashCode * 8191 + extent.hashCode(); + return hashCode; } @@ -48665,6 +48847,16 @@ public class TabletClientService { return lastComparison; } } + lastComparison = java.lang.Boolean.valueOf(isSetExtent()).compareTo(other.isSetExtent()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetExtent()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.extent, other.extent); + if (lastComparison != 0) { + return lastComparison; + } + } return 0; } @@ -48709,6 +48901,14 @@ public class TabletClientService { sb.append(this.externalCompactionId); } first = false; + if (!first) sb.append(", "); + sb.append("extent:"); + if (this.extent == null) { + sb.append("null"); + } else { + sb.append(this.extent); + } + first = false; sb.append(")"); return sb.toString(); } @@ -48722,6 +48922,9 @@ public class TabletClientService { if (credentials != null) { credentials.validate(); } + if (extent != null) { + extent.validate(); + } } private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { @@ -48784,6 +48987,15 @@ public class TabletClientService { org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } break; + case 4: // EXTENT + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.extent = new org.apache.accumulo.core.dataImpl.thrift.TKeyExtent(); + struct.extent.read(iprot); + struct.setExtentIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; default: org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } @@ -48814,6 +49026,11 @@ public class TabletClientService { oprot.writeString(struct.externalCompactionId); oprot.writeFieldEnd(); } + if (struct.extent != null) { + oprot.writeFieldBegin(EXTENT_FIELD_DESC); + struct.extent.write(oprot); + oprot.writeFieldEnd(); + } oprot.writeFieldStop(); oprot.writeStructEnd(); } @@ -48841,7 +49058,10 @@ public class TabletClientService { if (struct.isSetExternalCompactionId()) { optionals.set(2); } - oprot.writeBitSet(optionals, 3); + if (struct.isSetExtent()) { + optionals.set(3); + } + oprot.writeBitSet(optionals, 4); if (struct.isSetTinfo()) { struct.tinfo.write(oprot); } @@ -48851,12 +49071,15 @@ public class TabletClientService { if (struct.isSetExternalCompactionId()) { oprot.writeString(struct.externalCompactionId); } + if (struct.isSetExtent()) { + struct.extent.write(oprot); + } } @Override public void read(org.apache.thrift.protocol.TProtocol prot, compactionJobFailed_args struct) throws org.apache.thrift.TException { org.apache.thrift.protocol.TTupleProtocol iprot = (org.apache.thrift.protocol.TTupleProtocol) prot; - java.util.BitSet incoming = iprot.readBitSet(3); + java.util.BitSet incoming = iprot.readBitSet(4); if (incoming.get(0)) { struct.tinfo = new org.apache.accumulo.core.trace.thrift.TInfo(); struct.tinfo.read(iprot); @@ -48871,6 +49094,11 @@ public class TabletClientService { struct.externalCompactionId = iprot.readString(); struct.setExternalCompactionIdIsSet(true); } + if (incoming.get(3)) { + struct.extent = new org.apache.accumulo.core.dataImpl.thrift.TKeyExtent(); + struct.extent.read(iprot); + struct.setExtentIsSet(true); + } } } diff --git a/core/src/main/thrift/tabletserver.thrift b/core/src/main/thrift/tabletserver.thrift index 708962d..8c86917 100644 --- a/core/src/main/thrift/tabletserver.thrift +++ b/core/src/main/thrift/tabletserver.thrift @@ -540,14 +540,16 @@ service TabletClientService extends client.ClientService { 1:trace.TInfo tinfo 2:security.TCredentials credentials 3:string externalCompactionId - 4:i64 fileSize - 5:i64 entries + 4:data.TKeyExtent extent + 5:i64 fileSize + 6:i64 entries ) oneway void compactionJobFailed( 1:trace.TInfo tinfo 2:security.TCredentials credentials 3:string externalCompactionId + 4:data.TKeyExtent extent ) bool isRunningExternalCompaction( diff --git a/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionFinalizer.java b/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionFinalizer.java index 1b6c6b2..348ab43 100644 --- a/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionFinalizer.java +++ b/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionFinalizer.java @@ -114,11 +114,12 @@ public class CompactionFinalizer { if (ecfs.getFinalState() == FinalState.FINISHED) { LOG.info("Notifying tserver {} that compaction {} has finished.", loc, ecfs); client.compactionJobFinished(TraceUtil.traceInfo(), context.rpcCreds(), - ecfs.getExternalCompactionId().canonical(), ecfs.getFileSize(), ecfs.getEntries()); + ecfs.getExternalCompactionId().canonical(), ecfs.getExtent().toThrift(), + ecfs.getFileSize(), ecfs.getEntries()); } else if (ecfs.getFinalState() == FinalState.FAILED) { LOG.info("Notifying tserver {} that compaction {} has failed.", loc, ecfs); client.compactionJobFailed(TraceUtil.traceInfo(), context.rpcCreds(), - ecfs.getExternalCompactionId().canonical()); + ecfs.getExternalCompactionId().canonical(), ecfs.getExtent().toThrift()); } else { throw new IllegalArgumentException(ecfs.getFinalState().name()); } diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/ThriftClientHandler.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/ThriftClientHandler.java index 31e36e6..abb9968 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/ThriftClientHandler.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/ThriftClientHandler.java @@ -1694,7 +1694,7 @@ class ThriftClientHandler extends ClientServiceHandler implements TabletClientSe @Override public void compactionJobFinished(TInfo tinfo, TCredentials credentials, - String externalCompactionId, long fileSize, long entries) + String externalCompactionId, TKeyExtent extent, long fileSize, long entries) throws ThriftSecurityException, TException { if (!security.canPerformSystemActions(credentials)) { @@ -1703,20 +1703,20 @@ class ThriftClientHandler extends ClientServiceHandler implements TabletClientSe } server.getCompactionManager().commitExternalCompaction( - ExternalCompactionId.of(externalCompactionId), server.getOnlineTablets(), fileSize, + ExternalCompactionId.of(externalCompactionId), extent, server.getOnlineTablets(), fileSize, entries); } @Override public void compactionJobFailed(TInfo tinfo, TCredentials credentials, - String externalCompactionId) throws TException { + String externalCompactionId, TKeyExtent extent) throws TException { if (!security.canPerformSystemActions(credentials)) { throw new AccumuloSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED).asThriftException(); } server.getCompactionManager().externalCompactionFailed( - ExternalCompactionId.of(externalCompactionId), server.getOnlineTablets()); + ExternalCompactionId.of(externalCompactionId), extent, server.getOnlineTablets()); } @Override diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/compactions/CompactionManager.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/compactions/CompactionManager.java index e4a5596..44f0ccf 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/compactions/CompactionManager.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/compactions/CompactionManager.java @@ -33,6 +33,7 @@ import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.ConfigurationTypeHelper; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.dataImpl.KeyExtent; +import org.apache.accumulo.core.dataImpl.thrift.TKeyExtent; import org.apache.accumulo.core.metadata.schema.ExternalCompactionId; import org.apache.accumulo.core.spi.compaction.CompactionExecutorId; import org.apache.accumulo.core.spi.compaction.CompactionKind; @@ -446,8 +447,11 @@ public class CompactionManager { } public void commitExternalCompaction(ExternalCompactionId extCompactionId, - Map<KeyExtent,Tablet> currentTablets, long fileSize, long entries) { + TKeyExtent extentCompacted, Map<KeyExtent,Tablet> currentTablets, long fileSize, + long entries) { KeyExtent extent = runningExternalCompactions.get(extCompactionId); + // CBUG Use extentCompacted to perform additional validation that the extent has not + // merged, split, or otherwise changed. if (extent != null) { Tablet tablet = currentTablets.get(extent); if (tablet != null) { @@ -465,8 +469,10 @@ public class CompactionManager { return (null != extent && extent.compareTo(ke) == 0); } - public void externalCompactionFailed(ExternalCompactionId ecid, + public void externalCompactionFailed(ExternalCompactionId ecid, TKeyExtent extentCompacted, Map<KeyExtent,Tablet> currentTablets) { + // CBUG Use extentCompacted to perform additional validation that the extent has not + // merged, split, or otherwise changed. KeyExtent extent = runningExternalCompactions.get(ecid); if (extent != null) { Tablet tablet = currentTablets.get(extent); diff --git a/test/src/main/java/org/apache/accumulo/test/performance/NullTserver.java b/test/src/main/java/org/apache/accumulo/test/performance/NullTserver.java index 71d5f49..9abb4b0 100644 --- a/test/src/main/java/org/apache/accumulo/test/performance/NullTserver.java +++ b/test/src/main/java/org/apache/accumulo/test/performance/NullTserver.java @@ -298,7 +298,7 @@ public class NullTserver { @Override public void compactionJobFinished(TInfo tinfo, TCredentials credentials, - String externalCompactionId, long fileSize, long entries) + String externalCompactionId, TKeyExtent extent, long fileSize, long entries) throws ThriftSecurityException, TException { // TODO Auto-generated method stub @@ -306,7 +306,7 @@ public class NullTserver { @Override public void compactionJobFailed(TInfo tinfo, TCredentials credentials, - String externalCompactionId) throws TException { + String externalCompactionId, TKeyExtent extent) throws TException { // TODO Auto-generated method stub }