http://git-wip-us.apache.org/repos/asf/hbase-site/blob/47abd8e6/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.FlushResult.Result.html ---------------------------------------------------------------------- diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.FlushResult.Result.html b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.FlushResult.Result.html index 81d256e..73c1480 100644 --- a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.FlushResult.Result.html +++ b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/HRegion.FlushResult.Result.html @@ -2964,5322 +2964,5321 @@ <span class="sourceLineNo">2956</span> // This is expensive.<a name="line.2956"></a> <span class="sourceLineNo">2957</span> if (cell.getTimestamp() == HConstants.LATEST_TIMESTAMP && CellUtil.isDeleteType(cell)) {<a name="line.2957"></a> <span class="sourceLineNo">2958</span> byte[] qual = CellUtil.cloneQualifier(cell);<a name="line.2958"></a> -<span class="sourceLineNo">2959</span> if (qual == null) qual = HConstants.EMPTY_BYTE_ARRAY;<a name="line.2959"></a> -<span class="sourceLineNo">2960</span><a name="line.2960"></a> -<span class="sourceLineNo">2961</span> Integer count = kvCount.get(qual);<a name="line.2961"></a> -<span class="sourceLineNo">2962</span> if (count == null) {<a name="line.2962"></a> -<span class="sourceLineNo">2963</span> kvCount.put(qual, 1);<a name="line.2963"></a> -<span class="sourceLineNo">2964</span> } else {<a name="line.2964"></a> -<span class="sourceLineNo">2965</span> kvCount.put(qual, count + 1);<a name="line.2965"></a> -<span class="sourceLineNo">2966</span> }<a name="line.2966"></a> -<span class="sourceLineNo">2967</span> count = kvCount.get(qual);<a name="line.2967"></a> -<span class="sourceLineNo">2968</span><a name="line.2968"></a> -<span class="sourceLineNo">2969</span> Get get = new Get(CellUtil.cloneRow(cell));<a name="line.2969"></a> -<span class="sourceLineNo">2970</span> get.setMaxVersions(count);<a name="line.2970"></a> -<span class="sourceLineNo">2971</span> get.addColumn(family, qual);<a name="line.2971"></a> -<span class="sourceLineNo">2972</span> if (coprocessorHost != null) {<a name="line.2972"></a> -<span class="sourceLineNo">2973</span> if (!coprocessorHost.prePrepareTimeStampForDeleteVersion(mutation, cell,<a name="line.2973"></a> -<span class="sourceLineNo">2974</span> byteNow, get)) {<a name="line.2974"></a> -<span class="sourceLineNo">2975</span> updateDeleteLatestVersionTimeStamp(cell, get, count, byteNow);<a name="line.2975"></a> -<span class="sourceLineNo">2976</span> }<a name="line.2976"></a> -<span class="sourceLineNo">2977</span> } else {<a name="line.2977"></a> -<span class="sourceLineNo">2978</span> updateDeleteLatestVersionTimeStamp(cell, get, count, byteNow);<a name="line.2978"></a> -<span class="sourceLineNo">2979</span> }<a name="line.2979"></a> -<span class="sourceLineNo">2980</span> } else {<a name="line.2980"></a> -<span class="sourceLineNo">2981</span> CellUtil.updateLatestStamp(cell, byteNow, 0);<a name="line.2981"></a> -<span class="sourceLineNo">2982</span> }<a name="line.2982"></a> -<span class="sourceLineNo">2983</span> }<a name="line.2983"></a> -<span class="sourceLineNo">2984</span> }<a name="line.2984"></a> -<span class="sourceLineNo">2985</span> }<a name="line.2985"></a> -<span class="sourceLineNo">2986</span><a name="line.2986"></a> -<span class="sourceLineNo">2987</span> void updateDeleteLatestVersionTimeStamp(Cell cell, Get get, int count, byte[] byteNow)<a name="line.2987"></a> -<span class="sourceLineNo">2988</span> throws IOException {<a name="line.2988"></a> -<span class="sourceLineNo">2989</span> List<Cell> result = get(get, false);<a name="line.2989"></a> -<span class="sourceLineNo">2990</span><a name="line.2990"></a> -<span class="sourceLineNo">2991</span> if (result.size() < count) {<a name="line.2991"></a> -<span class="sourceLineNo">2992</span> // Nothing to delete<a name="line.2992"></a> -<span class="sourceLineNo">2993</span> CellUtil.updateLatestStamp(cell, byteNow, 0);<a name="line.2993"></a> -<span class="sourceLineNo">2994</span> return;<a name="line.2994"></a> -<span class="sourceLineNo">2995</span> }<a name="line.2995"></a> -<span class="sourceLineNo">2996</span> if (result.size() > count) {<a name="line.2996"></a> -<span class="sourceLineNo">2997</span> throw new RuntimeException("Unexpected size: " + result.size());<a name="line.2997"></a> -<span class="sourceLineNo">2998</span> }<a name="line.2998"></a> -<span class="sourceLineNo">2999</span> Cell getCell = result.get(count - 1);<a name="line.2999"></a> -<span class="sourceLineNo">3000</span> CellUtil.setTimestamp(cell, getCell.getTimestamp());<a name="line.3000"></a> -<span class="sourceLineNo">3001</span> }<a name="line.3001"></a> -<span class="sourceLineNo">3002</span><a name="line.3002"></a> -<span class="sourceLineNo">3003</span> @Override<a name="line.3003"></a> -<span class="sourceLineNo">3004</span> public void put(Put put) throws IOException {<a name="line.3004"></a> -<span class="sourceLineNo">3005</span> checkReadOnly();<a name="line.3005"></a> -<span class="sourceLineNo">3006</span><a name="line.3006"></a> -<span class="sourceLineNo">3007</span> // Do a rough check that we have resources to accept a write. The check is<a name="line.3007"></a> -<span class="sourceLineNo">3008</span> // 'rough' in that between the resource check and the call to obtain a<a name="line.3008"></a> -<span class="sourceLineNo">3009</span> // read lock, resources may run out. For now, the thought is that this<a name="line.3009"></a> -<span class="sourceLineNo">3010</span> // will be extremely rare; we'll deal with it when it happens.<a name="line.3010"></a> -<span class="sourceLineNo">3011</span> checkResources();<a name="line.3011"></a> -<span class="sourceLineNo">3012</span> startRegionOperation(Operation.PUT);<a name="line.3012"></a> -<span class="sourceLineNo">3013</span> try {<a name="line.3013"></a> -<span class="sourceLineNo">3014</span> // All edits for the given row (across all column families) must happen atomically.<a name="line.3014"></a> -<span class="sourceLineNo">3015</span> doBatchMutate(put);<a name="line.3015"></a> -<span class="sourceLineNo">3016</span> } finally {<a name="line.3016"></a> -<span class="sourceLineNo">3017</span> closeRegionOperation(Operation.PUT);<a name="line.3017"></a> -<span class="sourceLineNo">3018</span> }<a name="line.3018"></a> -<span class="sourceLineNo">3019</span> }<a name="line.3019"></a> -<span class="sourceLineNo">3020</span><a name="line.3020"></a> -<span class="sourceLineNo">3021</span> /**<a name="line.3021"></a> -<span class="sourceLineNo">3022</span> * Struct-like class that tracks the progress of a batch operation,<a name="line.3022"></a> -<span class="sourceLineNo">3023</span> * accumulating status codes and tracking the index at which processing<a name="line.3023"></a> -<span class="sourceLineNo">3024</span> * is proceeding.<a name="line.3024"></a> -<span class="sourceLineNo">3025</span> */<a name="line.3025"></a> -<span class="sourceLineNo">3026</span> private abstract static class BatchOperation<T> {<a name="line.3026"></a> -<span class="sourceLineNo">3027</span> T[] operations;<a name="line.3027"></a> -<span class="sourceLineNo">3028</span> int nextIndexToProcess = 0;<a name="line.3028"></a> -<span class="sourceLineNo">3029</span> OperationStatus[] retCodeDetails;<a name="line.3029"></a> -<span class="sourceLineNo">3030</span> WALEdit[] walEditsFromCoprocessors;<a name="line.3030"></a> -<span class="sourceLineNo">3031</span><a name="line.3031"></a> -<span class="sourceLineNo">3032</span> public BatchOperation(T[] operations) {<a name="line.3032"></a> -<span class="sourceLineNo">3033</span> this.operations = operations;<a name="line.3033"></a> -<span class="sourceLineNo">3034</span> this.retCodeDetails = new OperationStatus[operations.length];<a name="line.3034"></a> -<span class="sourceLineNo">3035</span> this.walEditsFromCoprocessors = new WALEdit[operations.length];<a name="line.3035"></a> -<span class="sourceLineNo">3036</span> Arrays.fill(this.retCodeDetails, OperationStatus.NOT_RUN);<a name="line.3036"></a> -<span class="sourceLineNo">3037</span> }<a name="line.3037"></a> -<span class="sourceLineNo">3038</span><a name="line.3038"></a> -<span class="sourceLineNo">3039</span> public abstract Mutation getMutation(int index);<a name="line.3039"></a> -<span class="sourceLineNo">3040</span> public abstract long getNonceGroup(int index);<a name="line.3040"></a> -<span class="sourceLineNo">3041</span> public abstract long getNonce(int index);<a name="line.3041"></a> -<span class="sourceLineNo">3042</span> /** This method is potentially expensive and should only be used for non-replay CP path. */<a name="line.3042"></a> -<span class="sourceLineNo">3043</span> public abstract Mutation[] getMutationsForCoprocs();<a name="line.3043"></a> -<span class="sourceLineNo">3044</span> public abstract boolean isInReplay();<a name="line.3044"></a> -<span class="sourceLineNo">3045</span> public abstract long getReplaySequenceId();<a name="line.3045"></a> -<span class="sourceLineNo">3046</span><a name="line.3046"></a> -<span class="sourceLineNo">3047</span> public boolean isDone() {<a name="line.3047"></a> -<span class="sourceLineNo">3048</span> return nextIndexToProcess == operations.length;<a name="line.3048"></a> -<span class="sourceLineNo">3049</span> }<a name="line.3049"></a> -<span class="sourceLineNo">3050</span> }<a name="line.3050"></a> -<span class="sourceLineNo">3051</span><a name="line.3051"></a> -<span class="sourceLineNo">3052</span> private static class MutationBatch extends BatchOperation<Mutation> {<a name="line.3052"></a> -<span class="sourceLineNo">3053</span> private long nonceGroup;<a name="line.3053"></a> -<span class="sourceLineNo">3054</span> private long nonce;<a name="line.3054"></a> -<span class="sourceLineNo">3055</span> public MutationBatch(Mutation[] operations, long nonceGroup, long nonce) {<a name="line.3055"></a> -<span class="sourceLineNo">3056</span> super(operations);<a name="line.3056"></a> -<span class="sourceLineNo">3057</span> this.nonceGroup = nonceGroup;<a name="line.3057"></a> -<span class="sourceLineNo">3058</span> this.nonce = nonce;<a name="line.3058"></a> -<span class="sourceLineNo">3059</span> }<a name="line.3059"></a> -<span class="sourceLineNo">3060</span><a name="line.3060"></a> -<span class="sourceLineNo">3061</span> @Override<a name="line.3061"></a> -<span class="sourceLineNo">3062</span> public Mutation getMutation(int index) {<a name="line.3062"></a> -<span class="sourceLineNo">3063</span> return this.operations[index];<a name="line.3063"></a> -<span class="sourceLineNo">3064</span> }<a name="line.3064"></a> -<span class="sourceLineNo">3065</span><a name="line.3065"></a> -<span class="sourceLineNo">3066</span> @Override<a name="line.3066"></a> -<span class="sourceLineNo">3067</span> public long getNonceGroup(int index) {<a name="line.3067"></a> -<span class="sourceLineNo">3068</span> return nonceGroup;<a name="line.3068"></a> -<span class="sourceLineNo">3069</span> }<a name="line.3069"></a> -<span class="sourceLineNo">3070</span><a name="line.3070"></a> -<span class="sourceLineNo">3071</span> @Override<a name="line.3071"></a> -<span class="sourceLineNo">3072</span> public long getNonce(int index) {<a name="line.3072"></a> -<span class="sourceLineNo">3073</span> return nonce;<a name="line.3073"></a> -<span class="sourceLineNo">3074</span> }<a name="line.3074"></a> -<span class="sourceLineNo">3075</span><a name="line.3075"></a> -<span class="sourceLineNo">3076</span> @Override<a name="line.3076"></a> -<span class="sourceLineNo">3077</span> public Mutation[] getMutationsForCoprocs() {<a name="line.3077"></a> -<span class="sourceLineNo">3078</span> return this.operations;<a name="line.3078"></a> -<span class="sourceLineNo">3079</span> }<a name="line.3079"></a> -<span class="sourceLineNo">3080</span><a name="line.3080"></a> -<span class="sourceLineNo">3081</span> @Override<a name="line.3081"></a> -<span class="sourceLineNo">3082</span> public boolean isInReplay() {<a name="line.3082"></a> -<span class="sourceLineNo">3083</span> return false;<a name="line.3083"></a> -<span class="sourceLineNo">3084</span> }<a name="line.3084"></a> -<span class="sourceLineNo">3085</span><a name="line.3085"></a> -<span class="sourceLineNo">3086</span> @Override<a name="line.3086"></a> -<span class="sourceLineNo">3087</span> public long getReplaySequenceId() {<a name="line.3087"></a> -<span class="sourceLineNo">3088</span> return 0;<a name="line.3088"></a> -<span class="sourceLineNo">3089</span> }<a name="line.3089"></a> -<span class="sourceLineNo">3090</span> }<a name="line.3090"></a> -<span class="sourceLineNo">3091</span><a name="line.3091"></a> -<span class="sourceLineNo">3092</span> private static class ReplayBatch extends BatchOperation<MutationReplay> {<a name="line.3092"></a> -<span class="sourceLineNo">3093</span> private long replaySeqId = 0;<a name="line.3093"></a> -<span class="sourceLineNo">3094</span> public ReplayBatch(MutationReplay[] operations, long seqId) {<a name="line.3094"></a> -<span class="sourceLineNo">3095</span> super(operations);<a name="line.3095"></a> -<span class="sourceLineNo">3096</span> this.replaySeqId = seqId;<a name="line.3096"></a> -<span class="sourceLineNo">3097</span> }<a name="line.3097"></a> -<span class="sourceLineNo">3098</span><a name="line.3098"></a> -<span class="sourceLineNo">3099</span> @Override<a name="line.3099"></a> -<span class="sourceLineNo">3100</span> public Mutation getMutation(int index) {<a name="line.3100"></a> -<span class="sourceLineNo">3101</span> return this.operations[index].mutation;<a name="line.3101"></a> -<span class="sourceLineNo">3102</span> }<a name="line.3102"></a> -<span class="sourceLineNo">3103</span><a name="line.3103"></a> -<span class="sourceLineNo">3104</span> @Override<a name="line.3104"></a> -<span class="sourceLineNo">3105</span> public long getNonceGroup(int index) {<a name="line.3105"></a> -<span class="sourceLineNo">3106</span> return this.operations[index].nonceGroup;<a name="line.3106"></a> -<span class="sourceLineNo">3107</span> }<a name="line.3107"></a> -<span class="sourceLineNo">3108</span><a name="line.3108"></a> -<span class="sourceLineNo">3109</span> @Override<a name="line.3109"></a> -<span class="sourceLineNo">3110</span> public long getNonce(int index) {<a name="line.3110"></a> -<span class="sourceLineNo">3111</span> return this.operations[index].nonce;<a name="line.3111"></a> -<span class="sourceLineNo">3112</span> }<a name="line.3112"></a> -<span class="sourceLineNo">3113</span><a name="line.3113"></a> -<span class="sourceLineNo">3114</span> @Override<a name="line.3114"></a> -<span class="sourceLineNo">3115</span> public Mutation[] getMutationsForCoprocs() {<a name="line.3115"></a> -<span class="sourceLineNo">3116</span> assert false;<a name="line.3116"></a> -<span class="sourceLineNo">3117</span> throw new RuntimeException("Should not be called for replay batch");<a name="line.3117"></a> -<span class="sourceLineNo">3118</span> }<a name="line.3118"></a> -<span class="sourceLineNo">3119</span><a name="line.3119"></a> -<span class="sourceLineNo">3120</span> @Override<a name="line.3120"></a> -<span class="sourceLineNo">3121</span> public boolean isInReplay() {<a name="line.3121"></a> -<span class="sourceLineNo">3122</span> return true;<a name="line.3122"></a> -<span class="sourceLineNo">3123</span> }<a name="line.3123"></a> -<span class="sourceLineNo">3124</span><a name="line.3124"></a> -<span class="sourceLineNo">3125</span> @Override<a name="line.3125"></a> -<span class="sourceLineNo">3126</span> public long getReplaySequenceId() {<a name="line.3126"></a> -<span class="sourceLineNo">3127</span> return this.replaySeqId;<a name="line.3127"></a> -<span class="sourceLineNo">3128</span> }<a name="line.3128"></a> -<span class="sourceLineNo">3129</span> }<a name="line.3129"></a> -<span class="sourceLineNo">3130</span><a name="line.3130"></a> -<span class="sourceLineNo">3131</span> public OperationStatus[] batchMutate(Mutation[] mutations, long nonceGroup, long nonce)<a name="line.3131"></a> -<span class="sourceLineNo">3132</span> throws IOException {<a name="line.3132"></a> -<span class="sourceLineNo">3133</span> // As it stands, this is used for 3 things<a name="line.3133"></a> -<span class="sourceLineNo">3134</span> // * batchMutate with single mutation - put/delete, separate or from checkAndMutate.<a name="line.3134"></a> -<span class="sourceLineNo">3135</span> // * coprocessor calls (see ex. BulkDeleteEndpoint).<a name="line.3135"></a> -<span class="sourceLineNo">3136</span> // So nonces are not really ever used by HBase. They could be by coprocs, and checkAnd...<a name="line.3136"></a> -<span class="sourceLineNo">3137</span> return batchMutate(new MutationBatch(mutations, nonceGroup, nonce));<a name="line.3137"></a> -<span class="sourceLineNo">3138</span> }<a name="line.3138"></a> -<span class="sourceLineNo">3139</span><a name="line.3139"></a> -<span class="sourceLineNo">3140</span> @Override<a name="line.3140"></a> -<span class="sourceLineNo">3141</span> public OperationStatus[] batchMutate(Mutation[] mutations) throws IOException {<a name="line.3141"></a> -<span class="sourceLineNo">3142</span> return batchMutate(mutations, HConstants.NO_NONCE, HConstants.NO_NONCE);<a name="line.3142"></a> -<span class="sourceLineNo">3143</span> }<a name="line.3143"></a> -<span class="sourceLineNo">3144</span><a name="line.3144"></a> -<span class="sourceLineNo">3145</span> public OperationStatus[] batchReplay(MutationReplay[] mutations, long replaySeqId)<a name="line.3145"></a> -<span class="sourceLineNo">3146</span> throws IOException {<a name="line.3146"></a> -<span class="sourceLineNo">3147</span> if (!RegionReplicaUtil.isDefaultReplica(getRegionInfo())<a name="line.3147"></a> -<span class="sourceLineNo">3148</span> && replaySeqId < lastReplayedOpenRegionSeqId) {<a name="line.3148"></a> -<span class="sourceLineNo">3149</span> // if it is a secondary replica we should ignore these entries silently<a name="line.3149"></a> -<span class="sourceLineNo">3150</span> // since they are coming out of order<a name="line.3150"></a> -<span class="sourceLineNo">3151</span> if (LOG.isTraceEnabled()) {<a name="line.3151"></a> -<span class="sourceLineNo">3152</span> LOG.trace(getRegionInfo().getEncodedName() + " : "<a name="line.3152"></a> -<span class="sourceLineNo">3153</span> + "Skipping " + mutations.length + " mutations with replaySeqId=" + replaySeqId<a name="line.3153"></a> -<span class="sourceLineNo">3154</span> + " which is < than lastReplayedOpenRegionSeqId=" + lastReplayedOpenRegionSeqId);<a name="line.3154"></a> -<span class="sourceLineNo">3155</span> for (MutationReplay mut : mutations) {<a name="line.3155"></a> -<span class="sourceLineNo">3156</span> LOG.trace(getRegionInfo().getEncodedName() + " : Skipping : " + mut.mutation);<a name="line.3156"></a> -<span class="sourceLineNo">3157</span> }<a name="line.3157"></a> -<span class="sourceLineNo">3158</span> }<a name="line.3158"></a> -<span class="sourceLineNo">3159</span><a name="line.3159"></a> -<span class="sourceLineNo">3160</span> OperationStatus[] statuses = new OperationStatus[mutations.length];<a name="line.3160"></a> -<span class="sourceLineNo">3161</span> for (int i = 0; i < statuses.length; i++) {<a name="line.3161"></a> -<span class="sourceLineNo">3162</span> statuses[i] = OperationStatus.SUCCESS;<a name="line.3162"></a> -<span class="sourceLineNo">3163</span> }<a name="line.3163"></a> -<span class="sourceLineNo">3164</span> return statuses;<a name="line.3164"></a> -<span class="sourceLineNo">3165</span> }<a name="line.3165"></a> -<span class="sourceLineNo">3166</span> return batchMutate(new ReplayBatch(mutations, replaySeqId));<a name="line.3166"></a> -<span class="sourceLineNo">3167</span> }<a name="line.3167"></a> -<span class="sourceLineNo">3168</span><a name="line.3168"></a> -<span class="sourceLineNo">3169</span> /**<a name="line.3169"></a> -<span class="sourceLineNo">3170</span> * Perform a batch of mutations.<a name="line.3170"></a> -<span class="sourceLineNo">3171</span> * It supports only Put and Delete mutations and will ignore other types passed.<a name="line.3171"></a> -<span class="sourceLineNo">3172</span> * @param batchOp contains the list of mutations<a name="line.3172"></a> -<span class="sourceLineNo">3173</span> * @return an array of OperationStatus which internally contains the<a name="line.3173"></a> -<span class="sourceLineNo">3174</span> * OperationStatusCode and the exceptionMessage if any.<a name="line.3174"></a> -<span class="sourceLineNo">3175</span> * @throws IOException<a name="line.3175"></a> -<span class="sourceLineNo">3176</span> */<a name="line.3176"></a> -<span class="sourceLineNo">3177</span> OperationStatus[] batchMutate(BatchOperation<?> batchOp) throws IOException {<a name="line.3177"></a> -<span class="sourceLineNo">3178</span> boolean initialized = false;<a name="line.3178"></a> -<span class="sourceLineNo">3179</span> Operation op = batchOp.isInReplay() ? Operation.REPLAY_BATCH_MUTATE : Operation.BATCH_MUTATE;<a name="line.3179"></a> -<span class="sourceLineNo">3180</span> startRegionOperation(op);<a name="line.3180"></a> -<span class="sourceLineNo">3181</span> try {<a name="line.3181"></a> -<span class="sourceLineNo">3182</span> while (!batchOp.isDone()) {<a name="line.3182"></a> -<span class="sourceLineNo">3183</span> if (!batchOp.isInReplay()) {<a name="line.3183"></a> -<span class="sourceLineNo">3184</span> checkReadOnly();<a name="line.3184"></a> -<span class="sourceLineNo">3185</span> }<a name="line.3185"></a> -<span class="sourceLineNo">3186</span> checkResources();<a name="line.3186"></a> -<span class="sourceLineNo">3187</span><a name="line.3187"></a> -<span class="sourceLineNo">3188</span> if (!initialized) {<a name="line.3188"></a> -<span class="sourceLineNo">3189</span> this.writeRequestsCount.add(batchOp.operations.length);<a name="line.3189"></a> -<span class="sourceLineNo">3190</span> if (!batchOp.isInReplay()) {<a name="line.3190"></a> -<span class="sourceLineNo">3191</span> doPreBatchMutateHook(batchOp);<a name="line.3191"></a> -<span class="sourceLineNo">3192</span> }<a name="line.3192"></a> -<span class="sourceLineNo">3193</span> initialized = true;<a name="line.3193"></a> -<span class="sourceLineNo">3194</span> }<a name="line.3194"></a> -<span class="sourceLineNo">3195</span> doMiniBatchMutate(batchOp);<a name="line.3195"></a> -<span class="sourceLineNo">3196</span> long newSize = this.getMemStoreSize();<a name="line.3196"></a> -<span class="sourceLineNo">3197</span> requestFlushIfNeeded(newSize);<a name="line.3197"></a> -<span class="sourceLineNo">3198</span> }<a name="line.3198"></a> -<span class="sourceLineNo">3199</span> } finally {<a name="line.3199"></a> -<span class="sourceLineNo">3200</span> closeRegionOperation(op);<a name="line.3200"></a> -<span class="sourceLineNo">3201</span> }<a name="line.3201"></a> -<span class="sourceLineNo">3202</span> return batchOp.retCodeDetails;<a name="line.3202"></a> -<span class="sourceLineNo">3203</span> }<a name="line.3203"></a> -<span class="sourceLineNo">3204</span><a name="line.3204"></a> -<span class="sourceLineNo">3205</span> private void doPreBatchMutateHook(BatchOperation<?> batchOp)<a name="line.3205"></a> -<span class="sourceLineNo">3206</span> throws IOException {<a name="line.3206"></a> -<span class="sourceLineNo">3207</span> /* Run coprocessor pre hook outside of locks to avoid deadlock */<a name="line.3207"></a> -<span class="sourceLineNo">3208</span> WALEdit walEdit = new WALEdit();<a name="line.3208"></a> -<span class="sourceLineNo">3209</span> if (coprocessorHost != null) {<a name="line.3209"></a> -<span class="sourceLineNo">3210</span> for (int i = 0 ; i < batchOp.operations.length; i++) {<a name="line.3210"></a> -<span class="sourceLineNo">3211</span> Mutation m = batchOp.getMutation(i);<a name="line.3211"></a> -<span class="sourceLineNo">3212</span> if (m instanceof Put) {<a name="line.3212"></a> -<span class="sourceLineNo">3213</span> if (coprocessorHost.prePut((Put) m, walEdit, m.getDurability())) {<a name="line.3213"></a> -<span class="sourceLineNo">3214</span> // pre hook says skip this Put<a name="line.3214"></a> -<span class="sourceLineNo">3215</span> // mark as success and skip in doMiniBatchMutation<a name="line.3215"></a> -<span class="sourceLineNo">3216</span> batchOp.retCodeDetails[i] = OperationStatus.SUCCESS;<a name="line.3216"></a> -<span class="sourceLineNo">3217</span> }<a name="line.3217"></a> -<span class="sourceLineNo">3218</span> } else if (m instanceof Delete) {<a name="line.3218"></a> -<span class="sourceLineNo">3219</span> Delete curDel = (Delete) m;<a name="line.3219"></a> -<span class="sourceLineNo">3220</span> if (curDel.getFamilyCellMap().isEmpty()) {<a name="line.3220"></a> -<span class="sourceLineNo">3221</span> // handle deleting a row case<a name="line.3221"></a> -<span class="sourceLineNo">3222</span> prepareDelete(curDel);<a name="line.3222"></a> -<span class="sourceLineNo">3223</span> }<a name="line.3223"></a> -<span class="sourceLineNo">3224</span> if (coprocessorHost.preDelete(curDel, walEdit, m.getDurability())) {<a name="line.3224"></a> -<span class="sourceLineNo">3225</span> // pre hook says skip this Delete<a name="line.3225"></a> -<span class="sourceLineNo">3226</span> // mark as success and skip in doMiniBatchMutation<a name="line.3226"></a> -<span class="sourceLineNo">3227</span> batchOp.retCodeDetails[i] = OperationStatus.SUCCESS;<a name="line.3227"></a> -<span class="sourceLineNo">3228</span> }<a name="line.3228"></a> -<span class="sourceLineNo">3229</span> } else {<a name="line.3229"></a> -<span class="sourceLineNo">3230</span> // In case of passing Append mutations along with the Puts and Deletes in batchMutate<a name="line.3230"></a> -<span class="sourceLineNo">3231</span> // mark the operation return code as failure so that it will not be considered in<a name="line.3231"></a> -<span class="sourceLineNo">3232</span> // the doMiniBatchMutation<a name="line.3232"></a> -<span class="sourceLineNo">3233</span> batchOp.retCodeDetails[i] = new OperationStatus(OperationStatusCode.FAILURE,<a name="line.3233"></a> -<span class="sourceLineNo">3234</span> "Put/Delete mutations only supported in batchMutate() now");<a name="line.3234"></a> -<span class="sourceLineNo">3235</span> }<a name="line.3235"></a> -<span class="sourceLineNo">3236</span> if (!walEdit.isEmpty()) {<a name="line.3236"></a> -<span class="sourceLineNo">3237</span> batchOp.walEditsFromCoprocessors[i] = walEdit;<a name="line.3237"></a> -<span class="sourceLineNo">3238</span> walEdit = new WALEdit();<a name="line.3238"></a> -<span class="sourceLineNo">3239</span> }<a name="line.3239"></a> -<span class="sourceLineNo">3240</span> }<a name="line.3240"></a> -<span class="sourceLineNo">3241</span> }<a name="line.3241"></a> -<span class="sourceLineNo">3242</span> }<a name="line.3242"></a> -<span class="sourceLineNo">3243</span><a name="line.3243"></a> -<span class="sourceLineNo">3244</span> /**<a name="line.3244"></a> -<span class="sourceLineNo">3245</span> * Called to do a piece of the batch that came in to {@link #batchMutate(Mutation[], long, long)}<a name="line.3245"></a> -<span class="sourceLineNo">3246</span> * In here we also handle replay of edits on region recover.<a name="line.3246"></a> -<span class="sourceLineNo">3247</span> * @return Change in size brought about by applying <code>batchOp</code><a name="line.3247"></a> -<span class="sourceLineNo">3248</span> */<a name="line.3248"></a> -<span class="sourceLineNo">3249</span> // TODO: This needs a rewrite. Doesn't have to be this long. St.Ack 20160120<a name="line.3249"></a> -<span class="sourceLineNo">3250</span> private void doMiniBatchMutate(BatchOperation<?> batchOp) throws IOException {<a name="line.3250"></a> -<span class="sourceLineNo">3251</span> boolean replay = batchOp.isInReplay();<a name="line.3251"></a> -<span class="sourceLineNo">3252</span> long currentNonceGroup = HConstants.NO_NONCE;<a name="line.3252"></a> -<span class="sourceLineNo">3253</span> long currentNonce = HConstants.NO_NONCE;<a name="line.3253"></a> -<span class="sourceLineNo">3254</span> WALEdit walEdit = null;<a name="line.3254"></a> -<span class="sourceLineNo">3255</span> boolean locked = false;<a name="line.3255"></a> -<span class="sourceLineNo">3256</span> // reference family maps directly so coprocessors can mutate them if desired<a name="line.3256"></a> -<span class="sourceLineNo">3257</span> Map<byte[], List<Cell>>[] familyMaps = new Map[batchOp.operations.length];<a name="line.3257"></a> -<span class="sourceLineNo">3258</span> // We try to set up a batch in the range [firstIndex,lastIndexExclusive)<a name="line.3258"></a> -<span class="sourceLineNo">3259</span> int firstIndex = batchOp.nextIndexToProcess;<a name="line.3259"></a> -<span class="sourceLineNo">3260</span> int lastIndexExclusive = firstIndex;<a name="line.3260"></a> -<span class="sourceLineNo">3261</span> boolean success = false;<a name="line.3261"></a> -<span class="sourceLineNo">3262</span> int noOfPuts = 0;<a name="line.3262"></a> -<span class="sourceLineNo">3263</span> int noOfDeletes = 0;<a name="line.3263"></a> -<span class="sourceLineNo">3264</span> WriteEntry writeEntry = null;<a name="line.3264"></a> -<span class="sourceLineNo">3265</span> int cellCount = 0;<a name="line.3265"></a> -<span class="sourceLineNo">3266</span> /** Keep track of the locks we hold so we can release them in finally clause */<a name="line.3266"></a> -<span class="sourceLineNo">3267</span> List<RowLock> acquiredRowLocks = Lists.newArrayListWithCapacity(batchOp.operations.length);<a name="line.3267"></a> -<span class="sourceLineNo">3268</span> MemStoreSize memstoreSize = new MemStoreSize();<a name="line.3268"></a> -<span class="sourceLineNo">3269</span> final ObservedExceptionsInBatch observedExceptions = new ObservedExceptionsInBatch();<a name="line.3269"></a> -<span class="sourceLineNo">3270</span> try {<a name="line.3270"></a> -<span class="sourceLineNo">3271</span> // STEP 1. Try to acquire as many locks as we can, and ensure we acquire at least one.<a name="line.3271"></a> -<span class="sourceLineNo">3272</span> int numReadyToWrite = 0;<a name="line.3272"></a> -<span class="sourceLineNo">3273</span> long now = EnvironmentEdgeManager.currentTime();<a name="line.3273"></a> -<span class="sourceLineNo">3274</span> while (lastIndexExclusive < batchOp.operations.length) {<a name="line.3274"></a> -<span class="sourceLineNo">3275</span> if (checkBatchOp(batchOp, lastIndexExclusive, familyMaps, now, observedExceptions)) {<a name="line.3275"></a> -<span class="sourceLineNo">3276</span> lastIndexExclusive++;<a name="line.3276"></a> -<span class="sourceLineNo">3277</span> continue;<a name="line.3277"></a> -<span class="sourceLineNo">3278</span> }<a name="line.3278"></a> -<span class="sourceLineNo">3279</span> Mutation mutation = batchOp.getMutation(lastIndexExclusive);<a name="line.3279"></a> -<span class="sourceLineNo">3280</span> // If we haven't got any rows in our batch, we should block to get the next one.<a name="line.3280"></a> -<span class="sourceLineNo">3281</span> RowLock rowLock = null;<a name="line.3281"></a> -<span class="sourceLineNo">3282</span> try {<a name="line.3282"></a> -<span class="sourceLineNo">3283</span> rowLock = getRowLockInternal(mutation.getRow(), true);<a name="line.3283"></a> -<span class="sourceLineNo">3284</span> } catch (TimeoutIOException e) {<a name="line.3284"></a> -<span class="sourceLineNo">3285</span> // We will retry when other exceptions, but we should stop if we timeout .<a name="line.3285"></a> -<span class="sourceLineNo">3286</span> throw e;<a name="line.3286"></a> -<span class="sourceLineNo">3287</span> } catch (IOException ioe) {<a name="line.3287"></a> -<span class="sourceLineNo">3288</span> LOG.warn("Failed getting lock, row=" + Bytes.toStringBinary(mutation.getRow()), ioe);<a name="line.3288"></a> -<span class="sourceLineNo">3289</span> }<a name="line.3289"></a> -<span class="sourceLineNo">3290</span> if (rowLock == null) {<a name="line.3290"></a> -<span class="sourceLineNo">3291</span> // We failed to grab another lock<a name="line.3291"></a> -<span class="sourceLineNo">3292</span> break; // Stop acquiring more rows for this batch<a name="line.3292"></a> -<span class="sourceLineNo">3293</span> } else {<a name="line.3293"></a> -<span class="sourceLineNo">3294</span> acquiredRowLocks.add(rowLock);<a name="line.3294"></a> -<span class="sourceLineNo">3295</span> }<a name="line.3295"></a> -<span class="sourceLineNo">3296</span><a name="line.3296"></a> -<span class="sourceLineNo">3297</span> lastIndexExclusive++;<a name="line.3297"></a> -<span class="sourceLineNo">3298</span> numReadyToWrite++;<a name="line.3298"></a> -<span class="sourceLineNo">3299</span> if (replay) {<a name="line.3299"></a> -<span class="sourceLineNo">3300</span> for (List<Cell> cells : mutation.getFamilyCellMap().values()) {<a name="line.3300"></a> -<span class="sourceLineNo">3301</span> cellCount += cells.size();<a name="line.3301"></a> -<span class="sourceLineNo">3302</span> }<a name="line.3302"></a> -<span class="sourceLineNo">3303</span> }<a name="line.3303"></a> -<span class="sourceLineNo">3304</span> }<a name="line.3304"></a> -<span class="sourceLineNo">3305</span><a name="line.3305"></a> -<span class="sourceLineNo">3306</span> // We've now grabbed as many mutations off the list as we can<a name="line.3306"></a> -<span class="sourceLineNo">3307</span><a name="line.3307"></a> -<span class="sourceLineNo">3308</span> // STEP 2. Update any LATEST_TIMESTAMP timestamps<a name="line.3308"></a> -<span class="sourceLineNo">3309</span> // We should record the timestamp only after we have acquired the rowLock,<a name="line.3309"></a> -<span class="sourceLineNo">3310</span> // otherwise, newer puts/deletes are not guaranteed to have a newer timestamp<a name="line.3310"></a> -<span class="sourceLineNo">3311</span> now = EnvironmentEdgeManager.currentTime();<a name="line.3311"></a> -<span class="sourceLineNo">3312</span> byte[] byteNow = Bytes.toBytes(now);<a name="line.3312"></a> -<span class="sourceLineNo">3313</span><a name="line.3313"></a> -<span class="sourceLineNo">3314</span> // Nothing to put/delete -- an exception in the above such as NoSuchColumnFamily?<a name="line.3314"></a> -<span class="sourceLineNo">3315</span> if (numReadyToWrite <= 0) {<a name="line.3315"></a> -<span class="sourceLineNo">3316</span> return;<a name="line.3316"></a> -<span class="sourceLineNo">3317</span> }<a name="line.3317"></a> -<span class="sourceLineNo">3318</span><a name="line.3318"></a> -<span class="sourceLineNo">3319</span> for (int i = firstIndex; !replay && i < lastIndexExclusive; i++) {<a name="line.3319"></a> -<span class="sourceLineNo">3320</span> // skip invalid<a name="line.3320"></a> -<span class="sourceLineNo">3321</span> if (batchOp.retCodeDetails[i].getOperationStatusCode()<a name="line.3321"></a> -<span class="sourceLineNo">3322</span> != OperationStatusCode.NOT_RUN) {<a name="line.3322"></a> -<span class="sourceLineNo">3323</span> // lastIndexExclusive was incremented above.<a name="line.3323"></a> -<span class="sourceLineNo">3324</span> continue;<a name="line.3324"></a> -<span class="sourceLineNo">3325</span> }<a name="line.3325"></a> -<span class="sourceLineNo">3326</span><a name="line.3326"></a> -<span class="sourceLineNo">3327</span> Mutation mutation = batchOp.getMutation(i);<a name="line.3327"></a> -<span class="sourceLineNo">3328</span> if (mutation instanceof Put) {<a name="line.3328"></a> -<span class="sourceLineNo">3329</span> updateCellTimestamps(familyMaps[i].values(), byteNow);<a name="line.3329"></a> -<span class="sourceLineNo">3330</span> noOfPuts++;<a name="line.3330"></a> -<span class="sourceLineNo">3331</span> } else {<a name="line.3331"></a> -<span class="sourceLineNo">3332</span> prepareDeleteTimestamps(mutation, familyMaps[i], byteNow);<a name="line.3332"></a> -<span class="sourceLineNo">3333</span> noOfDeletes++;<a name="line.3333"></a> -<span class="sourceLineNo">3334</span> }<a name="line.3334"></a> -<span class="sourceLineNo">3335</span> rewriteCellTags(familyMaps[i], mutation);<a name="line.3335"></a> -<span class="sourceLineNo">3336</span> WALEdit fromCP = batchOp.walEditsFromCoprocessors[i];<a name="line.3336"></a> -<span class="sourceLineNo">3337</span> if (fromCP != null) {<a name="line.3337"></a> -<span class="sourceLineNo">3338</span> cellCount += fromCP.size();<a name="line.3338"></a> -<span class="sourceLineNo">3339</span> }<a name="line.3339"></a> -<span class="sourceLineNo">3340</span> if (getEffectiveDurability(mutation.getDurability()) != Durability.SKIP_WAL) {<a name="line.3340"></a> -<span class="sourceLineNo">3341</span> for (List<Cell> cells : familyMaps[i].values()) {<a name="line.3341"></a> -<span class="sourceLineNo">3342</span> cellCount += cells.size();<a name="line.3342"></a> -<span class="sourceLineNo">3343</span> }<a name="line.3343"></a> -<span class="sourceLineNo">3344</span> }<a name="line.3344"></a> -<span class="sourceLineNo">3345</span> }<a name="line.3345"></a> -<span class="sourceLineNo">3346</span> lock(this.updatesLock.readLock(), numReadyToWrite);<a name="line.3346"></a> -<span class="sourceLineNo">3347</span> locked = true;<a name="line.3347"></a> -<span class="sourceLineNo">3348</span><a name="line.3348"></a> -<span class="sourceLineNo">3349</span> // calling the pre CP hook for batch mutation<a name="line.3349"></a> -<span class="sourceLineNo">3350</span> if (!replay && coprocessorHost != null) {<a name="line.3350"></a> -<span class="sourceLineNo">3351</span> MiniBatchOperationInProgress<Mutation> miniBatchOp =<a name="line.3351"></a> -<span class="sourceLineNo">3352</span> new MiniBatchOperationInProgress<>(batchOp.getMutationsForCoprocs(),<a name="line.3352"></a> -<span class="sourceLineNo">3353</span> batchOp.retCodeDetails, batchOp.walEditsFromCoprocessors, firstIndex, lastIndexExclusive);<a name="line.3353"></a> -<span class="sourceLineNo">3354</span> if (coprocessorHost.preBatchMutate(miniBatchOp)) {<a name="line.3354"></a> -<span class="sourceLineNo">3355</span> return;<a name="line.3355"></a> -<span class="sourceLineNo">3356</span> } else {<a name="line.3356"></a> -<span class="sourceLineNo">3357</span> for (int i = firstIndex; i < lastIndexExclusive; i++) {<a name="line.3357"></a> -<span class="sourceLineNo">3358</span> if (batchOp.retCodeDetails[i].getOperationStatusCode() != OperationStatusCode.NOT_RUN) {<a name="line.3358"></a> -<span class="sourceLineNo">3359</span> // lastIndexExclusive was incremented above.<a name="line.3359"></a> -<span class="sourceLineNo">3360</span> continue;<a name="line.3360"></a> -<span class="sourceLineNo">3361</span> }<a name="line.3361"></a> -<span class="sourceLineNo">3362</span> // we pass (i - firstIndex) below since the call expects a relative index<a name="line.3362"></a> -<span class="sourceLineNo">3363</span> Mutation[] cpMutations = miniBatchOp.getOperationsFromCoprocessors(i - firstIndex);<a name="line.3363"></a> -<span class="sourceLineNo">3364</span> if (cpMutations == null) {<a name="line.3364"></a> -<span class="sourceLineNo">3365</span> continue;<a name="line.3365"></a> -<span class="sourceLineNo">3366</span> }<a name="line.3366"></a> -<span class="sourceLineNo">3367</span> Mutation mutation = batchOp.getMutation(i);<a name="line.3367"></a> -<span class="sourceLineNo">3368</span> boolean skipWal = getEffectiveDurability(mutation.getDurability()) == Durability.SKIP_WAL;<a name="line.3368"></a> -<span class="sourceLineNo">3369</span> // Else Coprocessor added more Mutations corresponding to the Mutation at this index.<a name="line.3369"></a> -<span class="sourceLineNo">3370</span> for (int j = 0; j < cpMutations.length; j++) {<a name="line.3370"></a> -<span class="sourceLineNo">3371</span> Mutation cpMutation = cpMutations[j];<a name="line.3371"></a> -<span class="sourceLineNo">3372</span> Map<byte[], List<Cell>> cpFamilyMap = cpMutation.getFamilyCellMap();<a name="line.3372"></a> -<span class="sourceLineNo">3373</span> checkAndPrepareMutation(cpMutation, replay, cpFamilyMap, now);<a name="line.3373"></a> -<span class="sourceLineNo">3374</span><a name="line.3374"></a> -<span class="sourceLineNo">3375</span> // Acquire row locks. If not, the whole batch will fail.<a name="line.3375"></a> -<span class="sourceLineNo">3376</span> acquiredRowLocks.add(getRowLockInternal(cpMutation.getRow(), true));<a name="line.3376"></a> -<span class="sourceLineNo">3377</span><a name="line.3377"></a> -<span class="sourceLineNo">3378</span> // Returned mutations from coprocessor correspond to the Mutation at index i. We can<a name="line.3378"></a> -<span class="sourceLineNo">3379</span> // directly add the cells from those mutations to the familyMaps of this mutation.<a name="line.3379"></a> -<span class="sourceLineNo">3380</span> mergeFamilyMaps(familyMaps[i], cpFamilyMap); // will get added to the memstore later<a name="line.3380"></a> -<span class="sourceLineNo">3381</span><a name="line.3381"></a> -<span class="sourceLineNo">3382</span> // The durability of returned mutation is replaced by the corresponding mutation.<a name="line.3382"></a> -<span class="sourceLineNo">3383</span> // If the corresponding mutation contains the SKIP_WAL, we shouldn't count the<a name="line.3383"></a> -<span class="sourceLineNo">3384</span> // cells of returned mutation.<a name="line.3384"></a> -<span class="sourceLineNo">3385</span> if (!skipWal) {<a name="line.3385"></a> -<span class="sourceLineNo">3386</span> for (List<Cell> cells : cpFamilyMap.values()) {<a name="line.3386"></a> -<span class="sourceLineNo">3387</span> cellCount += cells.size();<a name="line.3387"></a> -<span class="sourceLineNo">3388</span> }<a name="line.3388"></a> -<span class="sourceLineNo">3389</span> }<a name="line.3389"></a> -<span class="sourceLineNo">3390</span> }<a name="line.3390"></a> -<span class="sourceLineNo">3391</span> }<a name="line.3391"></a> -<span class="sourceLineNo">3392</span> }<a name="line.3392"></a> -<span class="sourceLineNo">3393</span> }<a name="line.3393"></a> -<span class="sourceLineNo">3394</span><a name="line.3394"></a> -<span class="sourceLineNo">3395</span> // STEP 3. Build WAL edit<a name="line.3395"></a> -<span class="sourceLineNo">3396</span> walEdit = new WALEdit(cellCount, replay);<a name="line.3396"></a> -<span class="sourceLineNo">3397</span> Durability durability = Durability.USE_DEFAULT;<a name="line.3397"></a> -<span class="sourceLineNo">3398</span> for (int i = firstIndex; i < lastIndexExclusive; i++) {<a name="line.3398"></a> -<span class="sourceLineNo">3399</span> // Skip puts that were determined to be invalid during preprocessing<a name="line.3399"></a> -<span class="sourceLineNo">3400</span> if (batchOp.retCodeDetails[i].getOperationStatusCode() != OperationStatusCode.NOT_RUN) {<a name="line.3400"></a> -<span class="sourceLineNo">3401</span> continue;<a name="line.3401"></a> -<span class="sourceLineNo">3402</span> }<a name="line.3402"></a> -<span class="sourceLineNo">3403</span><a name="line.3403"></a> -<span class="sourceLineNo">3404</span> Mutation m = batchOp.getMutation(i);<a name="line.3404"></a> -<span class="sourceLineNo">3405</span> Durability tmpDur = getEffectiveDurability(m.getDurability());<a name="line.3405"></a> -<span class="sourceLineNo">3406</span> if (tmpDur.ordinal() > durability.ordinal()) {<a name="line.3406"></a> -<span class="sourceLineNo">3407</span> durability = tmpDur;<a name="line.3407"></a> -<span class="sourceLineNo">3408</span> }<a name="line.3408"></a> -<span class="sourceLineNo">3409</span> // we use durability of the original mutation for the mutation passed by CP.<a name="line.3409"></a> -<span class="sourceLineNo">3410</span> if (tmpDur == Durability.SKIP_WAL) {<a name="line.3410"></a> -<span class="sourceLineNo">3411</span> recordMutationWithoutWal(m.getFamilyCellMap());<a name="line.3411"></a> -<span class="sourceLineNo">3412</span> continue;<a name="line.3412"></a> -<span class="sourceLineNo">3413</span> }<a name="line.3413"></a> -<span class="sourceLineNo">3414</span><a name="line.3414"></a> -<span class="sourceLineNo">3415</span> long nonceGroup = batchOp.getNonceGroup(i);<a name="line.3415"></a> -<span class="sourceLineNo">3416</span> long nonce = batchOp.getNonce(i);<a name="line.3416"></a> -<span class="sourceLineNo">3417</span> // In replay, the batch may contain multiple nonces. If so, write WALEdit for each.<a name="line.3417"></a> -<span class="sourceLineNo">3418</span> // Given how nonces are originally written, these should be contiguous.<a name="line.3418"></a> -<span class="sourceLineNo">3419</span> // They don't have to be, it will still work, just write more WALEdits than needed.<a name="line.3419"></a> -<span class="sourceLineNo">3420</span> if (nonceGroup != currentNonceGroup || nonce != currentNonce) {<a name="line.3420"></a> -<span class="sourceLineNo">3421</span> // Write what we have so far for nonces out to WAL<a name="line.3421"></a> -<span class="sourceLineNo">3422</span> appendCurrentNonces(m, replay, walEdit, now, currentNonceGroup, currentNonce);<a name="line.3422"></a> -<span class="sourceLineNo">3423</span> walEdit = new WALEdit(cellCount, replay);<a name="line.3423"></a> -<span class="sourceLineNo">3424</span> currentNonceGroup = nonceGroup;<a name="line.3424"></a> -<span class="sourceLineNo">3425</span> currentNonce = nonce;<a name="line.3425"></a> -<span class="sourceLineNo">3426</span> }<a name="line.3426"></a> -<span class="sourceLineNo">3427</span><a name="line.3427"></a> -<span class="sourceLineNo">3428</span> // Add WAL edits by CP<a name="line.3428"></a> -<span class="sourceLineNo">3429</span> WALEdit fromCP = batchOp.walEditsFromCoprocessors[i];<a name="line.3429"></a> -<span class="sourceLineNo">3430</span> if (fromCP != null) {<a name="line.3430"></a> -<span class="sourceLineNo">3431</span> for (Cell cell : fromCP.getCells()) {<a name="line.3431"></a> -<span class="sourceLineNo">3432</span> walEdit.add(cell);<a name="line.3432"></a> -<span class="sourceLineNo">3433</span> }<a name="line.3433"></a> -<span class="sourceLineNo">3434</span> }<a name="line.3434"></a> -<span class="sourceLineNo">3435</span> addFamilyMapToWALEdit(familyMaps[i], walEdit);<a name="line.3435"></a> -<span class="sourceLineNo">3436</span> }<a name="line.3436"></a> -<span class="sourceLineNo">3437</span><a name="line.3437"></a> -<span class="sourceLineNo">3438</span> // STEP 4. Append the final edit to WAL and sync.<a name="line.3438"></a> -<span class="sourceLineNo">3439</span> Mutation mutation = batchOp.getMutation(firstIndex);<a name="line.3439"></a> -<span class="sourceLineNo">3440</span> WALKey walKey = null;<a name="line.3440"></a> -<span class="sourceLineNo">3441</span> long txid;<a name="line.3441"></a> -<span class="sourceLineNo">3442</span> if (replay) {<a name="line.3442"></a> -<span class="sourceLineNo">3443</span> // use wal key from the original<a name="line.3443"></a> -<span class="sourceLineNo">3444</span> walKey = new WALKey(this.getRegionInfo().getEncodedNameAsBytes(),<a name="line.3444"></a> -<span class="sourceLineNo">3445</span> this.htableDescriptor.getTableName(), WALKey.NO_SEQUENCE_ID, now,<a name="line.3445"></a> -<span class="sourceLineNo">3446</span> mutation.getClusterIds(), currentNonceGroup, currentNonce, mvcc);<a name="line.3446"></a> -<span class="sourceLineNo">3447</span> walKey.setOrigLogSeqNum(batchOp.getReplaySequenceId());<a name="line.3447"></a> -<span class="sourceLineNo">3448</span> if (!walEdit.isEmpty()) {<a name="line.3448"></a> -<span class="sourceLineNo">3449</span> txid = this.wal.append(this.getRegionInfo(), walKey, walEdit, true);<a name="line.3449"></a> -<span class="sourceLineNo">3450</span> if (txid != 0) {<a name="line.3450"></a> -<span class="sourceLineNo">3451</span> sync(txid, durability);<a name="line.3451"></a> -<span class="sourceLineNo">3452</span> }<a name="line.3452"></a> -<span class="sourceLineNo">3453</span> }<a name="line.3453"></a> -<span class="sourceLineNo">3454</span> } else {<a name="line.3454"></a> -<span class="sourceLineNo">3455</span> try {<a name="line.3455"></a> -<span class="sourceLineNo">3456</span> if (!walEdit.isEmpty()) {<a name="line.3456"></a> -<span class="sourceLineNo">3457</span> // we use HLogKey here instead of WALKey directly to support legacy coprocessors.<a name="line.3457"></a> -<span class="sourceLineNo">3458</span> walKey = new WALKey(this.getRegionInfo().getEncodedNameAsBytes(),<a name="line.3458"></a> -<span class="sourceLineNo">3459</span> this.htableDescriptor.getTableName(), WALKey.NO_SEQUENCE_ID, now,<a name="line.3459"></a> -<span class="sourceLineNo">3460</span> mutation.getClusterIds(), currentNonceGroup, currentNonce, mvcc,<a name="line.3460"></a> -<span class="sourceLineNo">3461</span> this.getReplicationScope());<a name="line.3461"></a> -<span class="sourceLineNo">3462</span> // TODO: Use the doAppend methods below... complicated by the replay stuff above.<a name="line.3462"></a> -<span class="sourceLineNo">3463</span> txid = this.wal.append(this.getRegionInfo(), walKey, walEdit, true);<a name="line.3463"></a> -<span class="sourceLineNo">3464</span> if (txid != 0) {<a name="line.3464"></a> -<span class="sourceLineNo">3465</span> sync(txid, durability);<a name="line.3465"></a> -<span class="sourceLineNo">3466</span> }<a name="line.3466"></a> -<span class="sourceLineNo">3467</span> if (writeEntry == null) {<a name="line.3467"></a> -<span class="sourceLineNo">3468</span> // if MVCC not preassigned, wait here until assigned<a name="line.3468"></a> -<span class="sourceLineNo">3469</span> writeEntry = walKey.getWriteEntry();<a name="line.3469"></a> -<span class="sourceLineNo">3470</span> }<a name="line.3470"></a> -<span class="sourceLineNo">3471</span> }<a name="line.3471"></a> -<span class="sourceLineNo">3472</span> } catch (IOException ioe) {<a name="line.3472"></a> -<span class="sourceLineNo">3473</span> if (walKey != null && writeEntry == null) {<a name="line.3473"></a> -<span class="sourceLineNo">3474</span> // the writeEntry is not preassigned and error occurred during append or sync<a name="line.3474"></a> -<span class="sourceLineNo">3475</span> mvcc.complete(walKey.getWriteEntry());<a name="line.3475"></a> -<span class="sourceLineNo">3476</span> }<a name="line.3476"></a> -<span class="sourceLineNo">3477</span> throw ioe;<a name="line.3477"></a> -<span class="sourceLineNo">3478</span> }<a name="line.3478"></a> -<span class="sourceLineNo">3479</span> }<a name="line.3479"></a> -<span class="sourceLineNo">3480</span> if (walKey == null) {<a name="line.3480"></a> -<span class="sourceLineNo">3481</span> // If no walKey, then not in replay and skipping WAL or some such. Begin an MVCC transaction<a name="line.3481"></a> -<span class="sourceLineNo">3482</span> // to get sequence id.<a name="line.3482"></a> -<span class="sourceLineNo">3483</span> writeEntry = mvcc.begin();<a name="line.3483"></a> -<span class="sourceLineNo">3484</span> }<a name="line.3484"></a> -<span class="sourceLineNo">3485</span><a name="line.3485"></a> -<span class="sourceLineNo">3486</span> // STEP 5. Write back to memstore<a name="line.3486"></a> -<span class="sourceLineNo">3487</span> for (int i = firstIndex; i < lastIndexExclusive; i++) {<a name="line.3487"></a> -<span class="sourceLineNo">3488</span> if (batchOp.retCodeDetails[i].getOperationStatusCode() != OperationStatusCode.NOT_RUN) {<a name="line.3488"></a> -<span class="sourceLineNo">3489</span> continue;<a name="line.3489"></a> -<span class="sourceLineNo">3490</span> }<a name="line.3490"></a> -<span class="sourceLineNo">3491</span> // We need to update the sequence id for following reasons.<a name="line.3491"></a> -<span class="sourceLineNo">3492</span> // 1) If the op is in replay mode, FSWALEntry#stampRegionSequenceId won't stamp sequence id.<a name="line.3492"></a> -<span class="sourceLineNo">3493</span> // 2) If no WAL, FSWALEntry won't be used<a name="line.3493"></a> -<span class="sourceLineNo">3494</span> // we use durability of the original mutation for the mutation passed by CP.<a name="line.3494"></a> -<span class="sourceLineNo">3495</span> boolean updateSeqId = replay || batchOp.getMutation(i).getDurability() == Durability.SKIP_WAL;<a name="line.3495"></a> -<span class="sourceLineNo">3496</span> if (updateSeqId) {<a name="line.3496"></a> -<span class="sourceLineNo">3497</span> this.updateSequenceId(familyMaps[i].values(),<a name="line.3497"></a> -<span class="sourceLineNo">3498</span> replay? batchOp.getReplaySequenceId(): writeEntry.getWriteNumber());<a name="line.3498"></a> -<span class="sourceLineNo">3499</span> }<a name="line.3499"></a> -<span class="sourceLineNo">3500</span> applyFamilyMapToMemStore(familyMaps[i], memstoreSize);<a name="line.3500"></a> -<span class="sourceLineNo">3501</span> }<a name="line.3501"></a> -<span class="sourceLineNo">3502</span><a name="line.3502"></a> -<span class="sourceLineNo">3503</span> // update memstore size<a name="line.3503"></a> -<span class="sourceLineNo">3504</span> this.addAndGetMemStoreSize(memstoreSize);<a name="line.3504"></a> -<span class="sourceLineNo">3505</span><a name="line.3505"></a> -<span class="sourceLineNo">3506</span> // calling the post CP hook for batch mutation<a name="line.3506"></a> -<span class="sourceLineNo">3507</span> if (!replay && coprocessorHost != null) {<a name="line.3507"></a> -<span class="sourceLineNo">3508</span> MiniBatchOperationInProgress<Mutation> miniBatchOp =<a name="line.3508"></a> -<span class="sourceLineNo">3509</span> new MiniBatchOperationInProgress<>(batchOp.getMutationsForCoprocs(),<a name="line.3509"></a> -<span class="sourceLineNo">3510</span> batchOp.retCodeDetails, batchOp.walEditsFromCoprocessors, firstIndex, lastIndexExclusive);<a name="line.3510"></a> -<span class="sourceLineNo">3511</span> coprocessorHost.postBatchMutate(miniBatchOp);<a name="line.3511"></a> -<span class="sourceLineNo">3512</span> }<a name="line.3512"></a> -<span class="sourceLineNo">3513</span><a name="line.3513"></a> -<span class="sourceLineNo">3514</span> // STEP 6. Complete mvcc.<a name="line.3514"></a> -<span class="sourceLineNo">3515</span> if (replay) {<a name="line.3515"></a> -<span class="sourceLineNo">3516</span> this.mvcc.advanceTo(batchOp.getReplaySequenceId());<a name="line.3516"></a> -<span class="sourceLineNo">3517</span> } else {<a name="line.3517"></a> -<span class="sourceLineNo">3518</span> // writeEntry won't be empty if not in replay mode<a name="line.3518"></a> -<span class="sourceLineNo">3519</span> mvcc.completeAndWait(writeEntry);<a name="line.3519"></a> -<span class="sourceLineNo">3520</span> writeEntry = null;<a name="line.3520"></a> -<span class="sourceLineNo">3521</span> }<a name="line.3521"></a> -<span class="sourceLineNo">3522</span><a name="line.3522"></a> -<span class="sourceLineNo">3523</span> // STEP 7. Release row locks, etc.<a name="line.3523"></a> -<span class="sourceLineNo">3524</span> if (locked) {<a name="line.3524"></a> -<span class="sourceLineNo">3525</span> this.updatesLock.readLock().unlock();<a name="line.3525"></a> -<span class="sourceLineNo">3526</span> locked = false;<a name="line.3526"></a> -<span class="sourceLineNo">3527</span> }<a name="line.3527"></a> -<span class="sourceLineNo">3528</span> releaseRowLocks(acquiredRowLocks);<a name="line.3528"></a> -<span class="sourceLineNo">3529</span><a name="line.3529"></a> -<span class="sourceLineNo">3530</span> for (int i = firstIndex; i < lastIndexExclusive; i ++) {<a name="line.3530"></a> -<span class="sourceLineNo">3531</span> if (batchOp.retCodeDetails[i] == OperationStatus.NOT_RUN) {<a name="line.3531"></a> -<span class="sourceLineNo">3532</span> batchOp.retCodeDetails[i] = OperationStatus.SUCCESS;<a name="line.3532"></a> -<span class="sourceLineNo">3533</span> }<a name="line.3533"></a> -<span class="sourceLineNo">3534</span> }<a name="line.3534"></a> -<span class="sourceLineNo">3535</span><a name="line.3535"></a> -<span class="sourceLineNo">3536</span> // STEP 8. Run coprocessor post hooks. This should be done after the wal is<a name="line.3536"></a> -<span class="sourceLineNo">3537</span> // synced so that the coprocessor contract is adhered to.<a name="line.3537"></a> -<span class="sourceLineNo">3538</span> if (!replay && coprocessorHost != null) {<a name="line.3538"></a> -<span class="sourceLineNo">3539</span> for (int i = firstIndex; i < lastIndexExclusive; i++) {<a name="line.3539"></a> -<span class="sourceLineNo">3540</span> // only for successful puts<a name="line.3540"></a> -<span class="sourceLineNo">3541</span> if (batchOp.retCodeDetails[i].getOperationStatusCode()<a name="line.3541"></a> -<span class="sourceLineNo">3542</span> != OperationStatusCode.SUCCESS) {<a name="line.3542"></a> -<span class="sourceLineNo">3543</span> continue;<a name="line.3543"></a> -<span class="sourceLineNo">3544</span> }<a name="line.3544"></a> -<span class="sourceLineNo">3545</span> Mutation m = batchOp.getMutation(i);<a name="line.3545"></a> -<span class="sourceLineNo">3546</span> if (m instanceof Put) {<a name="line.3546"></a> -<span class="sourceLineNo">3547</span> coprocessorHost.postPut((Put) m, walEdit, m.getDurability());<a name="line.3547"></a> -<span class="sourceLineNo">3548</span> } else {<a name="line.3548"></a> -<span class="sourceLineNo">3549</span> coprocessorHost.postDelete((Delete) m, walEdit, m.getDurability());<a name="line.3549"></a> -<span class="sourceLineNo">3550</span> }<a name="line.3550"></a> -<span class="sourceLineNo">3551</span> }<a name="line.3551"></a> -<span class="sourceLineNo">3552</span> }<a name="line.3552"></a> -<span class="sourceLineNo">3553</span><a name="line.3553"></a> -<span class="sourceLineNo">3554</span> success = true;<a name="line.3554"></a> -<span class="sourceLineNo">3555</span> } finally {<a name="line.3555"></a> -<span class="sourceLineNo">3556</span> // Call complete rather than completeAndWait because we probably had error if walKey != null<a name="line.3556"></a> -<span class="sourceLineNo">3557</span> if (writeEntry != null) mvcc.complete(writeEntry);<a name="line.3557"></a> -<span class="sourceLineNo">3558</span> if (locked) {<a name="line.3558"></a> -<span class="sourceLineNo">3559</span> this.updatesLock.readLock().unlock();<a name="line.3559"></a> -<span class="sourceLineNo">3560</span> }<a name="line.3560"></a> -<span class="sourceLineNo">3561</span> releaseRowLocks(acquiredRowLocks);<a name="line.3561"></a> -<span class="sourceLineNo">3562</span><a name="line.3562"></a> -<span class="sourceLineNo">3563</span> // See if the column families were consistent through the whole thing.<a name="line.3563"></a> -<span class="sourceLineNo">3564</span> // if they were then keep them. If they were not then pass a null.<a name="line.3564"></a> -<span class="sourceLineNo">3565</span> // null will be treated as unknown.<a name="line.3565"></a> -<span class="sourceLineNo">3566</span> // Total time taken might be involving Puts and Deletes.<a name="line.3566"></a> -<span class="sourceLineNo">3567</span> // Split the time for puts and deletes based on the total number of Puts and Deletes.<a name="line.3567"></a> -<span class="sourceLineNo">3568</span><a name="line.3568"></a> -<span class="sourceLineNo">3569</span> if (noOfPuts > 0) {<a name="line.3569"></a> -<span class="sourceLineNo">3570</span> // There were some Puts in the batch.<a name="line.3570"></a> -<span class="sourceLineNo">3571</span> if (this.metricsRegion != null) {<a name="line.3571"></a> -<span class="sourceLineNo">3572</span> this.metricsRegion.updatePut();<a name="line.3572"></a> -<span class="sourceLineNo">3573</span> }<a name="line.3573"></a> -<span class="sourceLineNo">3574</span> }<a name="line.3574"></a> -<span class="sourceLineNo">3575</span> if (noOfDeletes > 0) {<a name="line.3575"></a> -<span class="sourceLineNo">3576</span> // There were some Deletes in the batch.<a name="line.3576"></a> -<span class="sourceLineNo">3577</span> if (this.metricsRegion != null) {<a name="line.3577"></a> -<span class="sourceLineNo">3578</span> this.metricsRegion.updateDelete();<a name="line.3578"></a> -<span class="sourceLineNo">3579</span> }<a name="line.3579"></a> -<span class="sourceLineNo">3580</span> }<a name="line.3580"></a> -<span class="sourceLineNo">3581</span> if (!success) {<a name="line.3581"></a> -<span class="sourceLineNo">3582</span> for (int i = firstIndex; i < lastIndexExclusive; i++) {<a name="line.3582"></a> -<span class="sourceLineNo">3583</span> if (batchOp.retCodeDetails[i].getOperationStatusCode() == OperationStatusCode.NOT_RUN) {<a name="line.3583"></a> -<span class="sourceLineNo">3584</span> batchOp.retCodeDetails[i] = OperationStatus.FAILURE;<a name="line.3584"></a> -<span class="sourceLineNo">3585</span> }<a name="line.3585"></a> -<span class="sourceLineNo">3586</span> }<a name="line.3586"></a> -<span class="sourceLineNo">3587</span> }<a name="line.3587"></a> -<span class="sourceLineNo">3588</span> if (coprocessorHost != null && !batchOp.isInReplay()) {<a name="line.3588"></a> -<span class="sourceLineNo">3589</span> // call the coprocessor hook to do any finalization steps<a name="line.3589"></a> -<span class="sourceLineNo">3590</span> // after the put is done<a name="line.3590"></a> -<span class="sourceLineNo">3591</span> MiniBatchOperationInProgress<Mutation> miniBatchOp =<a name="line.3591"></a> -<span class="sourceLineNo">3592</span> new MiniBatchOperationInProgress<>(batchOp.getMutationsForCoprocs(),<a name="line.3592"></a> -<span class="sourceLineNo">3593</span> batchOp.retCodeDetails, batchOp.walEditsFromCoprocessors, firstIndex, lastIndexExclusive);<a name="line.3593"></a> -<span class="sourceLineNo">3594</span> coprocessorHost.postBatchMutateIndispensably(miniBatchOp, success);<a name="line.3594"></a> -<span class="sourceLineNo">3595</span> }<a name="line.3595"></a> -<span class="sourceLineNo">3596</span><a name="line.3596"></a> -<span class="sourceLineNo">3597</span> batchOp.nextIndexToProcess = lastIndexExclusive;<a name="line.3597"></a> -<span class="sourceLineNo">3598</span> }<a name="line.3598"></a> -<span class="sourceLineNo">3599</span> }<a name="line.3599"></a> -<span class="sourceLineNo">3600</span><a name="line.3600"></a> -<span class="sourceLineNo">3601</span> private void mergeFamilyMaps(Map<byte[], List<Cell>> familyMap,<a name="line.3601"></a> -<span class="sourceLineNo">3602</span> Map<byte[], List<Cell>> toBeMerged) {<a name="line.3602"></a> -<span class="sourceLineNo">3603</span> for (Map.Entry<byte[], List<Cell>> entry : toBeMerged.entrySet()) {<a name="line.3603"></a> -<span class="sourceLineNo">3604</span> List<Cell> cells = familyMap.get(entry.getKey());<a name="line.3604"></a> -<span class="sourceLineNo">3605</span> if (cells == null) {<a name="line.3605"></a> -<span class="sourceLineNo">3606</span> familyMap.put(entry.getKey(), entry.getValue());<a name="line.3606"></a> -<span class="sourceLineNo">3607</span> } else {<a name="line.3607"></a> -<span class="sourceLineNo">3608</span> cells.addAll(entry.getValue());<a name="line.3608"></a> -<span class="sourceLineNo">3609</span> }<a name="line.3609"></a> -<span class="sourceLineNo">3610</span> }<a name="line.3610"></a> -<span class="sourceLineNo">3611</span> }<a name="line.3611"></a> -<span class="sourceLineNo">3612</span><a name="line.3612"></a> -<span class="sourceLineNo">3613</span> private void appendCurrentNonces(final Mutation mutation, final boolean replay,<a name="line.3613"></a> -<span class="sourceLineNo">3614</span> final WALEdit walEdit, final long now, final long currentNonceGroup, final long currentNonce)<a name="line.3614"></a> -<span class="sourceLineNo">3615</span> throws IOException {<a name="line.3615"></a> -<span class="sourceLineNo">3616</span> if (walEdit.isEmpty()) return;<a name="line.3616"></a> -<span class="sourceLineNo">3617</span> if (!replay) throw new IOException("Multiple nonces per batch and not in replay");<a name="line.3617"></a> -<span class="sourceLineNo">3618</span> WALKey walKey = new WALKey(this.getRegionInfo().getEncodedNameAsBytes(),<a name="line.3618"></a> -<span class="sourceLineNo">3619</span> this.htableDescriptor.getTableName(), now, mutation.getClusterIds(),<a name="line.3619"></a> -<span class="sourceLineNo">3620</span> currentNonceGroup, currentNonce, mvcc, this.getReplicationScope());<a name="line.3620"></a> -<span class="sourceLineNo">3621</span> this.wal.append(this.getRegionInfo(), walKey, walEdit, true);<a name="line.3621"></a> -<span class="sourceLineNo">3622</span> // Complete the mvcc transaction started down in append else it will block others<a name="line.3622"></a> -<span class="sourceLineNo">3623</span> this.mvcc.complete(walKey.getWriteEntry());<a name="line.3623"></a> -<span class="sourceLineNo">3624</span> }<a name="line.3624"></a> -<span class="sourceLineNo">3625</span><a name="line.3625"></a> -<span class="sourceLineNo">3626</span> private boolean checkBatchOp(BatchOperation<?> batchOp, final int lastIndexExclusive,<a name="line.3626"></a> -<span class="sourceLineNo">3627</span> final Map<byte[], List<Cell>>[] familyMaps, final long now,<a name="line.3627"></a> -<span class="sourceLineNo">3628</span> final ObservedExceptionsInBatch observedExceptions)<a name="line.3628"></a> -<span class="sourceLineNo">3629</span> throws IOException {<a name="line.3629"></a> -<span class="sourceLineNo">3630</span> boolean skip = false;<a name="line.3630"></a> -<span class="sourceLineNo">3631</span> // Skip anything that "ran" already<a name="line.3631"></a> -<span class="sourceLineNo">3632</span> if (batchOp.retCodeDetails[lastIndexExclusive].getOperationStatusCode()<a name="line.3632"></a> -<span class="sourceLineNo">3633</span> != OperationStatusCode.NOT_RUN) {<a name="line.3633"></a> -<span class="sourceLineNo">3634</span> return true;<a name="line.3634"></a> -<span class="sourceLineNo">3635</span> }<a name="line.3635"></a> -<span class="sourceLineNo">3636</span> Mutation mutation = batchOp.getMutation(lastIndexExclusive);<a name="line.3636"></a> -<span class="sourceLineNo">3637</span> Map<byte[], List<Cell>> familyMap = mutation.getFamilyCellMap();<a name="line.3637"></a> -<span class="sourceLineNo">3638</span> // store the family map reference to allow for mutations<a name="line.3638"></a> -<span class="sourceLineNo">3639</span> familyMaps[lastIndexExclusive] = familyMap;<a name="line.3639"></a> -<span class="sourceLineNo">3640</span><a name="line.3640"></a> -<span class="sourceLineNo">3641</span> try {<a name="line.3641"></a> -<span class="sourceLineNo">3642</span> checkAndPrepareMutation(mutation, batchOp.isInReplay(), familyMap, now);<a name="line.3642"></a> -<span class="sourceLineNo">3643</span> } catch (NoSuchColumnFamilyException nscf) {<a name="line.3643"></a> -<span class="sourceLineNo">3644</span> final String msg = "No such column family in batch mutation. ";<a name="line.3644"></a> -<span class="sourceLineNo">3645</span> if (observedExceptions.hasSeenNoSuchFamily()) {<a name="line.3645"></a> -<span class="sourceLineNo">3646</span> LOG.warn(msg + nscf.getMessage());<a name="line.3646"></a> -<span class="sourceLineNo">3647</span> } else {<a name="line.3647"></a> -<span class="sourceLineNo">3648</span> LOG.warn(msg, nscf);<a name="line.3648"></a> -<span class="sourceLineNo">3649</span> observedExceptions.sawNoSuchFamily();<a name="line.3649"></a> -<span class="sourceLineNo">3650</span> }<a name="line.3650"></a> -<span class="sourceLineNo">3651</span> batchOp.retCodeDetails[lastIndexExclusive] = new OperationStatus(<a name="line.3651"></a> -<span class="sourceLineNo">3652</span> OperationStatusCode.BAD_FAMILY, nscf.getMessage());<a name="line.3652"></a> -<span class="sourceLineNo">3653</span> skip = true;<a name="line.3653"></a> -<span class="sourceLineNo">3654</span> } catch (FailedSanityCheckException fsce) {<a name="line.3654"></a> -<span class="sourceLineNo">3655</span> final String msg = "Batch Mutation did not pass sanity check. ";<a name="line.3655"></a> -<span class="sourceLineNo">3656</span> if (observedExceptions.hasSeenFailedSanityCheck()) {<a name="line.3656"></a> -<span class="sourceLineNo">3657</span> LOG.warn(msg + fsce.getMessage());<a name="line.3657"></a> -<span class="sourceLineNo">3658</span> } else {<a name="line.3658"></a> -<span class="sourceLineNo">3659</span> LOG.warn(msg, fsce);<a name="line.3659"></a> -<span class="sourceLineNo">3660</span> observedExceptions.sawFailedSanityCheck();<a name="line.3660"></a> -<span class="sourceLineNo">3661</span> }<a name="line.3661"></a> -<span class="sourceLineNo">3662</span> batchOp.retCodeDetails[lastIndexExclusive] = new OperationStatus(<a name="line.3662"></a> -<span class="sourceLineNo">3663</span> OperationStatusCode.SANITY_CHECK_FAILURE, fsce.getMessage());<a name="line.3663"></a> -<span class="sourceLineNo">3664</span> skip = true;<a name="line.3664"></a> -<span class="sourceLineNo">3665</span> } catch (WrongRegionException we) {<a name="line.3665"></a> -<span class="sourceLineNo">3666</span> final String msg = "Batch mutation had a row that does not belong to this region. ";<a name="line.3666"></a> -<span class="sourceLineNo">3667</span> if (observedExceptions.hasSeenWrongRegion()) {<a name="line.3667"></a> -<span class="sourceLineNo">3668</span> LOG.warn(msg + we.getMessage());<a name="line.3668"></a> -<span class="sourceLineNo">3669</span> } else {<a name="line.3669"></a> -<span class="sourceLineNo">3670</span> LOG.warn(msg, we);<a name="line.3670"></a> -<span class="sourceLineNo">3671</span> observedExceptions.sawWrongRegion();<a name="line.3671"></a> -<span class="sourceLineNo">3672</span> }<a name="line.3672"></a> -<span class="sourceLineNo">3673</span> batchOp.retCodeDetails[lastIndexExclusive] = new OperationStatus(<a name="line.3673"></a> -<span class="sourceLineNo">3674</span> OperationStatusCode.SANITY_CHECK_FAILURE, we.getMessage());<a name="line.3674"></a> -<span class="sourceLineNo">3675</span> skip = true;<a name="line.3675"></a> -<span class="sourceLineNo">3676</span> }<a name="line.3676"></a> -<span class="sourceLineNo">3677</span> return skip;<a name="line.3677"></a> -<span class="sourceLineNo">3678</span> }<a name="line.3678"></a> -<span class="sourceLineNo">3679</span><a name="line.3679"></a> -<span class="sourceLineNo">3680</span> private void checkAndPrepareMutation(Mutation mutation, boolean replay,<a name="line.3680"></a> -<span class="sourceLineNo">3681</span> final Map<byte[], List<Cell>> familyMap, final long now)<a name="line.3681"></a> -<span class="sourceLineNo">3682</span> throws IOException {<a name="line.3682"></a> -<span class="sourceLineNo">3683</span> if (mutation instanceof Put) {<a name="line.3683"></a> -<span class="sourceLineNo">3684</span> // Check the families in the put. If bad, skip this one.<a name="line.3684"></a> -<span class="sourceLineNo">3685</span> if (replay) {<a name="line.3685"></a> -<span class="sourceLineNo">3686</span> removeNonExistentColumnFamilyForReplay(familyMap);<a name="line.3686"></a> -<span class="sourceLineNo">3687</span> } else {<a name="line.3687"></a> -<span class="sourceLineNo">3688</span> checkFamilies(familyMap.keySet());<a name="line.3688"></a> -<span class="sourceLineNo">3689</span> }<a name="line.3689"></a> -<span class="sourceLineNo">3690</span> checkTimestamps(mutation.getFamilyCellMap(), now);<a name="line.3690"></a> -<span class="sourceLineNo">3691</span> } else {<a name="line.3691"></a> -<span class="sourceLineNo">3692</span> prepareDelete((Delete)mutation);<a name="line.3692"></a> -<span class="sourceLineNo">3693</span> }<a name="line.3693"></a> -<span class="sourceLineNo">3694</span> checkRow(mutation.getRow(), "doMiniBatchMutation");<a name="line.3694"></a> -<span class="sourceLineNo">3695</span> }<a name="line.3695"></a> -<span class="sourceLineNo">3696</span><a name="line.3696"></a> -<span class="sourceLineNo">3697</span> /**<a name="line.3697"></a> -<span class="sourceLineNo">3698</span> * During replay, there could exist column families which are removed between region server<a name="line.3698"></a> -<span class="sourceLineNo">3699</span> * failure and replay<a name="line.3699"></a> -<span class="sourceLineNo">3700</span> */<a name="line.3700"></a> -<span class="sourceLineNo">3701</span> private void removeNonExistentColumnFamilyForReplay(final Map<byte[], List<Cell>> familyMap) {<a name="line.3701"></a> -<span class="sourceLineNo">3702</span> List<byte[]> nonExistentList = null;<a name="line.3702"></a> -<span class="sourceLineNo">3703</span> for (byte[] family : familyMap.keySet()) {<a name="line.3703"></a> -<span class="sourceLineNo">3704</span> if (!this.htableDescriptor.hasColumnFamily(family)) {<a name="line.3704"></a> -<span class="sourceLineNo">3705</span> if (nonExistentList == null) {<a name="line.3705"></a> -<span class="sourceLineNo">3706</span> nonExistentList = new ArrayList<>();<a name="line.3706"></a> -<span class="sourceLineNo">3707</span> }<a name="line.3707"></a> -<span class="sourceLineNo">3708</span> nonExistentList.add(family);<a name="line.3708"></a> -<span class="sourceLineNo">3709</span> }<a name="line.3709"></a> -<span class="sourceLineNo">3710</span> }<a name="line.3710"></a> -<span class="sourceLineNo">3711</span> if (nonExistentList != null) {<a name="line.3711"></a> -<span class="sourceLineNo">3712</span> for (byte[] family : nonExistentList) {<a name="line.3712"></a> -<span class="sourceLineNo">3713</span> // Perhaps schema was changed between crash and replay<a name="line.3713"></a> -<span class="sourceLineNo">3714</span> LOG.info("No family for " + Bytes.toString(family) + " omit from reply.");<a name="line.3714"></a> -<span class="sourceLineNo">3715</span> familyMap.remove(family);<a name="line.3715"></a> -<span class="sourceLineNo">3716</span> }<a name="line.3716"></a> -<span class="sourceLineNo">3717</span> }<a name="line.3717"></a> -<span class="sourceLineNo">3718</span> }<a name="line.3718"></a> -<span class="sourceLineNo">3719</span><a name="line.3719"></a> -<span class="sourceLineNo">3720</span> /**<a name="line.3720"></a> -<span class="sourceLineNo">3721</span> * Returns effective durability from the passed durability and<a name="line.3721"></a> -<span class="sourceLineNo">3722</span> * the table descriptor.<a name="line.3722"></a> -<span class="sourceLineNo">3723</span> */<a name="line.3723"></a> -<span class="sourceLineNo">3724</span> protected Durability getEffectiveDurability(Durability d) {<a name="line.3724"></a> -<span class="sourceLineNo">3725</span> return d == Durability.USE_DEFAULT ? this.durability : d;<a name="line.3725"></a> -<span class="sourceLineNo">3726</span> }<a name="line.3726"></a> -<span class="sourceLineNo">3727</span><a name="line.3727"></a> -<span class="sourceLineNo">3728</span> @Override<a name="line.3728"></a> -<span class="sourceLineNo">3729</span> public boolean checkAndMutate(byte [] row, byte [] family, byte [] qualifier,<a name="line.3729"></a> -<span class="sourceLineNo">3730</span> CompareOperator op, ByteArrayComparable comparator, Mutation mutation, boolean writeToWAL)<a name="line.3730"></a> -<span class="sourceLineNo">3731</span> throws IOException{<a name="line.3731"></a> -<span class="sourceLineNo">3732</span> checkMutationType(mutation, row);<a name="line.3732"></a> -<span class="sourceLineNo">3733</span> return doCheckAndRowMutate(row, family, qualifier, op, comparator, null,<a name="line.3733"></a> -<span class="sourceLineNo">3734</span> mutation, writeToWAL);<a name="line.3734"></a> -<span class="sourceLineNo">3735</span> }<a name="line.3735"></a> -<span class="sourceLineNo">3736</span><a name="line.3736"></a> -<span class="sourceLineNo">3737</span> @Override<a name="line.3737"></a> -<span class="sourceLineNo">3738</span> public boolean checkAndRowMutate(byte [] row, byte [] family, byte [] qualifier,<a name="line.3738"></a> -<span class="sourceLineNo">3739</span> CompareOperator op, ByteArrayComparable comparator, RowMutations rm,<a name="line.3739"></a> -<span class="sourceLineNo">3740</span> boolean writeToWAL)<a name="line.3740"></a> -<span class="sourceLineNo">3741</span> throws IOException {<a name="line.3741"></a> -<span class="sourceLineNo">3742</span> return doCheckAndRowMutate(row, family, qualifier, op, comparator, rm, null,<a name="line.3742"></a> -<span class="sourceLineNo">3743</span> writeToWAL);<a name="line.3743"></a> -<span class="sourceLineNo">3744</span> }<a name="line.3744"></a> -<span class="sourceLineNo">3745</span><a name="line.3745"></a> -<span class="sourceLineNo">3746</span> /**<a name="line.3746"></a> -<span class="sourceLineNo">3747</span> * checkAndMutate and checkAndRowMutate are 90% the same. Rather than copy/paste, below has<a name="line.3747"></a> -<span class="sourceLineNo">3748</span> * switches in the few places where there is deviation.<a name="line.3748"></a> -<span class="sourceLineNo">3749</span> */<a name="line.3749"></a> -<span class="sourceLineNo">3750</span> private boolean doCheckAndRowMutate(byte [] row, byte [] family, byte [] qualifier,<a name="line.3750"></a> -<span class="sourceLineNo">3751</span> CompareOperator op, ByteArrayComparable comparator, RowMutations rowMutations,<a name="line.3751"></a> -<span class="sourceLineNo">3752</span> Mutation mutation, boolean writeToWAL)<a name="line.3752"></a> -<span class="sourceLineNo">3753</span> throws IOException {<a name="line.3753"></a> -<span class="sourceLineNo">3754</span> // Could do the below checks but seems wacky with two callers only. Just comment out for now.<a name="line.3754"></a> -<span class="sourceLineNo">3755</span> // One caller passes a Mutation, the other passes RowMutation. Presume all good so we don't<a name="line.3755"></a> -<span class="sourceLineNo">3756</span> // need these commented out checks.<a name="line.3756"></a> -<span class="sourceLineNo">3757</span> // if (rowMutations == null && mutation == null) throw new DoNotRetryIOException("Both null");<a name="line.3757"></a> -<span class="sourceLineNo">3758</span> // if (rowMutations != null && mutation != null) throw new DoNotRetryIOException("Both set");<a name="line.3758"></a> -<span class="sourceLineNo">3759</span> checkReadOnly();<a name="line.3759"></a> -<span class="sourceLineNo">3760</span> // TODO, add check for value length also move this check to the client<a name="line.3760"></a> -<span class="sourceLineNo">3761</span> checkResources();<a name="line.3761"></a> -<span class="sourceLineNo">3762</span> startRegionOperation();<a name="line.3762"></a> -<span class="sourceLineNo">3763</span> try {<a name="line.3763"></a> -<span class="sourceLineNo">3764</span> Get get = new Get(row);<a name="line.3764"></a> -<span class="sourceLineNo">3765</span> checkFamily(family);<a name="line.3765"></a> -<span class="sourceLineNo">3766</span> get.addColumn(family, qualifier);<a name="line.3766"></a> -<span class="sourceLineNo">3767</span> // Lock row - note that doBatchMutate will relock this row if called<a name="line.3767"></a> -<span class="sourceLineNo">3768</span> checkRow(row, "doCheckAndRowMutate");<a name="line.3768"></a> -<span class="sourceLineNo">3769</span> RowLock rowLock = getRowLockInternal(get.getRow(), false);<a name="line.3769"></a> -<span class="sourceLineNo">3770</span> try {<a name="line.3770"></a> -<span class="sourceLineNo">3771</span> if (mutation != null && this.getCoprocessorHost() != null) {<a name="line.3771"></a> -<span class="sourceLineNo">3772</span> // Call coprocessor.<a name="line.3772"></a> -<span class="sourceLineNo">3773</span> Boolean processed = null;<a name="line.3773"></a> -<span class="sourceLineNo">3774</span> if (mutation instanceof Put) {<a name="line.3774"></a> -<span class="sourceLineNo">3775</span> processed = this.getCoprocessorHost().preCheckAndPutAfterRowLock(row, family,<a name="line.3775"></a> -<span class="sourceLineNo">3776</span> qualifier, op, comparator, (Put)mutation);<a name="line.3776"></a> -<span class="sourceLineNo">3777</span> } else if (mutation instanceof Delete) {<a name="line.3777"></a> -<span class="sourceLineNo">3778</span> processed = this.getCoprocessorHost().preCheckAndDeleteAfterRowLock(row, family,<a name="line.3778"></a> -<span class="sourceLineNo">3779</span> qualifier, op, comparator, (Delete)mutation);<a name="line.3779"></a> -<span class="sourceLineNo">3780</span> }<a name="line.3780"></a> -<span class="sourceLineNo">3781</span> if (processed != null) {<a name="line.3781"></a> -<span class="sourceLineNo">3782</span> return processed;<a name="line.3782"></a> -<span class="sourceLineNo">3783</span> }<a name="line.3783"></a> -<span class="sourceLineNo">3784</span> }<a name="line.3784"></a> -<span class="sourceLineNo">3785</span> // NOTE: We used to wait here until mvcc caught up: mvcc.await();<a name="line.3785"></a> -<span class="sourceLineNo">3786</span> // Supposition is that now all changes are done under row locks, then when we go to read,<a name="line.3786"></a> -<span class="sourceLineNo">3787</span> // we'll get the latest on this row.<a name="line.3787"></a> -<span class="sourceLineNo">3788</span> List<Cell> result = get(get, false);<a name="line.3788"></a> -<span class="sourceLineNo">3789</span> boolean valueIsNull = comparator.getValue() == null || comparator.getValue().length == 0;<a name="line.3789"></a> -<span class="sourceLineNo">3790</span> boolean matches = false;<a name="line.3790"></a> -<span class="sourceLineNo">3791</span> long cellTs = 0;<a name="line.3791"></a> -<span class="sourceLineNo">3792</span> if (result.isEmpty() && valueIsNull) {<a name="line.3792"></a> -<span class="sourceLineNo">3793</span> matches = true;<a name="line.3793"></a> -<span class="sourceLineNo">3794</span> } else if (result.size() > 0 && result.get(0).getValueLength() == 0 && valueIsNull) {<a name="line.3794"></a> -<span class="sourceLineNo">3795</span> matches = true;<a name="line.3795"></a> -<span class="sourceLineNo">3796</span> cellTs = result.get(0).getTimestamp();<a name="line.3796"></a> -<span class="sourceLineNo">3797</span> } else if (result.size() == 1 && !valueIsNull) {<a name="line.3797"></a> -<span class="sourceLineNo">3798</span> Cell kv = result.get(0);<a name="line.3798"></a> -<span class="sourceLineNo">3799</span> cellTs = kv.getTimestamp();<a name="line.3799"></a> -<span class="sourceLineNo">3800</span> int compareResult = CellComparator.compareValue(kv, comparator);<a name="line.3800"></a> -<span class="sourceLineNo">3801</span> matches = matches(op, compareResult);<a name="line.3801"></a> -<span class="sourceLineNo">3802</span> }<a name="line.3802"></a> -<span class="sourceLineNo">3803</span> // If matches put the new put or delete the new delete<a name="line.3803"></a> -<span class="sourceLineNo">3804</span> if (matches) {<a name="line.3804"></a> -<span class="sourceLineNo">3805</span> // We have acquired the row lock already. If the system clock is NOT monotonically<a name="line.3805"></a> -<span class="sourceLineNo">3806</span> // non-decreasing (see HBASE-14070) we should make sure that the mutation has a<a name="line.3806"></a> -<span class="sourceLineNo">3807</span> // larger timestamp than what was observed via Get. doBatchMutate already does this, but<a name="line.3807"></a> -<span class="sourceLineNo">3808</span> // there is no way to pass the cellTs. See HBASE-14054.<a name="line.3808"></a> -<span class="sourceLineNo">3809</span> long now = EnvironmentEdgeManager.currentTime();<a name="line.3809"></a> -<span class="sourceLineNo">3810</span> long ts = Math.max(now, cellTs); // ensure write is not eclipsed<a name="line.3810"></a> -<span class="sourceLineNo">3811</span> byte[] byteTs = Bytes.toBytes(ts);<a name="line.3811"></a> -<span class="sourceLineNo">3812</span> if (mutation != null) {<a name="line.3812"></a> -<span class="sourceLineNo">3813</span> if (mutation instanceof Put) {<a name="line.3813"></a> -<span class="sou
<TRUNCATED>
