[
https://issues.apache.org/jira/browse/IMPALA-5031?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16562562#comment-16562562
]
Jim Apple commented on IMPALA-5031:
-----------------------------------
I run the tests with {{-ubsan}} and then {{grep}} the results:
https://jenkins.impala.io/view/Utility/job/ubuntu-16.04-from-scratch/2801/artifact/Impala/logs_static/logs/
{{grep}} for {{runtime error}} .
{noformat}
grep -r -A2 ': runtime error: ' | grep -o 'be/src/.*:[0-9]*:[0-9]*$' | sort |
uniq
{noformat}
{noformat}
be/src/exec/exec-node.cc:394:27
be/src/exec/exec-node.h:316:12
be/src/exec/hdfs-parquet-scanner.cc:1105:7
be/src/exec/hdfs-parquet-scanner.cc:1210:45
be/src/exec/hdfs-parquet-scanner.cc:167:26
be/src/exec/hdfs-scanner.h:510:14
be/src/exec/parquet-column-readers.cc:924:39
be/src/exec/parquet-metadata-utils.cc:162:26
be/src/exec/parquet-metadata-utils.cc:163:26
be/src/exec/parquet-metadata-utils.cc:164:26
be/src/exec/parquet-metadata-utils.cc:166:47
be/src/exec/parquet-metadata-utils.cc:386:22
be/src/exec/parquet-metadata-utils.cc:388:29
be/src/exec/parquet-metadata-utils.cc:402:41
be/src/exprs/agg-fn-evaluator.cc:322:7
be/src/exprs/aggregate-functions-ir.cc:396:3
be/src/exprs/aggregate-functions-ir.cc:438:22
be/src/exprs/aggregate-functions-ir.cc:463:25
be/src/exprs/aggregate-functions-ir.cc:531:3
be/src/exprs/aggregate-functions-ir.cc:572:16
be/src/exprs/aggregate-functions-ir.cc:584:14
be/src/exprs/decimal-operators-ir.cc:131:41
be/src/exprs/expr-test.cc:1480:3
be/src/exprs/expr-test.cc:1484:3
be/src/exprs/expr-test.cc:941:36
be/src/exprs/expr-test.cc:943:36
be/src/exprs/expr-test.cc:945:36
be/src/exprs/math-functions-ir.cc:327:10
be/src/exprs/math-functions-ir.cc:404:24
be/src/exprs/math-functions-ir.cc:405:13
be/src/exprs/operators-ir.cc:167:803
be/src/exprs/operators-ir.cc:168:823
be/src/exprs/operators-ir.cc:169:823
be/src/exprs/scalar-fn-call.cc:485:376
be/src/exprs/string-functions-ir.cc:143:3
be/src/exprs/string-functions-ir.cc:311:5
be/src/exprs/string-functions-ir.cc:868:3
be/src/exprs/string-functions-ir.cc:871:5
be/src/exprs/string-functions-ir.cc:873:5
be/src/runtime/decimal-value.inline.h:136:12
be/src/runtime/decimal-value.inline.h:253:17
be/src/runtime/decimal-value.inline.h:260:10
be/src/runtime/decimal-value.inline.h:260:59
be/src/runtime/decimal-value.inline.h:369:14
be/src/runtime/decimal-value.inline.h:426:16
be/src/runtime/decimal-value.inline.h:491:18
be/src/runtime/mem-pool.cc:190:62
be/src/runtime/mem-tracker.cc:400:31
be/src/runtime/multi-precision.h:106:20
be/src/runtime/multi-precision.h:107:38
be/src/runtime/multi-precision.h:108:12
be/src/runtime/multi-precision.h:109:11
be/src/runtime/raw-value.cc:159:9
be/src/runtime/row-batch-serialize-test.cc:243:9
be/src/runtime/row-batch-serialize-test.cc:300:13
be/src/runtime/sorter.cc:1085:5
be/src/runtime/string-buffer.h:54:5
be/src/runtime/tuple.cc:105:5
be/src/runtime/tuple.cc:146:5
be/src/runtime/tuple.h:98:5
be/src/thirdparty/mustache/mustache.cc:323:3
be/src/thirdparty/mustache/mustache.cc:363:81
be/src/udf/udf.cc:477:36
be/src/udf/udf.cc:507:28
be/src/udf/udf.cc:521:5
be/src/util/bitmap.h:78:5
be/src/util/bit-packing-test.cc:36:14
be/src/util/bit-packing-test.cc:60:25
be/src/util/coding-util-test.cc:45:3
be/src/util/decimal-util.h:49:18
be/src/util/decompress-test.cc:126:254
be/src/util/decompress-test.cc:148:254
be/src/util/decompress-test.cc:221:322
be/src/util/decompress-test.cc:269:254
be/src/util/runtime-profile.cc:1117:11
be/src/util/runtime-profile.cc:1117:3
be/src/util/streaming-sampler.h:110:5
be/src/util/streaming-sampler.h:55:5
be/src/util/string-parser.h:221:5
be/src/util/string-parser.h:397:14
{noformat}
{noformat}
for f in $(grep -r -A2 ': runtime error: ' | grep -o 'be/src/.*:[0-9]*:[0-9]*$'
| sort | uniq)
do
export g=${f//:/ }
export filename=$(echo $g | cut -d ' ' -f 1)
export line=$(echo $g | cut -d ' ' -f 2)
echo -n "$filename # $line ## "
head -n $line ~/Impala/$filename | tail -1 | tr -s ' '
done | sort | less
{noformat}
{noformat}
be/src/exec/exec-node.cc # 394 ## return
mem_tracker()->MemLimitExceeded(state, "Debug Action: MEM_LIMIT_EXCEEDED");
be/src/exec/exec-node.h # 316 ## return ExecDebugActionImpl(phase, state);
be/src/exec/hdfs-parquet-scanner.cc # 1105 ## InitTuple(tuple_desc,
template_tuple, tuple);
be/src/exec/hdfs-parquet-scanner.cc # 1210 ## uint8_t* metadata_ptr =
metadata_size_ptr - metadata_size;
be/src/exec/hdfs-parquet-scanner.cc # 167 ## Status footer_status =
ProcessFooter();
be/src/exec/hdfs-scanner.h # 510 ##
tuple->ClearNullBits(desc->null_bytes_offset(), desc->num_null_bytes());
be/src/exec/parquet-column-readers.cc # 924 ##
RETURN_IF_ERROR(ParquetMetadataUtils::ValidateRowGroupColumn(parent_->file_metadata_,
be/src/exec/parquet-metadata-utils.cc # 162 ## if (col_chunk_metadata.codec !=
parquet::CompressionCodec::UNCOMPRESSED &&
be/src/exec/parquet-metadata-utils.cc # 163 ## col_chunk_metadata.codec !=
parquet::CompressionCodec::SNAPPY &&
be/src/exec/parquet-metadata-utils.cc # 164 ## col_chunk_metadata.codec !=
parquet::CompressionCodec::GZIP) {
be/src/exec/parquet-metadata-utils.cc # 166 ## "'$2'.", filename,
col_chunk_metadata.codec, schema_element.name));
be/src/exec/parquet-metadata-utils.cc # 386 ## if
(node->element->repetition_type == parquet::FieldRepetitionType::OPTIONAL) {
be/src/exec/parquet-metadata-utils.cc # 388 ## } else if
(node->element->repetition_type == parquet::FieldRepetitionType::REPEATED &&
be/src/exec/parquet-metadata-utils.cc # 402 ##
RETURN_IF_ERROR(CreateSchemaTree(schema, max_def_level, max_rep_level,
ira_def_level,
be/src/exprs/agg-fn-evaluator.cc # 322 ##
reinterpret_cast<UpdateFn1>(fn)(agg_fn_ctx_.get(),
be/src/exprs/aggregate-functions-ir.cc # 396 ## DecimalAvgAddOrRemove(ctx,
src, dst, false);
be/src/exprs/aggregate-functions-ir.cc # 438 ## avg->sum_val16 += m *
src.val16;
be/src/exprs/aggregate-functions-ir.cc # 463 ## dst_struct->sum_val16 +=
src_struct->sum_val16;
be/src/exprs/aggregate-functions-ir.cc # 531 ## SumDecimalAddOrSubtract(ctx,
src, dst);
be/src/exprs/aggregate-functions-ir.cc # 572 ## dst->val16 += m * src.val16;
be/src/exprs/aggregate-functions-ir.cc # 584 ## dst->val16 += src.val16;
be/src/exprs/decimal-operators-ir.cc # 131 ## Decimal16Value scaled_val =
val16.ScaleTo(
be/src/exprs/expr-test.cc # 1480 ## TestFixedResultTypeOps<int64_t, int64_t,
int64_t>(numeric_limits<int64_t>::min()+1,
be/src/exprs/expr-test.cc # 1484 ## TestFixedResultTypeOps<int64_t, int64_t,
int64_t>(numeric_limits<int64_t>::min()+1,
be/src/exprs/expr-test.cc # 941 ## static_cast<Result>(cast_a + cast_b));
be/src/exprs/expr-test.cc # 943 ## static_cast<Result>(cast_a - cast_b));
be/src/exprs/expr-test.cc # 945 ## static_cast<Result>(cast_a * cast_b));
be/src/exprs/math-functions-ir.cc # 327 ## if
(!DecimalInBaseToDecimal(num.val, src_base.val, &decimal_num)) {
be/src/exprs/math-functions-ir.cc # 404 ## *result += digit * place;
be/src/exprs/math-functions-ir.cc # 405 ## place *= src_base;
be/src/exprs/operators-ir.cc # 167 ## BINARY_OP_NUMERIC_TYPES(Add, +);
be/src/exprs/operators-ir.cc # 168 ## BINARY_OP_NUMERIC_TYPES(Subtract, -);
be/src/exprs/operators-ir.cc # 169 ## BINARY_OP_NUMERIC_TYPES(Multiply, *);
be/src/exprs/scalar-fn-call.cc # 485 ## BOOST_PP_REPEAT_FROM_TO(0,
BOOST_PP_ADD(MAX_INTERP_ARGS, 1),
be/src/exprs/string-functions-ir.cc # 143 ## memcpy(result.ptr, str.ptr,
str.len);
be/src/exprs/string-functions-ir.cc # 311 ## memcpy(ptr, replace.ptr,
replace.len);
be/src/exprs/string-functions-ir.cc # 868 ## memcpy(ptr, strs[0].ptr,
strs[0].len);
be/src/exprs/string-functions-ir.cc # 871 ## memcpy(ptr, sep.ptr, sep.len);
be/src/exprs/string-functions-ir.cc # 873 ## memcpy(ptr, strs[i].ptr,
strs[i].len);
be/src/runtime/decimal-value.inline.h # 136 ## result *= mult;
be/src/runtime/decimal-value.inline.h # 253 ## left = x_left + y_left +
carry_to_left;
be/src/runtime/decimal-value.inline.h # 260 ## return
DecimalUtil::SafeMultiply(left, mult, *overflow) + right;
be/src/runtime/decimal-value.inline.h # 260 ## return
DecimalUtil::SafeMultiply(left, mult, *overflow) + right;
be/src/runtime/decimal-value.inline.h # 369 ## result = detail::AddLarge(
be/src/runtime/decimal-value.inline.h # 426 ## result = ConvertToInt128(
be/src/runtime/decimal-value.inline.h # 491 ## int128_t r = ConvertToInt128(x
/ y, DecimalUtil::MAX_UNSCALED_DECIMAL16, overflow);
be/src/runtime/mem-pool.cc # 190 ## vector<ChunkInfo>::iterator insert_chunk =
chunks_.begin() + current_chunk_idx_ + 1;
be/src/runtime/mem-tracker.cc # 400 ## const int64_t process_capacity =
process_tracker->SpareCapacity();
be/src/runtime/multi-precision.h # 106 ## int128_t n = v * scale;
be/src/runtime/multi-precision.h # 107 ## *overflow |= (result > max_value -
n);
be/src/runtime/multi-precision.h # 108 ## result += n;
be/src/runtime/multi-precision.h # 109 ## scale *= base;
be/src/runtime/raw-value.cc # 159 ## memcpy(dest->ptr, src->ptr, dest->len);
be/src/runtime/row-batch-serialize-test.cc # 243 ## char buf[len];
be/src/runtime/row-batch-serialize-test.cc # 300 ## WriteValue(tuple,
*slot_desc, batch->tuple_data_pool());
be/src/runtime/sorter.cc # 1085 ## memcpy(dest, string_val->ptr,
string_val->len);
be/src/runtime/string-buffer.h # 54 ## memcpy(buffer_ + len_, str, str_len);
be/src/runtime/tuple.cc # 105 ## memcpy(string_copy, string_v->ptr,
string_v->len);
be/src/runtime/tuple.cc # 146 ## memcpy(*data, string_v->ptr, string_v->len);
be/src/runtime/tuple.h # 98 ## memset(reinterpret_cast<uint8_t*>(this) +
null_bytes_offset, 0, num_null_bytes);
be/src/thirdparty/mustache/mustache.cc # 323 ## RenderTemplate(file_ss.str(),
document_root, *parent_context, out);
be/src/thirdparty/mustache/mustache.cc # 363 ## idx = EvaluateTag(document,
document_root, idx, &context, tag_op, tag_name, is_triple,
be/src/udf/udf.cc # 477 ## uint8_t* buffer =
results_pool_->Allocate(byte_size);
be/src/udf/udf.cc # 507 ## ptr = context->impl()->AllocateForResults(str_len);
be/src/udf/udf.cc # 521 ## memcpy(result.ptr, buf, len);
be/src/util/bitmap.h # 78 ## void SetAllBits(bool b) {
be/src/util/bit-packing-test.cc # 36 ## return (1U << bit_width) - 1;
be/src/util/bit-packing-test.cc # 60 ## const uint32_t mask =
ComputeMask(bit_width);
be/src/util/coding-util-test.cc # 45 ## memcpy(input_vector.data(),
input.c_str(), input.size());
be/src/util/decimal-util.h # 49 ## T result = a * b;
be/src/util/decompress-test.cc # 126 ## EXPECT_EQ(memcmp(input, output,
input_len), 0);
be/src/util/decompress-test.cc # 148 ## EXPECT_EQ(memcmp(input, output,
input_len), 0);
be/src/util/decompress-test.cc # 221 ## EXPECT_EQ(memcmp(uncompressed_input +
decompressed_len, output, output_len), 0);
be/src/util/decompress-test.cc # 269 ## EXPECT_EQ(memcmp(input, output,
input_len), 0);
be/src/util/runtime-profile.cc # 1117 ## memcpy(&counter->values[0], samples,
num * sizeof(int64_t));
be/src/util/runtime-profile.cc # 1117 ## memcpy(&counter->values[0], samples,
num * sizeof(int64_t));
be/src/util/streaming-sampler.h # 110 ## memcpy(samples_, samples.data(),
sizeof(T) * samples_collected_);
be/src/util/streaming-sampler.h # 55 ## memcpy(samples_,
initial_samples.data(), sizeof(T) * samples_collected_);
be/src/util/string-parser.h # 221 ## ApplyExponent(total_digits_count,
digits_after_dot_count,
be/src/util/string-parser.h # 397 ## *value *=
DecimalUtil::GetScaleMultiplier<T>(exponent - digits_after_dot_count);
{noformat}
> UBSAN clean and method for testing UBSAN cleanliness
> ----------------------------------------------------
>
> Key: IMPALA-5031
> URL: https://issues.apache.org/jira/browse/IMPALA-5031
> Project: IMPALA
> Issue Type: Task
> Components: Backend, Infrastructure
> Affects Versions: Impala 2.9.0
> Reporter: Jim Apple
> Assignee: Jim Apple
> Priority: Minor
>
> http://releases.llvm.org/3.8.0/tools/clang/docs/UndefinedBehaviorSanitizer.html
> builds are supported after https://gerrit.cloudera.org/#/c/6186/, but
> Impala's test suite triggers many errors under UBSAN. Those errors should be
> fixed and then there should be a way to run the test suite under UBSAN and
> fail if there were any errors detected.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
