http://git-wip-us.apache.org/repos/asf/impala/blob/fcf190c4/be/src/kudu/util/debug/trace_event_impl.h ---------------------------------------------------------------------- diff --git a/be/src/kudu/util/debug/trace_event_impl.h b/be/src/kudu/util/debug/trace_event_impl.h new file mode 100644 index 0000000..2650e8b --- /dev/null +++ b/be/src/kudu/util/debug/trace_event_impl.h @@ -0,0 +1,726 @@ +// Copyright (c) 2012 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef KUDU_UTIL_DEBUG_TRACE_EVENT_IMPL_H_ +#define KUDU_UTIL_DEBUG_TRACE_EVENT_IMPL_H_ + +#include <cstddef> +#include <cstdint> +#include <iosfwd> +#include <stack> +#include <string> +#include <unordered_map> +#include <vector> + +#include <glog/logging.h> +#include <gtest/gtest_prod.h> + +#include "kudu/gutil/atomicops.h" +#include "kudu/gutil/bind_helpers.h" +#include "kudu/gutil/callback.h" +#include "kudu/gutil/gscoped_ptr.h" +#include "kudu/gutil/macros.h" +#include "kudu/gutil/ref_counted.h" +#include "kudu/gutil/spinlock.h" +#include "kudu/gutil/walltime.h" +#include "kudu/util/mutex.h" + +// Older style trace macros with explicit id and extra data +// Only these macros result in publishing data to ETW as currently implemented. +#define TRACE_EVENT_BEGIN_ETW(name, id, extra) \ + base::debug::TraceLog::AddTraceEventEtw( \ + TRACE_EVENT_PHASE_BEGIN, \ + name, reinterpret_cast<const void*>(id), extra) + +#define TRACE_EVENT_END_ETW(name, id, extra) \ + base::debug::TraceLog::AddTraceEventEtw( \ + TRACE_EVENT_PHASE_END, \ + name, reinterpret_cast<const void*>(id), extra) + +#define TRACE_EVENT_INSTANT_ETW(name, id, extra) \ + base::debug::TraceLog::AddTraceEventEtw( \ + TRACE_EVENT_PHASE_INSTANT, \ + name, reinterpret_cast<const void*>(id), extra) + +template <typename Type> +class Singleton; + +#if defined(COMPILER_GCC) +namespace BASE_HASH_NAMESPACE { +template <> +struct hash<kudu::Thread*> { + std::size_t operator()(kudu::Thread* value) const { + return reinterpret_cast<std::size_t>(value); + } +}; +} // BASE_HASH_NAMESPACE +#endif + +namespace kudu { + +class RefCountedString; +class Thread; + +namespace debug { + +// For any argument of type TRACE_VALUE_TYPE_CONVERTABLE the provided +// class must implement this interface. +class ConvertableToTraceFormat : public kudu::RefCountedThreadSafe<ConvertableToTraceFormat> { + public: + // Append the class info to the provided |out| string. The appended + // data must be a valid JSON object. Strings must be properly quoted, and + // escaped. There is no processing applied to the content after it is + // appended. + virtual void AppendAsTraceFormat(std::string* out) const = 0; + + protected: + virtual ~ConvertableToTraceFormat() {} + + private: + friend class kudu::RefCountedThreadSafe<ConvertableToTraceFormat>; +}; + +struct TraceEventHandle { + uint32_t chunk_seq; + uint16_t chunk_index; + uint16_t event_index; +}; + +const int kTraceMaxNumArgs = 2; + +class BASE_EXPORT TraceEvent { + public: + union TraceValue { + bool as_bool; + uint64_t as_uint; + long long as_int; + double as_double; + const void* as_pointer; + const char* as_string; + }; + + TraceEvent(); + ~TraceEvent(); + + // We don't need to copy TraceEvent except when TraceEventBuffer is cloned. + // Use explicit copy method to avoid accidentally misuse of copy. + void CopyFrom(const TraceEvent& other); + + void Initialize( + int thread_id, + MicrosecondsInt64 timestamp, + MicrosecondsInt64 thread_timestamp, + char phase, + const unsigned char* category_group_enabled, + const char* name, + uint64_t id, + int num_args, + const char** arg_names, + const unsigned char* arg_types, + const uint64_t* arg_values, + const scoped_refptr<ConvertableToTraceFormat>* convertable_values, + unsigned char flags); + + void Reset(); + + void UpdateDuration(const MicrosecondsInt64& now, const MicrosecondsInt64& thread_now); + + // Serialize event data to JSON + void AppendAsJSON(std::string* out) const; + void AppendPrettyPrinted(std::ostringstream* out) const; + + static void AppendValueAsJSON(unsigned char type, + TraceValue value, + std::string* out); + + MicrosecondsInt64 timestamp() const { return timestamp_; } + MicrosecondsInt64 thread_timestamp() const { return thread_timestamp_; } + char phase() const { return phase_; } + int thread_id() const { return thread_id_; } + MicrosecondsInt64 duration() const { return duration_; } + MicrosecondsInt64 thread_duration() const { return thread_duration_; } + uint64_t id() const { return id_; } + unsigned char flags() const { return flags_; } + + // Exposed for unittesting: + + const kudu::RefCountedString* parameter_copy_storage() const { + return parameter_copy_storage_.get(); + } + + const unsigned char* category_group_enabled() const { + return category_group_enabled_; + } + + const char* name() const { return name_; } + +#if defined(OS_ANDROID) + void SendToATrace(); +#endif + + private: + // Note: these are ordered by size (largest first) for optimal packing. + MicrosecondsInt64 timestamp_; + MicrosecondsInt64 thread_timestamp_; + MicrosecondsInt64 duration_; + MicrosecondsInt64 thread_duration_; + // id_ can be used to store phase-specific data. + uint64_t id_; + TraceValue arg_values_[kTraceMaxNumArgs]; + const char* arg_names_[kTraceMaxNumArgs]; + scoped_refptr<ConvertableToTraceFormat> convertable_values_[kTraceMaxNumArgs]; + const unsigned char* category_group_enabled_; + const char* name_; + scoped_refptr<kudu::RefCountedString> parameter_copy_storage_; + int thread_id_; + char phase_; + unsigned char flags_; + unsigned char arg_types_[kTraceMaxNumArgs]; + + DISALLOW_COPY_AND_ASSIGN(TraceEvent); +}; + +// TraceBufferChunk is the basic unit of TraceBuffer. +class BASE_EXPORT TraceBufferChunk { + public: + explicit TraceBufferChunk(uint32_t seq) + : next_free_(0), + seq_(seq) { + } + + void Reset(uint32_t new_seq); + TraceEvent* AddTraceEvent(size_t* event_index); + bool IsFull() const { return next_free_ == kTraceBufferChunkSize; } + + uint32_t seq() const { return seq_; } + size_t capacity() const { return kTraceBufferChunkSize; } + size_t size() const { return next_free_; } + + TraceEvent* GetEventAt(size_t index) { + DCHECK(index < size()); + return &chunk_[index]; + } + const TraceEvent* GetEventAt(size_t index) const { + DCHECK(index < size()); + return &chunk_[index]; + } + + gscoped_ptr<TraceBufferChunk> Clone() const; + + static const size_t kTraceBufferChunkSize = 64; + + private: + size_t next_free_; + TraceEvent chunk_[kTraceBufferChunkSize]; + uint32_t seq_; +}; + +// TraceBuffer holds the events as they are collected. +class BASE_EXPORT TraceBuffer { + public: + virtual ~TraceBuffer() {} + + virtual gscoped_ptr<TraceBufferChunk> GetChunk(size_t *index) = 0; + virtual void ReturnChunk(size_t index, + gscoped_ptr<TraceBufferChunk> chunk) = 0; + + virtual bool IsFull() const = 0; + virtual size_t Size() const = 0; + virtual size_t Capacity() const = 0; + virtual TraceEvent* GetEventByHandle(TraceEventHandle handle) = 0; + + // For iteration. Each TraceBuffer can only be iterated once. + virtual const TraceBufferChunk* NextChunk() = 0; + + virtual gscoped_ptr<TraceBuffer> CloneForIteration() const = 0; +}; + +// TraceResultBuffer collects and converts trace fragments returned by TraceLog +// to JSON output. +class TraceResultBuffer { + public: + static std::string FlushTraceLogToString(); + static std::string FlushTraceLogToStringButLeaveBufferIntact(); + + private: + TraceResultBuffer(); + ~TraceResultBuffer(); + + static std::string DoFlush(bool leave_intact); + + // Callback for TraceLog::Flush + void Collect(const scoped_refptr<RefCountedString>& s, + bool has_more_events); + + bool first_; + std::string json_; +}; + +class BASE_EXPORT CategoryFilter { + public: + typedef std::vector<std::string> StringList; + + // The default category filter, used when none is provided. + // Allows all categories through, except if they end in the suffix 'Debug' or + // 'Test'. + static const char* kDefaultCategoryFilterString; + + // |filter_string| is a comma-delimited list of category wildcards. + // A category can have an optional '-' prefix to make it an excluded category. + // All the same rules apply above, so for example, having both included and + // excluded categories in the same list would not be supported. + // + // Example: CategoryFilter"test_MyTest*"); + // Example: CategoryFilter("test_MyTest*,test_OtherStuff"); + // Example: CategoryFilter("-excluded_category1,-excluded_category2"); + // Example: CategoryFilter("-*,webkit"); would disable everything but webkit. + // Example: CategoryFilter("-webkit"); would enable everything but webkit. + // + // Category filters can also be used to configure synthetic delays. + // + // Example: CategoryFilter("DELAY(gpu.PresentingFrame;16)"); would make swap + // buffers always take at least 16 ms. + // Example: CategoryFilter("DELAY(gpu.PresentingFrame;16;oneshot)"); would + // make swap buffers take at least 16 ms the first time it is + // called. + // Example: CategoryFilter("DELAY(gpu.PresentingFrame;16;alternating)"); + // would make swap buffers take at least 16 ms every other time it + // is called. + explicit CategoryFilter(const std::string& filter_string); + + CategoryFilter(const CategoryFilter& cf); + + ~CategoryFilter(); + + CategoryFilter& operator=(const CategoryFilter& rhs); + + // Writes the string representation of the CategoryFilter. This is a comma + // separated string, similar in nature to the one used to determine + // enabled/disabled category patterns, except here there is an arbitrary + // order, included categories go first, then excluded categories. Excluded + // categories are distinguished from included categories by the prefix '-'. + std::string ToString() const; + + // Determines whether category group would be enabled or + // disabled by this category filter. + bool IsCategoryGroupEnabled(const char* category_group) const; + + // Return a list of the synthetic delays specified in this category filter. + const StringList& GetSyntheticDelayValues() const; + + // Merges nested_filter with the current CategoryFilter + void Merge(const CategoryFilter& nested_filter); + + // Clears both included/excluded pattern lists. This would be equivalent to + // creating a CategoryFilter with an empty string, through the constructor. + // i.e: CategoryFilter(""). + // + // When using an empty filter, all categories are considered included as we + // are not excluding anything. + void Clear(); + + private: + FRIEND_TEST(TraceEventTestFixture, CategoryFilter); + + static bool IsEmptyOrContainsLeadingOrTrailingWhitespace( + const std::string& str); + + void Initialize(const std::string& filter_string); + void WriteString(const StringList& values, + std::string* out, + bool included) const; + void WriteString(const StringList& delays, std::string* out) const; + bool HasIncludedPatterns() const; + + bool DoesCategoryGroupContainCategory(const char* category_group, + const char* category) const; + + StringList included_; + StringList disabled_; + StringList excluded_; + StringList delays_; +}; + +class TraceSamplingThread; + +class BASE_EXPORT TraceLog { + public: + enum Mode { + DISABLED = 0, + RECORDING_MODE, + MONITORING_MODE, + }; + + // Options determines how the trace buffer stores data. + enum Options { + // Record until the trace buffer is full. + RECORD_UNTIL_FULL = 1 << 0, + + // Record until the user ends the trace. The trace buffer is a fixed size + // and we use it as a ring buffer during recording. + RECORD_CONTINUOUSLY = 1 << 1, + + // Enable the sampling profiler in the recording mode. + ENABLE_SAMPLING = 1 << 2, + + // Echo to console. Events are discarded. + ECHO_TO_CONSOLE = 1 << 3, + }; + + // The pointer returned from GetCategoryGroupEnabledInternal() points to a + // value with zero or more of the following bits. Used in this class only. + // The TRACE_EVENT macros should only use the value as a bool. + // These values must be in sync with macro values in TraceEvent.h in Blink. + enum CategoryGroupEnabledFlags { + // Category group enabled for the recording mode. + ENABLED_FOR_RECORDING = 1 << 0, + // Category group enabled for the monitoring mode. + ENABLED_FOR_MONITORING = 1 << 1, + // Category group enabled by SetEventCallbackEnabled(). + ENABLED_FOR_EVENT_CALLBACK = 1 << 2, + }; + + static TraceLog* GetInstance(); + + // Get set of known category groups. This can change as new code paths are + // reached. The known category groups are inserted into |category_groups|. + void GetKnownCategoryGroups(std::vector<std::string>* category_groups); + + // Retrieves a copy (for thread-safety) of the current CategoryFilter. + CategoryFilter GetCurrentCategoryFilter(); + + Options trace_options() const { + return static_cast<Options>(base::subtle::NoBarrier_Load(&trace_options_)); + } + + // Enables normal tracing (recording trace events in the trace buffer). + // See CategoryFilter comments for details on how to control what categories + // will be traced. If tracing has already been enabled, |category_filter| will + // be merged into the current category filter. + void SetEnabled(const CategoryFilter& category_filter, + Mode mode, Options options); + + // Disables normal tracing for all categories. + void SetDisabled(); + + bool IsEnabled() { return mode_ != DISABLED; } + + // The number of times we have begun recording traces. If tracing is off, + // returns -1. If tracing is on, then it returns the number of times we have + // recorded a trace. By watching for this number to increment, you can + // passively discover when a new trace has begun. This is then used to + // implement the TRACE_EVENT_IS_NEW_TRACE() primitive. + int GetNumTracesRecorded(); + +#if defined(OS_ANDROID) + void StartATrace(); + void StopATrace(); + void AddClockSyncMetadataEvent(); +#endif + + // Enabled state listeners give a callback when tracing is enabled or + // disabled. This can be used to tie into other library's tracing systems + // on-demand. + class EnabledStateObserver { + public: + // Called just after the tracing system becomes enabled, outside of the + // |lock_|. TraceLog::IsEnabled() is true at this point. + virtual void OnTraceLogEnabled() = 0; + + // Called just after the tracing system disables, outside of the |lock_|. + // TraceLog::IsEnabled() is false at this point. + virtual void OnTraceLogDisabled() = 0; + }; + void AddEnabledStateObserver(EnabledStateObserver* listener); + void RemoveEnabledStateObserver(EnabledStateObserver* listener); + bool HasEnabledStateObserver(EnabledStateObserver* listener) const; + + float GetBufferPercentFull() const; + bool BufferIsFull() const; + + // Not using kudu::Callback because of its limited by 7 parameters. + // Also, using primitive type allows directly passing callback from WebCore. + // WARNING: It is possible for the previously set callback to be called + // after a call to SetEventCallbackEnabled() that replaces or a call to + // SetEventCallbackDisabled() that disables the callback. + // This callback may be invoked on any thread. + // For TRACE_EVENT_PHASE_COMPLETE events, the client will still receive pairs + // of TRACE_EVENT_PHASE_BEGIN and TRACE_EVENT_PHASE_END events to keep the + // interface simple. + typedef void (*EventCallback)(MicrosecondsInt64 timestamp, + char phase, + const unsigned char* category_group_enabled, + const char* name, + uint64_t id, + int num_args, + const char* const arg_names[], + const unsigned char arg_types[], + const uint64_t arg_values[], + unsigned char flags); + + // Enable tracing for EventCallback. + void SetEventCallbackEnabled(const CategoryFilter& category_filter, + EventCallback cb); + void SetEventCallbackDisabled(); + + // Flush all collected events to the given output callback. The callback will + // be called one or more times synchronously from + // the current thread with IPC-bite-size chunks. The string format is + // undefined. Use TraceResultBuffer to convert one or more trace strings to + // JSON. The callback can be null if the caller doesn't want any data. + // Due to the implementation of thread-local buffers, flush can't be + // done when tracing is enabled. If called when tracing is enabled, the + // callback will be called directly with (empty_string, false) to indicate + // the end of this unsuccessful flush. + typedef kudu::Callback<void(const scoped_refptr<kudu::RefCountedString>&, + bool has_more_events)> OutputCallback; + void Flush(const OutputCallback& cb); + void FlushButLeaveBufferIntact(const OutputCallback& flush_output_callback); + + // Called by TRACE_EVENT* macros, don't call this directly. + // The name parameter is a category group for example: + // TRACE_EVENT0("renderer,webkit", "WebViewImpl::HandleInputEvent") + static const unsigned char* GetCategoryGroupEnabled(const char* name); + static const char* GetCategoryGroupName( + const unsigned char* category_group_enabled); + + // Called by TRACE_EVENT* macros, don't call this directly. + // If |copy| is set, |name|, |arg_name1| and |arg_name2| will be deep copied + // into the event; see "Memory scoping note" and TRACE_EVENT_COPY_XXX above. + TraceEventHandle AddTraceEvent( + char phase, + const unsigned char* category_group_enabled, + const char* name, + uint64_t id, + int num_args, + const char** arg_names, + const unsigned char* arg_types, + const uint64_t* arg_values, + const scoped_refptr<ConvertableToTraceFormat>* convertable_values, + unsigned char flags); + TraceEventHandle AddTraceEventWithThreadIdAndTimestamp( + char phase, + const unsigned char* category_group_enabled, + const char* name, + uint64_t id, + int thread_id, + const MicrosecondsInt64& timestamp, + int num_args, + const char** arg_names, + const unsigned char* arg_types, + const uint64_t* arg_values, + const scoped_refptr<ConvertableToTraceFormat>* convertable_values, + unsigned char flags); + static void AddTraceEventEtw(char phase, + const char* category_group, + const void* id, + const char* extra); + static void AddTraceEventEtw(char phase, + const char* category_group, + const void* id, + const std::string& extra); + + void UpdateTraceEventDuration(const unsigned char* category_group_enabled, + const char* name, + TraceEventHandle handle); + + // For every matching event, the callback will be called. + typedef kudu::Callback<void()> WatchEventCallback; + void SetWatchEvent(const std::string& category_name, + const std::string& event_name, + const WatchEventCallback& callback); + // Cancel the watch event. If tracing is enabled, this may race with the + // watch event notification firing. + void CancelWatchEvent(); + + int process_id() const { return process_id_; } + + // Allow tests to inspect TraceEvents. + size_t GetEventsSize() const { return logged_events_->Size(); } + TraceEvent* GetEventByHandle(TraceEventHandle handle); + + void SetProcessID(int process_id); + + // Process sort indices, if set, override the order of a process will appear + // relative to other processes in the trace viewer. Processes are sorted first + // on their sort index, ascending, then by their name, and then tid. + void SetProcessSortIndex(int sort_index); + + // Sets the name of the process. + void SetProcessName(const std::string& process_name); + + // Processes can have labels in addition to their names. Use labels, for + // instance, to list out the web page titles that a process is handling. + void UpdateProcessLabel(int label_id, const std::string& current_label); + void RemoveProcessLabel(int label_id); + + // Thread sort indices, if set, override the order of a thread will appear + // within its process in the trace viewer. Threads are sorted first on their + // sort index, ascending, then by their name, and then tid. + void SetThreadSortIndex(int64_t tid , int sort_index); + + // Allow setting an offset between the current MicrosecondsInt64 time and the time + // that should be reported. + void SetTimeOffset(MicrosecondsInt64 offset); + + size_t GetObserverCountForTest() const; + + + private: + FRIEND_TEST(TraceEventTestFixture, + TraceBufferRingBufferGetReturnChunk); + FRIEND_TEST(TraceEventTestFixture, + TraceBufferRingBufferHalfIteration); + FRIEND_TEST(TraceEventTestFixture, + TraceBufferRingBufferFullIteration); + + // This allows constructor and destructor to be private and usable only + // by the Singleton class. + friend class Singleton<TraceLog>; + + // Enable/disable each category group based on the current mode_, + // category_filter_, event_callback_ and event_callback_category_filter_. + // Enable the category group in the enabled mode if category_filter_ matches + // the category group, or event_callback_ is not null and + // event_callback_category_filter_ matches the category group. + void UpdateCategoryGroupEnabledFlags(); + void UpdateCategoryGroupEnabledFlag(int category_index); + + // Configure synthetic delays based on the values set in the current + // category filter. + void UpdateSyntheticDelaysFromCategoryFilter(); + + struct PerThreadInfo; + class OptionalAutoLock; + class ThreadLocalEventBuffer; + + TraceLog(); + ~TraceLog(); + const unsigned char* GetCategoryGroupEnabledInternal(const char* name); + void AddMetadataEventsWhileLocked(); + + TraceBuffer* trace_buffer() const { return logged_events_.get(); } + TraceBuffer* CreateTraceBuffer(); + + std::string EventToConsoleMessage(unsigned char phase, + const MicrosecondsInt64& timestamp, + TraceEvent* trace_event); + + TraceEvent* AddEventToThreadSharedChunkWhileLocked(TraceEventHandle* handle, + bool check_buffer_is_full); + void CheckIfBufferIsFullWhileLocked(); + void SetDisabledWhileLocked(); + + TraceEvent* GetEventByHandleInternal(TraceEventHandle handle, + OptionalAutoLock* lock); + + void ConvertTraceEventsToTraceFormat(gscoped_ptr<TraceBuffer> logged_events, + const OutputCallback& flush_output_callback); + void FinishFlush(int generation, + const OutputCallback& flush_output_callback); + + // Called when a thread which has registered trace events is about to exit. + void ThreadExiting(); + + // The static callback registered as a thread destructor. + static void ThreadExitingCB(void* arg); + + int generation() const { + return static_cast<int>(base::subtle::NoBarrier_Load(&generation_)); + } + bool CheckGeneration(int generation) const { + return generation == this->generation(); + } + void UseNextTraceBuffer(); + + MicrosecondsInt64 OffsetNow() const { + return OffsetTimestamp(GetMonoTimeMicros()); + } + MicrosecondsInt64 OffsetTimestamp(const MicrosecondsInt64& timestamp) const { + return timestamp - time_offset_; + } + + // Create a new PerThreadInfo object for the current thread, + // and register it in the active_threads_ list. + PerThreadInfo* SetupThreadLocalBuffer(); + + // This lock protects TraceLog member accesses (except for members protected + // by thread_info_lock_) from arbitrary threads. + mutable base::SpinLock lock_; + // This lock protects accesses to thread_names_, thread_event_start_times_ + // and thread_colors_. + base::SpinLock thread_info_lock_; + int locked_line_; + Mode mode_; + int num_traces_recorded_; + gscoped_ptr<TraceBuffer> logged_events_; + AtomicWord /* EventCallback */ event_callback_; + bool dispatching_to_observer_list_; + std::vector<EnabledStateObserver*> enabled_state_observer_list_; + + std::string process_name_; + std::unordered_map<int, std::string> process_labels_; + int process_sort_index_; + std::unordered_map<int, int> thread_sort_indices_; + std::unordered_map<int, std::string> thread_names_; + + // The following two maps are used only when ECHO_TO_CONSOLE. + std::unordered_map<int, std::stack<MicrosecondsInt64> > thread_event_start_times_; + std::unordered_map<std::string, int> thread_colors_; + + // XORed with TraceID to make it unlikely to collide with other processes. + uint64_t process_id_hash_; + + int process_id_; + + MicrosecondsInt64 time_offset_; + + // Allow tests to wake up when certain events occur. + WatchEventCallback watch_event_callback_; + AtomicWord /* const unsigned char* */ watch_category_; + std::string watch_event_name_; + + AtomicWord /* Options */ trace_options_; + + // Sampling thread handles. + gscoped_ptr<TraceSamplingThread> sampling_thread_; + scoped_refptr<kudu::Thread> sampling_thread_handle_; + + CategoryFilter category_filter_; + CategoryFilter event_callback_category_filter_; + + struct PerThreadInfo { + ThreadLocalEventBuffer* event_buffer_; + base::subtle::Atomic32 is_in_trace_event_; + + // Atomically take the event_buffer_ member, setting it to NULL. + // Returns the old value of the member. + ThreadLocalEventBuffer* AtomicTakeBuffer(); + }; + static __thread PerThreadInfo* thread_local_info_; + + Mutex active_threads_lock_; + // Map of PID -> PerThreadInfo + // Protected by active_threads_lock_. + typedef std::unordered_map<int64_t, PerThreadInfo*> ActiveThreadMap; + ActiveThreadMap active_threads_; + + // For events which can't be added into the thread local buffer, e.g. events + // from threads without a message loop. + gscoped_ptr<TraceBufferChunk> thread_shared_chunk_; + size_t thread_shared_chunk_index_; + + // The generation is incremented whenever tracing is enabled, and incremented + // again when the buffers are flushed. This ensures that trace events logged + // for a previous tracing session do not get accidentally flushed in the + // next tracing session. + AtomicWord generation_; + + DISALLOW_COPY_AND_ASSIGN(TraceLog); +}; + +} // namespace debug +} // namespace kudu + +#endif // KUDU_UTIL_DEBUG_TRACE_EVENT_IMPL_H_
http://git-wip-us.apache.org/repos/asf/impala/blob/fcf190c4/be/src/kudu/util/debug/trace_event_impl_constants.cc ---------------------------------------------------------------------- diff --git a/be/src/kudu/util/debug/trace_event_impl_constants.cc b/be/src/kudu/util/debug/trace_event_impl_constants.cc new file mode 100644 index 0000000..bf45ed7 --- /dev/null +++ b/be/src/kudu/util/debug/trace_event_impl_constants.cc @@ -0,0 +1,14 @@ +// Copyright (c) 2013 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "kudu/util/debug/trace_event_impl.h" + +namespace kudu { +namespace debug { + +// Enable everything but debug and test categories by default. +const char* CategoryFilter::kDefaultCategoryFilterString = "-*Debug,-*Test"; + +} // namespace debug +} // namespace kudu http://git-wip-us.apache.org/repos/asf/impala/blob/fcf190c4/be/src/kudu/util/debug/trace_event_memory.h ---------------------------------------------------------------------- diff --git a/be/src/kudu/util/debug/trace_event_memory.h b/be/src/kudu/util/debug/trace_event_memory.h new file mode 100644 index 0000000..6d9cf8d --- /dev/null +++ b/be/src/kudu/util/debug/trace_event_memory.h @@ -0,0 +1,28 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +#ifndef KUDU_DEBUG_TRACE_EVENT_MEMORY_H +#define KUDU_DEBUG_TRACE_EVENT_MEMORY_H + +// Stub for this part of chromium tracing we haven't yet +// imported. +// The Chromium code relies on a locally patch tcmalloc. +// See 5bc71bae28ea03689dbf50fe6baa15b574319091 in the Chromium +// repository. + +#define INTERNAL_TRACE_MEMORY(category_group, name) + +#endif /* KUDU_DEBUG_TRACE_EVENT_MEMORY_H */ http://git-wip-us.apache.org/repos/asf/impala/blob/fcf190c4/be/src/kudu/util/debug/trace_event_synthetic_delay.cc ---------------------------------------------------------------------- diff --git a/be/src/kudu/util/debug/trace_event_synthetic_delay.cc b/be/src/kudu/util/debug/trace_event_synthetic_delay.cc new file mode 100644 index 0000000..947ab88 --- /dev/null +++ b/be/src/kudu/util/debug/trace_event_synthetic_delay.cc @@ -0,0 +1,238 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "kudu/util/debug/trace_event_synthetic_delay.h" + +#include <cstring> +#include <ostream> + +#include <glog/logging.h> + +#include "kudu/gutil/dynamic_annotations.h" +#include "kudu/gutil/port.h" +#include "kudu/gutil/singleton.h" + +namespace { +const int kMaxSyntheticDelays = 32; +} // namespace + +namespace kudu { +namespace debug { + +TraceEventSyntheticDelayClock::TraceEventSyntheticDelayClock() {} +TraceEventSyntheticDelayClock::~TraceEventSyntheticDelayClock() {} + +class TraceEventSyntheticDelayRegistry : public TraceEventSyntheticDelayClock { + public: + static TraceEventSyntheticDelayRegistry* GetInstance(); + + TraceEventSyntheticDelay* GetOrCreateDelay(const char* name); + void ResetAllDelays(); + + // TraceEventSyntheticDelayClock implementation. + virtual MonoTime Now() OVERRIDE; + + private: + TraceEventSyntheticDelayRegistry(); + + friend class Singleton<TraceEventSyntheticDelayRegistry>; + + Mutex lock_; + TraceEventSyntheticDelay delays_[kMaxSyntheticDelays]; + TraceEventSyntheticDelay dummy_delay_; + base::subtle::Atomic32 delay_count_; + + DISALLOW_COPY_AND_ASSIGN(TraceEventSyntheticDelayRegistry); +}; + +TraceEventSyntheticDelay::TraceEventSyntheticDelay() + : mode_(STATIC), begin_count_(0), trigger_count_(0), clock_(nullptr) {} + +TraceEventSyntheticDelay::~TraceEventSyntheticDelay() {} + +TraceEventSyntheticDelay* TraceEventSyntheticDelay::Lookup( + const std::string& name) { + return TraceEventSyntheticDelayRegistry::GetInstance()->GetOrCreateDelay( + name.c_str()); +} + +void TraceEventSyntheticDelay::Initialize( + const std::string& name, + TraceEventSyntheticDelayClock* clock) { + name_ = name; + clock_ = clock; +} + +void TraceEventSyntheticDelay::SetTargetDuration(const MonoDelta& target_duration) { + MutexLock lock(lock_); + target_duration_ = target_duration; + trigger_count_ = 0; + begin_count_ = 0; +} + +void TraceEventSyntheticDelay::SetMode(Mode mode) { + MutexLock lock(lock_); + mode_ = mode; +} + +void TraceEventSyntheticDelay::SetClock(TraceEventSyntheticDelayClock* clock) { + MutexLock lock(lock_); + clock_ = clock; +} + +void TraceEventSyntheticDelay::Begin() { + // Note that we check for a non-zero target duration without locking to keep + // things quick for the common case when delays are disabled. Since the delay + // calculation is done with a lock held, it will always be correct. The only + // downside of this is that we may fail to apply some delays when the target + // duration changes. + ANNOTATE_BENIGN_RACE(&target_duration_, "Synthetic delay duration"); + if (!target_duration_.Initialized()) + return; + + MonoTime start_time = clock_->Now(); + { + MutexLock lock(lock_); + if (++begin_count_ != 1) + return; + end_time_ = CalculateEndTimeLocked(start_time); + } +} + +void TraceEventSyntheticDelay::BeginParallel(MonoTime* out_end_time) { + // See note in Begin(). + ANNOTATE_BENIGN_RACE(&target_duration_, "Synthetic delay duration"); + if (!target_duration_.Initialized()) { + *out_end_time = MonoTime(); + return; + } + + MonoTime start_time = clock_->Now(); + { + MutexLock lock(lock_); + *out_end_time = CalculateEndTimeLocked(start_time); + } +} + +void TraceEventSyntheticDelay::End() { + // See note in Begin(). + ANNOTATE_BENIGN_RACE(&target_duration_, "Synthetic delay duration"); + if (!target_duration_.Initialized()) + return; + + MonoTime end_time; + { + MutexLock lock(lock_); + if (!begin_count_ || --begin_count_ != 0) + return; + end_time = end_time_; + } + if (end_time.Initialized()) + ApplyDelay(end_time); +} + +void TraceEventSyntheticDelay::EndParallel(const MonoTime& end_time) { + if (end_time.Initialized()) + ApplyDelay(end_time); +} + +MonoTime TraceEventSyntheticDelay::CalculateEndTimeLocked( + const MonoTime& start_time) { + if (mode_ == ONE_SHOT && trigger_count_++) + return MonoTime(); + else if (mode_ == ALTERNATING && trigger_count_++ % 2) + return MonoTime(); + return start_time + target_duration_; +} + +void TraceEventSyntheticDelay::ApplyDelay(const MonoTime& end_time) { + TRACE_EVENT0("synthetic_delay", name_.c_str()); + while (clock_->Now() < end_time) { + // Busy loop. + } +} + +TraceEventSyntheticDelayRegistry* +TraceEventSyntheticDelayRegistry::GetInstance() { + return Singleton<TraceEventSyntheticDelayRegistry>::get(); +} + +TraceEventSyntheticDelayRegistry::TraceEventSyntheticDelayRegistry() + : delay_count_(0) {} + +TraceEventSyntheticDelay* TraceEventSyntheticDelayRegistry::GetOrCreateDelay( + const char* name) { + // Try to find an existing delay first without locking to make the common case + // fast. + int delay_count = base::subtle::Acquire_Load(&delay_count_); + for (int i = 0; i < delay_count; ++i) { + if (!strcmp(name, delays_[i].name_.c_str())) + return &delays_[i]; + } + + MutexLock lock(lock_); + delay_count = base::subtle::Acquire_Load(&delay_count_); + for (int i = 0; i < delay_count; ++i) { + if (!strcmp(name, delays_[i].name_.c_str())) + return &delays_[i]; + } + + DCHECK(delay_count < kMaxSyntheticDelays) + << "must increase kMaxSyntheticDelays"; + if (delay_count >= kMaxSyntheticDelays) + return &dummy_delay_; + + delays_[delay_count].Initialize(std::string(name), this); + base::subtle::Release_Store(&delay_count_, delay_count + 1); + return &delays_[delay_count]; +} + +MonoTime TraceEventSyntheticDelayRegistry::Now() { + return MonoTime::Now(); +} + +void TraceEventSyntheticDelayRegistry::ResetAllDelays() { + MutexLock lock(lock_); + int delay_count = base::subtle::Acquire_Load(&delay_count_); + for (int i = 0; i < delay_count; ++i) { + delays_[i].SetTargetDuration(MonoDelta()); + delays_[i].SetClock(this); + } +} + +void ResetTraceEventSyntheticDelays() { + TraceEventSyntheticDelayRegistry::GetInstance()->ResetAllDelays(); +} + +} // namespace debug +} // namespace kudu + +namespace trace_event_internal { + +ScopedSyntheticDelay::ScopedSyntheticDelay(const char* name, + AtomicWord* impl_ptr) + : delay_impl_(GetOrCreateDelay(name, impl_ptr)) { + delay_impl_->BeginParallel(&end_time_); +} + +ScopedSyntheticDelay::~ScopedSyntheticDelay() { + delay_impl_->EndParallel(end_time_); +} + +kudu::debug::TraceEventSyntheticDelay* GetOrCreateDelay( + const char* name, + AtomicWord* impl_ptr) { + kudu::debug::TraceEventSyntheticDelay* delay_impl = + reinterpret_cast<kudu::debug::TraceEventSyntheticDelay*>( + base::subtle::Acquire_Load(impl_ptr)); + if (!delay_impl) { + delay_impl = kudu::debug::TraceEventSyntheticDelayRegistry::GetInstance() + ->GetOrCreateDelay(name); + base::subtle::Release_Store( + impl_ptr, reinterpret_cast<AtomicWord>(delay_impl)); + } + return delay_impl; +} + +} // namespace trace_event_internal http://git-wip-us.apache.org/repos/asf/impala/blob/fcf190c4/be/src/kudu/util/debug/trace_event_synthetic_delay.h ---------------------------------------------------------------------- diff --git a/be/src/kudu/util/debug/trace_event_synthetic_delay.h b/be/src/kudu/util/debug/trace_event_synthetic_delay.h new file mode 100644 index 0000000..5866814 --- /dev/null +++ b/be/src/kudu/util/debug/trace_event_synthetic_delay.h @@ -0,0 +1,166 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// The synthetic delay framework makes it possible to dynamically inject +// arbitrary delays into into different parts of the codebase. This can be used, +// for instance, for testing various task scheduling algorithms. +// +// The delays are specified in terms of a target duration for a given block of +// code. If the code executes faster than the duration, the thread is made to +// sleep until the deadline is met. +// +// Code can be instrumented for delays with two sets of macros. First, for +// delays that should apply within a scope, use the following macro: +// +// TRACE_EVENT_SYNTHETIC_DELAY("cc.LayerTreeHost.DrawAndSwap"); +// +// For delaying operations that span multiple scopes, use: +// +// TRACE_EVENT_SYNTHETIC_DELAY_BEGIN("cc.Scheduler.BeginMainFrame"); +// ... +// TRACE_EVENT_SYNTHETIC_DELAY_END("cc.Scheduler.BeginMainFrame"); +// +// Here BEGIN establishes the start time for the delay and END executes the +// delay based on the remaining time. If BEGIN is called multiple times in a +// row, END should be called a corresponding number of times. Only the last +// call to END will have an effect. +// +// Note that a single delay may begin on one thread and end on another. This +// implies that a single delay cannot not be applied in several threads at once. + +#ifndef KUDU_UTIL_DEBUG_TRACE_EVENT_SYNTHETIC_DELAY_H_ +#define KUDU_UTIL_DEBUG_TRACE_EVENT_SYNTHETIC_DELAY_H_ + +#include <string> + +#include "kudu/gutil/atomicops.h" +#include "kudu/gutil/macros.h" +#include "kudu/util/debug/trace_event.h" +#include "kudu/util/monotime.h" +#include "kudu/util/mutex.h" + +// Apply a named delay in the current scope. +#define TRACE_EVENT_SYNTHETIC_DELAY(name) \ + static AtomicWord INTERNAL_TRACE_EVENT_UID(impl_ptr) = 0; \ + trace_event_internal::ScopedSyntheticDelay INTERNAL_TRACE_EVENT_UID(delay)( \ + name, &INTERNAL_TRACE_EVENT_UID(impl_ptr)); + +// Begin a named delay, establishing its timing start point. May be called +// multiple times as long as the calls to TRACE_EVENT_SYNTHETIC_DELAY_END are +// balanced. Only the first call records the timing start point. +#define TRACE_EVENT_SYNTHETIC_DELAY_BEGIN(name) \ + do { \ + static AtomicWord impl_ptr = 0; \ + trace_event_internal::GetOrCreateDelay(name, &impl_ptr)->Begin(); \ + } while (false) + +// End a named delay. The delay is applied only if this call matches the +// first corresponding call to TRACE_EVENT_SYNTHETIC_DELAY_BEGIN with the +// same delay. +#define TRACE_EVENT_SYNTHETIC_DELAY_END(name) \ + do { \ + static AtomicWord impl_ptr = 0; \ + trace_event_internal::GetOrCreateDelay(name, &impl_ptr)->End(); \ + } while (false) + +namespace kudu { +namespace debug { + +// Time source for computing delay durations. Used for testing. +class TRACE_EVENT_API_CLASS_EXPORT TraceEventSyntheticDelayClock { + public: + TraceEventSyntheticDelayClock(); + virtual ~TraceEventSyntheticDelayClock(); + virtual MonoTime Now() = 0; + + private: + DISALLOW_COPY_AND_ASSIGN(TraceEventSyntheticDelayClock); +}; + +// Single delay point instance. +class TRACE_EVENT_API_CLASS_EXPORT TraceEventSyntheticDelay { + public: + enum Mode { + STATIC, // Apply the configured delay every time. + ONE_SHOT, // Apply the configured delay just once. + ALTERNATING // Apply the configured delay every other time. + }; + + // Returns an existing named delay instance or creates a new one with |name|. + static TraceEventSyntheticDelay* Lookup(const std::string& name); + + void SetTargetDuration(const MonoDelta& target_duration); + void SetMode(Mode mode); + void SetClock(TraceEventSyntheticDelayClock* clock); + + // Begin the delay, establishing its timing start point. May be called + // multiple times as long as the calls to End() are balanced. Only the first + // call records the timing start point. + void Begin(); + + // End the delay. The delay is applied only if this call matches the first + // corresponding call to Begin() with the same delay. + void End(); + + // Begin a parallel instance of the delay. Several parallel instances may be + // active simultaneously and will complete independently. The computed end + // time for the delay is stored in |out_end_time|, which should later be + // passed to EndParallel(). + void BeginParallel(MonoTime* out_end_time); + + // End a previously started parallel delay. |end_time| is the delay end point + // computed by BeginParallel(). + void EndParallel(const MonoTime& end_time); + + private: + TraceEventSyntheticDelay(); + ~TraceEventSyntheticDelay(); + friend class TraceEventSyntheticDelayRegistry; + + void Initialize(const std::string& name, + TraceEventSyntheticDelayClock* clock); + MonoTime CalculateEndTimeLocked(const MonoTime& start_time); + void ApplyDelay(const MonoTime& end_time); + + Mutex lock_; + Mode mode_; + std::string name_; + int begin_count_; + int trigger_count_; + MonoTime end_time_; + MonoDelta target_duration_; + TraceEventSyntheticDelayClock* clock_; + + DISALLOW_COPY_AND_ASSIGN(TraceEventSyntheticDelay); +}; + +// Set the target durations of all registered synthetic delay points to zero. +TRACE_EVENT_API_CLASS_EXPORT void ResetTraceEventSyntheticDelays(); + +} // namespace debug +} // namespace kudu + +namespace trace_event_internal { + +// Helper class for scoped delays. Do not use directly. +class TRACE_EVENT_API_CLASS_EXPORT ScopedSyntheticDelay { + public: + explicit ScopedSyntheticDelay(const char* name, + AtomicWord* impl_ptr); + ~ScopedSyntheticDelay(); + + private: + kudu::debug::TraceEventSyntheticDelay* delay_impl_; + kudu::MonoTime end_time_; + + DISALLOW_COPY_AND_ASSIGN(ScopedSyntheticDelay); +}; + +// Helper for registering delays. Do not use directly. +TRACE_EVENT_API_CLASS_EXPORT kudu::debug::TraceEventSyntheticDelay* + GetOrCreateDelay(const char* name, AtomicWord* impl_ptr); + +} // namespace trace_event_internal + +#endif /* KUDU_UTIL_DEBUG_TRACE_EVENT_SYNTHETIC_DELAY_H_ */ http://git-wip-us.apache.org/repos/asf/impala/blob/fcf190c4/be/src/kudu/util/debug/trace_logging.h ---------------------------------------------------------------------- diff --git a/be/src/kudu/util/debug/trace_logging.h b/be/src/kudu/util/debug/trace_logging.h new file mode 100644 index 0000000..1a3b39e --- /dev/null +++ b/be/src/kudu/util/debug/trace_logging.h @@ -0,0 +1,132 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +// This header defines the following macros: +// +// VLOG_AND_TRACE(category, vlevel) +// +// Write a log message to VLOG(vlevel) as well as the current +// trace event buffer as an "INSTANT" trace event type. If the +// given vlog level is not enabled, this will still result in a +// trace buffer entry. +// +// The provided 'category' should be a trace event category, which +// allows the users to filter which trace events to enable. +// For example: +// +// VLOG_AND_TRACE("my_subsystem", 1) << "This always shows up in trace buffers " +// << "but only shows up in the log if VLOG(1) level logging is enabled."; +// +// Most VLOG(1) level log messages are reasonable to use this macro. +// Note that there is slightly more overhead to this macro as opposed +// to just using VLOG(1). +// +// Note that, like VLOG(n), this macro avoids evaluating its arguments unless +// either trace recording or VLOG(n) is enabled. In the case that both are enabled, +// the arguments are only evaluated once. +// +// +// LOG_AND_TRACE(category, severity) +// +// Same as the above, but always logs at the given severity level in addition +// to writing to the trace buffer. + +#ifndef KUDU_DEBUG_TRACE_LOGGING_H +#define KUDU_DEBUG_TRACE_LOGGING_H + +#include <glog/logging.h> +#include <string> + +#include "kudu/gutil/macros.h" +#include "kudu/util/debug/trace_event.h" + +// The inner workings of these macros are a bit arcane: +// - We make use of the fact that a block can be embedded within a ternary expression. +// This allows us to determine whether the trace event is enabled before we decide +// to evaluate the arguments. +// - We have to use google::LogMessageVoidify so that we can put 'void(0)' on one side +// of the ternary expression and the log stream on the other. This technique is +// cribbed from glog/logging.h. +#define VLOG_AND_TRACE_INTERNAL(category, vlevel) \ + kudu::debug::TraceGLog(__FILE__, __LINE__, category, google::GLOG_INFO, \ + /* send_to_log= */VLOG_IS_ON(vlevel)).stream() + +#define VLOG_AND_TRACE(category, vlevel) \ + !( { \ + bool enabled; \ + TRACE_EVENT_CATEGORY_GROUP_ENABLED(category, &enabled); \ + enabled || VLOG_IS_ON(vlevel); \ + } ) ? static_cast<void>(0) : \ + google::LogMessageVoidify() & VLOG_AND_TRACE_INTERNAL(category, vlevel) // NOLINT(*) + + +#define LOG_AND_TRACE(category, severity) \ + kudu::debug::TraceGLog(__FILE__, __LINE__, category, \ + google::GLOG_ ## severity, /* send_to_log= */true).stream() + +namespace kudu { +namespace debug { + +class TraceGLog { + public: + TraceGLog(const char* file, int line, const char* category, + google::LogSeverity severity, bool send_to_log) + : sink_(category), + google_msg_(file, line, severity, &sink_, send_to_log) { + } + + std::ostream& stream() { + return google_msg_.stream(); + } + + private: + class TraceLogSink : public google::LogSink { + public: + explicit TraceLogSink(const char* category) : category_(category) {} + void send(google::LogSeverity severity, const char* full_filename, + const char* base_filename, int line, + const struct ::tm* tm_time, const char* message, + size_t message_len) override { + // Rather than calling TRACE_EVENT_INSTANT here, we have to do it from + // the destructor. This is because glog holds its internal mutex while + // calling send(). So, if we try to use TRACE_EVENT here, and --trace_to_console + // is enabled, then we'd end up calling back into glog when its lock is already + // held. glog isn't re-entrant, so that causes a crash. + // + // By just storing the string here, and then emitting the trace in the dtor, + // we defer the tracing until the google::LogMessage has destructed and the + // glog lock is available again. + str_ = ToString(severity, base_filename, line, + tm_time, message, message_len); + } + virtual ~TraceLogSink() { + TRACE_EVENT_INSTANT1(category_, "vlog", TRACE_EVENT_SCOPE_THREAD, + "msg", str_); + } + + private: + const char* const category_; + std::string str_; + }; + + TraceLogSink sink_; + google::LogMessage google_msg_; +}; + +} // namespace debug +} // namespace kudu +#endif /* KUDU_DEBUG_TRACE_LOGGING_H */ http://git-wip-us.apache.org/repos/asf/impala/blob/fcf190c4/be/src/kudu/util/debug/unwind_safeness.cc ---------------------------------------------------------------------- diff --git a/be/src/kudu/util/debug/unwind_safeness.cc b/be/src/kudu/util/debug/unwind_safeness.cc new file mode 100644 index 0000000..c8e0adf --- /dev/null +++ b/be/src/kudu/util/debug/unwind_safeness.cc @@ -0,0 +1,164 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Override various libdl functions which can race with libunwind. +// The overridden versions set a threadlocal variable and our +// stack-tracing code checks the threadlocal before calling into +// libunwind. +// +// Based on public domain code by Aliaksey Kandratsenka at +// https://github.com/alk/unwind_safeness_helper + +#include "kudu/util/debug/unwind_safeness.h" + +#include <dlfcn.h> +#include <stddef.h> + +#include <ostream> + +#include <glog/logging.h> + +#define CALL_ORIG(func_name, ...) \ + ((decltype(&func_name))g_orig_ ## func_name)(__VA_ARGS__) + +// Don't hook dl_iterate_phdr in TSAN builds since TSAN already instruments this +// function and blocks signals while calling it. And skip it for macOS; it +// doesn't exist there. +#if !defined(THREAD_SANITIZER) && !defined(__APPLE__) +#define HOOK_DL_ITERATE_PHDR 1 +#endif + +typedef int (*dl_iterate_phdr_cbtype)(struct dl_phdr_info *, size_t, void *); + +namespace { + +// Whether InitializeIfNecessary() has been called. +bool g_initted; + +// The original versions of our wrapped functions. +void* g_orig_dlopen; +void* g_orig_dlclose; +#ifdef HOOK_DL_ITERATE_PHDR +void* g_orig_dl_iterate_phdr; +#endif + +// The depth of calls into libdl. +__thread int g_unsafeness_depth; + +// Scoped helper to track the recursion depth of calls into libdl +struct ScopedBumpDepth { + ScopedBumpDepth() { + g_unsafeness_depth++; + } + ~ScopedBumpDepth() { + g_unsafeness_depth--; + } +}; + +void *dlsym_or_die(const char *sym) { + dlerror(); + void* ret = dlsym(RTLD_NEXT, sym); + char* error = dlerror(); + CHECK(!error) << "failed to find symbol " << sym << ": " << error; + return ret; +} + +// Initialize the global variables which store the original references. This is +// set up as a constructor so that we're guaranteed to call this before main() +// while we are still single-threaded. +// +// NOTE: We _also_ call explicitly this from each of the wrappers, because +// there are some cases where the constructors of dynamic libraries may call +// dlopen, and there is no guarantee that our own constructor runs before +// the constructor of other libraries. +// +// A couple examples of the above: +// +// 1) In ASAN builds, the sanitizer runtime ends up calling dl_iterate_phdr from its +// initialization. +// 2) OpenSSL in FIPS mode calls dlopen() within its constructor. +__attribute__((constructor)) +void InitIfNecessary() { + // Dynamic library initialization is always single-threaded, so there's no + // need for any synchronization here. + if (g_initted) return; + + g_orig_dlopen = dlsym_or_die("dlopen"); + g_orig_dlclose = dlsym_or_die("dlclose"); +#ifdef HOOK_DL_ITERATE_PHDR + // Failing to hook dl_iterate_phdr is non-fatal. + // + // In toolchains where the linker is passed --as-needed by default, a + // dynamically linked binary that doesn't directly reference any kudu_util + // symbols will omit a DT_NEEDED entry for kudu_util. Such a binary will + // no doubt emit a DT_NEEDED entry for libc, which means libc will wind up + // _before_ kudu_util in dlsym's search order. The net effect: the dlsym() + // call below will fail. + // + // All Ubuntu releases since Natty[1] behave in this way, except that many + // of them are also vulnerable to a glibc bug[2] that'll cause such a + // failure to go unreported by dlerror(). In newer releases, the failure + // is reported and dlsym_or_die() crashes the process. + // + // Given that the subset of affected binaries is small, and given that + // dynamic linkage isn't used in production anyway, we'll just treat the + // hook attempt as a best effort. Affected binaries that actually attempt + // to invoke dl_iterate_phdr will dereference a null pointer and crash, so + // if this is ever becomes a problem, we'll know right away.a + // + // 1. https://wiki.ubuntu.com/NattyNarwhal/ToolchainTransition + // 2. https://sourceware.org/bugzilla/show_bug.cgi?id=19509 + g_orig_dl_iterate_phdr = dlsym(RTLD_NEXT, "dl_iterate_phdr"); +#endif + g_initted = true; +} + +} // anonymous namespace + +namespace kudu { +namespace debug { + +bool SafeToUnwindStack() { + return g_unsafeness_depth == 0; +} + +} // namespace debug +} // namespace kudu + +extern "C" { + +void *dlopen(const char *filename, int flag) { // NOLINT + InitIfNecessary(); + ScopedBumpDepth d; + return CALL_ORIG(dlopen, filename, flag); +} + +int dlclose(void *handle) { // NOLINT + InitIfNecessary(); + ScopedBumpDepth d; + return CALL_ORIG(dlclose, handle); +} + +#ifdef HOOK_DL_ITERATE_PHDR +int dl_iterate_phdr(dl_iterate_phdr_cbtype callback, void *data) { // NOLINT + InitIfNecessary(); + ScopedBumpDepth d; + return CALL_ORIG(dl_iterate_phdr, callback, data); +} +#endif + +} http://git-wip-us.apache.org/repos/asf/impala/blob/fcf190c4/be/src/kudu/util/debug/unwind_safeness.h ---------------------------------------------------------------------- diff --git a/be/src/kudu/util/debug/unwind_safeness.h b/be/src/kudu/util/debug/unwind_safeness.h new file mode 100644 index 0000000..4aab6f9 --- /dev/null +++ b/be/src/kudu/util/debug/unwind_safeness.h @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +#pragma once + +namespace kudu { +namespace debug { + +// Return true if it is currently safe to unwind the call stack. +// +// It's almost always safe unless we are in a signal handler context +// inside a call into libdl. +bool SafeToUnwindStack(); + +} // namespace debug +} // namespace kudu http://git-wip-us.apache.org/repos/asf/impala/blob/fcf190c4/be/src/kudu/util/debug_ref_counted.h ---------------------------------------------------------------------- diff --git a/be/src/kudu/util/debug_ref_counted.h b/be/src/kudu/util/debug_ref_counted.h new file mode 100644 index 0000000..7c2deca --- /dev/null +++ b/be/src/kudu/util/debug_ref_counted.h @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#ifndef KUDU_UTIL_DEBUG_REF_COUNTED_H_ +#define KUDU_UTIL_DEBUG_REF_COUNTED_H_ + +#include <glog/logging.h> + +#include "kudu/gutil/ref_counted.h" +#include "kudu/util/debug-util.h" + +namespace kudu { + +// For use in debugging. Change a ref-counted class to inherit from this, +// instead of RefCountedThreadSafe, and fill your logs with stack traces. +template <class T, typename Traits = DefaultRefCountedThreadSafeTraits<T> > +class DebugRefCountedThreadSafe : public RefCountedThreadSafe<T, Traits> { + public: + DebugRefCountedThreadSafe() {} + + void AddRef() const { + RefCountedThreadSafe<T, Traits>::AddRef(); + LOG(INFO) << "Incremented ref on " << this << ":\n" << GetStackTrace(); + } + + void Release() const { + LOG(INFO) << "Decrementing ref on " << this << ":\n" << GetStackTrace(); + RefCountedThreadSafe<T, Traits>::Release(); + } + + protected: + ~DebugRefCountedThreadSafe() {} + + private: + friend struct DefaultRefCountedThreadSafeTraits<T>; + + DISALLOW_COPY_AND_ASSIGN(DebugRefCountedThreadSafe); +}; + +} // namespace kudu + +#endif // KUDU_UTIL_DEBUG_REF_COUNTED_H_ http://git-wip-us.apache.org/repos/asf/impala/blob/fcf190c4/be/src/kudu/util/decimal_util-test.cc ---------------------------------------------------------------------- diff --git a/be/src/kudu/util/decimal_util-test.cc b/be/src/kudu/util/decimal_util-test.cc new file mode 100644 index 0000000..d7bfc35 --- /dev/null +++ b/be/src/kudu/util/decimal_util-test.cc @@ -0,0 +1,81 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include <string> + +#include <gtest/gtest.h> + +#include "kudu/util/decimal_util.h" + +using std::string; + +namespace kudu { + +TEST(TestDecimalUtil, TestMaxUnscaledDecimal) { + ASSERT_EQ(9, MaxUnscaledDecimal(1)); + ASSERT_EQ(99999, MaxUnscaledDecimal(5)); + ASSERT_EQ(kMaxUnscaledDecimal32, MaxUnscaledDecimal(kMaxDecimal32Precision)); + ASSERT_EQ(kMaxUnscaledDecimal64, MaxUnscaledDecimal(kMaxDecimal64Precision)); + ASSERT_EQ(kMaxUnscaledDecimal128, MaxUnscaledDecimal(kMaxDecimal128Precision)); +} + +TEST(TestDecimalUtil, TestMinUnscaledDecimal) { + ASSERT_EQ(-9, MinUnscaledDecimal(1)); + ASSERT_EQ(-99999, MinUnscaledDecimal(5)); + ASSERT_EQ(kMinUnscaledDecimal32, MinUnscaledDecimal(kMaxDecimal32Precision)); + ASSERT_EQ(kMinUnscaledDecimal64, MinUnscaledDecimal(kMaxDecimal64Precision)); + ASSERT_EQ(kMinUnscaledDecimal128, MinUnscaledDecimal(kMaxDecimal128Precision)); +} + +TEST(TestDecimalUtil, TestToString) { + ASSERT_EQ("999999999", + DecimalToString(kMaxUnscaledDecimal32, kDefaultDecimalScale)); + ASSERT_EQ("0.999999999", + DecimalToString(kMaxUnscaledDecimal32, kMaxDecimal32Precision)); + ASSERT_EQ("-999999999", + DecimalToString(kMinUnscaledDecimal32, kDefaultDecimalScale)); + ASSERT_EQ("-0.999999999", + DecimalToString(kMinUnscaledDecimal32, kMaxDecimal32Precision)); + + ASSERT_EQ("999999999999999999", + DecimalToString(kMaxUnscaledDecimal64, kDefaultDecimalScale)); + ASSERT_EQ("0.999999999999999999", + DecimalToString(kMaxUnscaledDecimal64, kMaxDecimal64Precision)); + ASSERT_EQ("-999999999999999999", + DecimalToString(kMinUnscaledDecimal64, kDefaultDecimalScale)); + ASSERT_EQ("-0.999999999999999999", + DecimalToString(kMinUnscaledDecimal64, kMaxDecimal64Precision)); + + ASSERT_EQ("99999999999999999999999999999999999999", + DecimalToString(kMaxUnscaledDecimal128, kDefaultDecimalScale)); + ASSERT_EQ("0.99999999999999999999999999999999999999", + DecimalToString(kMaxUnscaledDecimal128, kMaxDecimal128Precision)); + ASSERT_EQ("-99999999999999999999999999999999999999", + DecimalToString(kMinUnscaledDecimal128, kDefaultDecimalScale)); + ASSERT_EQ("-0.99999999999999999999999999999999999999", + DecimalToString(kMinUnscaledDecimal128, kMaxDecimal128Precision)); + + ASSERT_EQ("0", DecimalToString(0, 0)); + ASSERT_EQ("12345", DecimalToString(12345, 0)); + ASSERT_EQ("-12345", DecimalToString(-12345, 0)); + ASSERT_EQ("123.45", DecimalToString(12345, 2)); + ASSERT_EQ("-123.45", DecimalToString(-12345, 2)); + ASSERT_EQ("0.00012345", DecimalToString(12345, 8)); + ASSERT_EQ("-0.00012345", DecimalToString(-12345, 8)); +} + +} // namespace kudu http://git-wip-us.apache.org/repos/asf/impala/blob/fcf190c4/be/src/kudu/util/decimal_util.cc ---------------------------------------------------------------------- diff --git a/be/src/kudu/util/decimal_util.cc b/be/src/kudu/util/decimal_util.cc new file mode 100644 index 0000000..0e04494 --- /dev/null +++ b/be/src/kudu/util/decimal_util.cc @@ -0,0 +1,89 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "kudu/util/decimal_util.h" + +#include <string> + +#include <glog/logging.h> + +#include "kudu/gutil/port.h" + +namespace kudu { + +using std::string; + +// Workaround for an ASAN build issue documented here: +// https://bugs.llvm.org/show_bug.cgi?id=16404 +ATTRIBUTE_NO_SANITIZE_UNDEFINED +int128_t MaxUnscaledDecimal(int8_t precision) { + DCHECK_GE(precision, kMinDecimalPrecision); + DCHECK_LE(precision, kMaxDecimalPrecision); + int128_t result = 1; + for (; precision > 0; precision--) { + result = result * 10; + } + return result - 1; +} + +int128_t MinUnscaledDecimal(int8_t precision) { + return -MaxUnscaledDecimal(precision); +} + +// Workaround for an ASAN build issue documented here: +// https://bugs.llvm.org/show_bug.cgi?id=16404 +ATTRIBUTE_NO_SANITIZE_UNDEFINED +string DecimalToString(int128_t d, int8_t scale) { + // 38 digits, 1 extra leading zero, decimal point, + // and sign are good for 128-bit or smaller decimals. + char local[41]; + char *p = local + sizeof(local); + int128_t n = d < 0? -d : d; + int position = 0; + while (n) { + // Print the decimal in the scale position. + // No decimal is output when scale is 0. + if (scale != 0 && position == scale) { + *--p = '.'; + } + // Unroll the next digits. + *--p = '0' + n % 10; + n /= 10; + position++; + } + // True if the value is between 1 and -1. + bool fractional = position <= scale; + // Pad with zeros until the scale + while (position < scale) { + *--p = '0'; + position++; + } + // Add leading "0.". + if (fractional) { + if (d != 0) { + *--p = '.'; + } + *--p = '0'; + } + // Add sign for negative values. + if (d < 0) { + *--p = '-'; + } + return string(p, local + sizeof(local)); +} + +} // namespace kudu http://git-wip-us.apache.org/repos/asf/impala/blob/fcf190c4/be/src/kudu/util/decimal_util.h ---------------------------------------------------------------------- diff --git a/be/src/kudu/util/decimal_util.h b/be/src/kudu/util/decimal_util.h new file mode 100644 index 0000000..a465412 --- /dev/null +++ b/be/src/kudu/util/decimal_util.h @@ -0,0 +1,69 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include <stdint.h> +#include <string> + +#include "kudu/util/int128.h" + +namespace kudu { + // Maximum precision and absolute value of a Decimal that can be stored + // in 4 bytes. + static const int8_t kMaxDecimal32Precision = 9; + static const int32_t kMaxUnscaledDecimal32 = 999999999; // 9 9's + static const int32_t kMinUnscaledDecimal32 = -kMaxUnscaledDecimal32; // 9 9's + + // Maximum precision and absolute value of a valid Decimal can be + // stored in 8 bytes. + static const int8_t kMaxDecimal64Precision = 18; + static const int64_t kMaxUnscaledDecimal64 = 999999999999999999; // 18 9's + static const int64_t kMinUnscaledDecimal64 = -kMaxUnscaledDecimal64; // 18 9's + + // Maximum precision and absolute value of a valid Decimal can be + // stored in 16 bytes. + static const int8_t kMaxDecimal128Precision = 38; + // Hacky calculation because int128 literals are not supported. + static const int128_t kMaxUnscaledDecimal128 = + (((static_cast<int128_t>(999999999999999999) * 1000000000000000000) + + 999999999999999999) * 100) + 99; // 38 9's + static const int128_t kMinUnscaledDecimal128 = -kMaxUnscaledDecimal128; + + // Minimum and maximum precision for any Decimal. + static const int8_t kMinDecimalPrecision = 1; + static const int8_t kMaxDecimalPrecision = kMaxDecimal128Precision; + // Maximum absolute value for any Decimal. + static const int128_t kMaxUnscaledDecimal = kMaxUnscaledDecimal128; + static const int128_t kMinUnscaledDecimal = kMinUnscaledDecimal128; + + // Minimum scale for any Decimal. + static const int8_t kMinDecimalScale = 0; + static const int8_t kDefaultDecimalScale = 0; + // The maximum scale is the Decimal's precision. + + // Returns the maximum unscaled decimal value that can be stored + // based on the precision. + int128_t MaxUnscaledDecimal(int8_t precision); + + // Returns the maximum unscaled decimal value that can be stored + // based on the precision. + int128_t MinUnscaledDecimal(int8_t precision); + + std::string DecimalToString(int128_t value, int8_t scale); + +} // namespace kudu http://git-wip-us.apache.org/repos/asf/impala/blob/fcf190c4/be/src/kudu/util/easy_json-test.cc ---------------------------------------------------------------------- diff --git a/be/src/kudu/util/easy_json-test.cc b/be/src/kudu/util/easy_json-test.cc new file mode 100644 index 0000000..7074512 --- /dev/null +++ b/be/src/kudu/util/easy_json-test.cc @@ -0,0 +1,106 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include <string> + +#include <gtest/gtest.h> +#include <rapidjson/document.h> +#include <rapidjson/rapidjson.h> + +#include "kudu/gutil/integral_types.h" +#include "kudu/util/easy_json.h" +#include "kudu/util/test_util.h" + +using rapidjson::SizeType; +using rapidjson::Value; +using std::string; + +namespace kudu { + +class EasyJsonTest: public KuduTest {}; + +TEST_F(EasyJsonTest, TestNull) { + EasyJson ej; + ASSERT_TRUE(ej.value().IsNull()); +} + +TEST_F(EasyJsonTest, TestBasic) { + EasyJson ej; + ej.SetObject(); + ej.Set("1", true); + ej.Set("2", kint32min); + ej.Set("4", kint64min); + ej.Set("6", 1.0); + ej.Set("7", "string"); + + Value& v = ej.value(); + + ASSERT_EQ(v["1"].GetBool(), true); + ASSERT_EQ(v["2"].GetInt(), kint32min); + ASSERT_EQ(v["4"].GetInt64(), kint64min); + ASSERT_EQ(v["6"].GetDouble(), 1.0); + ASSERT_EQ(string(v["7"].GetString()), "string"); +} + +TEST_F(EasyJsonTest, TestNested) { + EasyJson ej; + ej.SetObject(); + ej.Get("nested").SetObject(); + ej.Get("nested").Set("nested_attr", true); + ASSERT_EQ(ej.value()["nested"]["nested_attr"].GetBool(), true); + + ej.Get("nested_array").SetArray(); + ej.Get("nested_array").PushBack(1); + ej.Get("nested_array").PushBack(2); + ASSERT_EQ(ej.value()["nested_array"][SizeType(0)].GetInt(), 1); + ASSERT_EQ(ej.value()["nested_array"][SizeType(1)].GetInt(), 2); +} + +TEST_F(EasyJsonTest, TestCompactSyntax) { + EasyJson ej; + ej["nested"]["nested_attr"] = true; + ASSERT_EQ(ej.value()["nested"]["nested_attr"].GetBool(), true); + + for (int i = 0; i < 2; i++) { + ej["nested_array"][i] = i + 1; + } + ASSERT_EQ(ej.value()["nested_array"][SizeType(0)].GetInt(), 1); + ASSERT_EQ(ej.value()["nested_array"][SizeType(1)].GetInt(), 2); +} + +TEST_F(EasyJsonTest, TestComplexInitializer) { + EasyJson ej; + ej = EasyJson::kObject; + ASSERT_TRUE(ej.value().IsObject()); + + EasyJson nested_arr = ej.Set("nested_arr", EasyJson::kArray); + ASSERT_TRUE(nested_arr.value().IsArray()); + + EasyJson nested_obj = nested_arr.PushBack(EasyJson::kObject); + ASSERT_TRUE(ej["nested_arr"][0].value().IsObject()); +} + +TEST_F(EasyJsonTest, TestAllocatorLifetime) { + EasyJson* root = new EasyJson; + EasyJson child = (*root)["child"]; + delete root; + + child["child_attr"] = 1; + ASSERT_EQ(child.value()["child_attr"].GetInt(), 1); +} + +} // namespace kudu http://git-wip-us.apache.org/repos/asf/impala/blob/fcf190c4/be/src/kudu/util/easy_json.cc ---------------------------------------------------------------------- diff --git a/be/src/kudu/util/easy_json.cc b/be/src/kudu/util/easy_json.cc new file mode 100644 index 0000000..9057b50 --- /dev/null +++ b/be/src/kudu/util/easy_json.cc @@ -0,0 +1,212 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "kudu/util/easy_json.h" + +#include <ostream> +#include <string> +#include <utility> + +#include <glog/logging.h> +#include <rapidjson/document.h> +#include <rapidjson/rapidjson.h> +#include <rapidjson/stringbuffer.h> +#include <rapidjson/writer.h> + +using rapidjson::SizeType; +using rapidjson::Value; +using std::string; + +namespace kudu { + +EasyJson::EasyJson() : alloc_(new EasyJsonAllocator), value_(&alloc_->value()) {} + +EasyJson::EasyJson(EasyJson::ComplexTypeInitializer type) + : alloc_(new EasyJsonAllocator), value_(&alloc_->value()) { + if (type == kObject) { + value_->SetObject(); + } else if (type == kArray) { + value_->SetArray(); + } +} + +EasyJson EasyJson::Get(const string& key) { + if (!value_->IsObject()) { + value_->SetObject(); + } + if (!value_->HasMember(key.c_str())) { + Value key_val(key.c_str(), alloc_->allocator()); + value_->AddMember(key_val, Value().SetNull(), alloc_->allocator()); + } + return EasyJson(&(*value_)[key.c_str()], alloc_); +} + +EasyJson EasyJson::Get(int index) { + if (!value_->IsArray()) { + value_->SetArray(); + } + while (SizeType(index) >= value_->Size()) { + value_->PushBack(Value().SetNull(), alloc_->allocator()); + } + return EasyJson(&(*value_)[index], alloc_); +} + +EasyJson EasyJson::operator[](const string& key) { + return Get(key); +} + +EasyJson EasyJson::operator[](int index) { + return Get(index); +} + +EasyJson& EasyJson::operator=(const string& val) { + value_->SetString(val.c_str(), alloc_->allocator()); + return *this; +} +template<typename T> +EasyJson& EasyJson::operator=(T val) { + *value_ = val; + return *this; +} +template EasyJson& EasyJson::operator=<bool>(bool val); +template EasyJson& EasyJson::operator=<int32_t>(int32_t val); +template EasyJson& EasyJson::operator=<int64_t>(int64_t val); +template EasyJson& EasyJson::operator=<uint32_t>(uint32_t val); +template EasyJson& EasyJson::operator=<uint64_t>(uint64_t val); +template EasyJson& EasyJson::operator=<double>(double val); +template<> EasyJson& EasyJson::operator=<const char*>(const char* val) { + value_->SetString(val, alloc_->allocator()); + return *this; +} +template<> EasyJson& EasyJson::operator=<EasyJson::ComplexTypeInitializer>( + EasyJson::ComplexTypeInitializer val) { + if (val == kObject) { + value_->SetObject(); + } else if (val == kArray) { + value_->SetArray(); + } + return (*this); +} + +EasyJson& EasyJson::SetObject() { + if (!value_->IsObject()) { + value_->SetObject(); + } + return *this; +} + +EasyJson& EasyJson::SetArray() { + if (!value_->IsArray()) { + value_->SetArray(); + } + return *this; +} + +EasyJson EasyJson::Set(const string& key, const string& val) { + return (Get(key) = val); +} +template<typename T> +EasyJson EasyJson::Set(const string& key, T val) { + return (Get(key) = val); +} +template EasyJson EasyJson::Set<bool>(const string& key, bool val); +template EasyJson EasyJson::Set<int32_t>(const string& key, int32_t val); +template EasyJson EasyJson::Set<int64_t>(const string& key, int64_t val); +template EasyJson EasyJson::Set<uint32_t>(const string& key, uint32_t val); +template EasyJson EasyJson::Set<uint64_t>(const string& key, uint64_t val); +template EasyJson EasyJson::Set<double>(const string& key, double val); +template EasyJson EasyJson::Set<const char*>(const string& key, const char* val); +template EasyJson EasyJson::Set<EasyJson::ComplexTypeInitializer>( + const string& key, + EasyJson::ComplexTypeInitializer val); + +EasyJson EasyJson::Set(int index, const string& val) { + return (Get(index) = val); +} +template<typename T> +EasyJson EasyJson::Set(int index, T val) { + return (Get(index) = val); +} +template EasyJson EasyJson::Set<bool>(int index, bool val); +template EasyJson EasyJson::Set<int32_t>(int index, int32_t val); +template EasyJson EasyJson::Set<int64_t>(int index, int64_t val); +template EasyJson EasyJson::Set<uint32_t>(int index, uint32_t val); +template EasyJson EasyJson::Set<uint64_t>(int index, uint64_t val); +template EasyJson EasyJson::Set<double>(int index, double val); +template EasyJson EasyJson::Set<const char*>(int index, const char* val); +template EasyJson EasyJson::Set<EasyJson::ComplexTypeInitializer>( + int index, + EasyJson::ComplexTypeInitializer val); + +EasyJson EasyJson::PushBack(const string& val) { + if (!value_->IsArray()) { + value_->SetArray(); + } + Value push_val(val.c_str(), alloc_->allocator()); + value_->PushBack(push_val, alloc_->allocator()); + return EasyJson(&(*value_)[value_->Size() - 1], alloc_); +} +template<typename T> +EasyJson EasyJson::PushBack(T val) { + if (!value_->IsArray()) { + value_->SetArray(); + } + value_->PushBack(val, alloc_->allocator()); + return EasyJson(&(*value_)[value_->Size() - 1], alloc_); +} +template EasyJson EasyJson::PushBack<bool>(bool val); +template EasyJson EasyJson::PushBack<int32_t>(int32_t val); +template EasyJson EasyJson::PushBack<int64_t>(int64_t val); +template EasyJson EasyJson::PushBack<uint32_t>(uint32_t val); +template EasyJson EasyJson::PushBack<uint64_t>(uint64_t val); +template EasyJson EasyJson::PushBack<double>(double val); +template<> EasyJson EasyJson::PushBack<const char*>(const char* val) { + if (!value_->IsArray()) { + value_->SetArray(); + } + Value push_val(val, alloc_->allocator()); + value_->PushBack(push_val, alloc_->allocator()); + return EasyJson(&(*value_)[value_->Size() - 1], alloc_); +} +template<> EasyJson EasyJson::PushBack<EasyJson::ComplexTypeInitializer>( + EasyJson::ComplexTypeInitializer val) { + if (!value_->IsArray()) { + value_->SetArray(); + } + Value push_val; + if (val == kObject) { + push_val.SetObject(); + } else if (val == kArray) { + push_val.SetArray(); + } else { + LOG(FATAL) << "Unknown initializer type"; + } + value_->PushBack(push_val, alloc_->allocator()); + return EasyJson(&(*value_)[value_->Size() - 1], alloc_); +} + +string EasyJson::ToString() const { + rapidjson::StringBuffer buffer; + rapidjson::Writer<rapidjson::StringBuffer> writer(buffer); + value_->Accept(writer); + return buffer.GetString(); +} + +EasyJson::EasyJson(Value* value, scoped_refptr<EasyJsonAllocator> alloc) + : alloc_(std::move(alloc)), value_(value) {} + +} // namespace kudu http://git-wip-us.apache.org/repos/asf/impala/blob/fcf190c4/be/src/kudu/util/easy_json.h ---------------------------------------------------------------------- diff --git a/be/src/kudu/util/easy_json.h b/be/src/kudu/util/easy_json.h new file mode 100644 index 0000000..bd0365a --- /dev/null +++ b/be/src/kudu/util/easy_json.h @@ -0,0 +1,190 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +#pragma once + +#include <string> + +#include <rapidjson/document.h> + +#include "kudu/gutil/ref_counted.h" + +namespace kudu { + +// A wrapper around rapidjson Value objects, to simplify usage. +// Intended solely for building json objects, not writing/parsing. +// +// Simplifies code like this: +// +// rapidjson::Document d; +// rapidjson::Value v; +// v.SetObject(); +// rapidjson::Value list; +// list.SetArray(); +// v.AddMember("list", list, d.GetAllocator()); +// v["list"].PushBack(rapidjson::Value().SetString("element"), d.GetAllocator()); +// +// To this: +// +// EasyJson ej; +// ej["list"][0] = "element"; +// +// Client code should build objects as demonstrated above, +// then call EasyJson::value() to obtain a reference to the +// built rapidjson Value. +class EasyJson { + public: + // Used for initializing EasyJson's with complex types. + // For example: + // + // EasyJson array; + // EasyJson nested = array.PushBack(EasyJson::kObject); + // nested["attr"] = "val"; + // // array = [ { "attr": "val" } ] + enum ComplexTypeInitializer { + kObject, + kArray + }; + + EasyJson(); + // Initializes the EasyJson object with the given type. + explicit EasyJson(ComplexTypeInitializer type); + ~EasyJson() = default; + + // Returns the child EasyJson associated with key. + // + // Note: this method can mutate the EasyJson object + // as follows: + // + // If this EasyJson's underlying Value is not an object + // (i.e. !this->value().IsObject()), then its Value is + // coerced to an object, overwriting the old Value. + // If the given key does not exist, a Null-valued + // EasyJson associated with key is created. + EasyJson Get(const std::string& key); + + // Returns the child EasyJson at index. + // + // Note: this method can mutate the EasyJson object + // as follows: + // + // If this EasyJson's underlying Value is not an array + // (i.e. !this->value().IsArray()), then its Value is + // coerced to an array, overwriting the old Value. + // If index >= this->value().Size(), then the underlying + // array's size is increased to index + 1 (new indices + // are filled with Null values). + EasyJson Get(int index); + + // Same as Get(key). + EasyJson operator[](const std::string& key); + // Same as Get(index). + EasyJson operator[](int index); + + // Sets the underlying Value equal to val. + // Returns a reference to the object itself. + // + // 'val' can be a bool, int32_t, int64_t, double, + // char*, string, or ComplexTypeInitializer. + EasyJson& operator=(const std::string& val); + template<typename T> + EasyJson& operator=(T val); + + // Sets the underlying Value to an object. + // Returns a reference to the object itself. + // + // i.e. after calling SetObject(), + // value().IsObject() == true + EasyJson& SetObject(); + // Sets the underlying Value to an array. + // Returns a reference to the object itself. + // + // i.e. after calling SetArray(), + // value().IsArray() == true + EasyJson& SetArray(); + + // Associates val with key. + // Returns the child object. + // + // If this EasyJson's underlying Value is not an object + // (i.e. !this->value().IsObject()), then its Value is + // coerced to an object, overwriting the old Value. + // If the given key does not exist, a new child entry + // is created with the given value. + EasyJson Set(const std::string& key, const std::string& val); + template<typename T> + EasyJson Set(const std::string& key, T val); + + // Stores val at index. + // Returns the child object. + // + // If this EasyJson's underlying Value is not an array + // (i.e. !this->value().IsArray()), then its Value is + // coerced to an array, overwriting the old Value. + // If index >= this->value().Size(), then the underlying + // array's size is increased to index + 1 (new indices + // are filled with Null values). + EasyJson Set(int index, const std::string& val); + template<typename T> + EasyJson Set(int index, T val); + + // Appends val to the underlying array. + // Returns a reference to the new child object. + // + // If this EasyJson's underlying Value is not an array + // (i.e. !this->value().IsArray()), then its Value is + // coerced to an array, overwriting the old Value. + EasyJson PushBack(const std::string& val); + template<typename T> + EasyJson PushBack(T val); + + // Returns a reference to the underlying Value. + rapidjson::Value& value() const { return *value_; } + + // Returns a string representation of the underlying json. + std::string ToString() const; + + private: + // One instance of EasyJsonAllocator is shared among a root + // EasyJson object and all of its descendants. The allocator + // owns the underlying rapidjson Value, and a rapidjson + // allocator (via a rapidjson::Document). + class EasyJsonAllocator : public RefCounted<EasyJsonAllocator> { + public: + rapidjson::Value& value() { return value_; } + rapidjson::Document::AllocatorType& allocator() { return value_.GetAllocator(); } + private: + friend class RefCounted<EasyJsonAllocator>; + ~EasyJsonAllocator() = default; + + // The underlying rapidjson::Value object (Document is + // a subclass of Value that has its own allocator). + rapidjson::Document value_; + }; + + // Used to instantiate descendant objects. + EasyJson(rapidjson::Value* value, scoped_refptr<EasyJsonAllocator> alloc); + + // One allocator is shared among an EasyJson object and + // all of its descendants. + scoped_refptr<EasyJsonAllocator> alloc_; + + // A pointer to the underlying Value in the object + // tree owned by alloc_. + rapidjson::Value* value_; +}; + +} // namespace kudu