Repository: incubator-quickstep Updated Branches: refs/heads/copy-to c7bd8c6d1 -> 9eae02253
Updates Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/9eae0225 Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/9eae0225 Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/9eae0225 Branch: refs/heads/copy-to Commit: 9eae022534bbb52f784a084c27c9aee360b046d6 Parents: c7bd8c6 Author: Jianqiao Zhu <jianq...@cs.wisc.edu> Authored: Fri Aug 4 19:59:07 2017 -0500 Committer: Jianqiao Zhu <jianq...@cs.wisc.edu> Committed: Fri Aug 4 19:59:07 2017 -0500 ---------------------------------------------------------------------- parser/ParseKeyValue.hpp | 4 +-- parser/ParseStatement.hpp | 23 +++++++++----- query_execution/QueryContext.hpp | 37 ++++++++++++++++------- relational_operators/TableExportOperator.cpp | 4 +-- 4 files changed, 45 insertions(+), 23 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/9eae0225/parser/ParseKeyValue.hpp ---------------------------------------------------------------------- diff --git a/parser/ParseKeyValue.hpp b/parser/ParseKeyValue.hpp index f2564b5..62cba7f 100644 --- a/parser/ParseKeyValue.hpp +++ b/parser/ParseKeyValue.hpp @@ -198,7 +198,7 @@ class ParseKeyStringList : public ParseKeyValue { }; /** - * @brief The parsed representation of a key-value pair. Value if of int type. + * @brief The parsed representation of a key-value pair. Value is of int type. **/ class ParseKeyIntegerValue : public ParseKeyValue { public: @@ -254,7 +254,7 @@ class ParseKeyIntegerValue : public ParseKeyValue { }; /** - * @brief The parsed representation of a key-value pair. Value if of bool type. + * @brief The parsed representation of a key-value pair. Value is of bool type. **/ class ParseKeyBoolValue : public ParseKeyValue { public: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/9eae0225/parser/ParseStatement.hpp ---------------------------------------------------------------------- diff --git a/parser/ParseStatement.hpp b/parser/ParseStatement.hpp index 60b2a08..c984b92 100644 --- a/parser/ParseStatement.hpp +++ b/parser/ParseStatement.hpp @@ -783,7 +783,7 @@ class ParseStatementInsertSelection : public ParseStatementInsert { class ParseStatementCopy : public ParseStatement { public: /** - * @brief Copy direction (FROM/TO). + * @brief Copy direction (FROM text file/TO text file). */ enum CopyDirection { kFrom, @@ -797,7 +797,7 @@ class ParseStatementCopy : public ParseStatement { * @param column_number Column number of the first token of this node in the SQL statement. * @param direction The copy direction (FROM/TO). * @param relation_name The name of the relation. - * @param file_name The name of the text file. + * @param file_name The name of the file. * @param params The optional parameters of the COPY statement. **/ ParseStatementCopy(const int line_number, @@ -818,9 +818,9 @@ class ParseStatementCopy : public ParseStatement { * * @param line_number Line number of the first token of this node in the SQL statement. * @param column_number Column number of the first token of this node in the SQL statement. - * @param set_operation_query The set operation query + * @param set_operation_query The set operation query. * @param with_clause The WITH clause of common table query expressions. - * @param file_name The name of the text file. + * @param file_name The name of the file. * @param params The optional parameters of the COPY statement. **/ ParseStatementCopy(const int line_number, @@ -852,9 +852,9 @@ class ParseStatementCopy : public ParseStatement { } /** - * @brief Get the direction (FROM/TO) of the COPY statement. + * @brief Get the direction (FROM text file/TO text file) of the COPY statement. * - * return The direction (FROM/TO) of the COPY statement. + * return The direction of the COPY statement. */ const CopyDirection getCopyDirection() const { return direction_; @@ -899,8 +899,7 @@ class ParseStatementCopy : public ParseStatement { /** * @brief Get the additional COPY parameters. * - * @return The string which terminates individual attribute values in the - * input file. + * @return The additional COPY parameters. **/ const PtrList<ParseKeyValue>* params() const { return params_.get(); @@ -957,7 +956,15 @@ class ParseStatementCopy : public ParseStatement { std::unique_ptr<ParseSetOperation> set_operation_query_; std::unique_ptr<PtrVector<ParseSubqueryTableReference>> with_clause_; + + // NOTE(jianqiao): For convenience of represenation, the file_name_ string + // contains a prefix character of '$' or '@' before its actual value, where + // (1) '$' indicates that the name is a standard I/O stream, + // e.g. $stdin, $stdout, $stderr ; + // (2) '@' indicates that the name is a regular file name, + // e.g. @data.csv, @result.tbl . std::unique_ptr<ParseString> file_name_; + std::unique_ptr<PtrList<ParseKeyValue>> params_; DISALLOW_COPY_AND_ASSIGN(ParseStatementCopy); http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/9eae0225/query_execution/QueryContext.hpp ---------------------------------------------------------------------- diff --git a/query_execution/QueryContext.hpp b/query_execution/QueryContext.hpp index a198a81..b146e13 100644 --- a/query_execution/QueryContext.hpp +++ b/query_execution/QueryContext.hpp @@ -572,19 +572,34 @@ class QueryContext { return window_aggregation_states_[id].release(); } - inline std::string* getBlockOutputBuffer(const block_id block) { - SpinSharedMutexExclusiveLock<false> lock(block_output_mutex_); - auto it = block_output_buffers_.find(block); - DCHECK(it != block_output_buffers_.end()); + /** + * @brief Release the output text buffer for the specified block. + * + * @param id The id of the block to release its output text buffer. + * + * @return The output text buffer for the specified block. Caller should take + * ownership of the returned object. + **/ + inline std::string* releaseBlockOutputTextBuffer(const block_id id) { + SpinSharedMutexExclusiveLock<false> lock(block_output_text_buffer_mutex_); + auto it = block_output_text_buffers_.find(id); + DCHECK(it != block_output_text_buffers_.end()); std::unique_ptr<std::string> output = std::move(it->second); - block_output_buffers_.erase(it); + block_output_text_buffers_.erase(it); return output.release(); } - inline void setBlockOutputBuffer(const block_id block, std::string *output) { - SpinSharedMutexExclusiveLock<false> lock(block_output_mutex_); - DCHECK(block_output_buffers_.find(block) == block_output_buffers_.end()); - block_output_buffers_.emplace(block, std::unique_ptr<std::string>(output)); + /** + * @brief Set the output text buffer for the specified block. + * + * @param id The id of the block. + * @param output The output buffer to set. Caller relinquishes ownership of + * \p output by calling this method. + **/ + inline void setBlockOutputTextBuffer(const block_id id, std::string *output) { + SpinSharedMutexExclusiveLock<false> lock(block_output_text_buffer_mutex_); + DCHECK(block_output_text_buffers_.find(id) == block_output_text_buffers_.end()); + block_output_text_buffers_.emplace(id, std::unique_ptr<std::string>(output)); } /** @@ -651,12 +666,12 @@ class QueryContext { std::vector<std::unique_ptr<Tuple>> tuples_; std::vector<std::unordered_map<attribute_id, std::unique_ptr<const Scalar>>> update_groups_; std::vector<std::unique_ptr<WindowAggregationOperationState>> window_aggregation_states_; - std::unordered_map<block_id, std::unique_ptr<std::string>> block_output_buffers_; + std::unordered_map<block_id, std::unique_ptr<std::string>> block_output_text_buffers_; mutable SpinSharedMutex<false> hash_tables_mutex_; mutable SpinSharedMutex<false> aggregation_states_mutex_; mutable SpinSharedMutex<false> insert_destinations_mutex_; - mutable SpinSharedMutex<false> block_output_mutex_; + mutable SpinSharedMutex<false> block_output_text_buffer_mutex_; DISALLOW_COPY_AND_ASSIGN(QueryContext); }; http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/9eae0225/relational_operators/TableExportOperator.cpp ---------------------------------------------------------------------- diff --git a/relational_operators/TableExportOperator.cpp b/relational_operators/TableExportOperator.cpp index 740c9da..03e260f 100644 --- a/relational_operators/TableExportOperator.cpp +++ b/relational_operators/TableExportOperator.cpp @@ -70,7 +70,7 @@ bool TableExportOperator::getAllWorkOrders( bus, output_buffer.get()), op_index_); - query_context->setBlockOutputBuffer(input_block_id, output_buffer.release()); + query_context->setBlockOutputTextBuffer(input_block_id, output_buffer.release()); }; if (input_relation_is_stored_) { @@ -118,7 +118,7 @@ void TableExportOperator::receiveFeedbackMessage( block_id completed_block_id = *static_cast<const block_id*>(msg.payload()); outputs_.emplace(completed_block_id, std::unique_ptr<std::string>( - query_context_->getBlockOutputBuffer(completed_block_id))); + query_context_->releaseBlockOutputTextBuffer(completed_block_id))); while (true) { block_id next_block_id;
