yinzhijian commented on a change in pull request #8765:
URL: https://github.com/apache/incubator-doris/pull/8765#discussion_r841006997
##########
File path: be/src/runtime/export_sink.cpp
##########
@@ -96,6 +97,13 @@ Status ExportSink::send(RuntimeState* state, RowBatch*
batch) {
int num_rows = batch->num_rows();
// we send at most 1024 rows at a time
int batch_send_rows = num_rows > 1024 ? 1024 : num_rows;
+ if (!_send_header && _t_export_sink.header.size() > 0) {
+ size_t written_len = 0;
+ RETURN_IF_ERROR(
+ _file_writer->write(reinterpret_cast<const
uint8_t*>(_t_export_sink.header.c_str()),
+ _t_export_sink.header.size(),
&written_len));
+ _send_header = true;
Review comment:
use _header_sent may be better
##########
File path: fe/fe-core/src/main/java/org/apache/doris/planner/ExportSink.java
##########
@@ -44,6 +45,15 @@ public ExportSink(String exportPath, String columnSeparator,
this.brokerDesc = brokerDesc;
}
+ public ExportSink(String exportPath, String columnSeparator,
+ String lineDelimiter, BrokerDesc brokerDesc, String
header) {
+ this.exportPath = exportPath;
+ this.columnSeparator = columnSeparator;
+ this.lineDelimiter = lineDelimiter;
+ this.brokerDesc = brokerDesc;
Review comment:
ExportSink(exportPath, columnSeparator, lineDelimiter, brokerDesc); may
be better
##########
File path: docs/zh-CN/sql-reference/sql-statements/Data Manipulation/STREAM
LOAD.md
##########
@@ -78,7 +78,7 @@ under the License.
exec_mem_limit: 导入内存限制。默认为 2GB。单位为字节。
- format: 指定导入数据格式，默认是csv，支持json格式。
+ format: 指定导入数据格式，默认是csv，支持json格和带header的csv格式:
csv_with_names、csv_with_names_and_types。
Review comment:
支持json格式， 少个式
##########
File path: be/src/runtime/export_sink.h
##########
@@ -77,6 +77,7 @@ class ExportSink : public DataSink {
RuntimeProfile::Counter* _bytes_written_counter;
RuntimeProfile::Counter* _rows_written_counter;
RuntimeProfile::Counter* _write_timer;
+ bool _send_header;
Review comment:
_header_sent may be better
##########
File path: fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java
##########
@@ -252,10 +253,40 @@ private void genExecFragment() throws UserException {
} catch (URISyntaxException e) {
throw new DdlException("Invalid export path: " + getExportPath());
}
- exportSink = new ExportSink(tmpExportPathStr, getColumnSeparator(),
getLineDelimiter(), brokerDesc);
+ String headerStr = genHeader(this.properties);
+ exportSink = new ExportSink(tmpExportPathStr, getColumnSeparator(),
getLineDelimiter(), brokerDesc, headerStr);
plan();
}
+ private String genHeader(Map<String, String> properties) {
+ String header = "";
+ if (properties.containsKey("format")) {
+ String headerType = properties.get("format");
+ if (headerType.equals(FeConstants.csv_with_names)) {
+ //names
+ for (SlotDescriptor slot : exportTupleDesc.getSlots()) {
+ header = header + slot.getColumn().getName() +
getColumnSeparator();
+ }
+ header = header.substring(0, header.length() -
getColumnSeparator().length());
+ header = header + getLineDelimiter();
+ } else if
(headerType.equals(FeConstants.csv_with_names_and_types)) {
+ //names
+ for (SlotDescriptor slot : exportTupleDesc.getSlots()) {
+ header = header + slot.getColumn().getName() +
getColumnSeparator();
+ }
+ header = header.substring(0, header.length() -
getColumnSeparator().length());
+ header = header + getLineDelimiter();
+ //types
Review comment:
use a common function to gen names may be better
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
