SteveYurongSu commented on a change in pull request #1914:
URL: https://github.com/apache/iotdb/pull/1914#discussion_r525236947
##########
File path: cli/src/main/java/org/apache/iotdb/tool/ImportCsv.java
##########
@@ -119,296 +88,78 @@ private static Options createOptions() {
/**
* Data from csv To tsfile.
*/
- private static void loadDataFromCSV(File file, int index) {
- statement = null;
+ private static void loadDataFromCSV(File file) {
int fileLine;
try {
fileLine = getFileLineCount(file);
} catch (IOException e) {
System.out.println("Failed to import file: " + file.getName());
return;
}
- File errorFile = new File(errorInsertInfo + index);
- if (!errorFile.exists()) {
- try {
- errorFile.createNewFile();
- } catch (IOException e) {
- System.out.println("Cannot create a errorFile because: " +
e.getMessage());
- return;
- }
- }
System.out.println("Start to import data from: " + file.getName());
- errorFlag = true;
try(BufferedReader br = new BufferedReader(new FileReader(file));
- BufferedWriter bw = new BufferedWriter(new FileWriter(errorFile));
ProgressBar pb = new ProgressBar("Import from: " + file.getName(),
fileLine)) {
pb.setExtraMessage("Importing...");
String header = br.readLine();
-
- bw.write("From " + file.getAbsolutePath());
- bw.newLine();
- bw.newLine();
- bw.write(header);
- bw.newLine();
- bw.newLine();
-
- // storage csv table head info
- Map<String, ArrayList<Integer>> deviceToColumn = new HashMap<>();
- // storage csv table head info
- List<String> colInfo = new ArrayList<>();
- // storage csv device sensor info, corresponding csv table head
- List<String> headInfo = new ArrayList<>();
-
- String[] strHeadInfo = header.split(",");
- if (strHeadInfo.length <= 1) {
+ String[] cols = splitCsvLine(header);
+ if (cols.length <= 1) {
System.out.println("The CSV file "+ file.getName() +" illegal, please
check first line");
return;
}
- long startTime = System.currentTimeMillis();
- Map<String, String> timeseriesDataType = new HashMap<>();
+ List<String> devices = new ArrayList<>();
+ List<Long> times = new ArrayList<>();
+ List<List<String>> measurementsList = new ArrayList<>();
+ List<List<String>> valuesList = new ArrayList<>();
+ Map<String, Map<String, Integer>> devicesToMeasurementsAndPositions =
new HashMap<>();
- boolean success = queryDatabaseMeta(strHeadInfo, file, bw,
timeseriesDataType, headInfo,
- deviceToColumn, colInfo);
- if (!success) {
- errorFlag = false;
- return;
+ for(int i = 1; i < cols.length; i++) {
+ splitColToDeviceAndMeasurement(cols[i],
devicesToMeasurementsAndPositions, i);
}
- statement = connection.createStatement();
-
-
- List<String> tmp = new ArrayList<>();
- success = readAndGenSqls(br, timeseriesDataType, deviceToColumn,
colInfo, headInfo,
- bw, tmp, pb);
- if (!success) {
- return;
+ String line;
+ while((line = br.readLine()) != null) {
+ cols = splitCsvLine(line);
+ for(Entry<String, Map<String, Integer>>
deviceToMeasurementsAndPositions: devicesToMeasurementsAndPositions.entrySet())
{
+ devices.add(deviceToMeasurementsAndPositions.getKey());
+ times.add(Long.parseLong(cols[0]));
+ Map<String, Integer> measurementsAndPositions =
deviceToMeasurementsAndPositions.getValue();
+ List<String> measurements = new ArrayList<>();
+ List<String> values = new ArrayList<>();
+ for(Entry<String, Integer> measurementAndPosition :
measurementsAndPositions.entrySet()) {
+ measurements.add(measurementAndPosition.getKey());
+ if(cols[measurementAndPosition.getValue()].equals("") &&
cols[measurementAndPosition.getValue()].equals("null")) {
+ values.add(null);
+ } else {
+ values.add(cols[measurementAndPosition.getValue()]);
+ }
+ }
+ measurementsList.add(measurements);
+ valuesList.add(values);
+ }
}
-
- executeSqls(bw, tmp, startTime, file);
+ session.insertRecords(devices, times, measurementsList, valuesList);
Review comment:
I think there may be a problem when importing millions of records at a
time...
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
