Changeset: 3afe172f23b3 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=3afe172f23b3 Modified Files: sql/backends/monet5/vaults/lidar/lidar.c Branch: data-vaults Log Message:
[LiDAR]: Refactor the reading of files So far files were being read column-wise. This meant that if we wanted to read 3 attributes (e.g. XYZ) we would need to open and close the file 3 times. In the case of big LAZ files this would become prohibitively expensive. This commit changes the implementation to row-wise read, meaning, that for each point we read all the attributes the user has requested. diffs (truncated from 718 to 300 lines): diff --git a/sql/backends/monet5/vaults/lidar/lidar.c b/sql/backends/monet5/vaults/lidar/lidar.c --- a/sql/backends/monet5/vaults/lidar/lidar.c +++ b/sql/backends/monet5/vaults/lidar/lidar.c @@ -27,7 +27,7 @@ #include <sys/stat.h> #include <dirent.h> -#include <lidar.h> +#include "lidar.h" #include <mutils.h> #include <sql_mvc.h> #include <sql_scenario.h> @@ -1286,112 +1286,11 @@ readValue(LASPointH p, ParameterValues p return ret; } -/* Given a BAT type, this macro defines a function that reads a column - * from the LiDAR file. - */ -#define READ_ARRAY(BAT_TYPE) \ -static BAT * \ -read_array_##BAT_TYPE(str fname, \ - ParameterValues val, long rows, \ - double scale, int *error_code) \ -{ \ - BAT *b; \ - BAT_TYPE *d = NULL; \ - LASPointH p = NULL; \ - LASReaderH reader = NULL; \ - int i; \ - RetVal value; \ - \ - b = COLnew(0, TYPE_##BAT_TYPE, rows, PERSISTENT); \ - \ - if (b == NULL) { \ - *error_code = 1; \ - goto cleanup##BAT_TYPE; \ - } \ - \ - MT_lock_set(&mt_lidar_lock); \ - LASError_Reset(); \ - reader = LASReader_Create(fname); \ - MT_lock_unset(&mt_lidar_lock); \ - if (LASError_GetErrorCount() != 0) { \ - *error_code = 2; \ - return NULL; \ - } \ - \ - d = (BAT_TYPE *) Tloc(b, 0); \ - \ - p = LASReader_GetNextPoint(reader); \ - i = 0; \ - while(p) { \ - switch(val) { \ - case PARAM_X_COORD: \ - case PARAM_Y_COORD: \ - case PARAM_Z_COORD: \ - value = readValue(p, val); \ - d[i] = value.val_dbl/scale; \ - break; \ - case PARAM_GPS_TIME: \ - value = readValue(p, val); \ - d[i] = value.val_dbl; \ - break; \ - case PARAM_SCAN_ANGLE: \ - case PARAM_CLASSIFICATION_NUMBER: \ - case PARAM_USER_DATA: \ - value = readValue(p, val); \ - d[i] = value.val_bte; \ - break; \ - case PARAM_INTENSITY: \ - case PARAM_N_RETURNS: \ - case PARAM_N_THIS_RETURN: \ - case PARAM_POINT_SOURCE_ID: \ - case PARAM_EDGE_OF_FLIGHT_LINE: \ - case PARAM_DIRECTION_OF_SCAN_FLAG: \ - case PARAM_RED_CHANNEL: \ - case PARAM_GREEN_CHANNEL: \ - case PARAM_BLUE_CHANNEL: \ - value = readValue(p, val); \ - d[i] = value.val_sht; \ - break; \ - case PARAM_VERTEX_INDEX: \ - d[i] = i; \ - break; \ - default: \ - fprintf(stderr, \ - "read_array_##BAT_TYPE: Unimplemented\n"); \ - } \ - p = LASReader_GetNextPoint(reader); \ - i++; \ - } \ -cleanup##BAT_TYPE: \ - MT_lock_set(&mt_lidar_lock); \ - if (p != NULL) LASPoint_Destroy(p); \ - if (reader != NULL) LASReader_Destroy(reader); \ - MT_lock_unset(&mt_lidar_lock); \ - \ - return b; \ -} - - -/* Define functions for reading into different typed columns */ -READ_ARRAY(bte) - -READ_ARRAY(sht) - -READ_ARRAY(int) - -READ_ARRAY(lng) - -#ifdef HAVE_HGE -READ_ARRAY(hge) -#endif - -READ_ARRAY(dbl) - static str LIDARloadTable_(mvc *m, sql_schema *sch, sql_table *lidar_tbl, str tname, sql_table *tbl, oid rid) { sql_table *lidar_fl, *lidar_cl; - sql_column *col; /*, *colx, *coly, *colz;*/ + sql_column *col, **columns; str fname; str msg = MAL_SUCCEED; oid frid = oid_nil, tid = oid_nil; @@ -1401,15 +1300,25 @@ LIDARloadTable_(mvc *m, sql_schema *sch, #endif int *tpcode = NULL; long *rep = NULL, *wid = NULL, rows = 0; - /* BAT *x = NULL, *y = NULL, *z = NULL; */ - BAT *bat = NULL; - sql_column *column; int precisionx = 0, precisiony = 0, precisionz = 0; int input_params; double scalex, scaley, scalez; int error_code; + int param_len, idx, i; + BAT **bats; + void **arr; + LASPointH p = NULL; + LASReaderH reader = NULL; + RetVal value; + bte *curr_bte; + sht *curr_sht; + int *curr_int; + lng *curr_lng; +#ifdef HAVE_HGE + hge *curr_hge; +#endif + dbl *curr_dbl; - (void) tname; /* col = mvc_bind_column(m, lidar_tbl, "name"); */ /* rid = table_funcs.column_find_row(m->session->tr, col, tname, NULL); */ /* if (rid == oid_nil) { */ @@ -1446,191 +1355,397 @@ LIDARloadTable_(mvc *m, sql_schema *sch, col = mvc_bind_column(m, lidar_tbl, "LoadParams"); input_params = *(int*)table_funcs.column_find_value(m->session->tr, col, rid); - for (prm = 1; prm <= PARAM_INTENSITY; prm <<= 1) { + param_len = 0; + for (prm = 1; prm <= PARAM_INTENSITY; prm <<= 1) + if (input_params & prm) + param_len++; + bats = (BAT **)malloc(param_len*sizeof(BAT *)); + columns = (sql_column **)malloc(param_len*sizeof(sql_column *)); + arr = malloc(param_len*sizeof(void *)); + if (bats == NULL) { + msg = createException(MAL, "lidar.loadtable", "malloc failed.\n"); + return msg; + } + + col = mvc_bind_column(m, lidar_cl, "file_id"); + tid = table_funcs.column_find_row(m->session->tr, col, (void *)&fid, NULL); + col = mvc_bind_column(m, lidar_cl, "ScaleX"); + scalex = *(double*)table_funcs.column_find_value(m->session->tr, col, tid); + col = mvc_bind_column(m, lidar_cl, "PrecisionX"); + precisionx = *(sht*)table_funcs.column_find_value(m->session->tr, col, tid); + + col = mvc_bind_column(m, lidar_cl, "file_id"); + tid = table_funcs.column_find_row(m->session->tr, col, (void *)&fid, NULL); + col = mvc_bind_column(m, lidar_cl, "ScaleY"); + scaley = *(double*)table_funcs.column_find_value(m->session->tr, col, tid); + col = mvc_bind_column(m, lidar_cl, "PrecisionY"); + precisiony = *(sht*)table_funcs.column_find_value(m->session->tr, col, tid); + + col = mvc_bind_column(m, lidar_cl, "file_id"); + tid = table_funcs.column_find_row(m->session->tr, col, (void *)&fid, NULL); + col = mvc_bind_column(m, lidar_cl, "ScaleZ"); + scalez = *(double*)table_funcs.column_find_value(m->session->tr, col, tid); + col = mvc_bind_column(m, lidar_cl, "PrecisionZ"); + precisionz = *(sht*)table_funcs.column_find_value(m->session->tr, col, tid); + + + /* This loop that creates the bats */ + error_code = 0; + for (prm = 1, idx=0; prm <= PARAM_INTENSITY; prm <<= 1) { if (input_params & prm) { - switch(prm) { - case PARAM_X_COORD: - col = mvc_bind_column(m, lidar_cl, "file_id"); - tid = table_funcs.column_find_row(m->session->tr, col, (void *)&fid, NULL); - col = mvc_bind_column(m, lidar_cl, "ScaleX"); - scalex = *(double*)table_funcs.column_find_value(m->session->tr, col, tid); - col = mvc_bind_column(m, lidar_cl, "PrecisionX"); - precisionx = *(sht*)table_funcs.column_find_value(m->session->tr, col, tid); - if (precisionx <= 2) - bat = read_array_bte(fname, prm, rows, scalex, &error_code); - else if (precisionx <= 4) - bat = read_array_sht(fname, prm, rows, scalex, &error_code); - else if (precisionx <= 8) - bat = read_array_int(fname, prm, rows, scalex, &error_code); - else if (precisionx <= 16) - bat = read_array_int(fname, prm, rows, scalex, &error_code); - else if (precisionx <= 32) - bat = read_array_lng(fname, prm, rows, scalex, &error_code); + switch (prm) { + case PARAM_X_COORD: + columns[idx] = mvc_bind_column(m, tbl, "x"); + if (precisionx <= 2) + bats[idx] = COLnew(0, TYPE_bte, rows, PERSISTENT); + else if (precisionx <= 4) + bats[idx] = COLnew(0, TYPE_sht, rows, PERSISTENT); + else if (precisionx <= 16) + bats[idx] = COLnew(0, TYPE_int, rows, PERSISTENT); + else if (precisionx <= 32) + bats[idx] = COLnew(0, TYPE_lng, rows, PERSISTENT); #ifdef HAVE_HGE - else if (precisionx <= 64) - bat = read_array_hge(fname, prm, rows, scalex, &error_code); + else if (precisionx <= 64) + bats[idx] = COLnew(0, TYPE_hge, rows, PERSISTENT); #endif - else { - bat = NULL; - error_code = 3; - } - column = mvc_bind_column(m, tbl, "x"); - break; - case PARAM_Y_COORD: - col = mvc_bind_column(m, lidar_cl, "file_id"); - tid = table_funcs.column_find_row(m->session->tr, col, (void *)&fid, NULL); - col = mvc_bind_column(m, lidar_cl, "ScaleY"); - scaley = *(double*)table_funcs.column_find_value(m->session->tr, col, tid); - col = mvc_bind_column(m, lidar_cl, "PrecisionY"); - precisiony = *(sht*)table_funcs.column_find_value(m->session->tr, col, tid); - if (precisiony <= 2) - bat = read_array_bte(fname, prm, rows, scaley, &error_code); - else if (precisiony <= 4) - bat = read_array_sht(fname, prm, rows, scaley, &error_code); - else if (precisiony <= 8) - bat = read_array_int(fname, prm, rows, scaley, &error_code); - else if (precisiony <= 16) - bat = read_array_int(fname, prm, rows, scaley, &error_code); - else if (precisiony <= 32) - bat = read_array_lng(fname, prm, rows, scaley, &error_code); + else { /* Unknown precision */ + bats[idx] = NULL; + error_code = 3; + } + if (bats[idx] == NULL && error_code == 0) { + error_code = 1; + } + break; + case PARAM_Y_COORD: + columns[idx] = mvc_bind_column(m, tbl, "y"); + if (precisiony <= 2) + bats[idx] = COLnew(0, TYPE_bte, rows, PERSISTENT); + else if (precisiony <= 4) + bats[idx] = COLnew(0, TYPE_sht, rows, PERSISTENT); + else if (precisiony <= 16) + bats[idx] = COLnew(0, TYPE_int, rows, PERSISTENT); + else if (precisiony <= 32) + bats[idx] = COLnew(0, TYPE_lng, rows, PERSISTENT); #ifdef HAVE_HGE - else if (precisiony <= 64) - bat = read_array_hge(fname, prm, rows, scaley, &error_code); + else if (precisiony <= 64) + bats[idx] = COLnew(0, TYPE_hge, rows, PERSISTENT); #endif - else { - bat = NULL; - error_code = 4; - } - column = mvc_bind_column(m, tbl, "y"); - break; - case PARAM_Z_COORD: - col = mvc_bind_column(m, lidar_cl, "file_id"); - tid = table_funcs.column_find_row(m->session->tr, col, (void *)&fid, NULL); - col = mvc_bind_column(m, lidar_cl, "ScaleZ"); - scalez = *(double*)table_funcs.column_find_value(m->session->tr, col, tid); - col = mvc_bind_column(m, lidar_cl, "PrecisionZ"); - precisionz = *(sht*)table_funcs.column_find_value(m->session->tr, col, tid); - if (precisionz <= 2) - bat = read_array_bte(fname, prm, rows, scalez, &error_code); - else if (precisionz <= 4) - bat = read_array_sht(fname, prm, rows, scalez, &error_code); - else if (precisionz <= 8) - bat = read_array_int(fname, prm, rows, scalez, &error_code); _______________________________________________ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list