Changeset: 3afe172f23b3 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=3afe172f23b3 Modified Files: sql/backends/monet5/vaults/lidar/lidar.c Branch: data-vaults Log Message:
[LiDAR]: Refactor the reading of files
So far files were being read column-wise. This meant that if we wanted
to read 3 attributes (e.g. XYZ) we would need to open and close the file
3 times. In the case of big LAZ files this would become prohibitively
expensive. This commit changes the implementation to row-wise read,
meaning, that for each point we read all the attributes the user has
requested.
diffs (truncated from 718 to 300 lines):
diff --git a/sql/backends/monet5/vaults/lidar/lidar.c
b/sql/backends/monet5/vaults/lidar/lidar.c
--- a/sql/backends/monet5/vaults/lidar/lidar.c
+++ b/sql/backends/monet5/vaults/lidar/lidar.c
@@ -27,7 +27,7 @@
#include <sys/stat.h>
#include <dirent.h>
-#include <lidar.h>
+#include "lidar.h"
#include <mutils.h>
#include <sql_mvc.h>
#include <sql_scenario.h>
@@ -1286,112 +1286,11 @@ readValue(LASPointH p, ParameterValues p
return ret;
}
-/* Given a BAT type, this macro defines a function that reads a column
- * from the LiDAR file.
- */
-#define READ_ARRAY(BAT_TYPE)
\
-static BAT *
\
-read_array_##BAT_TYPE(str fname,
\
- ParameterValues val, long rows,
\
- double scale, int *error_code)
\
-{
\
- BAT *b;
\
- BAT_TYPE *d = NULL;
\
- LASPointH p = NULL;
\
- LASReaderH reader = NULL;
\
- int i;
\
- RetVal value;
\
-
\
- b = COLnew(0, TYPE_##BAT_TYPE, rows, PERSISTENT);
\
-
\
- if (b == NULL) {
\
- *error_code = 1;
\
- goto cleanup##BAT_TYPE;
\
- }
\
-
\
- MT_lock_set(&mt_lidar_lock);
\
- LASError_Reset();
\
- reader = LASReader_Create(fname);
\
- MT_lock_unset(&mt_lidar_lock);
\
- if (LASError_GetErrorCount() != 0) {
\
- *error_code = 2;
\
- return NULL;
\
- }
\
-
\
- d = (BAT_TYPE *) Tloc(b, 0);
\
-
\
- p = LASReader_GetNextPoint(reader);
\
- i = 0;
\
- while(p) {
\
- switch(val) {
\
- case PARAM_X_COORD:
\
- case PARAM_Y_COORD:
\
- case PARAM_Z_COORD:
\
- value = readValue(p, val);
\
- d[i] = value.val_dbl/scale;
\
- break;
\
- case PARAM_GPS_TIME:
\
- value = readValue(p, val);
\
- d[i] = value.val_dbl;
\
- break;
\
- case PARAM_SCAN_ANGLE:
\
- case PARAM_CLASSIFICATION_NUMBER:
\
- case PARAM_USER_DATA:
\
- value = readValue(p, val);
\
- d[i] = value.val_bte;
\
- break;
\
- case PARAM_INTENSITY:
\
- case PARAM_N_RETURNS:
\
- case PARAM_N_THIS_RETURN:
\
- case PARAM_POINT_SOURCE_ID:
\
- case PARAM_EDGE_OF_FLIGHT_LINE:
\
- case PARAM_DIRECTION_OF_SCAN_FLAG:
\
- case PARAM_RED_CHANNEL:
\
- case PARAM_GREEN_CHANNEL:
\
- case PARAM_BLUE_CHANNEL:
\
- value = readValue(p, val);
\
- d[i] = value.val_sht;
\
- break;
\
- case PARAM_VERTEX_INDEX:
\
- d[i] = i;
\
- break;
\
- default:
\
- fprintf(stderr,
\
- "read_array_##BAT_TYPE:
Unimplemented\n"); \
- }
\
- p = LASReader_GetNextPoint(reader);
\
- i++;
\
- }
\
-cleanup##BAT_TYPE:
\
- MT_lock_set(&mt_lidar_lock);
\
- if (p != NULL) LASPoint_Destroy(p);
\
- if (reader != NULL) LASReader_Destroy(reader);
\
- MT_lock_unset(&mt_lidar_lock);
\
-
\
- return b;
\
-}
-
-
-/* Define functions for reading into different typed columns */
-READ_ARRAY(bte)
-
-READ_ARRAY(sht)
-
-READ_ARRAY(int)
-
-READ_ARRAY(lng)
-
-#ifdef HAVE_HGE
-READ_ARRAY(hge)
-#endif
-
-READ_ARRAY(dbl)
-
static str
LIDARloadTable_(mvc *m, sql_schema *sch, sql_table *lidar_tbl, str tname,
sql_table *tbl, oid rid)
{
sql_table *lidar_fl, *lidar_cl;
- sql_column *col; /*, *colx, *coly, *colz;*/
+ sql_column *col, **columns;
str fname;
str msg = MAL_SUCCEED;
oid frid = oid_nil, tid = oid_nil;
@@ -1401,15 +1300,25 @@ LIDARloadTable_(mvc *m, sql_schema *sch,
#endif
int *tpcode = NULL;
long *rep = NULL, *wid = NULL, rows = 0;
- /* BAT *x = NULL, *y = NULL, *z = NULL; */
- BAT *bat = NULL;
- sql_column *column;
int precisionx = 0, precisiony = 0, precisionz = 0;
int input_params;
double scalex, scaley, scalez;
int error_code;
+ int param_len, idx, i;
+ BAT **bats;
+ void **arr;
+ LASPointH p = NULL;
+ LASReaderH reader = NULL;
+ RetVal value;
+ bte *curr_bte;
+ sht *curr_sht;
+ int *curr_int;
+ lng *curr_lng;
+#ifdef HAVE_HGE
+ hge *curr_hge;
+#endif
+ dbl *curr_dbl;
- (void) tname;
/* col = mvc_bind_column(m, lidar_tbl, "name"); */
/* rid = table_funcs.column_find_row(m->session->tr, col, tname, NULL);
*/
/* if (rid == oid_nil) { */
@@ -1446,191 +1355,397 @@ LIDARloadTable_(mvc *m, sql_schema *sch,
col = mvc_bind_column(m, lidar_tbl, "LoadParams");
input_params = *(int*)table_funcs.column_find_value(m->session->tr,
col, rid);
- for (prm = 1; prm <= PARAM_INTENSITY; prm <<= 1) {
+ param_len = 0;
+ for (prm = 1; prm <= PARAM_INTENSITY; prm <<= 1)
+ if (input_params & prm)
+ param_len++;
+ bats = (BAT **)malloc(param_len*sizeof(BAT *));
+ columns = (sql_column **)malloc(param_len*sizeof(sql_column *));
+ arr = malloc(param_len*sizeof(void *));
+ if (bats == NULL) {
+ msg = createException(MAL, "lidar.loadtable", "malloc
failed.\n");
+ return msg;
+ }
+
+ col = mvc_bind_column(m, lidar_cl, "file_id");
+ tid = table_funcs.column_find_row(m->session->tr, col, (void *)&fid,
NULL);
+ col = mvc_bind_column(m, lidar_cl, "ScaleX");
+ scalex = *(double*)table_funcs.column_find_value(m->session->tr, col,
tid);
+ col = mvc_bind_column(m, lidar_cl, "PrecisionX");
+ precisionx = *(sht*)table_funcs.column_find_value(m->session->tr, col,
tid);
+
+ col = mvc_bind_column(m, lidar_cl, "file_id");
+ tid = table_funcs.column_find_row(m->session->tr, col, (void *)&fid,
NULL);
+ col = mvc_bind_column(m, lidar_cl, "ScaleY");
+ scaley = *(double*)table_funcs.column_find_value(m->session->tr, col,
tid);
+ col = mvc_bind_column(m, lidar_cl, "PrecisionY");
+ precisiony = *(sht*)table_funcs.column_find_value(m->session->tr, col,
tid);
+
+ col = mvc_bind_column(m, lidar_cl, "file_id");
+ tid = table_funcs.column_find_row(m->session->tr, col, (void *)&fid,
NULL);
+ col = mvc_bind_column(m, lidar_cl, "ScaleZ");
+ scalez = *(double*)table_funcs.column_find_value(m->session->tr, col,
tid);
+ col = mvc_bind_column(m, lidar_cl, "PrecisionZ");
+ precisionz = *(sht*)table_funcs.column_find_value(m->session->tr, col,
tid);
+
+
+ /* This loop that creates the bats */
+ error_code = 0;
+ for (prm = 1, idx=0; prm <= PARAM_INTENSITY; prm <<= 1) {
if (input_params & prm) {
- switch(prm) {
- case PARAM_X_COORD:
- col = mvc_bind_column(m, lidar_cl, "file_id");
- tid =
table_funcs.column_find_row(m->session->tr, col, (void *)&fid, NULL);
- col = mvc_bind_column(m, lidar_cl, "ScaleX");
- scalex =
*(double*)table_funcs.column_find_value(m->session->tr, col, tid);
- col = mvc_bind_column(m, lidar_cl,
"PrecisionX");
- precisionx =
*(sht*)table_funcs.column_find_value(m->session->tr, col, tid);
- if (precisionx <= 2)
- bat = read_array_bte(fname, prm, rows,
scalex, &error_code);
- else if (precisionx <= 4)
- bat = read_array_sht(fname, prm, rows,
scalex, &error_code);
- else if (precisionx <= 8)
- bat = read_array_int(fname, prm, rows,
scalex, &error_code);
- else if (precisionx <= 16)
- bat = read_array_int(fname, prm, rows,
scalex, &error_code);
- else if (precisionx <= 32)
- bat = read_array_lng(fname, prm, rows,
scalex, &error_code);
+ switch (prm) {
+ case PARAM_X_COORD:
+ columns[idx] = mvc_bind_column(m, tbl,
"x");
+ if (precisionx <= 2)
+ bats[idx] = COLnew(0, TYPE_bte,
rows, PERSISTENT);
+ else if (precisionx <= 4)
+ bats[idx] = COLnew(0, TYPE_sht,
rows, PERSISTENT);
+ else if (precisionx <= 16)
+ bats[idx] = COLnew(0, TYPE_int,
rows, PERSISTENT);
+ else if (precisionx <= 32)
+ bats[idx] = COLnew(0, TYPE_lng,
rows, PERSISTENT);
#ifdef HAVE_HGE
- else if (precisionx <= 64)
- bat = read_array_hge(fname, prm, rows,
scalex, &error_code);
+ else if (precisionx <= 64)
+ bats[idx] = COLnew(0, TYPE_hge,
rows, PERSISTENT);
#endif
- else {
- bat = NULL;
- error_code = 3;
- }
- column = mvc_bind_column(m, tbl, "x");
- break;
- case PARAM_Y_COORD:
- col = mvc_bind_column(m, lidar_cl, "file_id");
- tid =
table_funcs.column_find_row(m->session->tr, col, (void *)&fid, NULL);
- col = mvc_bind_column(m, lidar_cl, "ScaleY");
- scaley =
*(double*)table_funcs.column_find_value(m->session->tr, col, tid);
- col = mvc_bind_column(m, lidar_cl,
"PrecisionY");
- precisiony =
*(sht*)table_funcs.column_find_value(m->session->tr, col, tid);
- if (precisiony <= 2)
- bat = read_array_bte(fname, prm, rows,
scaley, &error_code);
- else if (precisiony <= 4)
- bat = read_array_sht(fname, prm, rows,
scaley, &error_code);
- else if (precisiony <= 8)
- bat = read_array_int(fname, prm, rows,
scaley, &error_code);
- else if (precisiony <= 16)
- bat = read_array_int(fname, prm, rows,
scaley, &error_code);
- else if (precisiony <= 32)
- bat = read_array_lng(fname, prm, rows,
scaley, &error_code);
+ else { /* Unknown precision */
+ bats[idx] = NULL;
+ error_code = 3;
+ }
+ if (bats[idx] == NULL && error_code ==
0) {
+ error_code = 1;
+ }
+ break;
+ case PARAM_Y_COORD:
+ columns[idx] = mvc_bind_column(m, tbl,
"y");
+ if (precisiony <= 2)
+ bats[idx] = COLnew(0, TYPE_bte,
rows, PERSISTENT);
+ else if (precisiony <= 4)
+ bats[idx] = COLnew(0, TYPE_sht,
rows, PERSISTENT);
+ else if (precisiony <= 16)
+ bats[idx] = COLnew(0, TYPE_int,
rows, PERSISTENT);
+ else if (precisiony <= 32)
+ bats[idx] = COLnew(0, TYPE_lng,
rows, PERSISTENT);
#ifdef HAVE_HGE
- else if (precisiony <= 64)
- bat = read_array_hge(fname, prm, rows,
scaley, &error_code);
+ else if (precisiony <= 64)
+ bats[idx] = COLnew(0, TYPE_hge,
rows, PERSISTENT);
#endif
- else {
- bat = NULL;
- error_code = 4;
- }
- column = mvc_bind_column(m, tbl, "y");
- break;
- case PARAM_Z_COORD:
- col = mvc_bind_column(m, lidar_cl, "file_id");
- tid =
table_funcs.column_find_row(m->session->tr, col, (void *)&fid, NULL);
- col = mvc_bind_column(m, lidar_cl, "ScaleZ");
- scalez =
*(double*)table_funcs.column_find_value(m->session->tr, col, tid);
- col = mvc_bind_column(m, lidar_cl,
"PrecisionZ");
- precisionz =
*(sht*)table_funcs.column_find_value(m->session->tr, col, tid);
- if (precisionz <= 2)
- bat = read_array_bte(fname, prm, rows,
scalez, &error_code);
- else if (precisionz <= 4)
- bat = read_array_sht(fname, prm, rows,
scalez, &error_code);
- else if (precisionz <= 8)
- bat = read_array_int(fname, prm, rows,
scalez, &error_code);
_______________________________________________
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list
