Handling hive nulls.
Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/f59b6bbe Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/f59b6bbe Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/f59b6bbe Branch: refs/heads/master Commit: f59b6bbeedd9b993ec08a40b5ae5dc243cddd5bb Parents: f2ab337 Author: Aaron McCurry <[email protected]> Authored: Tue Dec 17 08:11:01 2013 -0500 Committer: Aaron McCurry <[email protected]> Committed: Tue Dec 17 08:52:10 2013 -0500 ---------------------------------------------------------------------- .../apache/blur/mapreduce/lib/CsvBlurMapper.java | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/f59b6bbe/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/CsvBlurMapper.java ---------------------------------------------------------------------- diff --git a/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/CsvBlurMapper.java b/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/CsvBlurMapper.java index 8557dfc..8f59e31 100644 --- a/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/CsvBlurMapper.java +++ b/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/CsvBlurMapper.java @@ -47,7 +47,7 @@ import com.google.common.base.Splitter; */ public class CsvBlurMapper extends BaseBlurMapper<Writable, Text> { - private static final String UTF_8 = "UTF-8"; + public static final String UTF_8 = "UTF-8"; public static final String BLUR_CSV_AUTO_GENERATE_RECORD_ID_AS_HASH_OF_DATA = "blur.csv.auto.generate.record.id.as.hash.of.data"; public static final String BLUR_CSV_AUTO_GENERATE_ROW_ID_AS_HASH_OF_DATA = "blur.csv.auto.generate.row.id.as.hash.of.data"; public static final String BLUR_CSV_FAMILY_PATH_MAPPINGS_FAMILIES = "blur.csv.family.path.mappings.families"; @@ -55,6 +55,7 @@ public class CsvBlurMapper extends BaseBlurMapper<Writable, Text> { public static final String BLUR_CSV_SEPARATOR_BASE64 = "blur.csv.separator.base64"; public static final String BLUR_CSV_FAMILY_COLUMN_PREFIX = "blur.csv.family."; public static final String BLUR_CSV_FAMILIES = "blur.csv.families"; + public static final String HIVE_NULL = "\\N"; protected Map<String, List<String>> _columnNameMap; protected String _separator = Base64.encodeBase64String(",".getBytes()); @@ -440,8 +441,11 @@ public class CsvBlurMapper extends BaseBlurMapper<Writable, Text> { } for (int i = 0; i < columnNames.size(); i++) { - record.addColumn(columnNames.get(i), list.get(i + offset)); - _columnCounter.increment(1); + String val = handleHiveNulls(list.get(i + offset)); + if (val != null) { + record.addColumn(columnNames.get(i), val); + _columnCounter.increment(1); + } } _key.set(record.getRowId()); _mutate.setMutateType(MUTATE_TYPE.REPLACE); @@ -450,6 +454,13 @@ public class CsvBlurMapper extends BaseBlurMapper<Writable, Text> { context.progress(); } + protected String handleHiveNulls(String value) { + if (value.equals(HIVE_NULL)) { + return null; + } + return value; + } + public void setFamilyFromPath(String familyFromPath) { this._familyFromPath = familyFromPath; }
