Can you try increasing the Map memory settings in kylin_job_conf.xml or try to merge fewer segments at a time?
Thanks, Silas ________________________________ 发件人: yujunliang <[email protected]> 发送时间: 2018年9月4日 14:11 收件人: [email protected] 主题: Re: 答复: kylin merge it's stuck on MapReduce side.Every segment has about less than 1 MB data(source count:about 400 thousland) the logs below: Error: java.lang.OutOfMemoryError: Java heap space at java.util.Arrays.copyOf(Arrays.java:3236) at java.io.ByteArrayOutputStream.grow(ByteArrayOutputStream.java:118) at java.io.ByteArrayOutputStream.ensureCapacity(ByteArrayOutputStream.java:93) at java.io.ByteArrayOutputStream.write(ByteArrayOutputStream.java:153) at org.apache.commons.io.IOUtils.copyLarge(IOUtils.java:1793) at org.apache.commons.io.IOUtils.copyLarge(IOUtils.java:1769) at org.apache.commons.io.IOUtils.copy(IOUtils.java:1744) at org.apache.kylin.common.persistence.FileResourceStore.getResourceImpl(FileResourceStore.java:123) at org.apache.kylin.common.persistence.ResourceStore.getResource(ResourceStore.java:165) at org.apache.kylin.dict.DictionaryManager.load(DictionaryManager.java:404) at org.apache.kylin.dict.DictionaryManager$1.load(DictionaryManager.java:83) at org.apache.kylin.dict.DictionaryManager$1.load(DictionaryManager.java:80) at com.google.common.cache.LocalCache$LoadingValueReference.loadFuture(LocalCache.java:3568) at com.google.common.cache.LocalCache$Segment.loadSync(LocalCache.java:2350) at com.google.common.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2313) at com.google.common.cache.LocalCache$Segment.get(LocalCache.java:2228) at com.google.common.cache.LocalCache.get(LocalCache.java:3965) at com.google.common.cache.LocalCache.getOrLoad(LocalCache.java:3969) at com.google.common.cache.LocalCache$LocalManualCache.get(LocalCache.java:4829) at org.apache.kylin.dict.DictionaryManager.getDictionaryInfo(DictionaryManager.java:104) at org.apache.kylin.cube.CubeManager$DictionaryAssist.getDictionary(CubeManager.java:1028) at org.apache.kylin.cube.CubeManager.getDictionary(CubeManager.java:961) at org.apache.kylin.cube.CubeSegment.getDictionary(CubeSegment.java:324) at org.apache.kylin.cube.kv.CubeDimEncMap.getDictionary(CubeDimEncMap.java:86) at org.apache.kylin.cube.kv.CubeDimEncMap.get(CubeDimEncMap.java:65) at org.apache.kylin.cube.kv.RowKeyColumnIO.getColumnLength(RowKeyColumnIO.java:43) at org.apache.kylin.cube.kv.RowKeyEncoder.<init>(RowKeyEncoder.java:61) at org.apache.kylin.cube.kv.RowKeyEncoderProvider.getRowkeyEncoder(RowKeyEncoderProvider.java:41) at org.apache.kylin.engine.mr.steps.MergeCuboidMapper.doMap(MergeCuboidMapper.java:151) at org.apache.kylin.engine.mr.steps.MergeCuboidMapper.doMap(MergeCuboidMapper.java:61) at org.apache.kylin.engine.mr.KylinMapper.map(KylinMapper.java:77) at org.apache.hadoop.mapreduce.Mapper.run(Mapper.java:146) -- Sent from: https://apc01.safelinks.protection.outlook.com/?url=http%3A%2F%2Fapache-kylin.74782.x6.nabble.com%2F&data=02%7C01%7C%7C0ec43a0d0a2b46fb207708d6122e59e1%7C84df9e7fe9f640afb435aaaaaaaaaaaa%7C1%7C0%7C636716387613014969&sdata=HkzTxpuklmmmLYt1is2WiBk089Y3XHm%2B8Pxp25Jg2Oo%3D&reserved=0
