[
https://issues.apache.org/jira/browse/ATLAS-2198?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Ayub Pathan updated ATLAS-2198:
-------------------------------
Priority: Blocker (was: Critical)
> HiveHook fails with OOM when hive view with 500 columns are created.
> --------------------------------------------------------------------
>
> Key: ATLAS-2198
> URL: https://issues.apache.org/jira/browse/ATLAS-2198
> Project: Atlas
> Issue Type: Bug
> Components: atlas-intg
> Affects Versions: 0.8.2
> Reporter: Ayub Pathan
> Assignee: Apoorv Naik
> Priority: Blocker
> Fix For: 0.8.2
>
> Attachments: 0001-ATLAS-2198-Hive-Hook-OOM-fix.patch
>
>
> create below two commands with hivehook enabled, hiveHook fails with OOM.
> {noformat}
> create table t1_500(col0 string, col1 string, col2 string, col3 string, col4
> string, col5 string, col6 string, col7 string, col8 string, col9 string,
> col10 string, col11 string, col12 string, col13 string, col14 string, col15
> string, col16 string, col17 string, col18 string, col19 string, col20 string,
> col21 string, col22 string, col23 string, col24 string, col25 string, col26
> string, col27 string, col28 string, col29 string, col30 string, col31 string,
> col32 string, col33 string, col34 string, col35 string, col36 string, col37
> string, col38 string, col39 string, col40 string, col41 string, col42 string,
> col43 string, col44 string, col45 string, col46 string, col47 string, col48
> string, col49 string, col50 string, col51 string, col52 string, col53 string,
> col54 string, col55 string, col56 string, col57 string, col58 string, col59
> string, col60 string, col61 string, col62 string, col63 string, col64 string,
> col65 string, col66 string, col67 string, col68 string, col69 string, col70
> string, col71 string, col72 string, col73 string, col74 string, col75 string,
> col76 string, col77 string, col78 string, col79 string, col80 string, col81
> string, col82 string, col83 string, col84 string, col85 string, col86 string,
> col87 string, col88 string, col89 string, col90 string, col91 string, col92
> string, col93 string, col94 string, col95 string, col96 string, col97 string,
> col98 string, col99 string, col100 string, col101 string, col102 string,
> col103 string, col104 string, col105 string, col106 string, col107 string,
> col108 string, col109 string, col110 string, col111 string, col112 string,
> col113 string, col114 string, col115 string, col116 string, col117 string,
> col118 string, col119 string, col120 string, col121 string, col122 string,
> col123 string, col124 string, col125 string, col126 string, col127 string,
> col128 string, col129 string, col130 string, col131 string, col132 string,
> col133 string, col134 string, col135 string, col136 string, col137 string,
> col138 string, col139 string, col140 string, col141 string, col142 string,
> col143 string, col144 string, col145 string, col146 string, col147 string,
> col148 string, col149 string, col150 string, col151 string, col152 string,
> col153 string, col154 string, col155 string, col156 string, col157 string,
> col158 string, col159 string, col160 string, col161 string, col162 string,
> col163 string, col164 string, col165 string, col166 string, col167 string,
> col168 string, col169 string, col170 string, col171 string, col172 string,
> col173 string, col174 string, col175 string, col176 string, col177 string,
> col178 string, col179 string, col180 string, col181 string, col182 string,
> col183 string, col184 string, col185 string, col186 string, col187 string,
> col188 string, col189 string, col190 string, col191 string, col192 string,
> col193 string, col194 string, col195 string, col196 string, col197 string,
> col198 string, col199 string, col200 string, col201 string, col202 string,
> col203 string, col204 string, col205 string, col206 string, col207 string,
> col208 string, col209 string, col210 string, col211 string, col212 string,
> col213 string, col214 string, col215 string, col216 string, col217 string,
> col218 string, col219 string, col220 string, col221 string, col222 string,
> col223 string, col224 string, col225 string, col226 string, col227 string,
> col228 string, col229 string, col230 string, col231 string, col232 string,
> col233 string, col234 string, col235 string, col236 string, col237 string,
> col238 string, col239 string, col240 string, col241 string, col242 string,
> col243 string, col244 string, col245 string, col246 string, col247 string,
> col248 string, col249 string, col250 string, col251 string, col252 string,
> col253 string, col254 string, col255 string, col256 string, col257 string,
> col258 string, col259 string, col260 string, col261 string, col262 string,
> col263 string, col264 string, col265 string, col266 string, col267 string,
> col268 string, col269 string, col270 string, col271 string, col272 string,
> col273 string, col274 string, col275 string, col276 string, col277 string,
> col278 string, col279 string, col280 string, col281 string, col282 string,
> col283 string, col284 string, col285 string, col286 string, col287 string,
> col288 string, col289 string, col290 string, col291 string, col292 string,
> col293 string, col294 string, col295 string, col296 string, col297 string,
> col298 string, col299 string, col300 string, col301 string, col302 string,
> col303 string, col304 string, col305 string, col306 string, col307 string,
> col308 string, col309 string, col310 string, col311 string, col312 string,
> col313 string, col314 string, col315 string, col316 string, col317 string,
> col318 string, col319 string, col320 string, col321 string, col322 string,
> col323 string, col324 string, col325 string, col326 string, col327 string,
> col328 string, col329 string, col330 string, col331 string, col332 string,
> col333 string, col334 string, col335 string, col336 string, col337 string,
> col338 string, col339 string, col340 string, col341 string, col342 string,
> col343 string, col344 string, col345 string, col346 string, col347 string,
> col348 string, col349 string, col350 string, col351 string, col352 string,
> col353 string, col354 string, col355 string, col356 string, col357 string,
> col358 string, col359 string, col360 string, col361 string, col362 string,
> col363 string, col364 string, col365 string, col366 string, col367 string,
> col368 string, col369 string, col370 string, col371 string, col372 string,
> col373 string, col374 string, col375 string, col376 string, col377 string,
> col378 string, col379 string, col380 string, col381 string, col382 string,
> col383 string, col384 string, col385 string, col386 string, col387 string,
> col388 string, col389 string, col390 string, col391 string, col392 string,
> col393 string, col394 string, col395 string, col396 string, col397 string,
> col398 string, col399 string, col400 string, col401 string, col402 string,
> col403 string, col404 string, col405 string, col406 string, col407 string,
> col408 string, col409 string, col410 string, col411 string, col412 string,
> col413 string, col414 string, col415 string, col416 string, col417 string,
> col418 string, col419 string, col420 string, col421 string, col422 string,
> col423 string, col424 string, col425 string, col426 string, col427 string,
> col428 string, col429 string, col430 string, col431 string, col432 string,
> col433 string, col434 string, col435 string, col436 string, col437 string,
> col438 string, col439 string, col440 string, col441 string, col442 string,
> col443 string, col444 string, col445 string, col446 string, col447 string,
> col448 string, col449 string, col450 string, col451 string, col452 string,
> col453 string, col454 string, col455 string, col456 string, col457 string,
> col458 string, col459 string, col460 string, col461 string, col462 string,
> col463 string, col464 string, col465 string, col466 string, col467 string,
> col468 string, col469 string, col470 string, col471 string, col472 string,
> col473 string, col474 string, col475 string, col476 string, col477 string,
> col478 string, col479 string, col480 string, col481 string, col482 string,
> col483 string, col484 string, col485 string, col486 string, col487 string,
> col488 string, col489 string, col490 string, col491 string, col492 string,
> col493 string, col494 string, col495 string, col496 string, col497 string,
> col498 string, col499 string);
> create view v_t1_500 as select * from t1_500;
> {noformat}
> {noformat}
> 2017-10-10 09:11:35,098 INFO
> [org.apache.ranger.audit.queue.AuditBatchQueue0]: provider.BaseAuditHandler
> (BaseAuditHandler.java:logStatus(310)) - Audit Status Log:
> name=hiveServer2.async.multi_dest.batch,
> finalDestination=hiveServer2.async.multi_dest.batch.hdfs, interval=01:00.002
> minutes, events=4, succcessCount=2, totalEvents=624, totalSuccessCount=226
> 2017-10-10 09:12:24,697 INFO
> [org.apache.hadoop.hive.common.JvmPauseMonitor$Monitor@7eae55]:
> common.JvmPauseMonitor (JvmPauseMonitor.java:run(193)) - Detected pause in
> JVM or host machine (eg GC): pause of approximately 1005ms
> GC pool 'PS MarkSweep' had collection(s): count=1 time=450ms
> GC pool 'PS Scavenge' had collection(s): count=1 time=570ms
> 2017-10-10 09:12:51,726 INFO
> [org.apache.hadoop.hive.common.JvmPauseMonitor$Monitor@7eae55]:
> common.JvmPauseMonitor (JvmPauseMonitor.java:run(193)) - Detected pause in
> JVM or host machine (eg GC): pause of approximately 1851ms
> GC pool 'PS Scavenge' had collection(s): count=1 time=2277ms
> 2017-10-10 09:13:44,438 INFO
> [org.apache.hadoop.hive.common.JvmPauseMonitor$Monitor@7eae55]:
> common.JvmPauseMonitor (JvmPauseMonitor.java:run(193)) - Detected pause in
> JVM or host machine (eg GC): pause of approximately 1342ms
> No GCs detected
> 2017-10-10 09:13:49,732 ERROR [Atlas Logger 4]: hook.HiveHook
> (HiveHook.java:notifyAsPrivilegedAction(236)) - Error during notify CREATEVIEW
> java.lang.OutOfMemoryError: Requested array size exceeds VM limit
> at java.lang.StringCoding$StringEncoder.encode(StringCoding.java:300)
> at java.lang.StringCoding.encode(StringCoding.java:344)
> at java.lang.String.getBytes(String.java:918)
> at
> org.apache.commons.codec.binary.StringUtils.getBytesUnchecked(StringUtils.java:156)
> at
> org.apache.commons.codec.binary.StringUtils.getBytesUtf8(StringUtils.java:129)
> at
> org.apache.atlas.notification.AtlasNotificationBaseMessage.getBytesUtf8(AtlasNotificationBaseMessage.java:126)
> at
> org.apache.atlas.notification.AbstractNotification.createNotificationMessages(AbstractNotification.java:198)
> at
> org.apache.atlas.notification.AbstractNotification.send(AbstractNotification.java:111)
> at
> org.apache.atlas.hook.AtlasHook.notifyEntitiesInternal(AtlasHook.java:143)
> at org.apache.atlas.hook.AtlasHook.notifyEntities(AtlasHook.java:128)
> at org.apache.atlas.hook.AtlasHook.notifyEntities(AtlasHook.java:181)
> at org.apache.atlas.hive.hook.HiveHook.access$300(HiveHook.java:83)
> at org.apache.atlas.hive.hook.HiveHook$3.run(HiveHook.java:221)
> at java..AccessController.doPrivileged(Native Method)
> at javax..auth.Subject.doAs(Subject.java:422)
> at
> org.apache.hadoop..UserGroupInformation.doAs(UserGroupInformation.java:1866)
> at
> org.apache.atlas.hive.hook.HiveHook.notifyAsPrivilegedAction(HiveHook.java:230)
> at org.apache.atlas.hive.hook.HiveHook$2.run(HiveHook.java:203)
> at
> java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
> at java.util.concurrent.FutureTask.run(FutureTask.java:266)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> at java.lang.Thread.run(Thread.java:748)
> 2017-10-10 09:14:24,003 INFO [HiveServer2-HttpHandler-Pool: Thread-11656]:
> thrift.ThriftHttpServlet (ThriftHttpServlet.java:doPost(145)) - Could not
> validate cookie sent, will try to generate a new cookie
> {noformat}
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
