Ayub Pathan created ATLAS-2198:
----------------------------------
Summary: HiveHook fails with OOM when hive view with 500 columns
are created.
Key: ATLAS-2198
URL: https://issues.apache.org/jira/browse/ATLAS-2198
Project: Atlas
Issue Type: Bug
Components: atlas-intg
Affects Versions: 0.8.2
Reporter: Ayub Pathan
Priority: Critical
Fix For: 0.8.2
create below two commands with hivehook enabled, hiveHook fails with OOM.
{noformat}
create table t1_500(col0 string, col1 string, col2 string, col3 string, col4
string, col5 string, col6 string, col7 string, col8 string, col9 string, col10
string, col11 string, col12 string, col13 string, col14 string, col15 string,
col16 string, col17 string, col18 string, col19 string, col20 string, col21
string, col22 string, col23 string, col24 string, col25 string, col26 string,
col27 string, col28 string, col29 string, col30 string, col31 string, col32
string, col33 string, col34 string, col35 string, col36 string, col37 string,
col38 string, col39 string, col40 string, col41 string, col42 string, col43
string, col44 string, col45 string, col46 string, col47 string, col48 string,
col49 string, col50 string, col51 string, col52 string, col53 string, col54
string, col55 string, col56 string, col57 string, col58 string, col59 string,
col60 string, col61 string, col62 string, col63 string, col64 string, col65
string, col66 string, col67 string, col68 string, col69 string, col70 string,
col71 string, col72 string, col73 string, col74 string, col75 string, col76
string, col77 string, col78 string, col79 string, col80 string, col81 string,
col82 string, col83 string, col84 string, col85 string, col86 string, col87
string, col88 string, col89 string, col90 string, col91 string, col92 string,
col93 string, col94 string, col95 string, col96 string, col97 string, col98
string, col99 string, col100 string, col101 string, col102 string, col103
string, col104 string, col105 string, col106 string, col107 string, col108
string, col109 string, col110 string, col111 string, col112 string, col113
string, col114 string, col115 string, col116 string, col117 string, col118
string, col119 string, col120 string, col121 string, col122 string, col123
string, col124 string, col125 string, col126 string, col127 string, col128
string, col129 string, col130 string, col131 string, col132 string, col133
string, col134 string, col135 string, col136 string, col137 string, col138
string, col139 string, col140 string, col141 string, col142 string, col143
string, col144 string, col145 string, col146 string, col147 string, col148
string, col149 string, col150 string, col151 string, col152 string, col153
string, col154 string, col155 string, col156 string, col157 string, col158
string, col159 string, col160 string, col161 string, col162 string, col163
string, col164 string, col165 string, col166 string, col167 string, col168
string, col169 string, col170 string, col171 string, col172 string, col173
string, col174 string, col175 string, col176 string, col177 string, col178
string, col179 string, col180 string, col181 string, col182 string, col183
string, col184 string, col185 string, col186 string, col187 string, col188
string, col189 string, col190 string, col191 string, col192 string, col193
string, col194 string, col195 string, col196 string, col197 string, col198
string, col199 string, col200 string, col201 string, col202 string, col203
string, col204 string, col205 string, col206 string, col207 string, col208
string, col209 string, col210 string, col211 string, col212 string, col213
string, col214 string, col215 string, col216 string, col217 string, col218
string, col219 string, col220 string, col221 string, col222 string, col223
string, col224 string, col225 string, col226 string, col227 string, col228
string, col229 string, col230 string, col231 string, col232 string, col233
string, col234 string, col235 string, col236 string, col237 string, col238
string, col239 string, col240 string, col241 string, col242 string, col243
string, col244 string, col245 string, col246 string, col247 string, col248
string, col249 string, col250 string, col251 string, col252 string, col253
string, col254 string, col255 string, col256 string, col257 string, col258
string, col259 string, col260 string, col261 string, col262 string, col263
string, col264 string, col265 string, col266 string, col267 string, col268
string, col269 string, col270 string, col271 string, col272 string, col273
string, col274 string, col275 string, col276 string, col277 string, col278
string, col279 string, col280 string, col281 string, col282 string, col283
string, col284 string, col285 string, col286 string, col287 string, col288
string, col289 string, col290 string, col291 string, col292 string, col293
string, col294 string, col295 string, col296 string, col297 string, col298
string, col299 string, col300 string, col301 string, col302 string, col303
string, col304 string, col305 string, col306 string, col307 string, col308
string, col309 string, col310 string, col311 string, col312 string, col313
string, col314 string, col315 string, col316 string, col317 string, col318
string, col319 string, col320 string, col321 string, col322 string, col323
string, col324 string, col325 string, col326 string, col327 string, col328
string, col329 string, col330 string, col331 string, col332 string, col333
string, col334 string, col335 string, col336 string, col337 string, col338
string, col339 string, col340 string, col341 string, col342 string, col343
string, col344 string, col345 string, col346 string, col347 string, col348
string, col349 string, col350 string, col351 string, col352 string, col353
string, col354 string, col355 string, col356 string, col357 string, col358
string, col359 string, col360 string, col361 string, col362 string, col363
string, col364 string, col365 string, col366 string, col367 string, col368
string, col369 string, col370 string, col371 string, col372 string, col373
string, col374 string, col375 string, col376 string, col377 string, col378
string, col379 string, col380 string, col381 string, col382 string, col383
string, col384 string, col385 string, col386 string, col387 string, col388
string, col389 string, col390 string, col391 string, col392 string, col393
string, col394 string, col395 string, col396 string, col397 string, col398
string, col399 string, col400 string, col401 string, col402 string, col403
string, col404 string, col405 string, col406 string, col407 string, col408
string, col409 string, col410 string, col411 string, col412 string, col413
string, col414 string, col415 string, col416 string, col417 string, col418
string, col419 string, col420 string, col421 string, col422 string, col423
string, col424 string, col425 string, col426 string, col427 string, col428
string, col429 string, col430 string, col431 string, col432 string, col433
string, col434 string, col435 string, col436 string, col437 string, col438
string, col439 string, col440 string, col441 string, col442 string, col443
string, col444 string, col445 string, col446 string, col447 string, col448
string, col449 string, col450 string, col451 string, col452 string, col453
string, col454 string, col455 string, col456 string, col457 string, col458
string, col459 string, col460 string, col461 string, col462 string, col463
string, col464 string, col465 string, col466 string, col467 string, col468
string, col469 string, col470 string, col471 string, col472 string, col473
string, col474 string, col475 string, col476 string, col477 string, col478
string, col479 string, col480 string, col481 string, col482 string, col483
string, col484 string, col485 string, col486 string, col487 string, col488
string, col489 string, col490 string, col491 string, col492 string, col493
string, col494 string, col495 string, col496 string, col497 string, col498
string, col499 string);
create view v_t1_500 as select * from t1_500;
{noformat}
{noformat}
2017-10-10 09:11:35,098 INFO [org.apache.ranger.audit.queue.AuditBatchQueue0]:
provider.BaseAuditHandler (BaseAuditHandler.java:logStatus(310)) - Audit Status
Log: name=hiveServer2.async.multi_dest.batch,
finalDestination=hiveServer2.async.multi_dest.batch.hdfs, interval=01:00.002
minutes, events=4, succcessCount=2, totalEvents=624, totalSuccessCount=226
2017-10-10 09:12:24,697 INFO
[org.apache.hadoop.hive.common.JvmPauseMonitor$Monitor@7eae55]:
common.JvmPauseMonitor (JvmPauseMonitor.java:run(193)) - Detected pause in JVM
or host machine (eg GC): pause of approximately 1005ms
GC pool 'PS MarkSweep' had collection(s): count=1 time=450ms
GC pool 'PS Scavenge' had collection(s): count=1 time=570ms
2017-10-10 09:12:51,726 INFO
[org.apache.hadoop.hive.common.JvmPauseMonitor$Monitor@7eae55]:
common.JvmPauseMonitor (JvmPauseMonitor.java:run(193)) - Detected pause in JVM
or host machine (eg GC): pause of approximately 1851ms
GC pool 'PS Scavenge' had collection(s): count=1 time=2277ms
2017-10-10 09:13:44,438 INFO
[org.apache.hadoop.hive.common.JvmPauseMonitor$Monitor@7eae55]:
common.JvmPauseMonitor (JvmPauseMonitor.java:run(193)) - Detected pause in JVM
or host machine (eg GC): pause of approximately 1342ms
No GCs detected
2017-10-10 09:13:49,732 ERROR [Atlas Logger 4]: hook.HiveHook
(HiveHook.java:notifyAsPrivilegedAction(236)) - Error during notify CREATEVIEW
java.lang.OutOfMemoryError: Requested array size exceeds VM limit
at java.lang.StringCoding$StringEncoder.encode(StringCoding.java:300)
at java.lang.StringCoding.encode(StringCoding.java:344)
at java.lang.String.getBytes(String.java:918)
at
org.apache.commons.codec.binary.StringUtils.getBytesUnchecked(StringUtils.java:156)
at
org.apache.commons.codec.binary.StringUtils.getBytesUtf8(StringUtils.java:129)
at
org.apache.atlas.notification.AtlasNotificationBaseMessage.getBytesUtf8(AtlasNotificationBaseMessage.java:126)
at
org.apache.atlas.notification.AbstractNotification.createNotificationMessages(AbstractNotification.java:198)
at
org.apache.atlas.notification.AbstractNotification.send(AbstractNotification.java:111)
at
org.apache.atlas.hook.AtlasHook.notifyEntitiesInternal(AtlasHook.java:143)
at org.apache.atlas.hook.AtlasHook.notifyEntities(AtlasHook.java:128)
at org.apache.atlas.hook.AtlasHook.notifyEntities(AtlasHook.java:181)
at org.apache.atlas.hive.hook.HiveHook.access$300(HiveHook.java:83)
at org.apache.atlas.hive.hook.HiveHook$3.run(HiveHook.java:221)
at java..AccessController.doPrivileged(Native Method)
at javax..auth.Subject.doAs(Subject.java:422)
at
org.apache.hadoop..UserGroupInformation.doAs(UserGroupInformation.java:1866)
at
org.apache.atlas.hive.hook.HiveHook.notifyAsPrivilegedAction(HiveHook.java:230)
at org.apache.atlas.hive.hook.HiveHook$2.run(HiveHook.java:203)
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
2017-10-10 09:14:24,003 INFO [HiveServer2-HttpHandler-Pool: Thread-11656]:
thrift.ThriftHttpServlet (ThriftHttpServlet.java:doPost(145)) - Could not
validate cookie sent, will try to generate a new cookie
{noformat}
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
