Eywek opened a new issue #6717:
URL: https://github.com/apache/incubator-pinot/issues/6717
I'm using the 0.7.0-695ca390a8080409b3b76f250f2315b81b86b362.
I have the following table schema:
```json
{
"schemaName": "schema_605b02ec0eb00003003bfc41",
"dimensionFieldSpecs": [
{
"name": "key",
"dataType": "STRING"
},
{
"name": "name",
"dataType": "STRING"
},
{
"name": "url",
"dataType": "STRING"
},
{
"name": "tmpId",
"dataType": "STRING"
},
{
"name": "labels",
"dataType": "STRING",
"singleValueField": false
},
{
"name": "ids",
"dataType": "STRING",
"singleValueField": false
},
{
"name": "transId",
"dataType": "STRING"
},
{
"name": "clientId",
"dataType": "STRING"
},
{
"name": "blockId",
"dataType": "STRING"
},
{
"name": "intelliId",
"dataType": "STRING"
},
{
"name": "version",
"dataType": "STRING"
},
{
"name": "message",
"dataType": "STRING"
},
{
"name": "contentId",
"dataType": "STRING"
},
{
"name": "branchId",
"dataType": "STRING"
},
{
"name": "userId",
"dataType": "STRING"
},
{
"name": "eventId",
"dataType": "STRING"
},
{
"name": "v",
"dataType": "FLOAT",
"defaultNullValue": 0
},
{
"name": "ua",
"dataType": "STRING"
},
{
"name": "labels_keys",
"dataType": "STRING",
"singleValueField": false
},
{
"name": "agent_family",
"dataType": "STRING"
},
{
"name": "agent_major",
"dataType": "STRING"
},
{
"name": "agent_minor",
"dataType": "STRING"
},
{
"name": "agent_patch",
"dataType": "STRING"
},
{
"name": "agent_device_family",
"dataType": "STRING"
},
{
"name": "agent_device_major",
"dataType": "STRING"
},
{
"name": "agent_device_minor",
"dataType": "STRING"
},
{
"name": "agent_device_patch",
"dataType": "STRING"
},
{
"name": "agent_os_family",
"dataType": "STRING"
},
{
"name": "agent_os_major",
"dataType": "STRING"
},
{
"name": "agent_os_minor",
"dataType": "STRING"
},
{
"name": "agent_os_patch",
"dataType": "STRING"
}
],
"metricFieldSpecs": [
{
"name": "value",
"dataType": "FLOAT"
}
],
"dateTimeFieldSpecs": [
{
"name": "timestamp",
"dataType": "LONG",
"format": "1:MILLISECONDS:EPOCH",
"granularity": "1:HOURS"
}
]
}
```
On the following realtime table:
```json
{
"REALTIME": {
"tableName": "datasource_605b02ec0eb00003003bfc41_REALTIME",
"tableType": "REALTIME",
"segmentsConfig": {
"segmentPushFrequency": "HOURLY",
"segmentPushType": "APPEND",
"replicasPerPartition": "1",
"timeColumnName": "timestamp",
"timeType": "MILLISECONDS",
"replication": "1",
"schemaName": "schema_605b02ec0eb00003003bfc41"
},
"tenants": {
"broker": "DefaultTenant",
"server": "DefaultTenant"
},
"tableIndexConfig": {
"sortedColumn": [
"value",
"timestamp"
],
"invertedIndexColumns": [
"name",
"tmpId",
"labels",
"clientId",
"blockId",
"labels_keys"
],
"autoGeneratedInvertedIndex": false,
"createInvertedIndexDuringSegmentGeneration": false,
"loadMode": "mmap",
"streamConfigs": {
"streamType": "kafka",
"stream.kafka.consumer.type": "lowlevel",
"stream.kafka.topic.name":
"datasources-ingester-605b02ec0eb00003003bfc41",
"stream.kafka.decoder.class.name":
"org.apache.pinot.plugin.stream.kafka.KafkaJSONMessageDecoder",
"stream.kafka.consumer.factory.class.name":
"org.apache.pinot.plugin.stream.kafka20.KafkaConsumerFactory",
"stream.kafka.broker.list": "pulsar.internal.reelevant.io:9092",
"stream.kafka.consumer.prop.auto.offset.reset": "smallest",
"realtime.segment.flush.threshold.time": "1d",
"realtime.segment.flush.threshold.rows": "390000"
},
"enableDefaultStarTree": false,
"enableDynamicStarTreeCreation": false,
"aggregateMetrics": false,
"nullHandlingEnabled": true
},
"metadata": {},
"quota": {},
"task": {
"taskTypeConfigsMap": {
"RealtimeToOfflineSegmentsTask": {
"bucketTimePeriod": "1d",
"bufferTimePeriod": "1d",
"collectorType": "concat",
"maxNumRecordsPerSegment": "390000"
}
}
},
"routing": {},
"instanceAssignmentConfigMap": {
"CONSUMING": {
"tagPoolConfig": {
"tag": "DefaultTenant_REALTIME",
"poolBased": false,
"numPools": 0
},
"replicaGroupPartitionConfig": {
"replicaGroupBased": false,
"numInstances": 1,
"numReplicaGroups": 0,
"numInstancesPerReplicaGroup": 0,
"numPartitions": 0,
"numInstancesPerPartition": 0
}
},
"COMPLETED": {
"tagPoolConfig": {
"tag": "DefaultTenant_REALTIME",
"poolBased": false,
"numPools": 0
},
"replicaGroupPartitionConfig": {
"replicaGroupBased": false,
"numInstances": 2,
"numReplicaGroups": 0,
"numInstancesPerReplicaGroup": 0,
"numPartitions": 0,
"numInstancesPerPartition": 0
}
}
},
"isDimTable": false
}
}
```
But all of my segments (except for the consuming one) are offline with
following metadata:
```json
{
"segment.realtime.endOffset": "9223372036854775807",
"segment.start.time": "-1",
"segment.time.unit": null,
"segment.flush.threshold.size": "390000",
"segment.realtime.startOffset": "11329367812603904",
"segment.end.time": "-1",
"segment.total.docs": "-1",
"segment.table.name": "datasource_605b02ec0eb00003003bfc41",
"segment.realtime.numReplicas": "1",
"segment.creation.time": "1616612105480",
"segment.realtime.download.url": null,
"segment.name":
"datasource_605b02ec0eb00003003bfc41__0__4__20210324T1855Z",
"segment.index.version": null,
"segment.flush.threshold.time": null,
"segment.type": "REALTIME",
"segment.crc": "-1",
"segment.realtime.status": "IN_PROGRESS"
}
```
On the logs I can see:
```
2021/03/24 19:50:13.417 INFO [PinotLLCRealtimeSegmentManager]
[grizzly-http-server-3] Marking CONSUMING segment:
datasource_605b02ec0eb00003003bfc41__0__4__20210324T1855Z OFFLINE on instance:
Server_10.1.1.13_8098
```
```
2021/03/24 19:50:13.416 INFO [LLCSegmentCompletionHandlers]
[grizzly-http-server-3] Processing segmentStoppedConsuming:Offset:
11575658581037057,Segment name:
datasource_605b02ec0eb00003003bfc41__0__4__20210324T1855Z,Instance Id:
Server_10.1.1.13_8098,Reason: java.lang.NullPointerException,NumRows:
-1,BuildTimeMillis: -1,WaitTimeMillis: -1,ExtraTimeSec: -1,SegmentLocation:
null,MemoryUsedBytes: -1,SegmentSizeBytes: -1,StreamPartitionMsgOffset:
11575658581037057
```
```
Mar 25 08:50:30 pinot-hosts-3 bash[11610]: 2021/03/25 08:50:30.922 ERROR
[LLRealtimeSegmentDataManager_datasource_605b02ec0eb00003003bfc41__0__17__20210325T0755Z]
[datasource_605b02ec0eb00003003bfc41__0__17__20210325T0755Z] Exception while
in work
Mar 25 08:50:30 pinot-hosts-3 bash[11610]: java.lang.NullPointerException:
null
Mar 25 08:50:30 pinot-hosts-3 bash[11610]: at
org.apache.pinot.core.data.manager.realtime.SegmentBuildTimeLeaseExtender.addSegment(SegmentBuildTimeLeaseExtender.java:100)
~[pinot-all-0.7.0-jar-with-dependencies.jar:0.7.0-695ca390a8080409b3b76f250f2315b81b86b362]
Mar 25 08:50:30 pinot-hosts-3 bash[11610]: at
org.apache.pinot.core.data.manager.realtime.LLRealtimeSegmentDataManager.buildSegmentForCommit(LLRealtimeSegmentDataManager.java:698)
~[pinot-all-0.7.0-jar-with-dependencies.jar:0.7.0-695ca390a8080409b3b76f250f2315b81b86b362]
Mar 25 08:50:30 pinot-hosts-3 bash[11610]: at
org.apache.pinot.core.data.manager.realtime.LLRealtimeSegmentDataManager$PartitionConsumer.run(LLRealtimeSegmentDataManager.java:610)
[pinot-all-0.7.0-jar-with-dependencies.jar:0.7.0-695ca390a8080409b3b76f250f2315b81b86b362]
Mar 25 08:50:30 pinot-hosts-3 bash[11610]: at
java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
```
Do you have any idea why this happen? I don't really see why a
NullPointerException is throwed.
Thank you
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
