pzhdfy edited a comment on issue #7607: thetaSketch(with sketches-core-0.13.1) in groupBy always return value no more than 16384 URL: https://github.com/apache/incubator-druid/issues/7607#issuecomment-490337462 ### 1.use the newest Druid version( compiled from master branch, with sketches-core-0.13.1) ### 2.use this python generate test data ```import json f=open('test_theta_data','w') d={} for i in xrange(100000): #date_id,user_id d['date_id']='20190407' d['user_id']=i f.write('%s\n' % json.dumps(d)) ``` ### 3.use this Ingestion spec ```{ "type": "index_hadoop", "spec": { "dataSchema": { "dataSource": "test_theta", "parser": { "type": "hadoopyString", "parseSpec": { "format": "json", "timestampSpec": { "column": "date_id", "format": "yyyyMMdd", "missingValue": 0 }, "dimensionsSpec": { "dimensionExclusions": [], "dimensions": [ "date_id", "user_id" ] } } }, "metricsSpec": [], "granularitySpec": { "type": "uniform", "segmentGranularity": "DAY", "queryGranularity": "DAY", "rollup": true, "intervals": [ "2019-04-07/2019-04-08" ] } }, "ioConfig": { "type": "hadoop", "inputSpec": { "children": [ { "paths": "/home/druid/druid_test_histogram/test_theta_data", "type": "static" } ], "type": "multi" } }, "tuningConfig": { "type": "hadoop", "jobProperties": { "mapreduce.am.push.alter.metrics": "druid_indexer_config#druid.dataSource.name#druid.hive.table#mapreduce.input.multipleinputs.dir.formats:druidIndexerConfig#druidDataSource#druidHiveTable#multipleinputsDir", "mapreduce.job.classloader": "true", "mapreduce.job.classloader.system.classes": "-javax.validation.,java.,javax.,org.apache.commons.logging.,org.apache.log4j.,org.apache.hadoop.", "mapreduce.job.queuename": "root.offline.hdp.hdp.L_0", "mapreduce.map.java.opts": "-Xmx6G -Djava.net.preferIPv4Stack=true", "mapreduce.map.memory.mb": "10240", "mapreduce.reduce.java.opts": "-Xmx6G -Djava.net.preferIPv4Stack=true", "mapreduce.reduce.memory.mb": "10240", "mapreduce.reduce.speculative": "false" } } }, "context": { "druid.indexer.fork.property.druid.processing.buffer.sizeBytes": 1024, "druid.indexer.fork.property.druid.processing.numMergeBuffers": 1, "druid.indexer.fork.property.druid.processing.numThreads": 1 } } ``` 4.use groupby and topN ``` { "dimensions": [ "date_id" ], "aggregations": [ { "type": "thetaSketch", "fieldName": "user_id", "name": "count_distinct__user_id", } ], "intervals": "2019-04-07/2019-04-08", "limitSpec": { "limit": 5000, "type": "default", "columns": [ { "direction": "descending", "dimension": "count_distinct__user_id" } ] }, "granularity": "all", "queryType": "groupBy", "dataSource": "test_theta" } ``` { "dimension": "date_id", "aggregations": [ { "type": "thetaSketch", "fieldName": "user_id", "name": "count_distinct__user_id" } ], "intervals": "2019-04-07/2019-04-08", "metric": "count_distinct__user_id", "threshold": 5000, "granularity": "all", "queryType": "topN", "dataSource": "test_theta" } ``` ```
---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected] With regards, Apache Git Services --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
