Hey guys,
Finally i changed all my queries to constantscorequeries. It's way better,
but still, certain pages take a lot of time running... I don't understand
why, and i don't have anything in my ES logs...
Now the average time for search 20 users and their mentions/timeline +
scoring them is about 4s (and almost 4s for the search).
But when it takes time, it's still 60s for 1 page !!!
I tried reading the explain data i can't get after the query but there's no
response time. How can I find a way to understand why certain queries take
so much time ?
Thanks !
Le lundi 18 août 2014 12:29:10 UTC+2, Pierrick Boutruche a écrit :
>
> Hi everyone !
>
> I'm currently working on a tool with *ES and Twitter Streaming API*, in
> which I try to find interesting profiles on Twitter, based on what they
> tweet, RT and which of their interactions are shared/RT.
>
> Anyway, I use ES to index and search among tweets. To do that, I get
> Twitter stream data and put in a *single index users & tweets (2 types)*,
> linked by the user id via un parent-child relation. Actually, I thought of
> my indexing a lot and it is the best way to do it.
> - I need to update very often users (because i score them and because they
> update their profile quite often), so get the user nested in the tweet is
> not an option (too many replicas)
> - I could put user's tweets directly in the user object but I would have
> huge objects and I don't really want that.
>
> I work on a SoYouStart Server, 4c/4t 3.2GHz, 32Go RAM, 4To HDD.
>
> My settings for the index are :
>
> settings = {
>
> "index" : {
>
> "number_of_replicas" : 0,
>
> "refresh_interval" : '10s',
>
> "routing.allocation.disable_allocation": False
>
> },
>
> "analysis": {
>
> "analyzer": {
>
> "snowFrench":{
>
> "type": "snowball",
>
> "language": "French"
>
> },
>
> "snowEnglish":{
>
> "type": "snowball",
>
> "language": "English"
>
> },
>
> "snowGerman":{
>
> "type": "snowball",
>
> "language": "German"
>
> },
>
> "snowRussian":{
>
> "type": "snowball",
>
> "language": "Russian"
>
> },
>
> "snowSpanish":{
>
> "type": "snowball",
>
> "language": "Spanish"
>
> },
>
> "snowJapanese":{
>
> "type": "snowball",
>
> "language": "Japanese"
>
> },
>
> "edgeNGramAnalyzer":{
>
> "tokenizer": "myEdgeNGram"
>
> },
>
> "name_analyzer": {
>
> "tokenizer": "whitespace",
>
> "type": "custom",
>
> "filter": ["lowercase", "multi_words", "name_filter"]
>
> },
>
> "city_analyzer" : {
>
> "type" : "snowball",
>
> "language" : "English"
>
> }
>
> },
>
> "tokenizer" : {
>
> "myEdgeNGram" : {
>
> "type" : "edgeNGram",
>
> "min_gram" : 2,
>
> "max_gram" : 5
>
> },
>
> "name_tokenizer": {
>
> "type": "edgeNGram",
>
> "max_gram": 100,
>
> "min_gram": 4
>
> }
>
> },
>
> "filter": {
>
> "multi_words": {
>
> "type": "shingle",
>
> "min_shingle_size": 2,
>
> "max_shingle_size": 10
>
> },
>
> "name_filter": {
>
> "type": "edgeNGram",
>
> "max_gram": 100,
>
> "min_gram": 4
>
> }
>
> }
>
> }
>
> }
>
>
> And my mappings are :
>
> tweet_mapping = {
>
> "_all" : {
> "enabled" : False
> },
> "_ttl" : {
> "enabled" : True,
> "default" : "400d"
> },
> "_parent" : {
> "type" : 'user'
> },
> "properties": {
> "textfr": {
> 'type': 'string',
> '_analyzer': 'snowFrench',
> 'copy_to': 'text'
> },
> "texten": {
> 'type': 'string',
> '_analyzer': 'snowEnglish',
> 'copy_to': 'text'
> },
> "textde": {
> 'type': 'string',
> '_analyzer': 'snowGerman',
> 'copy_to': 'text'
> },
> "textja": {
> 'type': 'string',
> '_analyzer': 'snowJapanese',
> 'copy_to': 'text'
> },
> "textru": {
> 'type': 'string',
> '_analyzer': 'snowRussian',
> 'copy_to': 'text'
> },
> "textes": {
> 'type': 'string',
> '_analyzer': 'snowSpanish',
> 'copy_to': 'text'
> },
> "text": {
> 'type': 'string',
> 'null_value': '',
> 'index': 'analyzed',
> 'store': 'yes'
> },
> "entities": {
> 'type': 'object',
> 'index': 'analyzed',
> 'store': 'yes',
> 'properties': {
> "hashtags": {
> 'index': 'analyzed',
> 'store': 'yes',
> 'type': 'string',
> "_analyzer": "edgeNGramAnalyzer"
> },
> "mentions": {
> 'index': 'not_analyzed',
> 'store': 'yes',
> 'type': 'long',
> 'precision_step': 64
> }
> }
> },
> "lang": {
> 'index': 'not_analyzed',
> 'store': 'yes',
> 'type': 'string'
> },
> "created_at": {
> 'index': 'not_analyzed',
> 'store': 'yes',
> 'type': 'date',
> 'format' : 'dd-MM-YYYY HH:mm:ss'
> }
> }
> }
> user_mapping = {
> "_all" : {
> "enabled" : False
> },
> "_ttl" : {
> "enabled" : True,
> "default" : "600d"
> },
> "properties": {
> "lang": {
> 'index': 'not_analyzed',
> 'store': 'yes',
> 'type': 'string'
> },
> "name": {
> 'index': 'analyzed',
> 'store': 'yes',
> 'type': 'string',
> "_analyzer": "edgeNGramAnalyzer"
> },
> "screen_name": {
> 'index': 'analyzed',
> 'store': 'yes',
> 'type': 'string',
> "_analyzer": "edgeNGramAnalyzer"
> },
> "descfr": {
> 'type': 'string',
> '_analyzer': 'snowFrench',
> 'copy_to': 'description'
> },
> "descen": {
> 'type': 'string',
> '_analyzer': 'snowEnglish',
> 'copy_to': 'description'
> },
> "descde": {
> 'type': 'string',
> '_analyzer': 'snowGerman',
> 'copy_to': 'description'
> },
> "descja": {
> 'type': 'string',
> '_analyzer': 'snowJapanese',
> 'copy_to': 'description'
> },
> "descru": {
> 'type': 'string',
> '_analyzer': 'snowRussian',
> 'copy_to': 'description'
> },
> "desces": {
> 'type': 'string',
> '_analyzer': 'snowSpanish',
> 'copy_to': 'description'
> },
> "description": {
> 'type': 'string',
> 'null_value': '',
> 'index': 'analyzed',
> 'store': 'yes'
> },
> "created_at": {
> 'index': 'not_analyzed',
> 'store': 'yes',
> 'type': 'date',
> 'format' : 'dd-MM-YYYY HH:mm:ss'
> },
> "profile_image_url": {
> 'index': 'not_analyzed',
> 'store': 'yes',
> 'type': 'string'
> },
> "analysis": {
> 'type': 'object',
> 'index': 'analyzed',
> 'store': 'yes',
> 'properties': {
> "hashtags": {
> 'index': 'analyzed',
> 'store': 'yes',
> 'type': 'object'
> },
> "relations": {
> 'index': 'analyzed',
> 'store': 'yes',
> 'type': 'object'
> },
> "score": {
> 'index': 'analyzed',
> 'store': 'yes',
> 'type': 'object'
> }
> }
> },
> "location" : {
> 'type': 'object',
> 'index': 'analyzed',
> 'store': 'yes',
> "properties" : {
> "search_field": {
> 'index': 'analyzed',
> 'store': 'yes',
> 'type': 'string',
> 'analyzer': 'city_analyzer',
> 'null_value': ''
> },
> "name": {
> 'index': 'not_analyzed',
> 'store': 'yes',
> 'type': 'string',
> 'null_value': ''
> },
> "city": {
> 'index': 'analyzed',
> 'store': 'yes',
> 'type': 'object',
> 'properties': {
> 'name': {
> 'boost': 3.0,
> 'index': 'analyzed',
> 'store': 'yes',
> 'type': 'string',
> 'copy_to': 'location.search_field'
> },
> 'full_name': {
> 'boost': 3.0,
> 'index': 'analyzed',
> 'store': 'yes',
> 'type': 'string',
> 'copy_to': ['location.search_field', 'location.name']
> },
> 'alternate_names': {
> 'boost': 2.0,
> 'index': 'analyzed',
> 'store': 'yes',
> 'type': 'string',
> 'copy_to': 'location.search_field'
> }
> }
> },
> "admin2": {
> 'index': 'analyzed',
> 'store': 'yes',
> 'type': 'object',
> 'properties': {
> 'name': {
> 'boost': 1.5,
> 'index': 'analyzed',
> 'store': 'yes',
> 'type': 'string',
> 'copy_to': 'location.search_field'
> },
> 'full_name': {
> 'boost': 1.5,
> 'index': 'analyzed',
> 'store': 'yes',
> 'type': 'string',
> 'copy_to': ['location.search_field', 'location.name']
> }
> }
> },
> "admin1": {
> 'index': 'analyzed',
> 'store': 'yes',
> 'type': 'object',
> 'properties': {
> 'name': {
> 'boost': 1.2,
> 'index': 'analyzed',
> 'store': 'yes',
> 'type': 'string',
>
> ...
--
You received this message because you are subscribed to the Google Groups
"elasticsearch" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
To view this discussion on the web visit
https://groups.google.com/d/msgid/elasticsearch/7875dd94-8b2a-405e-aaf4-eeb3c21bd53b%40googlegroups.com.
For more options, visit https://groups.google.com/d/optout.
