Skip to content

Instantly share code, notes, and snippets.

@misablaha

misablaha/and_bool.txt

Last active August 29, 2015 14:21
Show Gist options
  • Save misablaha/6f67992dbc7409a10c34 to your computer and use it in GitHub Desktop.
Save misablaha/6f67992dbc7409a10c34 to your computer and use it in GitHub Desktop.
Download ZIP
Elasticsearch prformance test - random queries, random profile_ids and random sorting
Raw
and_bool.txt
filter AND vs BOOL
shards: 1 * 2 | docs: 59,925,838 | size: 23.51GB
=== and filter ===
filter cache: 4.8 GiB
┌────────────────────────────────┬───────┬─────┬─────┬────────┬───────┬─────────┐
│ type │ calls │ min │ max │ median │ mean │ 95 perc │
├────────────────────────────────┼───────┼─────┼─────┼────────┼───────┼─────────┤
│ total │ 1500 │ 1 │ 113 │ 4 │ 8.51 │ 26 │
├────────────────────────────────┼───────┼─────┼─────┼────────┼───────┼─────────┤
│ incoming │ 159 │ 1 │ 81 │ 6 │ 10.17 │ 28 │
│ incoming_replies │ 170 │ 1 │ 113 │ 3 │ 7.25 │ 18 │
│ mentions │ 176 │ 1 │ 106 │ 8 │ 9.97 │ 23 │
│ outgoing │ 160 │ 1 │ 52 │ 3 │ 7.10 │ 19 │
│ replies │ 162 │ 1 │ 78 │ 3 │ 6.40 │ 19 │
│ retweets │ 158 │ 1 │ 87 │ 3 │ 6.32 │ 22 │
│ retweets_of_your_tweet │ 182 │ 1 │ 75 │ 4 │ 8.01 │ 20 │
│ retweets_you_were_mentioned_in │ 162 │ 1 │ 108 │ 9 │ 14.79 │ 47 │
│ statuses │ 171 │ 1 │ 65 │ 3 │ 6.63 │ 19 │
└────────────────────────────────┴───────┴─────┴─────┴────────┴───────┴─────────┘
=== bool filter ===
filter cache: 4.8 GiB
┌────────────────────────────────┬───────┬─────┬─────┬────────┬───────┬─────────┐
│ type │ calls │ min │ max │ median │ mean │ 95 perc │
├────────────────────────────────┼───────┼─────┼─────┼────────┼───────┼─────────┤
│ total │ 1500 │ 1 │ 139 │ 6 │ 10.70 │ 29 │
├────────────────────────────────┼───────┼─────┼─────┼────────┼───────┼─────────┤
│ incoming │ 166 │ 1 │ 89 │ 5.5 │ 10.18 │ 25 │
│ incoming_replies │ 157 │ 1 │ 102 │ 6 │ 10.71 │ 34 │
│ mentions │ 144 │ 1 │ 139 │ 9 │ 14.28 │ 30 │
│ outgoing │ 177 │ 1 │ 77 │ 3 │ 8.44 │ 29 │
│ replies │ 165 │ 1 │ 59 │ 6 │ 8.88 │ 24 │
│ retweets │ 173 │ 1 │ 90 │ 5 │ 9.41 │ 29 │
│ retweets_of_your_tweet │ 177 │ 1 │ 102 │ 4 │ 9.41 │ 29 │
│ retweets_you_were_mentioned_in │ 171 │ 1 │ 99 │ 10 │ 15.27 │ 40 │
│ statuses │ 170 │ 1 │ 74 │ 7 │ 10.32 │ 25 │
└────────────────────────────────┴───────┴─────┴─────┴────────┴───────┴─────────┘
Raw
and_queries.js
queries = {
outgoing: {
"query": {
"filtered": {
"filter": {
"and": [
{"terms": {"author_id": profile_id}}
]
}
}
}
},
statuses: {
"query": {
"filtered": {
"filter": {
"and": [
{"terms": {"author_id": profile_id}},
{"missing": {"field": "twitter.reply_to_user_id"}},
// {"not": {"term": {"twitter.is_reply": true}}} this will be better than missing
{"not": {"term": {"twitter.is_retweet": true}}}
]
}
}
}
},
retweets: {
"query": {
"filtered": {
"filter": {
"and": [
{"terms": {"author_id": profile_id}},
{"term": {"twitter.is_retweet": true}}
]
}
}
}
}
replies: {
"query": {
"filtered": {
"filter": {
"and": [
{"terms": {"author_id": profile_id}},
{"exists": {"field": "twitter.reply_to_user_id"}}
// {"term": {"twitter.is_reply": true}} this will be better than exists
]
}
}
}
},
incoming: {
"query": {
"filtered": {
"filter": {
"and": [
{"terms": {"mentions": profile_id}}
]
}
}
}
},
incoming_replies: {
"query": {
"filtered": {
"filter": {
"and": [
{"terms": {"twitter.reply_to_user_id": profile_id}}
]
}
}
}
},
mentions: {
"query": {
"filtered": {
"filter": {
"and": [
{"terms": {"mentions": profile_id}},
{"not": {"term": {"twitter.is_retweet": true}}}
]
}
}
}
},
retweets_of_your_tweet: {
"query": {
"filtered": {
"filter": {
"and": [
{"terms": {"twitter.retweeted_user_id": profile_id}}
]
}
}
}
},
retweets_you_were_mentioned_in: {
"query": {
"filtered": {
"filter": {
"and": [
{"terms": {"mentions": profile_id}},
{"term": {"twitter.is_retweet": true}},
{"not": {"terms": {"twitter.retweeted_user_id": profile_id}}}
]
}
}
}
}
}
Raw
bool_queries.coffee
queries =
outgoing:
query:
filtered:
filter:
bool:
must: [
{terms: author_id: profileIds}
]
statuses:
query:
filtered:
filter:
bool:
must: [
{terms: author_id: profileIds}
{missing: field: "twitter.reply_to_user_id"}
{not: {term: "twitter.is_retweet": true}}
]
retweets:
query:
filtered:
filter:
bool:
must: [
{terms: author_id: profileIds}
{term: "twitter.is_retweet": true}
]
replies:
query:
filtered:
filter:
bool:
must: [
{terms: author_id: profileIds}
{exists: field: "twitter.reply_to_user_id"}
]
incoming:
query:
filtered:
filter:
bool:
must: [
{terms: mentions: profileIds}
]
incoming_replies:
query:
filtered:
filter:
bool:
must: [
{terms: "twitter.reply_to_user_id": profileIds}
]
mentions:
query:
filtered:
filter:
bool:
must: [
{terms: mentions: profileIds}
{not: {term: "twitter.is_retweet": true}}
]
retweets_of_your_tweet:
query:
filtered:
filter:
bool:
must: [
{terms: "twitter.retweeted_user_id": profileIds}
]
retweets_you_were_mentioned_in:
query:
filtered:
filter:
bool:
must: [
{terms: mentions: profileIds}
{term: "twitter.is_retweet": true}
{not: {terms: "twitter.retweeted_user_id": profileIds}}
]
Raw
doc_values.txt
DOC VALUES
https://www.elastic.co/guide/en/elasticsearch/guide/current/doc-values.html
=== doc_values TRUE ===
shards: 1 * 2 | docs: 59,925,838 | size: 23.51GB
first took: 500 ms
key influencers (aggregations):
first took: 1,400 ms
average took: 380 ms
┌────────────────────────────────┬───────┬─────┬─────┬────────┬───────┬─────────┐
│ type │ calls │ min │ max │ median │ mean │ 95 perc │
├────────────────────────────────┼───────┼─────┼─────┼────────┼───────┼─────────┤
│ total │ 1500 │ 1 │ 137 │ 4 │ 10.08 │ 32 │
├────────────────────────────────┼───────┼─────┼─────┼────────┼───────┼─────────┤
│ incoming │ 150 │ 1 │ 89 │ 8 │ 12.33 │ 33 │
│ incoming_replies │ 146 │ 1 │ 40 │ 3 │ 8.10 │ 26 │
│ mentions │ 177 │ 1 │ 46 │ 6 │ 11.55 │ 33 │
│ outgoing │ 183 │ 1 │ 132 │ 3 │ 9.43 │ 33 │
│ replies │ 167 │ 1 │ 127 │ 3 │ 9.21 │ 28 │
│ retweets │ 178 │ 1 │ 102 │ 3 │ 8.23 │ 29 │
│ retweets_of_your_tweet │ 175 │ 1 │ 42 │ 4 │ 8.80 │ 31 │
│ retweets_you_were_mentioned_in │ 163 │ 1 │ 90 │ 7 │ 13.13 │ 37 │
│ statuses │ 161 │ 1 │ 137 │ 3 │ 10.12 │ 30 │
└────────────────────────────────┴───────┴─────┴─────┴────────┴───────┴─────────┘
=== doc_values FALSE ===
shards: 1 * 2 | docs: 65,516,662 | size: 26.95GB
field cache: 469 MiB
first took: 15,500 ms !!!
key influencers (aggregations):
field cache: 969 MiB
first took: 35,000 ms !!!
average took: 220 ms
┌────────────────────────────────┬───────┬─────┬─────┬────────┬────────┬─────────┐
│ type │ calls │ min │ max │ median │ mean │ 95 perc │
├────────────────────────────────┼───────┼─────┼─────┼────────┼────────┼─────────┤
│ total │ 1500 │ 1 │ 438 │ 20 │ 49.19 │ 163 │
├────────────────────────────────┼───────┼─────┼─────┼────────┼────────┼─────────┤
│ incoming │ 150 │ 1 │ 438 │ 68 │ 81.21 │ 188 │
│ incoming_replies │ 160 │ 1 │ 280 │ 57 │ 69.37 │ 167 │
│ mentions │ 179 │ 1 │ 256 │ 72 │ 78.67 │ 175 │
│ outgoing │ 166 │ 1 │ 148 │ 12 │ 13.25 │ 28 │
│ replies │ 169 │ 1 │ 92 │ 11 │ 12.12 │ 30 │
│ retweets │ 190 │ 1 │ 110 │ 9 │ 11.38 │ 33 │
│ retweets_of_your_tweet │ 166 │ 1 │ 249 │ 47.5 │ 56.22 │ 150 │
│ retweets_you_were_mentioned_in │ 161 │ 1 │ 321 │ 105 │ 114.47 │ 252 │
│ statuses │ 159 │ 1 │ 109 │ 12 │ 14.14 │ 34 │
└────────────────────────────────┴───────┴─────┴─────┴────────┴────────┴─────────┘
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment