Friday, 28 December 2018

Inconsistent behavior of Elasticsearch with multiple must_not condition

I am working on NodeJs based API and it is build with Elasticsearch and NodeJs. I have to provide a functionality where API users can exclude some content based on the taxonomy terms.

freeform: Multi-value field - type : string

games: Multi-value field - type : string

players: Multi-value field - type : string

teams: Multi-value field - type : string

writer: Multi-value field - type : string

channels: Multi-value field - type : string

Format of API url will be something similar to below. https://api.test.net/2/xxxxx/article/?exclude=term1+term2

Above should return the content which is not tagged to term1 and term2.

Suppose we have 3 articles, first article is tagged to term1, second article is tagged to term2 and third article is tagged to term1 and term2.

Expected Result: https://api.test.net/2/xxxxx/article/?exclude=term1+term2 should return article first(tagged to term1), second article(tagged to term2) and not the third(Tagged to term1 and term2)

Actual Result: Sometimes https://api.test.net/2/xxxxx/article/?exclude=term1+term2 returns all three articles and sometime first(tagged to term1) and second article(tagged to term2).

Update: Below is the correct doc and I have added the query also but still I am seeing the inconsistency.

Doc:

{
   "team":{
      "mappings":{
         "article":{
            "properties":{
               "body":{
                  "properties":{
                     "content":{
                        "type":"string"
                     },
                     "html":{
                        "type":"string"
                     },
                     "type":{
                        "type":"string"
                     }
                  }
               },
               "brand":{
                  "type":"string"
               },
               "changed":{
                  "type":"date",
                  "format":"dateOptionalTime"
               },
               "changedUtc":{
                  "type":"long"
               },
               "content":{
                  "type":"string"
               },
               "created":{
                  "type":"date",
                  "format":"dateOptionalTime"
               },
               "createdUtc":{
                  "type":"long"
               },
               "domain":{
                  "type":"string"
               },
               "headline":{
                  "type":"string"
               },
               "langcode":{
                  "type":"string",
                  "index":"not_analyzed"
               },
               "nid":{
                  "type":"string"
               },
               "published":{
                  "type":"date",
                  "format":"dateOptionalTime"
               },
               "publishedUtc":{
                  "type":"long"
               },
               "raw":{
                  "type":"string",
                  "index":"no"
               },
               "revisionTimestamp":{
                  "type":"date",
                  "format":"dateOptionalTime"
               },
               "revisionTimestampUtc":{
                  "type":"long"
               },
               "status":{
                  "type":"string"
               },
               "subheadline":{
                  "type":"string"
               },
               "syndication":{
                  "properties":{
                     "brand":{
                        "type":"string"
                     },
                     "options":{
                        "properties":{
                           "actionFrom":{
                              "type":"string"
                           },
                           "publish":{
                              "type":"string"
                           },
                           "status":{
                              "type":"string"
                           }
                        }
                     },
                     "type":{
                        "type":"string"
                     }
                  }
               },
               "taxonomy":{
                  "properties":{
                     "coaches":{
                        "properties":{
                           "value":{
                              "type":"string"
                           }
                        }
                     },
                     "freeform":{
                        "properties":{
                           "value":{
                              "type":"string"
                           }
                        }
                     },
                     "games":{
                        "properties":{
                           "id":{
                              "type":"string"
                           },
                           "value":{
                              "type":"string"
                           }
                        }
                     },
                     "players":{
                        "properties":{
                           "id":{
                              "type":"string"
                           },
                           "value":{
                              "type":"string"
                           }
                        }
                     },
                     "section":{
                        "properties":{
                           "value":{
                              "type":"string"
                           }
                        }
                     },
                     "teams":{
                        "properties":{
                           "city":{
                              "type":"string"
                           },
                           "id":{
                              "type":"string"
                           },
                           "nickname":{
                              "type":"string"
                           },
                           "tricode":{
                              "type":"string"
                           },
                           "urlName":{
                              "type":"string"
                           },
                           "value":{
                              "type":"string"
                           }
                        }
                     },
                     "writer":{
                        "properties":{
                           "emailAddress":{
                              "type":"string"
                           },
                           "id":{
                              "type":"string"
                           },
                           "responsive_web":{
                              "type":"string"
                           },
                           "title":{
                              "type":"string"
                           },
                           "value":{
                              "type":"string"
                           }
                        }
                     }
                  }
               },
               "teaser":{
                  "type":"string"
               },
               "title":{
                  "type":"string"
               },
               "type":{
                  "type":"string"
               },
               "url":{
                  "type":"string",
                  "index":"not_analyzed"
               },
               "uuid":{
                  "type":"string",
                  "index":"not_analyzed"
               },
               "vid":{
                  "type":"string"
               }
            }
         }
      }
   }
}

Query:

{
   "filter":{
      "bool":{
         "should":[
            [
               {
                  "term":{
                     "status":1
                  }
               }
            ]
         ],
         "must":[

         ]
      }
   },
   "from":0,
   "size":10,
   "_source":{
      "include":[
         "uuid",
         "nid",
         "type",
         "title",
         "headline",
         "shortHeadline",
         "teaser",
         "url",
         "published",
         "changed",
         "subheadline",
         "listImage",
         "brand",
         "videoId",
         "videoSource",
         "duration",
         "taxonomy",
         "includeTerms",
         "excludeTerms",
         "media",
         "credit",
         "caption",
         "description",
         "videoCaptions",
         "franchiseId",
         "showAirTime",
         "slateImage",
         "mainImage",
         "thumbnailImage",
         "showTitle",
         "status",
         "pubstatus"
      ]
   },
   "query":{
      "bool":{
         "must":[
            {
               "query_string":{
                  "query":"langcode:\"en\""
               }
            },
            {
               "bool":{
                  "must_not":{
                     "query_string":{
                        "query":"(taxonomy.freeform.value:\"sociosqu\" OR taxonomy.freeform.value:\"nullam\") OR (taxonomy.games.id:\"sociosqu\" OR taxonomy.games.id:\"nullam\") OR (taxonomy.players.id:\"sociosqu\" OR taxonomy.players.id:\"nullam\") OR (taxonomy.teams.id:\"sociosqu\" OR taxonomy.teams.id:\"nullam\") OR (taxonomy.writer.value:\"sociosqu\" OR taxonomy.writer.value:\"nullam\") OR (taxonomy.channels.value:\"sociosqu\" OR taxonomy.channels.value:\"nullam\") OR (taxonomy.section.value:\"sociosqu\" OR taxonomy.section.value:\"nullam\") "
                     }
                  }
               }
            }
         ]
      }
   },
   "sort":[
      {
         "publishedUtc":{
            "order":"desc"
         }
      },
      {
         "_score":{
            "order":"desc"
         }
      }
   ]
}



from Inconsistent behavior of Elasticsearch with multiple must_not condition

No comments:

Post a Comment