Upgrade blevesearch to v0.8.1 (#9177)

For #1441

a91b427b59

vendor/github.com/blevesearch/bleve/search/searcher/search_boolean.go

@@ -45,6 +45,7 @@ type BooleanSearcher struct {
 	scorer          *scorer.ConjunctionQueryScorer
 	matches         []*search.DocumentMatch
 	initialized     bool
+	done            bool
 }
 
 func NewBooleanSearcher(indexReader index.IndexReader, mustSearcher search.Searcher, shouldSearcher search.Searcher, mustNotSearcher search.Searcher, options search.SearcherOptions) (*BooleanSearcher, error) {
@@ -207,6 +208,10 @@ func (s *BooleanSearcher) SetQueryNorm(qnorm float64) {
 
 func (s *BooleanSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) {
 
+	if s.done {
+		return nil, nil
+	}
+
 	if !s.initialized {
 		err := s.initSearchers(ctx)
 		if err != nil {
@@ -320,11 +325,19 @@ func (s *BooleanSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch
 		}
 	}
 
+	if rv == nil {
+		s.done = true
+	}
+
 	return rv, nil
 }
 
 func (s *BooleanSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) {
 
+	if s.done {
+		return nil, nil
+	}
+
 	if !s.initialized {
 		err := s.initSearchers(ctx)
 		if err != nil {
@@ -332,14 +345,8 @@ func (s *BooleanSearcher) Advance(ctx *search.SearchContext, ID index.IndexInter
 		}
 	}
 
-	// Advance the searchers only if the currentID cursor is trailing the lookup ID,
-	// additionally if the mustNotSearcher has been initialized, ensure that the
-	// cursor used to track the mustNotSearcher (currMustNot, which isn't tracked by
-	// currentID) is trailing the lookup ID as well - for in the case where currentID
-	// is nil and currMustNot is already at or ahead of the lookup ID, we MUST NOT
-	// advance the currentID or the currMustNot cursors.
-	if (s.currentID == nil || s.currentID.Compare(ID) < 0) &&
-		(s.currMustNot == nil || s.currMustNot.IndexInternalID.Compare(ID) < 0) {
+	// Advance the searcher only if the cursor is trailing the lookup ID
+	if s.currentID == nil || s.currentID.Compare(ID) < 0 {
 		var err error
 		if s.mustSearcher != nil {
 			if s.currMust != nil {
@@ -362,12 +369,17 @@ func (s *BooleanSearcher) Advance(ctx *search.SearchContext, ID index.IndexInter
 		}
 
 		if s.mustNotSearcher != nil {
-			if s.currMustNot != nil {
-				ctx.DocumentMatchPool.Put(s.currMustNot)
-			}
-			s.currMustNot, err = s.mustNotSearcher.Advance(ctx, ID)
-			if err != nil {
-				return nil, err
+			// Additional check for mustNotSearcher, whose cursor isn't tracked by
+			// currentID to prevent it from moving when the searcher's tracked
+			// position is already ahead of or at the requested ID.
+			if s.currMustNot == nil || s.currMustNot.IndexInternalID.Compare(ID) < 0 {
+				if s.currMustNot != nil {
+					ctx.DocumentMatchPool.Put(s.currMustNot)
+				}
+				s.currMustNot, err = s.mustNotSearcher.Advance(ctx, ID)
+				if err != nil {
+					return nil, err
+				}
 			}
 		}
 

vendor/github.com/blevesearch/bleve/search/searcher/search_geoboundingbox.go

@@ -22,6 +22,11 @@ import (
 	"github.com/blevesearch/bleve/search"
 )
 
+type filterFunc func(key []byte) bool
+
+var GeoBitsShift1 = (geo.GeoBits << 1)
+var GeoBitsShift1Minus1 = GeoBitsShift1 - 1
+
 func NewGeoBoundingBoxSearcher(indexReader index.IndexReader, minLon, minLat,
 	maxLon, maxLat float64, field string, boost float64,
 	options search.SearcherOptions, checkBoundaries bool) (
@@ -36,8 +41,11 @@ func NewGeoBoundingBoxSearcher(indexReader index.IndexReader, minLon, minLat,
 	}
 
 	// do math to produce list of terms needed for this search
-	onBoundaryTerms, notOnBoundaryTerms := ComputeGeoRange(0, (geo.GeoBits<<1)-1,
-		minLon, minLat, maxLon, maxLat, checkBoundaries)
+	onBoundaryTerms, notOnBoundaryTerms, err := ComputeGeoRange(0, GeoBitsShift1Minus1,
+		minLon, minLat, maxLon, maxLat, checkBoundaries, indexReader, field)
+	if err != nil {
+		return nil, err
+	}
 
 	var onBoundarySearcher search.Searcher
 	dvReader, err := indexReader.DocValueReader([]string{field})
@@ -94,59 +102,123 @@ var geoMaxShift = document.GeoPrecisionStep * 4
 var geoDetailLevel = ((geo.GeoBits << 1) - geoMaxShift) / 2
 
 func ComputeGeoRange(term uint64, shift uint,
-	sminLon, sminLat, smaxLon, smaxLat float64,
-	checkBoundaries bool) (
-	onBoundary [][]byte, notOnBoundary [][]byte) {
-	split := term | uint64(0x1)<<shift
-	var upperMax uint64
-	if shift < 63 {
-		upperMax = term | ((uint64(1) << (shift + 1)) - 1)
-	} else {
-		upperMax = 0xffffffffffffffff
-	}
-	lowerMax := split - 1
-	onBoundary, notOnBoundary = relateAndRecurse(term, lowerMax, shift,
-		sminLon, sminLat, smaxLon, smaxLat, checkBoundaries)
-	plusOnBoundary, plusNotOnBoundary := relateAndRecurse(split, upperMax, shift,
-		sminLon, sminLat, smaxLon, smaxLat, checkBoundaries)
-	onBoundary = append(onBoundary, plusOnBoundary...)
-	notOnBoundary = append(notOnBoundary, plusNotOnBoundary...)
-	return
-}
+	sminLon, sminLat, smaxLon, smaxLat float64, checkBoundaries bool,
+	indexReader index.IndexReader, field string) (
+	onBoundary [][]byte, notOnBoundary [][]byte, err error) {
+	preallocBytesLen := 32
+	preallocBytes := make([]byte, preallocBytesLen)
 
-func relateAndRecurse(start, end uint64, res uint,
-	sminLon, sminLat, smaxLon, smaxLat float64,
-	checkBoundaries bool) (
-	onBoundary [][]byte, notOnBoundary [][]byte) {
-	minLon := geo.MortonUnhashLon(start)
-	minLat := geo.MortonUnhashLat(start)
-	maxLon := geo.MortonUnhashLon(end)
-	maxLat := geo.MortonUnhashLat(end)
-
-	level := ((geo.GeoBits << 1) - res) >> 1
-
-	within := res%document.GeoPrecisionStep == 0 &&
-		geo.RectWithin(minLon, minLat, maxLon, maxLat,
-			sminLon, sminLat, smaxLon, smaxLat)
-	if within || (level == geoDetailLevel &&
-		geo.RectIntersects(minLon, minLat, maxLon, maxLat,
-			sminLon, sminLat, smaxLon, smaxLat)) {
-		if !within && checkBoundaries {
-			return [][]byte{
-				numeric.MustNewPrefixCodedInt64(int64(start), res),
-			}, nil
+	makePrefixCoded := func(in int64, shift uint) (rv numeric.PrefixCoded) {
+		if len(preallocBytes) <= 0 {
+			preallocBytesLen = preallocBytesLen * 2
+			preallocBytes = make([]byte, preallocBytesLen)
 		}
-		return nil,
-			[][]byte{
-				numeric.MustNewPrefixCodedInt64(int64(start), res),
-			}
-	} else if level < geoDetailLevel &&
-		geo.RectIntersects(minLon, minLat, maxLon, maxLat,
-			sminLon, sminLat, smaxLon, smaxLat) {
-		return ComputeGeoRange(start, res-1, sminLon, sminLat, smaxLon, smaxLat,
-			checkBoundaries)
+
+		rv, preallocBytes, err =
+			numeric.NewPrefixCodedInt64Prealloc(in, shift, preallocBytes)
+
+		return rv
 	}
-	return nil, nil
+
+	var fieldDict index.FieldDictContains
+	var isIndexed filterFunc
+	if irr, ok := indexReader.(index.IndexReaderContains); ok {
+		fieldDict, err = irr.FieldDictContains(field)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		isIndexed = func(term []byte) bool {
+			found, err := fieldDict.Contains(term)
+			return err == nil && found
+		}
+	}
+
+	defer func() {
+		if fieldDict != nil {
+			if fd, ok := fieldDict.(index.FieldDict); ok {
+				cerr := fd.Close()
+				if cerr != nil {
+					err = cerr
+				}
+			}
+		}
+	}()
+
+	if isIndexed == nil {
+		isIndexed = func(term []byte) bool {
+			if indexReader != nil {
+				reader, err := indexReader.TermFieldReader(term, field, false, false, false)
+				if err != nil || reader == nil {
+					return false
+				}
+				if reader.Count() == 0 {
+					_ = reader.Close()
+					return false
+				}
+				_ = reader.Close()
+			}
+			return true
+		}
+	}
+
+	var computeGeoRange func(term uint64, shift uint) // declare for recursion
+
+	relateAndRecurse := func(start, end uint64, res, level uint) {
+		minLon := geo.MortonUnhashLon(start)
+		minLat := geo.MortonUnhashLat(start)
+		maxLon := geo.MortonUnhashLon(end)
+		maxLat := geo.MortonUnhashLat(end)
+
+		within := res%document.GeoPrecisionStep == 0 &&
+			geo.RectWithin(minLon, minLat, maxLon, maxLat,
+				sminLon, sminLat, smaxLon, smaxLat)
+		if within || (level == geoDetailLevel &&
+			geo.RectIntersects(minLon, minLat, maxLon, maxLat,
+				sminLon, sminLat, smaxLon, smaxLat)) {
+			codedTerm := makePrefixCoded(int64(start), res)
+			if isIndexed(codedTerm) {
+				if !within && checkBoundaries {
+					onBoundary = append(onBoundary, codedTerm)
+				} else {
+					notOnBoundary = append(notOnBoundary, codedTerm)
+				}
+			}
+		} else if level < geoDetailLevel &&
+			geo.RectIntersects(minLon, minLat, maxLon, maxLat,
+				sminLon, sminLat, smaxLon, smaxLat) {
+			computeGeoRange(start, res-1)
+		}
+	}
+
+	computeGeoRange = func(term uint64, shift uint) {
+		if err != nil {
+			return
+		}
+
+		split := term | uint64(0x1)<<shift
+		var upperMax uint64
+		if shift < 63 {
+			upperMax = term | ((uint64(1) << (shift + 1)) - 1)
+		} else {
+			upperMax = 0xffffffffffffffff
+		}
+
+		lowerMax := split - 1
+
+		level := (GeoBitsShift1 - shift) >> 1
+
+		relateAndRecurse(term, lowerMax, shift, level)
+		relateAndRecurse(split, upperMax, shift, level)
+	}
+
+	computeGeoRange(term, shift)
+
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return onBoundary, notOnBoundary, err
 }
 
 func buildRectFilter(dvReader index.DocValueReader, field string,

vendor/github.com/blevesearch/bleve/search/searcher/search_geopointdistance.go

@@ -34,7 +34,7 @@ func NewGeoPointDistanceSearcher(indexReader index.IndexReader, centerLon,
 	// build a searcher for the box
 	boxSearcher, err := boxSearcher(indexReader,
 		topLeftLon, topLeftLat, bottomRightLon, bottomRightLat,
-		field, boost, options)
+		field, boost, options, false)
 	if err != nil {
 		return nil, err
 	}
@@ -54,19 +54,20 @@ func NewGeoPointDistanceSearcher(indexReader index.IndexReader, centerLon,
 // two boxes joined through a disjunction searcher
 func boxSearcher(indexReader index.IndexReader,
 	topLeftLon, topLeftLat, bottomRightLon, bottomRightLat float64,
-	field string, boost float64, options search.SearcherOptions) (
+	field string, boost float64, options search.SearcherOptions, checkBoundaries bool) (
 	search.Searcher, error) {
 	if bottomRightLon < topLeftLon {
 		// cross date line, rewrite as two parts
 
 		leftSearcher, err := NewGeoBoundingBoxSearcher(indexReader,
 			-180, bottomRightLat, bottomRightLon, topLeftLat,
-			field, boost, options, false)
+			field, boost, options, checkBoundaries)
 		if err != nil {
 			return nil, err
 		}
 		rightSearcher, err := NewGeoBoundingBoxSearcher(indexReader,
-			topLeftLon, bottomRightLat, 180, topLeftLat, field, boost, options, false)
+			topLeftLon, bottomRightLat, 180, topLeftLat, field, boost, options,
+			checkBoundaries)
 		if err != nil {
 			_ = leftSearcher.Close()
 			return nil, err
@@ -85,7 +86,7 @@ func boxSearcher(indexReader index.IndexReader,
 	// build geoboundinggox searcher for that bounding box
 	boxSearcher, err := NewGeoBoundingBoxSearcher(indexReader,
 		topLeftLon, bottomRightLat, bottomRightLon, topLeftLat, field, boost,
-		options, false)
+		options, checkBoundaries)
 	if err != nil {
 		return nil, err
 	}

vendor/github.com/blevesearch/bleve/search/searcher/search_geopolygon.go

@@ -0,0 +1,110 @@
+//  Copyright (c) 2019 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package searcher
+
+import (
+	"github.com/blevesearch/bleve/geo"
+	"github.com/blevesearch/bleve/index"
+	"github.com/blevesearch/bleve/numeric"
+	"github.com/blevesearch/bleve/search"
+	"math"
+)
+
+func NewGeoBoundedPolygonSearcher(indexReader index.IndexReader,
+	polygon []geo.Point, field string, boost float64,
+	options search.SearcherOptions) (search.Searcher, error) {
+
+	// compute the bounding box enclosing the polygon
+	topLeftLon, topLeftLat, bottomRightLon, bottomRightLat, err :=
+		geo.BoundingRectangleForPolygon(polygon)
+	if err != nil {
+		return nil, err
+	}
+
+	// build a searcher for the bounding box on the polygon
+	boxSearcher, err := boxSearcher(indexReader,
+		topLeftLon, topLeftLat, bottomRightLon, bottomRightLat,
+		field, boost, options, true)
+	if err != nil {
+		return nil, err
+	}
+
+	dvReader, err := indexReader.DocValueReader([]string{field})
+	if err != nil {
+		return nil, err
+	}
+
+	// wrap it in a filtering searcher that checks for the polygon inclusivity
+	return NewFilteringSearcher(boxSearcher,
+		buildPolygonFilter(dvReader, field, polygon)), nil
+}
+
+const float64EqualityThreshold = 1e-6
+
+func almostEqual(a, b float64) bool {
+	return math.Abs(a-b) <= float64EqualityThreshold
+}
+
+// buildPolygonFilter returns true if the point lies inside the
+// polygon. It is based on the ray-casting technique as referred
+// here: https://wrf.ecse.rpi.edu/nikola/pubdetails/pnpoly.html
+func buildPolygonFilter(dvReader index.DocValueReader, field string,
+	polygon []geo.Point) FilterFunc {
+	return func(d *search.DocumentMatch) bool {
+		var lon, lat float64
+		var found bool
+
+		err := dvReader.VisitDocValues(d.IndexInternalID, func(field string, term []byte) {
+			// only consider the values which are shifted 0
+			prefixCoded := numeric.PrefixCoded(term)
+			shift, err := prefixCoded.Shift()
+			if err == nil && shift == 0 {
+				i64, err := prefixCoded.Int64()
+				if err == nil {
+					lon = geo.MortonUnhashLon(uint64(i64))
+					lat = geo.MortonUnhashLat(uint64(i64))
+					found = true
+				}
+			}
+		})
+
+		// Note: this approach works for points which are strictly inside
+		// the polygon. ie it might fail for certain points on the polygon boundaries.
+		if err == nil && found {
+			nVertices := len(polygon)
+			var inside bool
+			// check for a direct vertex match
+			if almostEqual(polygon[0].Lat, lat) &&
+				almostEqual(polygon[0].Lon, lon) {
+				return true
+			}
+
+			for i := 1; i < nVertices; i++ {
+				if almostEqual(polygon[i].Lat, lat) &&
+					almostEqual(polygon[i].Lon, lon) {
+					return true
+				}
+				if (polygon[i].Lat > lat) != (polygon[i-1].Lat > lat) &&
+					lon < (polygon[i-1].Lon-polygon[i].Lon)*(lat-polygon[i].Lat)/
+						(polygon[i-1].Lat-polygon[i].Lat)+polygon[i].Lon {
+					inside = !inside
+				}
+			}
+			return inside
+
+		}
+		return false
+	}
+}

vendor/github.com/blevesearch/bleve/search/searcher/search_numeric_range.go

@@ -53,20 +53,49 @@ func NewNumericRangeSearcher(indexReader index.IndexReader,
 	if !*inclusiveMax && maxInt64 != math.MinInt64 {
 		maxInt64--
 	}
+
+	var fieldDict index.FieldDictContains
+	var isIndexed filterFunc
+	var err error
+	if irr, ok := indexReader.(index.IndexReaderContains); ok {
+		fieldDict, err = irr.FieldDictContains(field)
+		if err != nil {
+			return nil, err
+		}
+
+		isIndexed = func(term []byte) bool {
+			found, err := fieldDict.Contains(term)
+			return err == nil && found
+		}
+	}
+
 	// FIXME hard-coded precision, should match field declaration
 	termRanges := splitInt64Range(minInt64, maxInt64, 4)
-	terms := termRanges.Enumerate()
+	terms := termRanges.Enumerate(isIndexed)
+	if fieldDict != nil {
+		if fd, ok := fieldDict.(index.FieldDict); ok {
+			cerr := fd.Close()
+			if cerr != nil {
+				err = cerr
+			}
+		}
+	}
+
 	if len(terms) < 1 {
 		// cannot return MatchNoneSearcher because of interaction with
 		// commit f391b991c20f02681bacd197afc6d8aed444e132
 		return NewMultiTermSearcherBytes(indexReader, terms, field, boost, options,
 			true)
 	}
-	var err error
-	terms, err = filterCandidateTerms(indexReader, terms, field)
-	if err != nil {
-		return nil, err
+
+	// for upside_down
+	if isIndexed == nil {
+		terms, err = filterCandidateTerms(indexReader, terms, field)
+		if err != nil {
+			return nil, err
+		}
 	}
+
 	if tooManyClauses(len(terms)) {
 		return nil, tooManyClausesErr(len(terms))
 	}
@@ -125,11 +154,17 @@ type termRange struct {
 	endTerm   []byte
 }
 
-func (t *termRange) Enumerate() [][]byte {
+func (t *termRange) Enumerate(filter filterFunc) [][]byte {
 	var rv [][]byte
 	next := t.startTerm
 	for bytes.Compare(next, t.endTerm) <= 0 {
-		rv = append(rv, next)
+		if filter != nil {
+			if filter(next) {
+				rv = append(rv, next)
+			}
+		} else {
+			rv = append(rv, next)
+		}
 		next = incrementBytes(next)
 	}
 	return rv
@@ -150,10 +185,10 @@ func incrementBytes(in []byte) []byte {
 
 type termRanges []*termRange
 
-func (tr termRanges) Enumerate() [][]byte {
+func (tr termRanges) Enumerate(filter filterFunc) [][]byte {
 	var rv [][]byte
 	for _, tri := range tr {
-		trie := tri.Enumerate()
+		trie := tri.Enumerate(filter)
 		rv = append(rv, trie...)
 	}
 	return rv