feat: add roaring bitmaps in posing lists

cmd/index_analyzer/main.go

@@ -200,10 +200,19 @@ func analyzeIndex(
 			logger.Fatal("error unpacking lids block", zap.Error(err))
 		}
 
-		last := len(block.Offsets) - 2
-		for i := 0; i <= last; i++ {
-			tokenLIDs = append(tokenLIDs, block.LIDs[block.Offsets[i]:block.Offsets[i+1]]...)
-			if i < last || block.IsLastLID { // the end of token lids
+		listsCount := block.GetCount()
+		for i := 0; i < listsCount; i++ {
+			lidsBatch := block.GetLIDs(i)
+			iter := lidsBatch.Iter()
+			for {
+				lid, ok := iter.Next()
+				if !ok {
+					break
+				}
+				tokenLIDs = append(tokenLIDs, lid)
+			}
+
+			if i < listsCount || block.IsLastLID() { // the end of token lids
 				lidsTotal += len(tokenLIDs)
 				lidsLens[tid] = len(tokenLIDs)
 				lidsUniq[getLIDsHash(tokenLIDs)] = len(tokenLIDs)

cmd/seq-db/seq-db.go

@@ -273,6 +273,7 @@ func startStore(
 				TokenTableZstdLevel:    cfg.Compression.SealedZstdCompressionLevel,
 				DocBlocksZstdLevel:     cfg.Compression.DocBlockZstdCompressionLevel,
 				DocBlockSize:           int(cfg.DocsSorting.DocBlockSize),
+				LIDsBitmapThreshold:    cfg.Sealing.Lids.BitmapThreshold,
 			},
 			Fraction: frac.Config{
 				Search: frac.SearchConfig{

config/config.go

@@ -75,6 +75,10 @@ type Config struct {
 		Lids struct {
 			// BlockSize sets max lids (postings) saved per LIDs block.
 			BlockSize int `config:"block_size" default:"65536"`
+			// BitmapThreshold specifies minimum number of LIDs in the lid list
+			// which are serialized as bitmap. LIDs lists with more elements use bitmap encoding,
+			// while smaller lists use delta encoding.
+			BitmapThreshold int `config:"bitmap_threshold" default:"65536"`
 		} `config:"lids"`
 	} `config:"sealing"`
 

config/frac_version.go

@@ -21,6 +21,12 @@ const (
 
 	// BinaryDataV4 - delta bitpack encoded MIDs and LIDs
 	BinaryDataV4
+
+	// BinaryDataV5 - token frequencies stored in token blocks for large tokens
+	BinaryDataV5
+
+	// BinaryDataV6 - bitmap for sufficiently large LID lists, mixed LIDs block format
+	BinaryDataV6
 )
 
-const CurrentFracVersion = BinaryDataV4
+const CurrentFracVersion = BinaryDataV6

frac/common/seal_params.go

@@ -10,8 +10,9 @@ type SealParams struct {
 	TokenListZstdLevel     int
 	DocsPositionsZstdLevel int
 	TokenTableZstdLevel    int
-	DocBlocksZstdLevel     int // DocBlocksZstdLevel is the zstd compress level of each document block.
 
-	LIDBlockSize int
-	DocBlockSize int // DocBlockSize is decompressed payload size of document block.
+	DocBlocksZstdLevel  int // DocBlocksZstdLevel is the zstd compress level of each document block.
+	LIDBlockSize        int
+	LIDsBitmapThreshold int // LIDsBitmapThreshold is the minimum number of LIDs in the lid list to serialize as bitmap.
+	DocBlockSize        int // DocBlockSize is decompressed payload size of document block.
 }

frac/fraction_test.go

@@ -104,6 +104,7 @@ func (s *FractionTestSuite) SetupTestCommon() {
 		DocBlocksZstdLevel:     1,
 		LIDBlockSize:           512,
 		DocBlockSize:           128 * int(units.KiB),
+		LIDsBitmapThreshold:    25,
 	}
 
 	var err error
@@ -1385,6 +1386,18 @@ func (s *FractionTestSuite) TestSearchLargeFrac() {
 			fromTime: fromTime,
 			toTime:   toTime,
 		},
+		{
+			name: "complex AND+OR",
+			query: "(service:gateway OR service:proxy OR service:scheduler) AND " +
+				"(message:request OR message:failed) AND (level:1 OR level:2 OR level:3)",
+			filter: func(doc *testDoc) bool {
+				return (doc.service == gateway || doc.service == proxy || doc.service == "scheduler") &&
+					(strings.Contains(doc.message, "request") || strings.Contains(doc.message, "failed")) &&
+					(doc.level >= 1 && doc.level <= 3)
+			},
+			fromTime: fromTime,
+			toTime:   toTime,
+		},
 		{
 			name:  "service:gateway AND NOT (message:request OR message:timed OR level:[0 to 3])",
 			query: "service:gateway AND NOT (message:request OR message:timed OR level:[0 to 3])",

frac/processor/search.go

@@ -175,10 +175,10 @@ func IndexSearch(
 	return qpr, nil
 }
 
-func batcher(evalTree node.Node, buf []node.LID) func(need int) []node.LID {
+func batcher(evalTree node.Node, buf []node.LID, desc bool) func(need int) []node.LID {
 	if batchNode, ok := tryConvertToBatchedTree(evalTree); ok {
 		return func(need int) []node.LID {
-			buf = batchNode.NextBatch().LIDs(buf[:0])
+			buf = batchNode.NextBatch(need).CopyLIDs(desc, buf[:0])
 			if len(buf) > need {
 				buf = buf[:need]
 			}
@@ -227,7 +227,7 @@ func iterateEvalTree(
 	mids := buffers.mids
 	rids := buffers.rids
 
-	batchedEvalTree := batcher(evalTree, buffers.lids)
+	batchedEvalTree := batcher(evalTree, buffers.lids, params.Order.IsDesc())
 
 	timerEval := sw.Timer("eval_tree_next")
 	timerMID := sw.Timer("get_mid")
@@ -353,9 +353,9 @@ func sampler(n uint32) func(in []node.LID) []node.LID {
 func tryConvertToBatchedTree(evalTree node.Node) (node.BatchedNode, bool) {
 	switch it := evalTree.(type) {
 	case *lids.IteratorDesc:
-		return it, true
+		return lids.NewBatchedIteratorDesc(it), true
 	case *lids.IteratorAsc:
-		return it, true
+		return lids.NewBatchedIteratorAsc(it), true
 	default:
 		return nil, false
 	}