You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

segment_plugin.go 4.4KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133
  1. // Copyright (c) 2019 Couchbase, Inc.
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. package scorch
  15. import (
  16. "fmt"
  17. "github.com/RoaringBitmap/roaring"
  18. index "github.com/blevesearch/bleve_index_api"
  19. segment "github.com/blevesearch/scorch_segment_api/v2"
  20. zapv11 "github.com/blevesearch/zapx/v11"
  21. zapv12 "github.com/blevesearch/zapx/v12"
  22. zapv13 "github.com/blevesearch/zapx/v13"
  23. zapv14 "github.com/blevesearch/zapx/v14"
  24. zapv15 "github.com/blevesearch/zapx/v15"
  25. )
  26. // SegmentPlugin represents the essential functions required by a package to plug in
  27. // it's segment implementation
  28. type SegmentPlugin interface {
  29. // Type is the name for this segment plugin
  30. Type() string
  31. // Version is a numeric value identifying a specific version of this type.
  32. // When incompatible changes are made to a particular type of plugin, the
  33. // version must be incremented.
  34. Version() uint32
  35. // New takes a set of Documents and turns them into a new Segment
  36. New(results []index.Document) (segment.Segment, uint64, error)
  37. // Open attempts to open the file at the specified path and
  38. // return the corresponding Segment
  39. Open(path string) (segment.Segment, error)
  40. // Merge takes a set of Segments, and creates a new segment on disk at
  41. // the specified path.
  42. // Drops is a set of bitmaps (one for each segment) indicating which
  43. // documents can be dropped from the segments during the merge.
  44. // If the closeCh channel is closed, Merge will cease doing work at
  45. // the next opportunity, and return an error (closed).
  46. // StatsReporter can optionally be provided, in which case progress
  47. // made during the merge is reported while operation continues.
  48. // Returns:
  49. // A slice of new document numbers (one for each input segment),
  50. // this allows the caller to know a particular document's new
  51. // document number in the newly merged segment.
  52. // The number of bytes written to the new segment file.
  53. // An error, if any occurred.
  54. Merge(segments []segment.Segment, drops []*roaring.Bitmap, path string,
  55. closeCh chan struct{}, s segment.StatsReporter) (
  56. [][]uint64, uint64, error)
  57. }
  58. var supportedSegmentPlugins map[string]map[uint32]SegmentPlugin
  59. var defaultSegmentPlugin SegmentPlugin
  60. func init() {
  61. ResetSegmentPlugins()
  62. RegisterSegmentPlugin(&zapv15.ZapPlugin{}, true)
  63. RegisterSegmentPlugin(&zapv14.ZapPlugin{}, false)
  64. RegisterSegmentPlugin(&zapv13.ZapPlugin{}, false)
  65. RegisterSegmentPlugin(&zapv12.ZapPlugin{}, false)
  66. RegisterSegmentPlugin(&zapv11.ZapPlugin{}, false)
  67. }
  68. func ResetSegmentPlugins() {
  69. supportedSegmentPlugins = map[string]map[uint32]SegmentPlugin{}
  70. }
  71. func RegisterSegmentPlugin(plugin SegmentPlugin, makeDefault bool) {
  72. if _, ok := supportedSegmentPlugins[plugin.Type()]; !ok {
  73. supportedSegmentPlugins[plugin.Type()] = map[uint32]SegmentPlugin{}
  74. }
  75. supportedSegmentPlugins[plugin.Type()][plugin.Version()] = plugin
  76. if makeDefault {
  77. defaultSegmentPlugin = plugin
  78. }
  79. }
  80. func SupportedSegmentTypes() (rv []string) {
  81. for k := range supportedSegmentPlugins {
  82. rv = append(rv, k)
  83. }
  84. return
  85. }
  86. func SupportedSegmentTypeVersions(typ string) (rv []uint32) {
  87. for k := range supportedSegmentPlugins[typ] {
  88. rv = append(rv, k)
  89. }
  90. return rv
  91. }
  92. func chooseSegmentPlugin(forcedSegmentType string,
  93. forcedSegmentVersion uint32) (SegmentPlugin, error) {
  94. if versions, ok := supportedSegmentPlugins[forcedSegmentType]; ok {
  95. if segPlugin, ok := versions[uint32(forcedSegmentVersion)]; ok {
  96. return segPlugin, nil
  97. }
  98. return nil, fmt.Errorf(
  99. "unsupported version %d for segment type: %s, supported: %v",
  100. forcedSegmentVersion, forcedSegmentType,
  101. SupportedSegmentTypeVersions(forcedSegmentType))
  102. }
  103. return nil, fmt.Errorf("unsupported segment type: %s, supported: %v",
  104. forcedSegmentType, SupportedSegmentTypes())
  105. }
  106. func (s *Scorch) loadSegmentPlugin(forcedSegmentType string,
  107. forcedSegmentVersion uint32) error {
  108. segPlugin, err := chooseSegmentPlugin(forcedSegmentType,
  109. forcedSegmentVersion)
  110. if err != nil {
  111. return err
  112. }
  113. s.segPlugin = segPlugin
  114. return nil
  115. }