util.go 6.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305
  1. package roaring
  2. import (
  3. "math"
  4. "math/rand"
  5. "sort"
  6. )
  7. const (
  8. arrayDefaultMaxSize = 4096 // containers with 4096 or fewer integers should be array containers.
  9. arrayLazyLowerBound = 1024
  10. maxCapacity = 1 << 16
  11. serialCookieNoRunContainer = 12346 // only arrays and bitmaps
  12. invalidCardinality = -1
  13. serialCookie = 12347 // runs, arrays, and bitmaps
  14. noOffsetThreshold = 4
  15. // MaxUint32 is the largest uint32 value.
  16. MaxUint32 = math.MaxUint32
  17. // MaxRange is One more than the maximum allowed bitmap bit index. For use as an upper
  18. // bound for ranges.
  19. MaxRange uint64 = MaxUint32 + 1
  20. // MaxUint16 is the largest 16 bit unsigned int.
  21. // This is the largest value an interval16 can store.
  22. MaxUint16 = math.MaxUint16
  23. // Compute wordSizeInBytes, the size of a word in bytes.
  24. _m = ^uint64(0)
  25. _logS = _m>>8&1 + _m>>16&1 + _m>>32&1
  26. wordSizeInBytes = 1 << _logS
  27. // other constants used in ctz_generic.go
  28. wordSizeInBits = wordSizeInBytes << 3 // word size in bits
  29. )
  30. const maxWord = 1<<wordSizeInBits - 1
  31. // doesn't apply to runContainers
  32. func getSizeInBytesFromCardinality(card int) int {
  33. if card > arrayDefaultMaxSize {
  34. // bitmapContainer
  35. return maxCapacity / 8
  36. }
  37. // arrayContainer
  38. return 2 * card
  39. }
  40. func fill(arr []uint64, val uint64) {
  41. for i := range arr {
  42. arr[i] = val
  43. }
  44. }
  45. func fillRange(arr []uint64, start, end int, val uint64) {
  46. for i := start; i < end; i++ {
  47. arr[i] = val
  48. }
  49. }
  50. func fillArrayAND(container []uint16, bitmap1, bitmap2 []uint64) {
  51. if len(bitmap1) != len(bitmap2) {
  52. panic("array lengths don't match")
  53. }
  54. // TODO: rewrite in assembly
  55. pos := 0
  56. for k := range bitmap1 {
  57. bitset := bitmap1[k] & bitmap2[k]
  58. for bitset != 0 {
  59. t := bitset & -bitset
  60. container[pos] = uint16((k*64 + int(popcount(t-1))))
  61. pos = pos + 1
  62. bitset ^= t
  63. }
  64. }
  65. }
  66. func fillArrayANDNOT(container []uint16, bitmap1, bitmap2 []uint64) {
  67. if len(bitmap1) != len(bitmap2) {
  68. panic("array lengths don't match")
  69. }
  70. // TODO: rewrite in assembly
  71. pos := 0
  72. for k := range bitmap1 {
  73. bitset := bitmap1[k] &^ bitmap2[k]
  74. for bitset != 0 {
  75. t := bitset & -bitset
  76. container[pos] = uint16((k*64 + int(popcount(t-1))))
  77. pos = pos + 1
  78. bitset ^= t
  79. }
  80. }
  81. }
  82. func fillArrayXOR(container []uint16, bitmap1, bitmap2 []uint64) {
  83. if len(bitmap1) != len(bitmap2) {
  84. panic("array lengths don't match")
  85. }
  86. // TODO: rewrite in assembly
  87. pos := 0
  88. for k := 0; k < len(bitmap1); k++ {
  89. bitset := bitmap1[k] ^ bitmap2[k]
  90. for bitset != 0 {
  91. t := bitset & -bitset
  92. container[pos] = uint16((k*64 + int(popcount(t-1))))
  93. pos = pos + 1
  94. bitset ^= t
  95. }
  96. }
  97. }
  98. func highbits(x uint32) uint16 {
  99. return uint16(x >> 16)
  100. }
  101. func lowbits(x uint32) uint16 {
  102. return uint16(x & maxLowBit)
  103. }
  104. const maxLowBit = 0xFFFF
  105. func flipBitmapRange(bitmap []uint64, start int, end int) {
  106. if start >= end {
  107. return
  108. }
  109. firstword := start / 64
  110. endword := (end - 1) / 64
  111. bitmap[firstword] ^= ^(^uint64(0) << uint(start%64))
  112. for i := firstword; i < endword; i++ {
  113. bitmap[i] = ^bitmap[i]
  114. }
  115. bitmap[endword] ^= ^uint64(0) >> (uint(-end) % 64)
  116. }
  117. func resetBitmapRange(bitmap []uint64, start int, end int) {
  118. if start >= end {
  119. return
  120. }
  121. firstword := start / 64
  122. endword := (end - 1) / 64
  123. if firstword == endword {
  124. bitmap[firstword] &= ^((^uint64(0) << uint(start%64)) & (^uint64(0) >> (uint(-end) % 64)))
  125. return
  126. }
  127. bitmap[firstword] &= ^(^uint64(0) << uint(start%64))
  128. for i := firstword + 1; i < endword; i++ {
  129. bitmap[i] = 0
  130. }
  131. bitmap[endword] &= ^(^uint64(0) >> (uint(-end) % 64))
  132. }
  133. func setBitmapRange(bitmap []uint64, start int, end int) {
  134. if start >= end {
  135. return
  136. }
  137. firstword := start / 64
  138. endword := (end - 1) / 64
  139. if firstword == endword {
  140. bitmap[firstword] |= (^uint64(0) << uint(start%64)) & (^uint64(0) >> (uint(-end) % 64))
  141. return
  142. }
  143. bitmap[firstword] |= ^uint64(0) << uint(start%64)
  144. for i := firstword + 1; i < endword; i++ {
  145. bitmap[i] = ^uint64(0)
  146. }
  147. bitmap[endword] |= ^uint64(0) >> (uint(-end) % 64)
  148. }
  149. func flipBitmapRangeAndCardinalityChange(bitmap []uint64, start int, end int) int {
  150. before := wordCardinalityForBitmapRange(bitmap, start, end)
  151. flipBitmapRange(bitmap, start, end)
  152. after := wordCardinalityForBitmapRange(bitmap, start, end)
  153. return int(after - before)
  154. }
  155. func resetBitmapRangeAndCardinalityChange(bitmap []uint64, start int, end int) int {
  156. before := wordCardinalityForBitmapRange(bitmap, start, end)
  157. resetBitmapRange(bitmap, start, end)
  158. after := wordCardinalityForBitmapRange(bitmap, start, end)
  159. return int(after - before)
  160. }
  161. func setBitmapRangeAndCardinalityChange(bitmap []uint64, start int, end int) int {
  162. before := wordCardinalityForBitmapRange(bitmap, start, end)
  163. setBitmapRange(bitmap, start, end)
  164. after := wordCardinalityForBitmapRange(bitmap, start, end)
  165. return int(after - before)
  166. }
  167. func wordCardinalityForBitmapRange(bitmap []uint64, start int, end int) uint64 {
  168. answer := uint64(0)
  169. if start >= end {
  170. return answer
  171. }
  172. firstword := start / 64
  173. endword := (end - 1) / 64
  174. for i := firstword; i <= endword; i++ {
  175. answer += popcount(bitmap[i])
  176. }
  177. return answer
  178. }
  179. func selectBitPosition(w uint64, j int) int {
  180. seen := 0
  181. // Divide 64bit
  182. part := w & 0xFFFFFFFF
  183. n := popcount(part)
  184. if n <= uint64(j) {
  185. part = w >> 32
  186. seen += 32
  187. j -= int(n)
  188. }
  189. w = part
  190. // Divide 32bit
  191. part = w & 0xFFFF
  192. n = popcount(part)
  193. if n <= uint64(j) {
  194. part = w >> 16
  195. seen += 16
  196. j -= int(n)
  197. }
  198. w = part
  199. // Divide 16bit
  200. part = w & 0xFF
  201. n = popcount(part)
  202. if n <= uint64(j) {
  203. part = w >> 8
  204. seen += 8
  205. j -= int(n)
  206. }
  207. w = part
  208. // Lookup in final byte
  209. var counter uint
  210. for counter = 0; counter < 8; counter++ {
  211. j -= int((w >> counter) & 1)
  212. if j < 0 {
  213. break
  214. }
  215. }
  216. return seen + int(counter)
  217. }
  218. func panicOn(err error) {
  219. if err != nil {
  220. panic(err)
  221. }
  222. }
  223. type ph struct {
  224. orig int
  225. rand int
  226. }
  227. type pha []ph
  228. func (p pha) Len() int { return len(p) }
  229. func (p pha) Less(i, j int) bool { return p[i].rand < p[j].rand }
  230. func (p pha) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
  231. func getRandomPermutation(n int) []int {
  232. r := make([]ph, n)
  233. for i := 0; i < n; i++ {
  234. r[i].orig = i
  235. r[i].rand = rand.Intn(1 << 29)
  236. }
  237. sort.Sort(pha(r))
  238. m := make([]int, n)
  239. for i := range m {
  240. m[i] = r[i].orig
  241. }
  242. return m
  243. }
  244. func minOfInt(a, b int) int {
  245. if a < b {
  246. return a
  247. }
  248. return b
  249. }
  250. func maxOfInt(a, b int) int {
  251. if a > b {
  252. return a
  253. }
  254. return b
  255. }
  256. func maxOfUint16(a, b uint16) uint16 {
  257. if a > b {
  258. return a
  259. }
  260. return b
  261. }
  262. func minOfUint16(a, b uint16) uint16 {
  263. if a < b {
  264. return a
  265. }
  266. return b
  267. }