flag.go 4.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160
  1. // Unless explicitly stated otherwise all files in this repository are licensed
  2. // under the Apache License 2.0.
  3. // This product includes software developed at Datadog (https://www.datadoghq.com/).
  4. // Copyright 2021 Datadog, Inc.
  5. package encoding
  6. import (
  7. "io"
  8. )
  9. // An encoded DDSketch comprises multiple contiguous blocks (sequences of
  10. // bytes). Each block is prefixed with a flag that indicates what the block
  11. // contains and how the data is encoded in the block.
  12. //
  13. // A flag is a single byte, which itself contains two parts:
  14. // - the flag type (the 2 least significant bits),
  15. // - the subflag (the 6 most significant bits).
  16. //
  17. // There are four flag types, for:
  18. // - sketch features,
  19. // - index mapping,
  20. // - positive value store,
  21. // - negative value store.
  22. //
  23. // The meaning of the subflag depends on the flag type:
  24. // - for the sketch feature flag type, it indicates what feature is encoded,
  25. // - for the index mapping flag type, it indicates what mapping is encoded and
  26. // how,
  27. // - for the store flag types, it indicates how bins are encoded.
  28. const (
  29. numBitsForType byte = 2
  30. flagTypeMask byte = (1 << numBitsForType) - 1
  31. subFlagMask byte = ^flagTypeMask
  32. )
  33. type Flag struct{ byte }
  34. type FlagType struct{ byte } // mask: 0b00000011
  35. type SubFlag struct{ byte } // mask: 0b11111100
  36. var (
  37. // FLAG TYPES
  38. flagTypeSketchFeatures = FlagType{0b00}
  39. FlagTypeIndexMapping = FlagType{0b10}
  40. FlagTypePositiveStore = FlagType{0b01}
  41. FlagTypeNegativeStore = FlagType{0b11}
  42. // SKETCH FEATURES
  43. // Encodes the count of the zero bin.
  44. // Encoding format:
  45. // - [byte] flag
  46. // - [varfloat64] count of the zero bin
  47. FlagZeroCountVarFloat = NewFlag(flagTypeSketchFeatures, newSubFlag(1))
  48. // Encode the total count.
  49. // Encoding format:
  50. // - [byte] flag
  51. // - [varfloat64] total count
  52. FlagCount = NewFlag(flagTypeSketchFeatures, newSubFlag(0x28))
  53. // Encode the summary statistics.
  54. // Encoding format:
  55. // - [byte] flag
  56. // - [float64LE] summary stat
  57. FlagSum = NewFlag(flagTypeSketchFeatures, newSubFlag(0x21))
  58. FlagMin = NewFlag(flagTypeSketchFeatures, newSubFlag(0x22))
  59. FlagMax = NewFlag(flagTypeSketchFeatures, newSubFlag(0x23))
  60. // INDEX MAPPING
  61. // Encodes log-like index mappings, specifying the base (gamma) and the index offset
  62. // The subflag specifies the interpolation method.
  63. // Encoding format:
  64. // - [byte] flag
  65. // - [float64LE] gamma
  66. // - [float64LE] index offset
  67. FlagIndexMappingBaseLogarithmic = NewFlag(FlagTypeIndexMapping, newSubFlag(0))
  68. FlagIndexMappingBaseLinear = NewFlag(FlagTypeIndexMapping, newSubFlag(1))
  69. FlagIndexMappingBaseQuadratic = NewFlag(FlagTypeIndexMapping, newSubFlag(2))
  70. FlagIndexMappingBaseCubic = NewFlag(FlagTypeIndexMapping, newSubFlag(3))
  71. FlagIndexMappingBaseQuartic = NewFlag(FlagTypeIndexMapping, newSubFlag(4))
  72. // BINS
  73. // Encodes N bins, each one with its index and its count.
  74. // Indexes are delta-encoded.
  75. // Encoding format:
  76. // - [byte] flag
  77. // - [uvarint64] number of bins N
  78. // - [varint64] index of first bin
  79. // - [varfloat64] count of first bin
  80. // - [varint64] difference between the index of the second bin and the index
  81. // of the first bin
  82. // - [varfloat64] count of second bin
  83. // - ...
  84. // - [varint64] difference between the index of the N-th bin and the index
  85. // of the (N-1)-th bin
  86. // - [varfloat64] count of N-th bin
  87. BinEncodingIndexDeltasAndCounts = newSubFlag(1)
  88. // Encodes N bins whose counts are each equal to 1.
  89. // Indexes are delta-encoded.
  90. // Encoding format:
  91. // - [byte] flag
  92. // - [uvarint64] number of bins N
  93. // - [varint64] index of first bin
  94. // - [varint64] difference between the index of the second bin and the index
  95. // of the first bin
  96. // - ...
  97. // - [varint64] difference between the index of the N-th bin and the index
  98. // of the (N-1)-th bin
  99. BinEncodingIndexDeltas = newSubFlag(2)
  100. // Encodes N contiguous bins, specifiying the count of each one
  101. // Encoding format:
  102. // - [byte] flag
  103. // - [uvarint64] number of bins N
  104. // - [varint64] index of first bin
  105. // - [varint64] difference between two successive indexes
  106. // - [varfloat64] count of first bin
  107. // - [varfloat64] count of second bin
  108. // - ...
  109. // - [varfloat64] count of N-th bin
  110. BinEncodingContiguousCounts = newSubFlag(3)
  111. )
  112. func NewFlag(t FlagType, s SubFlag) Flag {
  113. return Flag{t.byte | s.byte}
  114. }
  115. func (f Flag) Type() FlagType {
  116. return FlagType{f.byte & flagTypeMask}
  117. }
  118. func (f Flag) SubFlag() SubFlag {
  119. return SubFlag{f.byte & subFlagMask}
  120. }
  121. func newSubFlag(b byte) SubFlag {
  122. return SubFlag{b << numBitsForType}
  123. }
  124. // EncodeFlag encodes a flag and appends its content to the provided []byte.
  125. func EncodeFlag(b *[]byte, f Flag) {
  126. *b = append(*b, f.byte)
  127. }
  128. // DecodeFlag decodes a flag and updates the provided []byte so that it starts
  129. // immediately after the encoded flag.
  130. func DecodeFlag(b *[]byte) (Flag, error) {
  131. if len(*b) == 0 {
  132. return Flag{}, io.EOF
  133. }
  134. flag := Flag{(*b)[0]}
  135. *b = (*b)[1:]
  136. return flag, nil
  137. }