encoding.go 6.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208
  1. // Unless explicitly stated otherwise all files in this repository are licensed
  2. // under the Apache License 2.0.
  3. // This product includes software developed at Datadog (https://www.datadoghq.com/).
  4. // Copyright 2021 Datadog, Inc.
  5. package encoding
  6. import (
  7. "encoding/binary"
  8. "errors"
  9. "io"
  10. "math"
  11. "math/bits"
  12. )
  13. // Encoding functions append bytes to the provided *[]byte, allowing avoiding
  14. // allocations if the slice initially has a large enough capacity.
  15. // Decoding functions also take *[]byte as input, and when they do not return an
  16. // error, advance the slice so that it starts at the immediate byte after the
  17. // decoded part (or so that it is empty if there is no such byte).
  18. const (
  19. MaxVarLen64 = 9
  20. varfloat64Rotate = 6
  21. )
  22. var uvarint64Sizes = initUvarint64Sizes()
  23. var varfloat64Sizes = initVarfloat64Sizes()
  24. // EncodeUvarint64 serializes 64-bit unsigned integers 7 bits at a time,
  25. // starting with the least significant bits. The most significant bit in each
  26. // output byte is the continuation bit and indicates whether there are
  27. // additional non-zero bits encoded in following bytes. There are at most 9
  28. // output bytes and the last one does not have a continuation bit, allowing for
  29. // it to encode 8 bits (8*7+8 = 64).
  30. func EncodeUvarint64(b *[]byte, v uint64) {
  31. for i := 0; i < MaxVarLen64-1; i++ {
  32. if v < 0x80 {
  33. break
  34. }
  35. *b = append(*b, byte(v)|byte(0x80))
  36. v >>= 7
  37. }
  38. *b = append(*b, byte(v))
  39. }
  40. // DecodeUvarint64 deserializes 64-bit unsigned integers that have been encoded
  41. // using EncodeUvarint64.
  42. func DecodeUvarint64(b *[]byte) (uint64, error) {
  43. x := uint64(0)
  44. s := uint(0)
  45. for i := 0; ; i++ {
  46. if len(*b) <= i {
  47. return 0, io.EOF
  48. }
  49. n := (*b)[i]
  50. if n < 0x80 || i == MaxVarLen64-1 {
  51. *b = (*b)[i+1:]
  52. return x | uint64(n)<<s, nil
  53. }
  54. x |= uint64(n&0x7F) << s
  55. s += 7
  56. }
  57. }
  58. // Uvarint64Size returns the number of bytes that EncodeUvarint64 encodes a
  59. // 64-bit unsigned integer into.
  60. func Uvarint64Size(v uint64) int {
  61. return uvarint64Sizes[bits.LeadingZeros64(v)]
  62. }
  63. func initUvarint64Sizes() [65]int {
  64. var sizes [65]int
  65. b := []byte{}
  66. for i := 0; i <= 64; i++ {
  67. b = b[:0]
  68. EncodeUvarint64(&b, ^uint64(0)>>i)
  69. sizes[i] = len(b)
  70. }
  71. return sizes
  72. }
  73. // EncodeVarint64 serializes 64-bit signed integers using zig-zag encoding,
  74. // which ensures small-scale integers are turned into unsigned integers that
  75. // have leading zeros, whether they are positive or negative, hence allows for
  76. // space-efficient varuint encoding of those values.
  77. func EncodeVarint64(b *[]byte, v int64) {
  78. EncodeUvarint64(b, uint64(v>>(64-1)^(v<<1)))
  79. }
  80. // DecodeVarint64 deserializes 64-bit signed integers that have been encoded
  81. // using EncodeVarint32.
  82. func DecodeVarint64(b *[]byte) (int64, error) {
  83. v, err := DecodeUvarint64(b)
  84. return int64((v >> 1) ^ -(v & 1)), err
  85. }
  86. // Varint64Size returns the number of bytes that EncodeVarint64 encodes a 64-bit
  87. // signed integer into.
  88. func Varint64Size(v int64) int {
  89. return Uvarint64Size(uint64(v>>(64-1) ^ (v << 1)))
  90. }
  91. var errVarint32Overflow = errors.New("varint overflows a 32-bit integer")
  92. // DecodeVarint32 deserializes 32-bit signed integers that have been encoded
  93. // using EncodeVarint64.
  94. func DecodeVarint32(b *[]byte) (int32, error) {
  95. v, err := DecodeVarint64(b)
  96. if err != nil {
  97. return 0, err
  98. }
  99. if v > math.MaxInt32 || v < math.MinInt32 {
  100. return 0, errVarint32Overflow
  101. }
  102. return int32(v), nil
  103. }
  104. // EncodeFloat64LE serializes 64-bit floating-point values, starting with the
  105. // least significant bytes.
  106. func EncodeFloat64LE(b *[]byte, v float64) {
  107. *b = append(*b, make([]byte, 8)...)
  108. binary.LittleEndian.PutUint64((*b)[len(*b)-8:], math.Float64bits(v))
  109. }
  110. // DecodeFloat64LE deserializes 64-bit floating-point values that have been
  111. // encoded with EncodeFloat64LE.
  112. func DecodeFloat64LE(b *[]byte) (float64, error) {
  113. if len(*b) < 8 {
  114. return 0, io.EOF
  115. }
  116. v := math.Float64frombits(binary.LittleEndian.Uint64(*b))
  117. *b = (*b)[8:]
  118. return v, nil
  119. }
  120. // EncodeVarfloat64 serializes 64-bit floating-point values using a method that
  121. // is similar to the varuint encoding and that is space-efficient for
  122. // non-negative integer values. The output takes at most 9 bytes.
  123. // Input values are first shifted as floating-point values (+1), then transmuted
  124. // to integer values, then shifted again as integer values (-Float64bits(1)).
  125. // That is in order to minimize the number of non-zero bits when dealing with
  126. // non-negative integer values.
  127. // After that transformation, any input integer value no greater than 2^53 (the
  128. // largest integer value that can be encoded exactly as a 64-bit floating-point
  129. // value) will have at least 6 leading zero bits. By rotating bits to the left,
  130. // those bits end up at the right of the binary representation.
  131. // The resulting bits are then encoded similarly to the varuint method, but
  132. // starting with the most significant bits.
  133. func EncodeVarfloat64(b *[]byte, v float64) {
  134. x := bits.RotateLeft64(math.Float64bits(v+1)-math.Float64bits(1), varfloat64Rotate)
  135. for i := 0; i < MaxVarLen64-1; i++ {
  136. n := byte(x >> (8*8 - 7))
  137. x <<= 7
  138. if x == 0 {
  139. *b = append(*b, n)
  140. return
  141. }
  142. *b = append(*b, n|byte(0x80))
  143. }
  144. n := byte(x >> (8 * 7))
  145. *b = append(*b, n)
  146. }
  147. // DecodeVarfloat64 deserializes 64-bit floating-point values that have been
  148. // encoded with EncodeVarfloat64.
  149. func DecodeVarfloat64(b *[]byte) (float64, error) {
  150. x := uint64(0)
  151. i := int(0)
  152. s := uint(8*8 - 7)
  153. for {
  154. if len(*b) <= i {
  155. return 0, io.EOF
  156. }
  157. n := (*b)[i]
  158. if i == MaxVarLen64-1 {
  159. x |= uint64(n)
  160. break
  161. }
  162. if n < 0x80 {
  163. x |= uint64(n) << s
  164. break
  165. }
  166. x |= uint64(n&0x7F) << s
  167. i++
  168. s -= 7
  169. }
  170. *b = (*b)[i+1:]
  171. return math.Float64frombits(bits.RotateLeft64(x, -varfloat64Rotate)+math.Float64bits(1)) - 1, nil
  172. }
  173. // Varfloat64Size returns the number of bytes that EncodeVarfloat64 encodes a
  174. // 64-bit floating-point value into.
  175. func Varfloat64Size(v float64) int {
  176. x := bits.RotateLeft64(math.Float64bits(v+1)-math.Float64bits(1), varfloat64Rotate)
  177. return varfloat64Sizes[bits.TrailingZeros64(x)]
  178. }
  179. func initVarfloat64Sizes() [65]int {
  180. var sizes [65]int
  181. b := []byte{}
  182. for i := 0; i <= 64; i++ {
  183. b = b[:0]
  184. EncodeVarfloat64(&b, math.Float64frombits(bits.RotateLeft64(^uint64(0)<<i, -varfloat64Rotate)+math.Float64bits(1))-1)
  185. sizes[i] = len(b)
  186. }
  187. return sizes
  188. }