retry.go 3.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144
  1. /*
  2. *
  3. * Copyright 2023 Google LLC
  4. *
  5. * Licensed under the Apache License, Version 2.0 (the "License");
  6. * you may not use this file except in compliance with the License.
  7. * You may obtain a copy of the License at
  8. *
  9. * https://www.apache.org/licenses/LICENSE-2.0
  10. *
  11. * Unless required by applicable law or agreed to in writing, software
  12. * distributed under the License is distributed on an "AS IS" BASIS,
  13. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14. * See the License for the specific language governing permissions and
  15. * limitations under the License.
  16. *
  17. */
  18. // Package retry provides a retry helper for talking to S2A gRPC server.
  19. // The implementation is modeled after
  20. // https://github.com/googleapis/google-cloud-go/blob/main/compute/metadata/retry.go
  21. package retry
  22. import (
  23. "context"
  24. "math/rand"
  25. "time"
  26. "google.golang.org/grpc/grpclog"
  27. )
  28. const (
  29. maxRetryAttempts = 5
  30. maxRetryForLoops = 10
  31. )
  32. type defaultBackoff struct {
  33. max time.Duration
  34. mul float64
  35. cur time.Duration
  36. }
  37. // Pause returns a duration, which is used as the backoff wait time
  38. // before the next retry.
  39. func (b *defaultBackoff) Pause() time.Duration {
  40. d := time.Duration(1 + rand.Int63n(int64(b.cur)))
  41. b.cur = time.Duration(float64(b.cur) * b.mul)
  42. if b.cur > b.max {
  43. b.cur = b.max
  44. }
  45. return d
  46. }
  47. // Sleep will wait for the specified duration or return on context
  48. // expiration.
  49. func Sleep(ctx context.Context, d time.Duration) error {
  50. t := time.NewTimer(d)
  51. select {
  52. case <-ctx.Done():
  53. t.Stop()
  54. return ctx.Err()
  55. case <-t.C:
  56. return nil
  57. }
  58. }
  59. // NewRetryer creates an instance of S2ARetryer using the defaultBackoff
  60. // implementation.
  61. var NewRetryer = func() *S2ARetryer {
  62. return &S2ARetryer{bo: &defaultBackoff{
  63. cur: 100 * time.Millisecond,
  64. max: 30 * time.Second,
  65. mul: 2,
  66. }}
  67. }
  68. type backoff interface {
  69. Pause() time.Duration
  70. }
  71. // S2ARetryer implements a retry helper for talking to S2A gRPC server.
  72. type S2ARetryer struct {
  73. bo backoff
  74. attempts int
  75. }
  76. // Attempts return the number of retries attempted.
  77. func (r *S2ARetryer) Attempts() int {
  78. return r.attempts
  79. }
  80. // Retry returns a boolean indicating whether retry should be performed
  81. // and the backoff duration.
  82. func (r *S2ARetryer) Retry(err error) (time.Duration, bool) {
  83. if err == nil {
  84. return 0, false
  85. }
  86. if r.attempts >= maxRetryAttempts {
  87. return 0, false
  88. }
  89. r.attempts++
  90. return r.bo.Pause(), true
  91. }
  92. // Run uses S2ARetryer to execute the function passed in, until success or reaching
  93. // max number of retry attempts.
  94. func Run(ctx context.Context, f func() error) {
  95. retryer := NewRetryer()
  96. forLoopCnt := 0
  97. var err error
  98. for {
  99. err = f()
  100. if bo, shouldRetry := retryer.Retry(err); shouldRetry {
  101. if grpclog.V(1) {
  102. grpclog.Infof("will attempt retry: %v", err)
  103. }
  104. if ctx.Err() != nil {
  105. if grpclog.V(1) {
  106. grpclog.Infof("exit retry loop due to context error: %v", ctx.Err())
  107. }
  108. break
  109. }
  110. if errSleep := Sleep(ctx, bo); errSleep != nil {
  111. if grpclog.V(1) {
  112. grpclog.Infof("exit retry loop due to sleep error: %v", errSleep)
  113. }
  114. break
  115. }
  116. // This shouldn't happen, just make sure we are not stuck in the for loops.
  117. forLoopCnt++
  118. if forLoopCnt > maxRetryForLoops {
  119. if grpclog.V(1) {
  120. grpclog.Infof("exit the for loop after too many retries")
  121. }
  122. break
  123. }
  124. continue
  125. }
  126. if grpclog.V(1) {
  127. grpclog.Infof("retry conditions not met, exit the loop")
  128. }
  129. break
  130. }
  131. }