GitOps for k8s
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

await.go 3.2KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119
  1. package main
  2. import (
  3. "context"
  4. "errors"
  5. "fmt"
  6. "io"
  7. "time"
  8. "github.com/fluxcd/flux/api"
  9. "github.com/fluxcd/flux/job"
  10. "github.com/fluxcd/flux/update"
  11. )
  12. var ErrTimeout = errors.New("timeout")
  13. // await polls for a job to complete, then for the resulting commit to
  14. // be applied
  15. func await(ctx context.Context, stdout, stderr io.Writer, client api.Server, jobID job.ID, apply bool, verbosity int, timeout time.Duration) error {
  16. result, err := awaitJob(ctx, client, jobID, timeout)
  17. if err != nil {
  18. if err == ErrTimeout {
  19. fmt.Fprintln(stderr, `
  20. We timed out waiting for the result of the operation. This does not
  21. necessarily mean it has failed. You can check the state of the
  22. cluster, or commit logs, to see if there was a result. In general, it
  23. is safe to retry operations.`)
  24. // because the outcome is unknown, still return the err to indicate an exceptional exit
  25. }
  26. return err
  27. }
  28. if result.Result != nil {
  29. update.PrintResults(stdout, result.Result, verbosity)
  30. }
  31. if result.Revision != "" {
  32. fmt.Fprintf(stderr, "Commit pushed:\t%s\n", result.Revision[:7])
  33. }
  34. if result.Result == nil {
  35. fmt.Fprintf(stderr, "Nothing to do\n")
  36. return nil
  37. }
  38. if apply && result.Revision != "" {
  39. if err := awaitSync(ctx, client, result.Revision, timeout); err != nil {
  40. if err == ErrTimeout {
  41. fmt.Fprintln(stderr, `
  42. The operation succeeded, but we timed out waiting for the commit to be
  43. applied. This does not necessarily mean there is a problem. Use
  44. fluxctl sync
  45. to run a sync interactively.`)
  46. return nil
  47. }
  48. return err
  49. }
  50. fmt.Fprintf(stderr, "Commit applied:\t%s\n", result.Revision[:7])
  51. }
  52. return nil
  53. }
  54. // await polls for a job to have been completed, with exponential backoff.
  55. func awaitJob(ctx context.Context, client api.Server, jobID job.ID, timeout time.Duration) (job.Result, error) {
  56. var result job.Result
  57. err := backoff(100*time.Millisecond, 2, 50, timeout, func() (bool, error) {
  58. j, err := client.JobStatus(ctx, jobID)
  59. if err != nil {
  60. return false, err
  61. }
  62. switch j.StatusString {
  63. case job.StatusFailed:
  64. result = j.Result
  65. return false, j
  66. case job.StatusSucceeded:
  67. if j.Err != "" {
  68. // How did we succeed but still get an error!?
  69. return false, j
  70. }
  71. result = j.Result
  72. return true, nil
  73. }
  74. return false, nil
  75. })
  76. return result, err
  77. }
  78. // await polls for a commit to have been applied, with exponential backoff.
  79. func awaitSync(ctx context.Context, client api.Server, revision string, timeout time.Duration) error {
  80. return backoff(1*time.Second, 2, 10, timeout, func() (bool, error) {
  81. refs, err := client.SyncStatus(ctx, revision)
  82. return err == nil && len(refs) == 0, err
  83. })
  84. }
  85. // backoff polls for f() to have been completed, with exponential backoff.
  86. func backoff(initialDelay, factor, maxFactor, timeout time.Duration, f func() (bool, error)) error {
  87. maxDelay := initialDelay * maxFactor
  88. finish := time.Now().Add(timeout)
  89. for delay := initialDelay; time.Now().Before(finish); delay = min(delay*factor, maxDelay) {
  90. ok, err := f()
  91. if ok || err != nil {
  92. return err
  93. }
  94. // If we don't have time to try again, stop
  95. if time.Now().Add(delay).After(finish) {
  96. break
  97. }
  98. time.Sleep(delay)
  99. }
  100. return ErrTimeout
  101. }
  102. func min(t1, t2 time.Duration) time.Duration {
  103. if t1 < t2 {
  104. return t1
  105. }
  106. return t2
  107. }