GitOps for k8s
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

operations.go 14KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472
  1. package git
  2. import (
  3. "bufio"
  4. "bytes"
  5. "context"
  6. "encoding/json"
  7. "fmt"
  8. "io"
  9. "os"
  10. "os/exec"
  11. "strings"
  12. "sync"
  13. "github.com/pkg/errors"
  14. )
  15. // If true, every git invocation will be echoed to stdout (with the exception of those added to `exemptedTraceCommands`)
  16. const trace = false
  17. // Whilst debugging or developing, you may wish to filter certain git commands out of the logs when tracing is on.
  18. var exemptedTraceCommands = []string{
  19. // To filter out a certain git subcommand add it here, e.g.:
  20. // "config",
  21. }
  22. // Env vars that are allowed to be inherited from the OS
  23. var allowedEnvVars = []string{
  24. // these are for people using (no) proxies
  25. "http_proxy", "https_proxy", "no_proxy", "GIT_PROXY_COMMAND",
  26. // these are needed for GPG to find its files
  27. "HOME", "GNUPGHOME",
  28. // these for the git-secrets helper
  29. "SECRETS_DIR", "SECRETS_EXTENSION",
  30. // these are for Google Cloud SDK to find its files (which will
  31. // have to be mounted, if running in a container)
  32. "CLOUDSDK_CONFIG", "CLOUDSDK_PYTHON",
  33. }
  34. type gitCmdConfig struct {
  35. dir string
  36. env []string
  37. out io.Writer
  38. }
  39. func config(ctx context.Context, workingDir, user, email string) error {
  40. for k, v := range map[string]string{
  41. "user.name": user,
  42. "user.email": email,
  43. } {
  44. args := []string{"config", k, v}
  45. if err := execGitCmd(ctx, args, gitCmdConfig{dir: workingDir}); err != nil {
  46. return errors.Wrap(err, "setting git config")
  47. }
  48. }
  49. return nil
  50. }
  51. func clone(ctx context.Context, workingDir, repoURL, repoBranch string) (path string, err error) {
  52. repoPath := workingDir
  53. args := []string{"clone"}
  54. if repoBranch != "" {
  55. args = append(args, "--branch", repoBranch)
  56. }
  57. args = append(args, repoURL, repoPath)
  58. if err := execGitCmd(ctx, args, gitCmdConfig{dir: workingDir}); err != nil {
  59. return "", errors.Wrap(err, "git clone")
  60. }
  61. return repoPath, nil
  62. }
  63. func mirror(ctx context.Context, workingDir, repoURL string) (path string, err error) {
  64. repoPath := workingDir
  65. args := []string{"clone", "--mirror"}
  66. args = append(args, repoURL, repoPath)
  67. if err := execGitCmd(ctx, args, gitCmdConfig{dir: workingDir}); err != nil {
  68. return "", errors.Wrap(err, "git clone --mirror")
  69. }
  70. return repoPath, nil
  71. }
  72. func checkout(ctx context.Context, workingDir, ref string) error {
  73. args := []string{"checkout", ref, "--"}
  74. return execGitCmd(ctx, args, gitCmdConfig{dir: workingDir})
  75. }
  76. func add(ctx context.Context, workingDir, path string) error {
  77. args := []string{"add", "--", path}
  78. return execGitCmd(ctx, args, gitCmdConfig{dir: workingDir})
  79. }
  80. // checkPush sanity-checks that we can write to the upstream repo
  81. // (being able to `clone` is an adequate check that we can read the
  82. // upstream).
  83. func checkPush(ctx context.Context, workingDir, upstream, branch string) error {
  84. // --force just in case we fetched the tag from upstream when cloning
  85. args := []string{"tag", "--force", CheckPushTag}
  86. if branch != "" {
  87. args = append(args, branch)
  88. }
  89. if err := execGitCmd(ctx, args, gitCmdConfig{dir: workingDir}); err != nil {
  90. return errors.Wrap(err, "tag for write check")
  91. }
  92. args = []string{"push", "--force", upstream, "tag", CheckPushTag}
  93. if err := execGitCmd(ctx, args, gitCmdConfig{dir: workingDir}); err != nil {
  94. return errors.Wrap(err, "attempt to push tag")
  95. }
  96. return deleteTag(ctx, workingDir, CheckPushTag, upstream)
  97. }
  98. // deleteTag deletes the given git tag
  99. // See https://git-scm.com/docs/git-tag and https://git-scm.com/docs/git-push for more info.
  100. func deleteTag(ctx context.Context, workingDir, tag, upstream string) error {
  101. args := []string{"push", "--delete", upstream, "tag", tag}
  102. return execGitCmd(ctx, args, gitCmdConfig{dir: workingDir})
  103. }
  104. func secretUnseal(ctx context.Context, workingDir string) error {
  105. args := []string{"secret", "reveal", "-f"}
  106. if err := execGitCmd(ctx, args, gitCmdConfig{dir: workingDir}); err != nil {
  107. return errors.Wrap(err, "git secret reveal -f")
  108. }
  109. return nil
  110. }
  111. func commit(ctx context.Context, workingDir string, commitAction CommitAction) error {
  112. args := []string{"commit", "--no-verify", "-a", "-m", commitAction.Message}
  113. var env []string
  114. if commitAction.Author != "" {
  115. args = append(args, "--author", commitAction.Author)
  116. }
  117. if commitAction.SigningKey != "" {
  118. args = append(args, fmt.Sprintf("--gpg-sign=%s", commitAction.SigningKey))
  119. }
  120. args = append(args, "--")
  121. if err := execGitCmd(ctx, args, gitCmdConfig{dir: workingDir, env: env}); err != nil {
  122. return errors.Wrap(err, "git commit")
  123. }
  124. return nil
  125. }
  126. // push the refs given to the upstream repo
  127. func push(ctx context.Context, workingDir, upstream string, refs []string) error {
  128. args := append([]string{"push", upstream}, refs...)
  129. if err := execGitCmd(ctx, args, gitCmdConfig{dir: workingDir}); err != nil {
  130. return errors.Wrap(err, fmt.Sprintf("git push %s %s", upstream, refs))
  131. }
  132. return nil
  133. }
  134. // fetch updates refs from the upstream.
  135. func fetch(ctx context.Context, workingDir, upstream string, refspec ...string) error {
  136. args := append([]string{"fetch", "--tags", upstream}, refspec...)
  137. // In git <=2.20 the error started with an uppercase, in 2.21 this
  138. // was changed to be consistent with all other die() and error()
  139. // messages, cast to lowercase to support both versions.
  140. // Ref: https://github.com/git/git/commit/0b9c3afdbfb62936337efc52b4007a446939b96b
  141. if err := execGitCmd(ctx, args, gitCmdConfig{dir: workingDir}); err != nil &&
  142. !strings.Contains(strings.ToLower(err.Error()), "couldn't find remote ref") {
  143. return errors.Wrap(err, fmt.Sprintf("git fetch --tags %s %s", upstream, refspec))
  144. }
  145. return nil
  146. }
  147. func refExists(ctx context.Context, workingDir, ref string) (bool, error) {
  148. args := []string{"rev-list", ref, "--"}
  149. if err := execGitCmd(ctx, args, gitCmdConfig{dir: workingDir}); err != nil {
  150. if strings.Contains(err.Error(), "bad revision") {
  151. return false, nil
  152. }
  153. return false, err
  154. }
  155. return true, nil
  156. }
  157. // Get the full ref for a shorthand notes ref.
  158. func getNotesRef(ctx context.Context, workingDir, ref string) (string, error) {
  159. out := &bytes.Buffer{}
  160. args := []string{"notes", "--ref", ref, "get-ref"}
  161. if err := execGitCmd(ctx, args, gitCmdConfig{dir: workingDir, out: out}); err != nil {
  162. return "", err
  163. }
  164. return strings.TrimSpace(out.String()), nil
  165. }
  166. func addNote(ctx context.Context, workingDir, rev, notesRef string, note interface{}) error {
  167. b, err := json.Marshal(note)
  168. if err != nil {
  169. return err
  170. }
  171. args := []string{"notes", "--ref", notesRef, "add", "-m", string(b), rev}
  172. return execGitCmd(ctx, args, gitCmdConfig{dir: workingDir})
  173. }
  174. func getNote(ctx context.Context, workingDir, notesRef, rev string, note interface{}) (ok bool, err error) {
  175. out := &bytes.Buffer{}
  176. args := []string{"notes", "--ref", notesRef, "show", rev}
  177. if err := execGitCmd(ctx, args, gitCmdConfig{dir: workingDir, out: out}); err != nil {
  178. if strings.Contains(strings.ToLower(err.Error()), "no note found for object") {
  179. return false, nil
  180. }
  181. return false, err
  182. }
  183. if err := json.NewDecoder(out).Decode(note); err != nil {
  184. return false, err
  185. }
  186. return true, nil
  187. }
  188. // Get all revisions with a note (NB: DO NOT RELY ON THE ORDERING)
  189. // It appears to be ordered by ascending git object ref, not by time.
  190. // Return a map to make it easier to do "if in" type queries.
  191. func noteRevList(ctx context.Context, workingDir, notesRef string) (map[string]struct{}, error) {
  192. out := &bytes.Buffer{}
  193. args := []string{"notes", "--ref", notesRef, "list"}
  194. if err := execGitCmd(ctx, args, gitCmdConfig{dir: workingDir, out: out}); err != nil {
  195. return nil, err
  196. }
  197. noteList := splitList(out.String())
  198. result := make(map[string]struct{}, len(noteList))
  199. for _, l := range noteList {
  200. split := strings.Fields(l)
  201. if len(split) > 0 {
  202. result[split[1]] = struct{}{} // First field contains the object ref (commit id in our case)
  203. }
  204. }
  205. return result, nil
  206. }
  207. // Get the commit hash for a reference
  208. func refRevision(ctx context.Context, workingDir, ref string) (string, error) {
  209. out := &bytes.Buffer{}
  210. args := []string{"rev-list", "--max-count", "1", ref, "--"}
  211. if err := execGitCmd(ctx, args, gitCmdConfig{dir: workingDir, out: out}); err != nil {
  212. return "", err
  213. }
  214. return strings.TrimSpace(out.String()), nil
  215. }
  216. // Return the revisions and one-line log commit messages
  217. func onelinelog(ctx context.Context, workingDir, refspec string, subdirs []string) ([]Commit, error) {
  218. out := &bytes.Buffer{}
  219. args := []string{"log", "--pretty=format:%GK|%G?|%H|%s", refspec}
  220. args = append(args, "--")
  221. if len(subdirs) > 0 {
  222. args = append(args, subdirs...)
  223. }
  224. if err := execGitCmd(ctx, args, gitCmdConfig{dir: workingDir, out: out}); err != nil {
  225. return nil, err
  226. }
  227. return splitLog(out.String())
  228. }
  229. func splitLog(s string) ([]Commit, error) {
  230. lines := splitList(s)
  231. commits := make([]Commit, len(lines))
  232. for i, m := range lines {
  233. parts := strings.SplitN(m, "|", 4)
  234. commits[i].Signature = Signature{
  235. Key: parts[0],
  236. Status: parts[1],
  237. }
  238. commits[i].Revision = parts[2]
  239. commits[i].Message = parts[3]
  240. }
  241. return commits, nil
  242. }
  243. func splitList(s string) []string {
  244. if strings.TrimSpace(s) == "" {
  245. return []string{}
  246. }
  247. outStr := strings.TrimSuffix(s, "\n")
  248. return strings.Split(outStr, "\n")
  249. }
  250. // Move the tag to the ref given and push that tag upstream
  251. func moveTagAndPush(ctx context.Context, workingDir, upstream string, action TagAction) error {
  252. args := []string{"tag", "--force", "-a", "-m", action.Message}
  253. var env []string
  254. if action.SigningKey != "" {
  255. args = append(args, fmt.Sprintf("--local-user=%s", action.SigningKey))
  256. }
  257. args = append(args, action.Tag, action.Revision)
  258. if err := execGitCmd(ctx, args, gitCmdConfig{dir: workingDir, env: env}); err != nil {
  259. return errors.Wrap(err, "moving tag "+action.Tag)
  260. }
  261. args = []string{"push", "--force", upstream, "tag", action.Tag}
  262. if err := execGitCmd(ctx, args, gitCmdConfig{dir: workingDir}); err != nil {
  263. return errors.Wrap(err, "pushing tag to origin")
  264. }
  265. return nil
  266. }
  267. // Verify tag signature and return the revision it points to
  268. func verifyTag(ctx context.Context, workingDir, tag string) (string, error) {
  269. out := &bytes.Buffer{}
  270. args := []string{"verify-tag", "--format", "%(object)", tag}
  271. if err := execGitCmd(ctx, args, gitCmdConfig{dir: workingDir, out: out}); err != nil {
  272. return "", errors.Wrap(err, "verifying tag "+tag)
  273. }
  274. return strings.TrimSpace(out.String()), nil
  275. }
  276. // Verify commit signature
  277. func verifyCommit(ctx context.Context, workingDir, commit string) error {
  278. args := []string{"verify-commit", commit}
  279. if err := execGitCmd(ctx, args, gitCmdConfig{dir: workingDir}); err != nil {
  280. return fmt.Errorf("failed to verify commit %s", commit)
  281. }
  282. return nil
  283. }
  284. func changed(ctx context.Context, workingDir, ref string, subPaths []string) ([]string, error) {
  285. out := &bytes.Buffer{}
  286. // This uses --diff-filter to only look at changes for file _in
  287. // the working dir_; i.e, we do not report on things that no
  288. // longer appear.
  289. args := []string{"diff", "--name-only", "--diff-filter=ACMRT", ref}
  290. args = append(args, "--")
  291. if len(subPaths) > 0 {
  292. args = append(args, subPaths...)
  293. }
  294. if err := execGitCmd(ctx, args, gitCmdConfig{dir: workingDir, out: out}); err != nil {
  295. return nil, err
  296. }
  297. return splitList(out.String()), nil
  298. }
  299. // traceGitCommand returns a log line that can be useful when debugging and developing git activity
  300. func traceGitCommand(args []string, config gitCmdConfig, stdOutAndStdErr string) string {
  301. for _, exemptedCommand := range exemptedTraceCommands {
  302. if exemptedCommand == args[0] {
  303. return ""
  304. }
  305. }
  306. prepare := func(input string) string {
  307. output := strings.Trim(input, "\x00")
  308. output = strings.TrimSuffix(output, "\n")
  309. output = strings.Replace(output, "\n", "\\n", -1)
  310. return output
  311. }
  312. command := `git ` + strings.Join(args, " ")
  313. out := prepare(stdOutAndStdErr)
  314. return fmt.Sprintf(
  315. "TRACE: command=%q out=%q dir=%q env=%q",
  316. command,
  317. out,
  318. config.dir,
  319. strings.Join(config.env, ","),
  320. )
  321. }
  322. type threadSafeBuffer struct {
  323. buf bytes.Buffer
  324. mu sync.Mutex
  325. }
  326. func (b *threadSafeBuffer) Write(p []byte) (n int, err error) {
  327. b.mu.Lock()
  328. defer b.mu.Unlock()
  329. return b.buf.Write(p)
  330. }
  331. func (b *threadSafeBuffer) Read(p []byte) (n int, err error) {
  332. b.mu.Lock()
  333. defer b.mu.Unlock()
  334. return b.buf.Read(p)
  335. }
  336. func (b *threadSafeBuffer) Bytes() []byte {
  337. b.mu.Lock()
  338. defer b.mu.Unlock()
  339. return b.buf.Bytes()
  340. }
  341. func (b *threadSafeBuffer) String() string {
  342. b.mu.Lock()
  343. defer b.mu.Unlock()
  344. return b.buf.String()
  345. }
  346. // execGitCmd runs a `git` command with the supplied arguments.
  347. func execGitCmd(ctx context.Context, args []string, config gitCmdConfig) error {
  348. c := exec.CommandContext(ctx, "git", args...)
  349. if config.dir != "" {
  350. c.Dir = config.dir
  351. }
  352. c.Env = append(env(), config.env...)
  353. stdOutAndStdErr := &threadSafeBuffer{}
  354. c.Stdout = stdOutAndStdErr
  355. c.Stderr = stdOutAndStdErr
  356. if config.out != nil {
  357. c.Stdout = io.MultiWriter(c.Stdout, config.out)
  358. }
  359. err := c.Run()
  360. if err != nil {
  361. if len(stdOutAndStdErr.Bytes()) > 0 {
  362. err = errors.New(stdOutAndStdErr.String())
  363. msg := findErrorMessage(stdOutAndStdErr)
  364. if msg != "" {
  365. err = fmt.Errorf("%s, full output:\n %s", msg, err.Error())
  366. }
  367. }
  368. }
  369. if trace {
  370. if traceCommand := traceGitCommand(args, config, stdOutAndStdErr.String()); traceCommand != "" {
  371. println(traceCommand)
  372. }
  373. }
  374. if ctx.Err() == context.DeadlineExceeded {
  375. return errors.Wrap(ctx.Err(), fmt.Sprintf("running git command: %s %v", "git", args))
  376. } else if ctx.Err() == context.Canceled {
  377. return errors.Wrap(ctx.Err(), fmt.Sprintf("context was unexpectedly cancelled when running git command: %s %v", "git", args))
  378. }
  379. return err
  380. }
  381. func env() []string {
  382. env := []string{"GIT_TERMINAL_PROMPT=0"}
  383. // include allowed env vars from os
  384. for _, k := range allowedEnvVars {
  385. if v, ok := os.LookupEnv(k); ok {
  386. env = append(env, k+"="+v)
  387. }
  388. }
  389. return env
  390. }
  391. // check returns true if there are any local changes.
  392. func check(ctx context.Context, workingDir string, subdirs []string, checkFullRepo bool) bool {
  393. // `--quiet` means "exit with 1 if there are changes"
  394. args := []string{"diff", "--quiet"}
  395. if checkFullRepo {
  396. args = append(args, "HEAD", "--")
  397. } else {
  398. args = append(args, "--")
  399. if len(subdirs) > 0 {
  400. args = append(args, subdirs...)
  401. }
  402. }
  403. return execGitCmd(ctx, args, gitCmdConfig{dir: workingDir}) != nil
  404. }
  405. func findErrorMessage(output io.Reader) string {
  406. sc := bufio.NewScanner(output)
  407. for sc.Scan() {
  408. switch {
  409. case strings.HasPrefix(sc.Text(), "fatal: "):
  410. return sc.Text()
  411. case strings.HasPrefix(sc.Text(), "ERROR fatal: "): // Saw this error on ubuntu systems
  412. return sc.Text()
  413. case strings.HasPrefix(sc.Text(), "error:"):
  414. return strings.Trim(sc.Text(), "error: ")
  415. }
  416. }
  417. return ""
  418. }