GitOps for k8s
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

daemon_test.go 15KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604
  1. package daemon
  2. import (
  3. "bufio"
  4. "context"
  5. "fmt"
  6. "os"
  7. "path/filepath"
  8. "strings"
  9. "sync"
  10. "testing"
  11. "time"
  12. "github.com/go-kit/kit/log"
  13. "github.com/weaveworks/flux"
  14. "github.com/weaveworks/flux/api/v6"
  15. "github.com/weaveworks/flux/api/v9"
  16. "github.com/weaveworks/flux/cluster"
  17. "github.com/weaveworks/flux/cluster/kubernetes"
  18. kresource "github.com/weaveworks/flux/cluster/kubernetes/resource"
  19. "github.com/weaveworks/flux/cluster/kubernetes/testfiles"
  20. "github.com/weaveworks/flux/event"
  21. "github.com/weaveworks/flux/git"
  22. "github.com/weaveworks/flux/git/gittest"
  23. "github.com/weaveworks/flux/image"
  24. "github.com/weaveworks/flux/job"
  25. "github.com/weaveworks/flux/policy"
  26. "github.com/weaveworks/flux/registry"
  27. registryMock "github.com/weaveworks/flux/registry/mock"
  28. "github.com/weaveworks/flux/resource"
  29. "github.com/weaveworks/flux/update"
  30. )
  31. const (
  32. // These have to match the values in cluster/kubernetes/testfiles/data.go
  33. svc = "default:deployment/helloworld"
  34. container = "greeter"
  35. ns = "default"
  36. newHelloImage = "quay.io/weaveworks/helloworld:2"
  37. currentHelloImage = "quay.io/weaveworks/helloworld:master-a000001"
  38. invalidNS = "adsajkfldsa"
  39. testVersion = "test"
  40. )
  41. var (
  42. testBytes = []byte(`{}`)
  43. timeout = 5 * time.Second
  44. )
  45. // When I ping, I should get a response
  46. func TestDaemon_Ping(t *testing.T) {
  47. d, start, clean, _, _ := mockDaemon(t)
  48. start()
  49. defer clean()
  50. ctx := context.Background()
  51. if d.Ping(ctx) != nil {
  52. t.Fatal("Cluster did not return valid nil ping")
  53. }
  54. }
  55. // When I ask a version, I should get a version
  56. func TestDaemon_Version(t *testing.T) {
  57. d, start, clean, _, _ := mockDaemon(t)
  58. start()
  59. defer clean()
  60. ctx := context.Background()
  61. v, err := d.Version(ctx)
  62. if err != nil {
  63. t.Fatalf("Error: %s", err.Error())
  64. }
  65. if v != testVersion {
  66. t.Fatalf("Expected %v but got %v", testVersion, v)
  67. }
  68. }
  69. // When I export it should export the current (mocked) k8s cluster
  70. func TestDaemon_Export(t *testing.T) {
  71. d, start, clean, _, _ := mockDaemon(t)
  72. start()
  73. defer clean()
  74. ctx := context.Background()
  75. bytes, err := d.Export(ctx)
  76. if err != nil {
  77. t.Fatalf("Error: %s", err.Error())
  78. }
  79. if string(testBytes) != string(bytes) {
  80. t.Fatalf("Expected %v but got %v", string(testBytes), string(bytes))
  81. }
  82. }
  83. // When I call list services, it should list all the services
  84. func TestDaemon_ListServices(t *testing.T) {
  85. d, start, clean, _, _ := mockDaemon(t)
  86. start()
  87. defer clean()
  88. ctx := context.Background()
  89. // No namespace
  90. s, err := d.ListServices(ctx, "")
  91. if err != nil {
  92. t.Fatalf("Error: %s", err.Error())
  93. }
  94. if len(s) != 2 {
  95. t.Fatalf("Expected %v but got %v", 2, len(s))
  96. }
  97. // Just namespace
  98. s, err = d.ListServices(ctx, ns)
  99. if err != nil {
  100. t.Fatalf("Error: %s", err.Error())
  101. }
  102. if 1 != len(s) {
  103. t.Fatalf("Expected %v but got %v", 1, len(s))
  104. }
  105. // Invalid NS
  106. s, err = d.ListServices(ctx, invalidNS)
  107. if err != nil {
  108. t.Fatalf("Error: %s", err.Error())
  109. }
  110. if len(s) != 0 {
  111. t.Fatalf("Expected %v but got %v", 0, len(s))
  112. }
  113. }
  114. // When I call list images for a service, it should return images
  115. func TestDaemon_ListImages(t *testing.T) {
  116. d, start, clean, _, _ := mockDaemon(t)
  117. start()
  118. defer clean()
  119. ctx := context.Background()
  120. // List all images for services
  121. ss := update.ResourceSpec(update.ResourceSpecAll)
  122. is, err := d.ListImages(ctx, ss)
  123. if err != nil {
  124. t.Fatalf("Error: %s", err.Error())
  125. }
  126. ids := imageIDs(is)
  127. if 3 != len(ids) {
  128. t.Fatalf("Expected %v but got %v", 3, len(ids))
  129. }
  130. // List images for specific service
  131. ss = update.ResourceSpec(svc)
  132. is, err = d.ListImages(ctx, ss)
  133. if err != nil {
  134. t.Fatalf("Error: %s", err.Error())
  135. }
  136. ids = imageIDs(is)
  137. if 2 != len(ids) {
  138. t.Fatalf("Expected %v but got %v", 2, len(ids))
  139. }
  140. }
  141. // When I call notify, it should cause a sync
  142. func TestDaemon_NotifyChange(t *testing.T) {
  143. d, start, clean, mockK8s, events := mockDaemon(t)
  144. w := newWait(t)
  145. ctx := context.Background()
  146. var syncCalled int
  147. var syncDef *cluster.SyncDef
  148. var syncMu sync.Mutex
  149. mockK8s.SyncFunc = func(def cluster.SyncDef) error {
  150. syncMu.Lock()
  151. syncCalled++
  152. syncDef = &def
  153. syncMu.Unlock()
  154. return nil
  155. }
  156. start()
  157. defer clean()
  158. d.NotifyChange(ctx, v9.Change{Kind: v9.GitChange, Source: v9.GitUpdate{}})
  159. w.Eventually(func() bool {
  160. syncMu.Lock()
  161. defer syncMu.Unlock()
  162. return syncCalled == 1
  163. }, "Waiting for sync called")
  164. // Check that sync was called
  165. syncMu.Lock()
  166. defer syncMu.Unlock()
  167. if syncCalled != 1 {
  168. t.Errorf("Sync was not called once, was called %d times", syncCalled)
  169. } else if syncDef == nil {
  170. t.Errorf("Sync was called with a nil syncDef")
  171. } else if len(syncDef.Actions) != len(testfiles.ServiceMap("unimportant")) {
  172. t.Errorf("Sync was not called with the %d actions, was called with: %d", len(testfiles.Files), len(syncDef.Actions))
  173. }
  174. // Check that history was written to
  175. var e []event.Event
  176. w.Eventually(func() bool {
  177. e, _ = events.AllEvents(time.Time{}, -1, time.Time{})
  178. return len(e) > 0
  179. }, "Waiting for new events")
  180. if 1 != len(e) {
  181. t.Fatal("Expected one log event from the sync, but got", len(e))
  182. } else if event.EventSync != e[0].Type {
  183. t.Fatalf("Expected event with type %s but got %s", event.EventSync, e[0].Type)
  184. }
  185. }
  186. // When I perform a release, it should add a job to update git to the queue
  187. // When I ask about a Job, it should tell me about a job
  188. // When I perform a release, it should update the git repo
  189. func TestDaemon_Release(t *testing.T) {
  190. d, start, clean, _, _ := mockDaemon(t)
  191. start()
  192. defer clean()
  193. w := newWait(t)
  194. ctx := context.Background()
  195. // Perform a release
  196. id := updateImage(ctx, d, t)
  197. // Check that job is queued
  198. stat, err := d.JobStatus(ctx, id)
  199. if err != nil {
  200. t.Fatalf("Error: %s", err.Error())
  201. } else if stat.Err != "" {
  202. t.Fatal("Job status error should be empty")
  203. } else if stat.StatusString != job.StatusQueued {
  204. t.Fatalf("Expected %v but got %v", job.StatusQueued, stat.StatusString)
  205. }
  206. // Wait for job to succeed
  207. w.ForJobSucceeded(d, id)
  208. // Wait and check that the git manifest has been altered
  209. w.Eventually(func() bool {
  210. co, err := d.Repo.Clone(ctx, d.GitConfig)
  211. if err != nil {
  212. return false
  213. }
  214. defer co.Clean()
  215. // open a file
  216. if file, err := os.Open(filepath.Join(co.ManifestDir(), "helloworld-deploy.yaml")); err == nil {
  217. // make sure it gets closed
  218. defer file.Close()
  219. // create a new scanner and read the file line by line
  220. scanner := bufio.NewScanner(file)
  221. for scanner.Scan() {
  222. if strings.Contains(scanner.Text(), newHelloImage) {
  223. return true
  224. }
  225. }
  226. } else {
  227. t.Fatal(err)
  228. }
  229. // If we get here we haven't found the line we are looking for.
  230. return false
  231. }, "Waiting for new manifest")
  232. }
  233. // When I update a policy, I expect it to add to the queue
  234. // When I update a policy, it should add an annotation to the manifest
  235. func TestDaemon_PolicyUpdate(t *testing.T) {
  236. d, start, clean, _, _ := mockDaemon(t)
  237. start()
  238. defer clean()
  239. w := newWait(t)
  240. ctx := context.Background()
  241. // Push an update to a policy
  242. id := updatePolicy(ctx, t, d)
  243. // Wait for job to succeed
  244. w.ForJobSucceeded(d, id)
  245. // Wait and check for new annotation
  246. w.Eventually(func() bool {
  247. co, err := d.Repo.Clone(ctx, d.GitConfig)
  248. if err != nil {
  249. t.Error(err)
  250. return false
  251. }
  252. defer co.Clean()
  253. m, err := d.Manifests.LoadManifests(co.Dir(), co.ManifestDir())
  254. if err != nil {
  255. t.Fatalf("Error: %s", err.Error())
  256. }
  257. return len(m[svc].Policy()) > 0
  258. }, "Waiting for new annotation")
  259. }
  260. // When I call sync status, it should return a commit showing the sync
  261. // that is about to take place. Then it should return empty once it is
  262. // complete
  263. func TestDaemon_SyncStatus(t *testing.T) {
  264. d, start, clean, _, _ := mockDaemon(t)
  265. start()
  266. defer clean()
  267. w := newWait(t)
  268. ctx := context.Background()
  269. // Perform a release
  270. id := updateImage(ctx, d, t)
  271. // Get the commit id
  272. stat := w.ForJobSucceeded(d, id)
  273. // Note: I can't test for an expected number of commits > 0
  274. // because I can't control how fast the sync loop updates the cluster
  275. // Once sync'ed to the cluster, it should empty
  276. w.ForSyncStatus(d, stat.Result.Revision, 0)
  277. }
  278. // When I restart fluxd, there won't be any jobs in the cache
  279. func TestDaemon_JobStatusWithNoCache(t *testing.T) {
  280. d, start, clean, _, _ := mockDaemon(t)
  281. start()
  282. defer clean()
  283. w := newWait(t)
  284. ctx := context.Background()
  285. // Perform update
  286. id := updatePolicy(ctx, t, d)
  287. // Make sure the job finishes first
  288. w.ForJobSucceeded(d, id)
  289. // Clear the cache like we've just restarted
  290. d.JobStatusCache = &job.StatusCache{Size: 100}
  291. // Now check if we can get the job status from the commit
  292. w.ForJobSucceeded(d, id)
  293. }
  294. func makeImageInfo(ref string, t time.Time) image.Info {
  295. return image.Info{ID: mustParseImageRef(ref), CreatedAt: t}
  296. }
  297. func mustParseImageRef(ref string) image.Ref {
  298. r, err := image.ParseRef(ref)
  299. if err != nil {
  300. panic(err)
  301. }
  302. return r
  303. }
  304. func mockDaemon(t *testing.T) (*Daemon, func(), func(), *cluster.Mock, *mockEventWriter) {
  305. logger := log.NewNopLogger()
  306. singleService := cluster.Controller{
  307. ID: flux.MustParseResourceID(svc),
  308. Containers: cluster.ContainersOrExcuse{
  309. Containers: []resource.Container{
  310. {
  311. Name: container,
  312. Image: mustParseImageRef(currentHelloImage),
  313. },
  314. },
  315. },
  316. }
  317. multiService := []cluster.Controller{
  318. singleService,
  319. cluster.Controller{
  320. ID: flux.MakeResourceID("another", "deployment", "service"),
  321. Containers: cluster.ContainersOrExcuse{
  322. Containers: []resource.Container{
  323. {
  324. Name: "it-doesn't-matter",
  325. Image: mustParseImageRef("another/service:latest"),
  326. },
  327. },
  328. },
  329. },
  330. }
  331. repo, repoCleanup := gittest.Repo(t)
  332. params := git.Config{
  333. Branch: "master",
  334. UserName: "example",
  335. UserEmail: "example@example.com",
  336. SyncTag: "flux-test",
  337. NotesRef: "fluxtest",
  338. }
  339. var k8s *cluster.Mock
  340. {
  341. k8s = &cluster.Mock{}
  342. k8s.AllServicesFunc = func(maybeNamespace string) ([]cluster.Controller, error) {
  343. if maybeNamespace == ns {
  344. return []cluster.Controller{
  345. singleService,
  346. }, nil
  347. } else if maybeNamespace == "" {
  348. return multiService, nil
  349. }
  350. return []cluster.Controller{}, nil
  351. }
  352. k8s.ExportFunc = func() ([]byte, error) { return testBytes, nil }
  353. k8s.FindDefinedServicesFunc = (&kubernetes.Manifests{}).FindDefinedServices
  354. k8s.LoadManifestsFunc = kresource.Load
  355. k8s.ParseManifestsFunc = func(allDefs []byte) (map[string]resource.Resource, error) {
  356. return kresource.ParseMultidoc(allDefs, "test")
  357. }
  358. k8s.PingFunc = func() error { return nil }
  359. k8s.ServicesWithPoliciesFunc = (&kubernetes.Manifests{}).ServicesWithPolicies
  360. k8s.SomeServicesFunc = func([]flux.ResourceID) ([]cluster.Controller, error) {
  361. return []cluster.Controller{
  362. singleService,
  363. }, nil
  364. }
  365. k8s.SyncFunc = func(def cluster.SyncDef) error { return nil }
  366. k8s.UpdatePoliciesFunc = (&kubernetes.Manifests{}).UpdatePolicies
  367. k8s.UpdateDefinitionFunc = (&kubernetes.Manifests{}).UpdateDefinition
  368. }
  369. var imageRegistry registry.Registry
  370. {
  371. img1 := makeImageInfo(currentHelloImage, time.Now())
  372. img2 := makeImageInfo(newHelloImage, time.Now().Add(1*time.Second))
  373. img3 := makeImageInfo("another/service:latest", time.Now().Add(1*time.Second))
  374. imageRegistry = &registryMock.Registry{
  375. Images: []image.Info{
  376. img1,
  377. img2,
  378. img3,
  379. },
  380. }
  381. }
  382. events := &mockEventWriter{}
  383. // Shutdown chan and waitgroups
  384. shutdown := make(chan struct{})
  385. wg := &sync.WaitGroup{}
  386. // Jobs queue (starts itself)
  387. jobs := job.NewQueue(shutdown, wg)
  388. // Finally, the daemon
  389. d := &Daemon{
  390. Repo: repo,
  391. GitConfig: params,
  392. Cluster: k8s,
  393. Manifests: &kubernetes.Manifests{},
  394. Registry: imageRegistry,
  395. V: testVersion,
  396. Jobs: jobs,
  397. JobStatusCache: &job.StatusCache{Size: 100},
  398. EventWriter: events,
  399. Logger: logger,
  400. LoopVars: &LoopVars{},
  401. }
  402. start := func() {
  403. wg.Add(1)
  404. go repo.Start(shutdown, wg)
  405. gittest.WaitForRepoReady(repo, t)
  406. wg.Add(1)
  407. go d.Loop(shutdown, wg, logger)
  408. }
  409. stop := func() {
  410. // Close daemon first so we don't get errors if the queue closes before the daemon
  411. close(shutdown)
  412. wg.Wait()
  413. repoCleanup()
  414. }
  415. return d, start, stop, k8s, events
  416. }
  417. type mockEventWriter struct {
  418. events []event.Event
  419. sync.Mutex
  420. }
  421. func (w *mockEventWriter) LogEvent(e event.Event) error {
  422. w.Lock()
  423. defer w.Unlock()
  424. w.events = append(w.events, e)
  425. return nil
  426. }
  427. func (w *mockEventWriter) AllEvents(_ time.Time, _ int64, _ time.Time) ([]event.Event, error) {
  428. w.Lock()
  429. defer w.Unlock()
  430. return w.events, nil
  431. }
  432. // DAEMON TEST HELPERS
  433. type wait struct {
  434. t *testing.T
  435. timeout time.Duration
  436. }
  437. func newWait(t *testing.T) wait {
  438. return wait{
  439. t: t,
  440. timeout: timeout,
  441. }
  442. }
  443. const interval = 10 * time.Millisecond
  444. func (w *wait) Eventually(f func() bool, msg string) {
  445. stop := time.Now().Add(w.timeout)
  446. for time.Now().Before(stop) {
  447. if f() {
  448. return
  449. }
  450. time.Sleep(interval)
  451. }
  452. w.t.Fatal(msg)
  453. }
  454. func (w *wait) ForJobSucceeded(d *Daemon, jobID job.ID) job.Status {
  455. var stat job.Status
  456. var err error
  457. ctx := context.Background()
  458. w.Eventually(func() bool {
  459. stat, err = d.JobStatus(ctx, jobID)
  460. if err != nil {
  461. return false
  462. }
  463. switch stat.StatusString {
  464. case job.StatusSucceeded:
  465. return true
  466. case job.StatusFailed:
  467. w.t.Fatal(stat.Err)
  468. return true
  469. default:
  470. return false
  471. }
  472. }, "Waiting for job to succeed")
  473. return stat
  474. }
  475. func (w *wait) ForSyncStatus(d *Daemon, rev string, expectedNumCommits int) []string {
  476. var revs []string
  477. var err error
  478. w.Eventually(func() bool {
  479. ctx := context.Background()
  480. revs, err = d.SyncStatus(ctx, rev)
  481. return err == nil && len(revs) == expectedNumCommits
  482. }, fmt.Sprintf("Waiting for sync status to have %d commits", expectedNumCommits))
  483. return revs
  484. }
  485. func imageIDs(status []v6.ImageStatus) []image.Info {
  486. var availableImgs []image.Info
  487. for _, i := range status {
  488. for _, c := range i.Containers {
  489. availableImgs = append(availableImgs, c.Available...)
  490. }
  491. }
  492. return availableImgs
  493. }
  494. func updateImage(ctx context.Context, d *Daemon, t *testing.T) job.ID {
  495. return updateManifest(ctx, t, d, update.Spec{
  496. Type: update.Images,
  497. Spec: update.ReleaseSpec{
  498. Kind: update.ReleaseKindExecute,
  499. ServiceSpecs: []update.ResourceSpec{update.ResourceSpecAll},
  500. ImageSpec: newHelloImage,
  501. },
  502. })
  503. }
  504. func updatePolicy(ctx context.Context, t *testing.T, d *Daemon) job.ID {
  505. return updateManifest(ctx, t, d, update.Spec{
  506. Type: update.Policy,
  507. Spec: policy.Updates{
  508. flux.MustParseResourceID("default:deployment/helloworld"): {
  509. Add: policy.Set{
  510. policy.Locked: "true",
  511. },
  512. },
  513. },
  514. })
  515. }
  516. func updateManifest(ctx context.Context, t *testing.T, d *Daemon, spec update.Spec) job.ID {
  517. id, err := d.UpdateManifests(ctx, spec)
  518. if err != nil {
  519. t.Fatalf("Error: %s", err.Error())
  520. }
  521. if id == "" {
  522. t.Fatal("id should not be empty")
  523. }
  524. return id
  525. }