GitOps for k8s
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

daemon_test.go 22KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913
  1. package daemon
  2. import (
  3. "bufio"
  4. "context"
  5. "fmt"
  6. "os"
  7. "path/filepath"
  8. "strings"
  9. "sync"
  10. "testing"
  11. "time"
  12. "github.com/go-kit/kit/log"
  13. "github.com/stretchr/testify/assert"
  14. "github.com/fluxcd/flux/api/v10"
  15. "github.com/fluxcd/flux/api/v11"
  16. "github.com/fluxcd/flux/api/v6"
  17. "github.com/fluxcd/flux/api/v9"
  18. "github.com/fluxcd/flux/cluster"
  19. "github.com/fluxcd/flux/cluster/kubernetes"
  20. "github.com/fluxcd/flux/cluster/mock"
  21. "github.com/fluxcd/flux/event"
  22. "github.com/fluxcd/flux/git"
  23. "github.com/fluxcd/flux/git/gittest"
  24. "github.com/fluxcd/flux/image"
  25. "github.com/fluxcd/flux/job"
  26. "github.com/fluxcd/flux/manifests"
  27. "github.com/fluxcd/flux/policy"
  28. "github.com/fluxcd/flux/registry"
  29. registryMock "github.com/fluxcd/flux/registry/mock"
  30. "github.com/fluxcd/flux/resource"
  31. fluxsync "github.com/fluxcd/flux/sync"
  32. "github.com/fluxcd/flux/update"
  33. )
  34. const (
  35. // These have to match the values in cluster/kubernetes/testfiles/data.go
  36. wl = "default:deployment/helloworld"
  37. container = "greeter"
  38. ns = "default"
  39. oldHelloImage = "quay.io/weaveworks/helloworld:3" // older in time but newer version!
  40. newHelloImage = "quay.io/weaveworks/helloworld:2"
  41. currentHelloImage = "quay.io/weaveworks/helloworld:master-a000001"
  42. anotherWl = "another:deployment/service"
  43. anotherContainer = "it-doesn't-matter"
  44. anotherImage = "another/service:latest"
  45. invalidNS = "adsajkfldsa"
  46. testVersion = "test"
  47. )
  48. var testBytes = []byte(`{}`)
  49. const timeout = 10 * time.Second
  50. // When I ping, I should get a response
  51. func TestDaemon_Ping(t *testing.T) {
  52. d, start, clean, _, _, _ := mockDaemon(t)
  53. start()
  54. defer clean()
  55. ctx := context.Background()
  56. if d.Ping(ctx) != nil {
  57. t.Fatal("Cluster did not return valid nil ping")
  58. }
  59. }
  60. // When I ask a version, I should get a version
  61. func TestDaemon_Version(t *testing.T) {
  62. d, start, clean, _, _, _ := mockDaemon(t)
  63. start()
  64. defer clean()
  65. ctx := context.Background()
  66. v, err := d.Version(ctx)
  67. if err != nil {
  68. t.Fatalf("Error: %s", err.Error())
  69. }
  70. if v != testVersion {
  71. t.Fatalf("Expected %v but got %v", testVersion, v)
  72. }
  73. }
  74. // When I export it should export the current (mocked) k8s cluster
  75. func TestDaemon_Export(t *testing.T) {
  76. d, start, clean, _, _, _ := mockDaemon(t)
  77. start()
  78. defer clean()
  79. ctx := context.Background()
  80. bytes, err := d.Export(ctx)
  81. if err != nil {
  82. t.Fatalf("Error: %s", err.Error())
  83. }
  84. if string(testBytes) != string(bytes) {
  85. t.Fatalf("Expected %v but got %v", string(testBytes), string(bytes))
  86. }
  87. }
  88. // When I call list workloads, it should list all the workloads
  89. func TestDaemon_ListWorkloads(t *testing.T) {
  90. d, start, clean, _, _, _ := mockDaemon(t)
  91. start()
  92. defer clean()
  93. ctx := context.Background()
  94. // No namespace
  95. s, err := d.ListServices(ctx, "")
  96. if err != nil {
  97. t.Fatalf("Error: %s", err.Error())
  98. }
  99. if len(s) != 2 {
  100. t.Fatalf("Expected %v but got %v", 2, len(s))
  101. }
  102. // Just namespace
  103. s, err = d.ListServices(ctx, ns)
  104. if err != nil {
  105. t.Fatalf("Error: %s", err.Error())
  106. }
  107. if 1 != len(s) {
  108. t.Fatalf("Expected %v but got %v", 1, len(s))
  109. }
  110. // Invalid NS
  111. s, err = d.ListServices(ctx, invalidNS)
  112. if err != nil {
  113. t.Fatalf("Error: %s", err.Error())
  114. }
  115. if len(s) != 0 {
  116. t.Fatalf("Expected %v but got %v", 0, len(s))
  117. }
  118. }
  119. // When I call list workloads with options, it should list all the requested workloads
  120. func TestDaemon_ListWorkloadsWithOptions(t *testing.T) {
  121. d, start, clean, _, _, _ := mockDaemon(t)
  122. start()
  123. defer clean()
  124. ctx := context.Background()
  125. t.Run("no filter", func(t *testing.T) {
  126. s, err := d.ListServicesWithOptions(ctx, v11.ListServicesOptions{})
  127. if err != nil {
  128. t.Fatalf("Error: %s", err.Error())
  129. }
  130. if len(s) != 2 {
  131. t.Fatalf("Expected %v but got %v", 2, len(s))
  132. }
  133. })
  134. t.Run("filter id", func(t *testing.T) {
  135. s, err := d.ListServicesWithOptions(ctx, v11.ListServicesOptions{
  136. Namespace: "",
  137. Services: []resource.ID{resource.MustParseID(wl)}})
  138. if err != nil {
  139. t.Fatalf("Error: %s", err.Error())
  140. }
  141. if len(s) != 1 {
  142. t.Fatalf("Expected %v but got %v", 1, len(s))
  143. }
  144. })
  145. t.Run("filter id and namespace", func(t *testing.T) {
  146. _, err := d.ListServicesWithOptions(ctx, v11.ListServicesOptions{
  147. Namespace: "foo",
  148. Services: []resource.ID{resource.MustParseID(wl)}})
  149. if err == nil {
  150. t.Fatal("Expected error but got nil")
  151. }
  152. })
  153. t.Run("filter unsupported id kind", func(t *testing.T) {
  154. _, err := d.ListServicesWithOptions(ctx, v11.ListServicesOptions{
  155. Namespace: "foo",
  156. Services: []resource.ID{resource.MustParseID("default:unsupportedkind/goodbyeworld")}})
  157. if err == nil {
  158. t.Fatal("Expected error but got nil")
  159. }
  160. })
  161. }
  162. // When I call list images for a workload, it should return images
  163. func TestDaemon_ListImagesWithOptions(t *testing.T) {
  164. d, start, clean, _, _, _ := mockDaemon(t)
  165. start()
  166. defer clean()
  167. ctx := context.Background()
  168. specAll := update.ResourceSpec(update.ResourceSpecAll)
  169. // Service 1
  170. svcID, err := resource.ParseID(wl)
  171. assert.NoError(t, err)
  172. currentImageRef, err := image.ParseRef(currentHelloImage)
  173. assert.NoError(t, err)
  174. newImageRef, err := image.ParseRef(newHelloImage)
  175. assert.NoError(t, err)
  176. oldImageRef, err := image.ParseRef(oldHelloImage)
  177. assert.NoError(t, err)
  178. // Service 2
  179. anotherSvcID, err := resource.ParseID(anotherWl)
  180. assert.NoError(t, err)
  181. anotherImageRef, err := image.ParseRef(anotherImage)
  182. assert.NoError(t, err)
  183. tests := []struct {
  184. name string
  185. opts v10.ListImagesOptions
  186. expectedImages []v6.ImageStatus
  187. expectedNumImages int
  188. shouldError bool
  189. }{
  190. {
  191. name: "All services",
  192. opts: v10.ListImagesOptions{Spec: specAll},
  193. expectedImages: []v6.ImageStatus{
  194. {
  195. ID: svcID,
  196. Containers: []v6.Container{
  197. {
  198. Name: container,
  199. Current: image.Info{ID: currentImageRef},
  200. LatestFiltered: image.Info{ID: newImageRef},
  201. Available: []image.Info{
  202. {ID: newImageRef},
  203. {ID: currentImageRef},
  204. {ID: oldImageRef},
  205. },
  206. AvailableImagesCount: 3,
  207. NewAvailableImagesCount: 1,
  208. FilteredImagesCount: 3,
  209. NewFilteredImagesCount: 1,
  210. },
  211. },
  212. },
  213. {
  214. ID: anotherSvcID,
  215. Containers: []v6.Container{
  216. {
  217. Name: anotherContainer,
  218. Current: image.Info{ID: anotherImageRef},
  219. LatestFiltered: image.Info{},
  220. Available: []image.Info{
  221. {ID: anotherImageRef},
  222. },
  223. AvailableImagesCount: 1,
  224. NewAvailableImagesCount: 0,
  225. FilteredImagesCount: 0, // Excludes latest
  226. NewFilteredImagesCount: 0,
  227. },
  228. },
  229. },
  230. },
  231. shouldError: false,
  232. },
  233. {
  234. name: "Specific service",
  235. opts: v10.ListImagesOptions{Spec: update.ResourceSpec(wl)},
  236. expectedImages: []v6.ImageStatus{
  237. {
  238. ID: svcID,
  239. Containers: []v6.Container{
  240. {
  241. Name: container,
  242. Current: image.Info{ID: currentImageRef},
  243. LatestFiltered: image.Info{ID: newImageRef},
  244. Available: []image.Info{
  245. {ID: newImageRef},
  246. {ID: currentImageRef},
  247. {ID: oldImageRef},
  248. },
  249. AvailableImagesCount: 3,
  250. NewAvailableImagesCount: 1,
  251. FilteredImagesCount: 3,
  252. NewFilteredImagesCount: 1,
  253. },
  254. },
  255. },
  256. },
  257. shouldError: false,
  258. },
  259. {
  260. name: "Override container field selection",
  261. opts: v10.ListImagesOptions{
  262. Spec: specAll,
  263. OverrideContainerFields: []string{"Name", "Current", "NewAvailableImagesCount"},
  264. },
  265. expectedImages: []v6.ImageStatus{
  266. {
  267. ID: svcID,
  268. Containers: []v6.Container{
  269. {
  270. Name: container,
  271. Current: image.Info{ID: currentImageRef},
  272. NewAvailableImagesCount: 1,
  273. },
  274. },
  275. },
  276. {
  277. ID: anotherSvcID,
  278. Containers: []v6.Container{
  279. {
  280. Name: anotherContainer,
  281. Current: image.Info{ID: anotherImageRef},
  282. NewAvailableImagesCount: 0,
  283. },
  284. },
  285. },
  286. },
  287. shouldError: false,
  288. },
  289. {
  290. name: "Override container field selection with invalid field",
  291. opts: v10.ListImagesOptions{
  292. Spec: specAll,
  293. OverrideContainerFields: []string{"InvalidField"},
  294. },
  295. expectedImages: nil,
  296. shouldError: true,
  297. },
  298. {
  299. name: "Specific namespace",
  300. opts: v10.ListImagesOptions{
  301. Spec: specAll,
  302. Namespace: ns,
  303. },
  304. expectedImages: []v6.ImageStatus{
  305. {
  306. ID: svcID,
  307. Containers: []v6.Container{
  308. {
  309. Name: container,
  310. Current: image.Info{ID: currentImageRef},
  311. LatestFiltered: image.Info{ID: newImageRef},
  312. Available: []image.Info{
  313. {ID: newImageRef},
  314. {ID: currentImageRef},
  315. {ID: oldImageRef},
  316. },
  317. AvailableImagesCount: 3,
  318. NewAvailableImagesCount: 1,
  319. FilteredImagesCount: 3,
  320. NewFilteredImagesCount: 1,
  321. },
  322. },
  323. },
  324. },
  325. shouldError: false,
  326. },
  327. {
  328. name: "Specific namespace and service",
  329. opts: v10.ListImagesOptions{
  330. Spec: update.ResourceSpec(wl),
  331. Namespace: ns,
  332. },
  333. expectedImages: nil,
  334. shouldError: true,
  335. },
  336. }
  337. for _, tt := range tests {
  338. t.Run(tt.name, func(t *testing.T) {
  339. is, err := d.ListImagesWithOptions(ctx, tt.opts)
  340. assert.Equal(t, tt.shouldError, err != nil)
  341. // Clear CreatedAt fields for testing
  342. for ri, r := range is {
  343. for ci, c := range r.Containers {
  344. is[ri].Containers[ci].Current.CreatedAt = time.Time{}
  345. is[ri].Containers[ci].LatestFiltered.CreatedAt = time.Time{}
  346. for ai := range c.Available {
  347. is[ri].Containers[ci].Available[ai].CreatedAt = time.Time{}
  348. }
  349. }
  350. }
  351. assert.Equal(t, tt.expectedImages, is)
  352. })
  353. }
  354. }
  355. // When I call notify, it should cause a sync
  356. func TestDaemon_NotifyChange(t *testing.T) {
  357. d, start, clean, mockK8s, events, _ := mockDaemon(t)
  358. w := newWait(t)
  359. ctx := context.Background()
  360. var syncCalled int
  361. var syncDef *cluster.SyncSet
  362. var syncMu sync.Mutex
  363. mockK8s.SyncFunc = func(def cluster.SyncSet) error {
  364. syncMu.Lock()
  365. syncCalled++
  366. syncDef = &def
  367. syncMu.Unlock()
  368. return nil
  369. }
  370. start()
  371. defer clean()
  372. d.NotifyChange(ctx, v9.Change{Kind: v9.GitChange, Source: v9.GitUpdate{}})
  373. w.Eventually(func() bool {
  374. syncMu.Lock()
  375. defer syncMu.Unlock()
  376. return syncCalled == 1
  377. }, "Waiting for sync called")
  378. // Check that sync was called
  379. syncMu.Lock()
  380. defer syncMu.Unlock()
  381. if syncCalled != 1 {
  382. t.Errorf("Sync was not called once, was called %d times", syncCalled)
  383. } else if syncDef == nil {
  384. t.Errorf("Sync was called with a nil syncDef")
  385. }
  386. // Check that history was written to
  387. w.Eventually(func() bool {
  388. es, _ := events.AllEvents(time.Time{}, -1, time.Time{})
  389. for _, e := range es {
  390. if e.Type == event.EventSync {
  391. return true
  392. }
  393. }
  394. return false
  395. }, "Waiting for new sync events")
  396. }
  397. // When I perform a release, it should add a job to update git to the queue
  398. // When I ask about a Job, it should tell me about a job
  399. // When I perform a release, it should update the git repo
  400. func TestDaemon_Release(t *testing.T) {
  401. d, start, clean, _, _, _ := mockDaemon(t)
  402. start()
  403. defer clean()
  404. w := newWait(t)
  405. ctx := context.Background()
  406. // Perform a release
  407. id := updateImage(ctx, d, t)
  408. // Check that job is queued
  409. stat, err := d.JobStatus(ctx, id)
  410. if err != nil {
  411. t.Fatalf("Error: %s", err.Error())
  412. } else if stat.Err != "" {
  413. t.Fatal("Job status error should be empty")
  414. } else if stat.StatusString != job.StatusQueued {
  415. t.Fatalf("Expected %v but got %v", job.StatusQueued, stat.StatusString)
  416. }
  417. // Wait for job to succeed
  418. w.ForJobSucceeded(d, id)
  419. // Wait and check that the git manifest has been altered
  420. w.Eventually(func() bool {
  421. co, err := d.Repo.Clone(ctx, d.GitConfig)
  422. if err != nil {
  423. return false
  424. }
  425. defer co.Clean()
  426. // open a file
  427. dirs := co.AbsolutePaths()
  428. if file, err := os.Open(filepath.Join(dirs[0], "helloworld-deploy.yaml")); err == nil {
  429. // make sure it gets closed
  430. defer file.Close()
  431. // create a new scanner and read the file line by line
  432. scanner := bufio.NewScanner(file)
  433. for scanner.Scan() {
  434. if strings.Contains(scanner.Text(), newHelloImage) {
  435. return true
  436. }
  437. }
  438. } else {
  439. t.Fatal(err)
  440. }
  441. // If we get here we haven't found the line we are looking for.
  442. return false
  443. }, "Waiting for new manifest")
  444. }
  445. // When I update a policy, I expect it to add to the queue
  446. // When I update a policy, it should add an annotation to the manifest
  447. func TestDaemon_PolicyUpdate(t *testing.T) {
  448. d, start, clean, _, _, _ := mockDaemon(t)
  449. start()
  450. defer clean()
  451. w := newWait(t)
  452. ctx := context.Background()
  453. // Push an update to a policy
  454. id := updatePolicy(ctx, t, d)
  455. // Wait for job to succeed
  456. w.ForJobSucceeded(d, id)
  457. // Wait and check for new annotation
  458. w.Eventually(func() bool {
  459. co, err := d.Repo.Clone(ctx, d.GitConfig)
  460. if err != nil {
  461. t.Error(err)
  462. return false
  463. }
  464. defer co.Clean()
  465. cm := manifests.NewRawFiles(co.Dir(), co.AbsolutePaths(), d.Manifests)
  466. m, err := cm.GetAllResourcesByID(context.TODO())
  467. if err != nil {
  468. t.Fatalf("Error: %s", err.Error())
  469. }
  470. return len(m[wl].Policies()) > 0
  471. }, "Waiting for new annotation")
  472. }
  473. // When I call sync status, it should return a commit showing the sync
  474. // that is about to take place. Then it should return empty once it is
  475. // complete
  476. func TestDaemon_SyncStatus(t *testing.T) {
  477. d, start, clean, _, _, _ := mockDaemon(t)
  478. start()
  479. defer clean()
  480. w := newWait(t)
  481. ctx := context.Background()
  482. // Perform a release
  483. id := updateImage(ctx, d, t)
  484. // Get the commit id
  485. stat := w.ForJobSucceeded(d, id)
  486. // Note: I can't test for an expected number of commits > 0
  487. // because I can't control how fast the sync loop updates the cluster
  488. // Once sync'ed to the cluster, it should empty
  489. w.ForSyncStatus(d, stat.Result.Revision, 0)
  490. }
  491. // When I restart fluxd, there won't be any jobs in the cache
  492. func TestDaemon_JobStatusWithNoCache(t *testing.T) {
  493. d, start, clean, _, _, restart := mockDaemon(t)
  494. start()
  495. defer clean()
  496. w := newWait(t)
  497. ctx := context.Background()
  498. // Perform update
  499. id := updatePolicy(ctx, t, d)
  500. // Make sure the job finishes first
  501. w.ForJobSucceeded(d, id)
  502. // Clear the cache like we've just restarted
  503. restart(func() {
  504. d.JobStatusCache = &job.StatusCache{Size: 100}
  505. })
  506. // Now check if we can get the job status from the commit
  507. w.ForJobSucceeded(d, id)
  508. }
  509. func TestDaemon_Automated(t *testing.T) {
  510. d, start, clean, k8s, _, _ := mockDaemon(t)
  511. defer clean()
  512. w := newWait(t)
  513. workload := cluster.Workload{
  514. ID: resource.MakeID(ns, "deployment", "helloworld"),
  515. Containers: cluster.ContainersOrExcuse{
  516. Containers: []resource.Container{
  517. {
  518. Name: container,
  519. Image: mustParseImageRef(currentHelloImage),
  520. },
  521. },
  522. },
  523. }
  524. k8s.SomeWorkloadsFunc = func(ctx context.Context, ids []resource.ID) ([]cluster.Workload, error) {
  525. return []cluster.Workload{workload}, nil
  526. }
  527. start()
  528. // updates from helloworld:master-xxx to helloworld:2
  529. w.ForImageTag(t, d, wl, container, "2")
  530. }
  531. func TestDaemon_Automated_semver(t *testing.T) {
  532. d, start, clean, k8s, _, _ := mockDaemon(t)
  533. defer clean()
  534. w := newWait(t)
  535. resid := resource.MustParseID("default:deployment/semver")
  536. workload := cluster.Workload{
  537. ID: resid,
  538. Containers: cluster.ContainersOrExcuse{
  539. Containers: []resource.Container{
  540. {
  541. Name: container,
  542. Image: mustParseImageRef(currentHelloImage),
  543. },
  544. },
  545. },
  546. }
  547. k8s.SomeWorkloadsFunc = func(ctx context.Context, ids []resource.ID) ([]cluster.Workload, error) {
  548. return []cluster.Workload{workload}, nil
  549. }
  550. start()
  551. // helloworld:3 is older than helloworld:2 but semver orders by version
  552. w.ForImageTag(t, d, resid.String(), container, "3")
  553. }
  554. func makeImageInfo(ref string, t time.Time) image.Info {
  555. return image.Info{ID: mustParseImageRef(ref), CreatedAt: t}
  556. }
  557. func mustParseImageRef(ref string) image.Ref {
  558. r, err := image.ParseRef(ref)
  559. if err != nil {
  560. panic(err)
  561. }
  562. return r
  563. }
  564. func mockDaemon(t *testing.T) (*Daemon, func(), func(), *mock.Mock, *mockEventWriter, func(func())) {
  565. logger := log.NewNopLogger()
  566. singleService := cluster.Workload{
  567. ID: resource.MustParseID(wl),
  568. Containers: cluster.ContainersOrExcuse{
  569. Containers: []resource.Container{
  570. {
  571. Name: container,
  572. Image: mustParseImageRef(currentHelloImage),
  573. },
  574. },
  575. },
  576. }
  577. multiService := []cluster.Workload{
  578. singleService,
  579. {
  580. ID: resource.MakeID("another", "deployment", "service"),
  581. Containers: cluster.ContainersOrExcuse{
  582. Containers: []resource.Container{
  583. {
  584. Name: anotherContainer,
  585. Image: mustParseImageRef(anotherImage),
  586. },
  587. },
  588. },
  589. },
  590. }
  591. repo, repoCleanup := gittest.Repo(t)
  592. syncTag := "flux-test"
  593. params := git.Config{
  594. Branch: "master",
  595. UserName: "example",
  596. UserEmail: "example@example.com",
  597. NotesRef: "fluxtest",
  598. }
  599. var k8s *mock.Mock
  600. {
  601. k8s = &mock.Mock{}
  602. k8s.AllWorkloadsFunc = func(ctx context.Context, maybeNamespace string) ([]cluster.Workload, error) {
  603. if maybeNamespace == ns {
  604. return []cluster.Workload{
  605. singleService,
  606. }, nil
  607. } else if maybeNamespace == "" {
  608. return multiService, nil
  609. }
  610. return []cluster.Workload{}, nil
  611. }
  612. k8s.IsAllowedResourceFunc = func(resource.ID) bool { return true }
  613. k8s.ExportFunc = func(ctx context.Context) ([]byte, error) { return testBytes, nil }
  614. k8s.PingFunc = func() error { return nil }
  615. k8s.SomeWorkloadsFunc = func(ctx context.Context, ids []resource.ID) ([]cluster.Workload, error) {
  616. return []cluster.Workload{
  617. singleService,
  618. }, nil
  619. }
  620. k8s.SyncFunc = func(def cluster.SyncSet) error { return nil }
  621. }
  622. var imageRegistry registry.Registry
  623. {
  624. img0 := makeImageInfo(oldHelloImage, time.Now().Add(-1*time.Second))
  625. img1 := makeImageInfo(currentHelloImage, time.Now())
  626. img2 := makeImageInfo(newHelloImage, time.Now().Add(1*time.Second))
  627. img3 := makeImageInfo("another/service:latest", time.Now().Add(1*time.Second))
  628. imageRegistry = &registryMock.Registry{
  629. Images: []image.Info{
  630. img1,
  631. img2,
  632. img3,
  633. img0,
  634. },
  635. }
  636. }
  637. events := &mockEventWriter{}
  638. // Shutdown chan and waitgroups
  639. jshutdown := make(chan struct{})
  640. dshutdown := make(chan struct{})
  641. jwg := &sync.WaitGroup{}
  642. dwg := &sync.WaitGroup{}
  643. // Jobs queue (starts itself)
  644. jobs := job.NewQueue(jshutdown, jwg)
  645. manifests := kubernetes.NewManifests(kubernetes.ConstNamespacer("default"), log.NewLogfmtLogger(os.Stdout))
  646. gitSync, _ := fluxsync.NewGitTagSyncProvider(repo, syncTag, "", false, params)
  647. // Finally, the daemon
  648. d := &Daemon{
  649. Repo: repo,
  650. GitConfig: params,
  651. Cluster: k8s,
  652. Manifests: manifests,
  653. Registry: imageRegistry,
  654. V: testVersion,
  655. Jobs: jobs,
  656. JobStatusCache: &job.StatusCache{Size: 100},
  657. EventWriter: events,
  658. Logger: logger,
  659. LoopVars: &LoopVars{GitTimeout: timeout, SyncState: gitSync},
  660. }
  661. start := func() {
  662. if err := repo.Ready(context.Background()); err != nil {
  663. t.Fatal(err)
  664. }
  665. dwg.Add(1)
  666. go d.Loop(dshutdown, dwg, logger)
  667. }
  668. stop := func() {
  669. // Close daemon first so any outstanding jobs are picked up by the queue, otherwise
  670. // calls to Queue#Enqueue() will block forever. Jobs may be enqueued if the daemon's
  671. // image polling picks up automated updates.
  672. close(dshutdown)
  673. dwg.Wait()
  674. close(jshutdown)
  675. jwg.Wait()
  676. repoCleanup()
  677. }
  678. restart := func(f func()) {
  679. close(dshutdown)
  680. dwg.Wait()
  681. f()
  682. dshutdown = make(chan struct{})
  683. start()
  684. }
  685. return d, start, stop, k8s, events, restart
  686. }
  687. type mockEventWriter struct {
  688. events []event.Event
  689. sync.Mutex
  690. }
  691. func (w *mockEventWriter) LogEvent(e event.Event) error {
  692. w.Lock()
  693. defer w.Unlock()
  694. w.events = append(w.events, e)
  695. return nil
  696. }
  697. func (w *mockEventWriter) AllEvents(_ time.Time, _ int64, _ time.Time) ([]event.Event, error) {
  698. w.Lock()
  699. defer w.Unlock()
  700. return w.events, nil
  701. }
  702. // DAEMON TEST HELPERS
  703. type wait struct {
  704. t *testing.T
  705. timeout time.Duration
  706. }
  707. func newWait(t *testing.T) wait {
  708. return wait{
  709. t: t,
  710. timeout: timeout,
  711. }
  712. }
  713. const interval = 10 * time.Millisecond
  714. func (w *wait) Eventually(f func() bool, msg string) {
  715. stop := time.Now().Add(w.timeout)
  716. for time.Now().Before(stop) {
  717. if f() {
  718. return
  719. }
  720. time.Sleep(interval)
  721. }
  722. w.t.Fatal(msg)
  723. }
  724. func (w *wait) ForJobSucceeded(d *Daemon, jobID job.ID) job.Status {
  725. var stat job.Status
  726. var err error
  727. ctx := context.Background()
  728. w.Eventually(func() bool {
  729. stat, err = d.JobStatus(ctx, jobID)
  730. if err != nil {
  731. return false
  732. }
  733. switch stat.StatusString {
  734. case job.StatusSucceeded:
  735. return true
  736. case job.StatusFailed:
  737. w.t.Fatal(stat.Err)
  738. return true
  739. default:
  740. return false
  741. }
  742. }, "Waiting for job to succeed")
  743. return stat
  744. }
  745. func (w *wait) ForSyncStatus(d *Daemon, rev string, expectedNumCommits int) []string {
  746. var revs []string
  747. var err error
  748. w.Eventually(func() bool {
  749. ctx := context.Background()
  750. revs, err = d.SyncStatus(ctx, rev)
  751. return err == nil && len(revs) == expectedNumCommits
  752. }, fmt.Sprintf("Waiting for sync status to have %d commits", expectedNumCommits))
  753. return revs
  754. }
  755. func (w *wait) ForImageTag(t *testing.T, d *Daemon, workload, container, tag string) {
  756. w.Eventually(func() bool {
  757. co, err := d.Repo.Clone(context.TODO(), d.GitConfig)
  758. if err != nil {
  759. return false
  760. }
  761. defer co.Clean()
  762. cm := manifests.NewRawFiles(co.Dir(), co.AbsolutePaths(), d.Manifests)
  763. resources, err := cm.GetAllResourcesByID(context.TODO())
  764. assert.NoError(t, err)
  765. workload, ok := resources[workload].(resource.Workload)
  766. assert.True(t, ok)
  767. for _, c := range workload.Containers() {
  768. if c.Name == container && c.Image.Tag == tag {
  769. return true
  770. }
  771. }
  772. return false
  773. }, fmt.Sprintf("Waiting for image tag: %q", tag))
  774. }
  775. func updateImage(ctx context.Context, d *Daemon, t *testing.T) job.ID {
  776. return updateManifest(ctx, t, d, update.Spec{
  777. Type: update.Images,
  778. Spec: update.ReleaseImageSpec{
  779. Kind: update.ReleaseKindExecute,
  780. ServiceSpecs: []update.ResourceSpec{update.ResourceSpecAll},
  781. ImageSpec: newHelloImage,
  782. },
  783. })
  784. }
  785. func updatePolicy(ctx context.Context, t *testing.T, d *Daemon) job.ID {
  786. return updateManifest(ctx, t, d, update.Spec{
  787. Type: update.Policy,
  788. Spec: resource.PolicyUpdates{
  789. resource.MustParseID("default:deployment/helloworld"): {
  790. Add: policy.Set{
  791. policy.Locked: "true",
  792. },
  793. },
  794. },
  795. })
  796. }
  797. func updateManifest(ctx context.Context, t *testing.T, d *Daemon, spec update.Spec) job.ID {
  798. id, err := d.UpdateManifests(ctx, spec)
  799. if err != nil {
  800. t.Fatalf("Error: %s", err.Error())
  801. }
  802. if id == "" {
  803. t.Fatal("id should not be empty")
  804. }
  805. return id
  806. }