Browse Source

Sort resources to apply into dependency order

Kubernetes resource kinds have a partial order relation of (loosely)
"may refer to": a Deployment may mount a ConfigMap as a volume; most
resources are scoped to a namespace; a RoleBinding refers to a Role or
ClusterRole; and so on.

Usually, you want the referenced resources to be created or changed
before the referring resources. In general, Kubernetes is designed so
that it will sort itself out eventually, but things go more smoothly
if things are present before they are needed.

Here I've boiled the ordering down to a small number of ranks, with
each rank containing kinds that refer only to kinds in the ranks
before. When applying resources, they are sorted by rank, so that
resources that depend on other resources will get updated or created
after those others.
Michael Bridgen 1 year ago
parent
commit
8ad9943f37
3 changed files with 99 additions and 28 deletions
  1. 7
    5
      cluster/kubernetes/kubernetes.go
  2. 60
    22
      cluster/kubernetes/sync.go
  3. 32
    1
      cluster/kubernetes/sync_test.go

+ 7
- 5
cluster/kubernetes/kubernetes.go View File

@@ -43,13 +43,15 @@ type extendedClient struct {
43 43
 
44 44
 // --- internal types for keeping track of syncing
45 45
 
46
+type metadata struct {
47
+	Name      string `yaml:"name"`
48
+	Namespace string `yaml:"namespace"`
49
+}
50
+
46 51
 type apiObject struct {
47 52
 	resource.Resource
48
-	Kind     string `yaml:"kind"`
49
-	Metadata struct {
50
-		Name      string `yaml:"name"`
51
-		Namespace string `yaml:"namespace"`
52
-	} `yaml:"metadata"`
53
+	Kind     string   `yaml:"kind"`
54
+	Metadata metadata `yaml:"metadata"`
53 55
 }
54 56
 
55 57
 // A convenience for getting an minimal object from some bytes.

+ 60
- 22
cluster/kubernetes/sync.go View File

@@ -5,6 +5,7 @@ import (
5 5
 	"fmt"
6 6
 	"io"
7 7
 	"os/exec"
8
+	"sort"
8 9
 	"strings"
9 10
 	"time"
10 11
 
@@ -16,23 +17,15 @@ import (
16 17
 )
17 18
 
18 19
 type changeSet struct {
19
-	nsObjs   map[string][]*apiObject
20
-	noNsObjs map[string][]*apiObject
20
+	objs map[string][]*apiObject
21 21
 }
22 22
 
23 23
 func makeChangeSet() changeSet {
24
-	return changeSet{
25
-		nsObjs:   make(map[string][]*apiObject),
26
-		noNsObjs: make(map[string][]*apiObject),
27
-	}
24
+	return changeSet{objs: make(map[string][]*apiObject)}
28 25
 }
29 26
 
30 27
 func (c *changeSet) stage(cmd string, o *apiObject) {
31
-	if o.hasNamespace() {
32
-		c.nsObjs[cmd] = append(c.nsObjs[cmd], o)
33
-	} else {
34
-		c.noNsObjs[cmd] = append(c.noNsObjs[cmd], o)
35
-	}
28
+	c.objs[cmd] = append(c.objs[cmd], o)
36 29
 }
37 30
 
38 31
 // Applier is something that will apply a changeset to the cluster.
@@ -78,9 +71,50 @@ func (c *Kubectl) connectArgs() []string {
78 71
 	return args
79 72
 }
80 73
 
74
+// rankOfKind returns an int denoting the position of the given kind
75
+// in the partial ordering of Kubernetes resources, according to which
76
+// kinds depend on which (derived by hand).
77
+func rankOfKind(kind string) int {
78
+	switch kind {
79
+	// Namespaces answer to NOONE
80
+	case "Namespace":
81
+		return 0
82
+	// These don't go in namespaces; or do, but don't depend on anything else
83
+	case "ServiceAccount", "ClusterRole", "Role", "PersistentVolume", "Service":
84
+		return 1
85
+	// These depend on something above, but not each other
86
+	case "ResourceQuota", "LimitRange", "Secret", "ConfigMap", "RoleBinding", "ClusterRoleBinding", "PersistentVolumeClaim", "Ingress":
87
+		return 2
88
+	// Same deal, next layer
89
+	case "DaemonSet", "Deployment", "ReplicationController", "ReplicaSet", "Job", "CronJob", "StatefulSet":
90
+		return 3
91
+	// Assumption: anything not mentioned isn't depended _upon_, so
92
+	// can come last.
93
+	default:
94
+		return 4
95
+	}
96
+}
97
+
98
+type applyOrder []*apiObject
99
+
100
+func (objs applyOrder) Len() int {
101
+	return len(objs)
102
+}
103
+
104
+func (objs applyOrder) Swap(i, j int) {
105
+	objs[i], objs[j] = objs[j], objs[i]
106
+}
107
+
108
+func (objs applyOrder) Less(i, j int) bool {
109
+	ranki, rankj := rankOfKind(objs[i].Kind), rankOfKind(objs[j].Kind)
110
+	if ranki == rankj {
111
+		return objs[i].Metadata.Name < objs[j].Metadata.Name
112
+	}
113
+	return ranki < rankj
114
+}
115
+
81 116
 func (c *Kubectl) apply(logger log.Logger, cs changeSet) (errs cluster.SyncError) {
82
-	f := func(m map[string][]*apiObject, cmd string, args ...string) {
83
-		objs := m[cmd]
117
+	f := func(objs []*apiObject, cmd string, args ...string) {
84 118
 		if len(objs) == 0 {
85 119
 			return
86 120
 		}
@@ -96,15 +130,19 @@ func (c *Kubectl) apply(logger log.Logger, cs changeSet) (errs cluster.SyncError
96 130
 		}
97 131
 	}
98 132
 
99
-	// When deleting resources we must ensure any resource in a non-default
100
-	// namespace is deleted before the namespace that it is in. Since namespace
101
-	// resources don't specify a namespace, this ordering guarantees that.
102
-	f(cs.nsObjs, "delete")
103
-	f(cs.noNsObjs, "delete", "--namespace", "default")
104
-	// Likewise, when applying resources we must ensure the namespace is applied
105
-	// first, so we run the commands the other way round.
106
-	f(cs.noNsObjs, "apply", "--namespace", "default")
107
-	f(cs.nsObjs, "apply")
133
+	// When deleting objects, the only real concern is that we don't
134
+	// try to delete things that have already been deleted by
135
+	// Kubernete's GC -- most notably, resources in a namespace which
136
+	// is also being deleted. GC does not have the dependency ranking,
137
+	// but we can use it as a shortcut to avoid the above problem at
138
+	// least.
139
+	objs := cs.objs["delete"]
140
+	sort.Sort(sort.Reverse(applyOrder(objs)))
141
+	f(objs, "delete")
142
+
143
+	objs = cs.objs["apply"]
144
+	sort.Sort(applyOrder(objs))
145
+	f(objs, "apply")
108 146
 	return errs
109 147
 }
110 148
 

cluster/kubernetes/kubernetes_test.go → cluster/kubernetes/sync_test.go View File

@@ -1,6 +1,7 @@
1 1
 package kubernetes
2 2
 
3 3
 import (
4
+	"sort"
4 5
 	"testing"
5 6
 
6 7
 	"github.com/go-kit/kit/log"
@@ -15,7 +16,7 @@ type mockApplier struct {
15 16
 }
16 17
 
17 18
 func (m *mockApplier) apply(_ log.Logger, c changeSet) cluster.SyncError {
18
-	if len(c.nsObjs) != 0 || len(c.noNsObjs) != 0 {
19
+	if len(c.objs) != 0 {
19 20
 		m.commandRun = true
20 21
 	}
21 22
 	return nil
@@ -79,3 +80,33 @@ func TestSyncMalformed(t *testing.T) {
79 80
 		t.Error("expected no commands run")
80 81
 	}
81 82
 }
83
+
84
+// TestApplyOrder checks that applyOrder works as expected.
85
+func TestApplyOrder(t *testing.T) {
86
+	objs := []*apiObject{
87
+		{
88
+			Kind: "Deployment",
89
+			Metadata: metadata{
90
+				Name: "deploy",
91
+			},
92
+		},
93
+		{
94
+			Kind: "Secret",
95
+			Metadata: metadata{
96
+				Name: "secret",
97
+			},
98
+		},
99
+		{
100
+			Kind: "Namespace",
101
+			Metadata: metadata{
102
+				Name: "namespace",
103
+			},
104
+		},
105
+	}
106
+	sort.Sort(applyOrder(objs))
107
+	for i, name := range []string{"namespace", "secret", "deploy"} {
108
+		if objs[i].Metadata.Name != name {
109
+			t.Errorf("Expected %q at position %d, got %q", name, i, objs[i].Metadata.Name)
110
+		}
111
+	}
112
+}

Loading…
Cancel
Save