GitOps for k8s
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

kubernetes.go 10KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366
  1. package kubernetes
  2. import (
  3. "bytes"
  4. "context"
  5. "encoding/json"
  6. "fmt"
  7. "sync"
  8. hrclient "github.com/fluxcd/helm-operator/pkg/client/clientset/versioned"
  9. "github.com/go-kit/kit/log"
  10. "github.com/pkg/errors"
  11. "gopkg.in/yaml.v2"
  12. apiv1 "k8s.io/api/core/v1"
  13. apierrors "k8s.io/apimachinery/pkg/api/errors"
  14. meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
  15. "k8s.io/client-go/discovery"
  16. k8sclientdynamic "k8s.io/client-go/dynamic"
  17. k8sclient "k8s.io/client-go/kubernetes"
  18. "github.com/fluxcd/flux/cluster"
  19. kresource "github.com/fluxcd/flux/cluster/kubernetes/resource"
  20. fhrclient "github.com/fluxcd/flux/integrations/client/clientset/versioned"
  21. "github.com/fluxcd/flux/resource"
  22. "github.com/fluxcd/flux/ssh"
  23. )
  24. type coreClient k8sclient.Interface
  25. type dynamicClient k8sclientdynamic.Interface
  26. type fluxHelmClient fhrclient.Interface
  27. type helmOperatorClient hrclient.Interface
  28. type discoveryClient discovery.DiscoveryInterface
  29. type ExtendedClient struct {
  30. coreClient
  31. dynamicClient
  32. fluxHelmClient
  33. helmOperatorClient
  34. discoveryClient
  35. }
  36. func MakeClusterClientset(core coreClient, dyn dynamicClient, fluxhelm fluxHelmClient,
  37. helmop helmOperatorClient, disco discoveryClient) ExtendedClient {
  38. return ExtendedClient{
  39. coreClient: core,
  40. dynamicClient: dyn,
  41. fluxHelmClient: fluxhelm,
  42. helmOperatorClient: helmop,
  43. discoveryClient: disco,
  44. }
  45. }
  46. // --- add-ons
  47. // Kubernetes has a mechanism of "Add-ons", whereby manifest files
  48. // left in a particular directory on the Kubernetes master will be
  49. // applied. We can recognise these, because they:
  50. // 1. Must be in the namespace `kube-system`; and,
  51. // 2. Must have one of the labels below set, else the addon manager will ignore them.
  52. //
  53. // We want to ignore add-ons, since they are managed by the add-on
  54. // manager, and attempts to control them via other means will fail.
  55. // k8sObject represents an value from which you can obtain typical
  56. // Kubernetes metadata. These methods are implemented by the
  57. // Kubernetes API resource types.
  58. type k8sObject interface {
  59. GetName() string
  60. GetNamespace() string
  61. GetLabels() map[string]string
  62. GetAnnotations() map[string]string
  63. }
  64. func isAddon(obj k8sObject) bool {
  65. if obj.GetNamespace() != "kube-system" {
  66. return false
  67. }
  68. labels := obj.GetLabels()
  69. if labels["kubernetes.io/cluster-service"] == "true" ||
  70. labels["addonmanager.kubernetes.io/mode"] == "EnsureExists" ||
  71. labels["addonmanager.kubernetes.io/mode"] == "Reconcile" {
  72. return true
  73. }
  74. return false
  75. }
  76. // --- /add ons
  77. // Cluster is a handle to a Kubernetes API server.
  78. // (Typically, this code is deployed into the same cluster.)
  79. type Cluster struct {
  80. // Do garbage collection when syncing resources
  81. GC bool
  82. // dry run garbage collection without syncing
  83. DryGC bool
  84. client ExtendedClient
  85. applier Applier
  86. version string // string response for the version command.
  87. logger log.Logger
  88. sshKeyRing ssh.KeyRing
  89. // syncErrors keeps a record of all per-resource errors during
  90. // the sync from Git repo to the cluster.
  91. syncErrors map[resource.ID]error
  92. muSyncErrors sync.RWMutex
  93. allowedNamespaces []string
  94. loggedAllowedNS map[string]bool // to keep track of whether we've logged a problem with seeing an allowed namespace
  95. imageExcludeList []string
  96. mu sync.Mutex
  97. }
  98. // NewCluster returns a usable cluster.
  99. func NewCluster(client ExtendedClient, applier Applier, sshKeyRing ssh.KeyRing, logger log.Logger, allowedNamespaces []string, imageExcludeList []string) *Cluster {
  100. c := &Cluster{
  101. client: client,
  102. applier: applier,
  103. logger: logger,
  104. sshKeyRing: sshKeyRing,
  105. allowedNamespaces: allowedNamespaces,
  106. loggedAllowedNS: map[string]bool{},
  107. imageExcludeList: imageExcludeList,
  108. }
  109. return c
  110. }
  111. // --- cluster.Cluster
  112. // SomeWorkloads returns the workloads named, missing out any that don't
  113. // exist in the cluster or aren't in an allowed namespace.
  114. // They do not necessarily have to be returned in the order requested.
  115. func (c *Cluster) SomeWorkloads(ctx context.Context, ids []resource.ID) (res []cluster.Workload, err error) {
  116. var workloads []cluster.Workload
  117. for _, id := range ids {
  118. if !c.IsAllowedResource(id) {
  119. continue
  120. }
  121. ns, kind, name := id.Components()
  122. resourceKind, ok := resourceKinds[kind]
  123. if !ok {
  124. c.logger.Log("warning", "unsupported kind", "resource", id)
  125. continue
  126. }
  127. workload, err := resourceKind.getWorkload(ctx, c, ns, name)
  128. if err != nil {
  129. if apierrors.IsForbidden(err) || apierrors.IsNotFound(err) {
  130. continue
  131. }
  132. return nil, err
  133. }
  134. if !isAddon(workload) {
  135. c.muSyncErrors.RLock()
  136. workload.syncError = c.syncErrors[id]
  137. c.muSyncErrors.RUnlock()
  138. workloads = append(workloads, workload.toClusterWorkload(id))
  139. }
  140. }
  141. return workloads, nil
  142. }
  143. // AllWorkloads returns all workloads in allowed namespaces matching the criteria; that is, in
  144. // the namespace (or any namespace if that argument is empty)
  145. func (c *Cluster) AllWorkloads(ctx context.Context, namespace string) (res []cluster.Workload, err error) {
  146. namespaces, err := c.getAllowedAndExistingNamespaces(ctx)
  147. if err != nil {
  148. return nil, errors.Wrap(err, "getting namespaces")
  149. }
  150. var allworkloads []cluster.Workload
  151. for _, ns := range namespaces {
  152. if namespace != "" && ns.Name != namespace {
  153. continue
  154. }
  155. for kind, resourceKind := range resourceKinds {
  156. workloads, err := resourceKind.getWorkloads(ctx, c, ns.Name)
  157. if err != nil {
  158. switch {
  159. case apierrors.IsNotFound(err):
  160. // Kind not supported by API server, skip
  161. continue
  162. case apierrors.IsForbidden(err):
  163. // K8s can return forbidden instead of not found for non super admins
  164. c.logger.Log("warning", "not allowed to list resources", "err", err)
  165. continue
  166. default:
  167. return nil, err
  168. }
  169. }
  170. for _, workload := range workloads {
  171. if !isAddon(workload) {
  172. id := resource.MakeID(ns.Name, kind, workload.GetName())
  173. c.muSyncErrors.RLock()
  174. workload.syncError = c.syncErrors[id]
  175. c.muSyncErrors.RUnlock()
  176. allworkloads = append(allworkloads, workload.toClusterWorkload(id))
  177. }
  178. }
  179. }
  180. }
  181. return allworkloads, nil
  182. }
  183. func (c *Cluster) setSyncErrors(errs cluster.SyncError) {
  184. c.muSyncErrors.Lock()
  185. defer c.muSyncErrors.Unlock()
  186. c.syncErrors = make(map[resource.ID]error)
  187. for _, e := range errs {
  188. c.syncErrors[e.ResourceID] = e.Error
  189. }
  190. }
  191. func (c *Cluster) Ping() error {
  192. _, err := c.client.coreClient.Discovery().ServerVersion()
  193. return err
  194. }
  195. // Export exports cluster resources
  196. func (c *Cluster) Export(ctx context.Context) ([]byte, error) {
  197. var config bytes.Buffer
  198. namespaces, err := c.getAllowedAndExistingNamespaces(ctx)
  199. if err != nil {
  200. return nil, errors.Wrap(err, "getting namespaces")
  201. }
  202. encoder := yaml.NewEncoder(&config)
  203. defer encoder.Close()
  204. for _, ns := range namespaces {
  205. // kind & apiVersion must be set, since TypeMeta is not populated
  206. ns.Kind = "Namespace"
  207. ns.APIVersion = "v1"
  208. err := encoder.Encode(yamlThroughJSON{ns})
  209. if err != nil {
  210. return nil, errors.Wrap(err, "marshalling namespace to YAML")
  211. }
  212. for _, resourceKind := range resourceKinds {
  213. workloads, err := resourceKind.getWorkloads(ctx, c, ns.Name)
  214. if err != nil {
  215. switch {
  216. case apierrors.IsNotFound(err):
  217. // Kind not supported by API server, skip
  218. continue
  219. case apierrors.IsForbidden(err):
  220. // K8s can return forbidden instead of not found for non super admins
  221. c.logger.Log("warning", "not allowed to list resources", "err", err)
  222. continue
  223. default:
  224. return nil, err
  225. }
  226. }
  227. for _, pc := range workloads {
  228. if !isAddon(pc) {
  229. if err := encoder.Encode(yamlThroughJSON{pc.k8sObject}); err != nil {
  230. return nil, err
  231. }
  232. }
  233. }
  234. }
  235. }
  236. return config.Bytes(), nil
  237. }
  238. func (c *Cluster) PublicSSHKey(regenerate bool) (ssh.PublicKey, error) {
  239. if regenerate {
  240. if err := c.sshKeyRing.Regenerate(); err != nil {
  241. return ssh.PublicKey{}, err
  242. }
  243. }
  244. publicKey, _ := c.sshKeyRing.KeyPair()
  245. return publicKey, nil
  246. }
  247. // getAllowedAndExistingNamespaces returns a list of existing namespaces that
  248. // the Flux instance is expected to have access to and can look for resources inside of.
  249. // It returns a list of all namespaces unless an explicit list of allowed namespaces
  250. // has been set on the Cluster instance.
  251. func (c *Cluster) getAllowedAndExistingNamespaces(ctx context.Context) ([]apiv1.Namespace, error) {
  252. if len(c.allowedNamespaces) > 0 {
  253. nsList := []apiv1.Namespace{}
  254. for _, name := range c.allowedNamespaces {
  255. if err := ctx.Err(); err != nil {
  256. return nil, err
  257. }
  258. ns, err := c.client.CoreV1().Namespaces().Get(name, meta_v1.GetOptions{})
  259. switch {
  260. case err == nil:
  261. c.loggedAllowedNS[name] = false // reset, so if the namespace goes away we'll log it again
  262. nsList = append(nsList, *ns)
  263. case apierrors.IsUnauthorized(err) || apierrors.IsForbidden(err) || apierrors.IsNotFound(err):
  264. if !c.loggedAllowedNS[name] {
  265. c.logger.Log("warning", "cannot access allowed namespace",
  266. "namespace", name, "err", err)
  267. c.loggedAllowedNS[name] = true
  268. }
  269. default:
  270. return nil, err
  271. }
  272. }
  273. return nsList, nil
  274. }
  275. if err := ctx.Err(); err != nil {
  276. return nil, err
  277. }
  278. namespaces, err := c.client.CoreV1().Namespaces().List(meta_v1.ListOptions{})
  279. if err != nil {
  280. return nil, err
  281. }
  282. return namespaces.Items, nil
  283. }
  284. func (c *Cluster) IsAllowedResource(id resource.ID) bool {
  285. if len(c.allowedNamespaces) == 0 {
  286. // All resources are allowed when all namespaces are allowed
  287. return true
  288. }
  289. namespace, kind, name := id.Components()
  290. namespaceToCheck := namespace
  291. if namespace == kresource.ClusterScope {
  292. // All cluster-scoped resources (not namespaced) are allowed ...
  293. if kind != "namespace" {
  294. return true
  295. }
  296. // ... except namespaces themselves, whose name needs to be explicitly allowed
  297. namespaceToCheck = name
  298. }
  299. for _, allowedNS := range c.allowedNamespaces {
  300. if namespaceToCheck == allowedNS {
  301. return true
  302. }
  303. }
  304. return false
  305. }
  306. type yamlThroughJSON struct {
  307. toMarshal interface{}
  308. }
  309. func (y yamlThroughJSON) MarshalYAML() (interface{}, error) {
  310. rawJSON, err := json.Marshal(y.toMarshal)
  311. if err != nil {
  312. return nil, fmt.Errorf("error marshaling into JSON: %s", err)
  313. }
  314. var jsonObj interface{}
  315. if err = yaml.Unmarshal(rawJSON, &jsonObj); err != nil {
  316. return nil, fmt.Errorf("error unmarshaling from JSON: %s", err)
  317. }
  318. return jsonObj, nil
  319. }