GitOps for k8s
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

main.go 29KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774
  1. package main
  2. import (
  3. "flag"
  4. "fmt"
  5. "io/ioutil"
  6. "net/http"
  7. _ "net/http/pprof"
  8. "os"
  9. "os/exec"
  10. "os/signal"
  11. "path/filepath"
  12. "runtime"
  13. "strconv"
  14. "strings"
  15. "sync"
  16. "syscall"
  17. "time"
  18. helmopclient "github.com/fluxcd/helm-operator/pkg/client/clientset/versioned"
  19. "github.com/go-kit/kit/log"
  20. "github.com/prometheus/client_golang/prometheus/promhttp"
  21. "github.com/spf13/pflag"
  22. crd "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
  23. k8serrors "k8s.io/apimachinery/pkg/api/errors"
  24. k8sruntime "k8s.io/apimachinery/pkg/util/runtime"
  25. k8sclientdynamic "k8s.io/client-go/dynamic"
  26. k8sclient "k8s.io/client-go/kubernetes"
  27. "k8s.io/client-go/rest"
  28. "k8s.io/client-go/tools/clientcmd"
  29. "k8s.io/klog"
  30. "github.com/fluxcd/flux/checkpoint"
  31. "github.com/fluxcd/flux/cluster"
  32. "github.com/fluxcd/flux/cluster/kubernetes"
  33. "github.com/fluxcd/flux/daemon"
  34. "github.com/fluxcd/flux/git"
  35. "github.com/fluxcd/flux/gpg"
  36. transport "github.com/fluxcd/flux/http"
  37. "github.com/fluxcd/flux/http/client"
  38. daemonhttp "github.com/fluxcd/flux/http/daemon"
  39. "github.com/fluxcd/flux/image"
  40. hrclient "github.com/fluxcd/flux/integrations/client/clientset/versioned"
  41. "github.com/fluxcd/flux/job"
  42. "github.com/fluxcd/flux/manifests"
  43. "github.com/fluxcd/flux/registry"
  44. "github.com/fluxcd/flux/registry/cache"
  45. registryMemcache "github.com/fluxcd/flux/registry/cache/memcached"
  46. registryMiddleware "github.com/fluxcd/flux/registry/middleware"
  47. "github.com/fluxcd/flux/remote"
  48. "github.com/fluxcd/flux/ssh"
  49. fluxsync "github.com/fluxcd/flux/sync"
  50. )
  51. var version = "unversioned"
  52. const (
  53. product = "weave-flux"
  54. // This is used as the "burst" value for rate limiting, and
  55. // therefore also as the limit to the number of concurrent fetches
  56. // and memcached connections, since these in general can't do any
  57. // more work than is allowed by the burst amount.
  58. defaultRemoteConnections = 10
  59. // There are running systems that assume these defaults (by not
  60. // supplying a value for one or both). Don't change them.
  61. defaultGitSyncTag = "flux-sync"
  62. defaultGitNotesRef = "flux"
  63. defaultGitSkipMessage = "\n\n[ci skip]"
  64. RequireECR = "ecr"
  65. )
  66. var (
  67. RequireValues = []string{RequireECR}
  68. )
  69. func optionalVar(fs *pflag.FlagSet, value ssh.OptionalValue, name, usage string) ssh.OptionalValue {
  70. fs.Var(value, name, usage)
  71. return value
  72. }
  73. type stringset []string
  74. func (set stringset) has(possible string) bool {
  75. for _, s := range set {
  76. if s == possible {
  77. return true
  78. }
  79. }
  80. return false
  81. }
  82. func main() {
  83. // Flag domain.
  84. fs := pflag.NewFlagSet("default", pflag.ContinueOnError)
  85. fs.Usage = func() {
  86. fmt.Fprintf(os.Stderr, "DESCRIPTION\n")
  87. fmt.Fprintf(os.Stderr, " fluxd is the agent of flux.\n")
  88. fmt.Fprintf(os.Stderr, "\n")
  89. fmt.Fprintf(os.Stderr, "FLAGS\n")
  90. fs.PrintDefaults()
  91. }
  92. // This mirrors how kubectl extracts information from the environment.
  93. var (
  94. logFormat = fs.String("log-format", "fmt", "change the log format.")
  95. listenAddr = fs.StringP("listen", "l", ":3030", "listen address where /metrics and API will be served")
  96. listenMetricsAddr = fs.String("listen-metrics", "", "listen address for /metrics endpoint")
  97. kubernetesKubectl = fs.String("kubernetes-kubectl", "", "optional, explicit path to kubectl tool")
  98. versionFlag = fs.Bool("version", false, "get version number")
  99. // Git repo & key etc.
  100. gitURL = fs.String("git-url", "", "URL of git repo with Kubernetes manifests; e.g., git@github.com:weaveworks/flux-get-started")
  101. gitBranch = fs.String("git-branch", "master", "branch of git repo to use for Kubernetes manifests")
  102. gitPath = fs.StringSlice("git-path", []string{}, "relative paths within the git repo to locate Kubernetes manifests")
  103. gitReadonly = fs.Bool("git-readonly", false, fmt.Sprintf("use to prevent Flux from pushing changes to git; implies --sync-state=%s", fluxsync.NativeStateMode))
  104. gitUser = fs.String("git-user", "Weave Flux", "username to use as git committer")
  105. gitEmail = fs.String("git-email", "support@weave.works", "email to use as git committer")
  106. gitSetAuthor = fs.Bool("git-set-author", false, "if set, the author of git commits will reflect the user who initiated the commit and will differ from the git committer.")
  107. gitLabel = fs.String("git-label", "", "label to keep track of sync progress; overrides both --git-sync-tag and --git-notes-ref")
  108. gitSecret = fs.Bool("git-secret", false, `if set, git-secret will be run on every git checkout. A gpg key must be imported using --git-gpg-key-import or by mounting a keyring containing it directly`)
  109. // Old git config; still used if --git-label is not supplied, but --git-label is preferred.
  110. gitSyncTag = fs.String("git-sync-tag", defaultGitSyncTag, fmt.Sprintf("tag to use to mark sync progress for this cluster (only relevant when --sync-state=%s)", fluxsync.GitTagStateMode))
  111. gitNotesRef = fs.String("git-notes-ref", defaultGitNotesRef, "ref to use for keeping commit annotations in git notes")
  112. gitSkip = fs.Bool("git-ci-skip", false, `append "[ci skip]" to commit messages so that CI will skip builds`)
  113. gitSkipMessage = fs.String("git-ci-skip-message", "", "additional text for commit messages, useful for skipping builds in CI. Use this to supply specific text, or set --git-ci-skip")
  114. gitPollInterval = fs.Duration("git-poll-interval", 5*time.Minute, "period at which to poll git repo for new commits")
  115. gitTimeout = fs.Duration("git-timeout", 20*time.Second, "duration after which git operations time out")
  116. // GPG commit signing
  117. gitImportGPG = fs.StringSlice("git-gpg-key-import", []string{}, "keys at the paths given will be imported for use of signing and verifying commits")
  118. gitSigningKey = fs.String("git-signing-key", "", "if set, commits Flux makes will be signed with this GPG key")
  119. gitVerifySignatures = fs.Bool("git-verify-signatures", false, "if set, the signature of commits will be verified before Flux applies them")
  120. // syncing
  121. syncInterval = fs.Duration("sync-interval", 5*time.Minute, "apply config in git to cluster at least this often, even if there are no new commits")
  122. syncGC = fs.Bool("sync-garbage-collection", false, "experimental; delete resources that were created by fluxd, but are no longer in the git repo")
  123. dryGC = fs.Bool("sync-garbage-collection-dry", false, "experimental; only log what would be garbage collected, rather than deleting. Implies --sync-garbage-collection")
  124. syncState = fs.String("sync-state", fluxsync.GitTagStateMode, fmt.Sprintf("method used by flux for storing state (one of {%s})", strings.Join([]string{fluxsync.GitTagStateMode, fluxsync.NativeStateMode}, ",")))
  125. // registry
  126. memcachedHostname = fs.String("memcached-hostname", "memcached", "hostname for memcached service.")
  127. memcachedPort = fs.Int("memcached-port", 11211, "memcached service port.")
  128. memcachedTimeout = fs.Duration("memcached-timeout", time.Second, "maximum time to wait before giving up on memcached requests.")
  129. memcachedService = fs.String("memcached-service", "memcached", "SRV service used to discover memcache servers.")
  130. automationInterval = fs.Duration("automation-interval", 5*time.Minute, "period at which to check for image updates for automated workloads")
  131. registryPollInterval = fs.Duration("registry-poll-interval", 5*time.Minute, "period at which to check for updated images")
  132. registryRPS = fs.Float64("registry-rps", 50, "maximum registry requests per second per host")
  133. registryBurst = fs.Int("registry-burst", defaultRemoteConnections, "maximum number of warmer connections to remote and memcache")
  134. registryTrace = fs.Bool("registry-trace", false, "output trace of image registry requests to log")
  135. registryInsecure = fs.StringSlice("registry-insecure-host", []string{}, "let these registry hosts skip TLS host verification and fall back to using HTTP instead of HTTPS; this allows man-in-the-middle attacks, so use with extreme caution")
  136. registryExcludeImage = fs.StringSlice("registry-exclude-image", []string{"k8s.gcr.io/*"}, "do not scan images that match these glob expressions; the default is to exclude the 'k8s.gcr.io/*' images")
  137. registryUseLabels = fs.StringSlice("registry-use-labels", []string{"index.docker.io/weaveworks/*", "index.docker.io/fluxcd/*"}, "use the timestamp (RFC3339) from labels for (canonical) image refs that match these glob expression")
  138. // AWS authentication
  139. registryAWSRegions = fs.StringSlice("registry-ecr-region", nil, "include just these AWS regions when scanning images in ECR; when not supplied, the cluster's region will included if it can be detected through the AWS API")
  140. registryAWSAccountIDs = fs.StringSlice("registry-ecr-include-id", nil, "restrict ECR scanning to these AWS account IDs; if not supplied, all account IDs that aren't excluded may be scanned")
  141. registryAWSBlockAccountIDs = fs.StringSlice("registry-ecr-exclude-id", []string{registry.EKS_SYSTEM_ACCOUNT}, "do not scan ECR for images in these AWS account IDs; the default is to exclude the EKS system account")
  142. registryRequire = fs.StringSlice("registry-require", nil, fmt.Sprintf(`exit with an error if auto-authentication with any of the given registries is not possible (possible values: {%s})`, strings.Join(RequireValues, ",")))
  143. // k8s-secret backed ssh keyring configuration
  144. k8sInCluster = fs.Bool("k8s-in-cluster", true, "set this to true if fluxd is deployed as a container inside Kubernetes")
  145. k8sSecretName = fs.String("k8s-secret-name", "flux-git-deploy", "name of the k8s secret used to store the private SSH key")
  146. k8sSecretVolumeMountPath = fs.String("k8s-secret-volume-mount-path", "/etc/fluxd/ssh", "mount location of the k8s secret storing the private SSH key")
  147. k8sSecretDataKey = fs.String("k8s-secret-data-key", "identity", "data key holding the private SSH key within the k8s secret")
  148. k8sNamespaceWhitelist = fs.StringSlice("k8s-namespace-whitelist", []string{}, "experimental, optional: restrict the view of the cluster to the namespaces listed. All namespaces are included if this is not set")
  149. k8sAllowNamespace = fs.StringSlice("k8s-allow-namespace", []string{}, "experimental: restrict all operations to the provided namespaces")
  150. k8sVerbosity = fs.Int("k8s-verbosity", 0, "klog verbosity level")
  151. // SSH key generation
  152. sshKeyBits = optionalVar(fs, &ssh.KeyBitsValue{}, "ssh-keygen-bits", "-b argument to ssh-keygen (default unspecified)")
  153. sshKeyType = optionalVar(fs, &ssh.KeyTypeValue{}, "ssh-keygen-type", "-t argument to ssh-keygen (default unspecified)")
  154. sshKeygenDir = fs.String("ssh-keygen-dir", "", "directory, ideally on a tmpfs volume, in which to generate new SSH keys when necessary")
  155. // manifest generation
  156. manifestGeneration = fs.Bool("manifest-generation", false, "experimental; search for .flux.yaml files to generate manifests")
  157. // upstream connection settings
  158. upstreamURL = fs.String("connect", "", "connect to an upstream service e.g., Weave Cloud, at this base address")
  159. token = fs.String("token", "", "authentication token for upstream service")
  160. rpcTimeout = fs.Duration("rpc-timeout", 10*time.Second, "maximum time an operation requested by the upstream may take")
  161. dockerConfig = fs.String("docker-config", "", "path to a docker config to use for image registry credentials")
  162. _ = fs.Duration("registry-cache-expiry", 0, "")
  163. )
  164. fs.MarkDeprecated("registry-cache-expiry", "no longer used; cache entries are expired adaptively according to how often they change")
  165. fs.MarkDeprecated("k8s-namespace-whitelist", "changed to --k8s-allow-namespace, use that instead")
  166. fs.MarkDeprecated("registry-poll-interval", "changed to --automation-interval, use that instead")
  167. var kubeConfig *string
  168. {
  169. // Set the default kube config
  170. if home := homeDir(); home != "" {
  171. kubeConfig = fs.String("kube-config", filepath.Join(home, ".kube", "config"), "the absolute path of the k8s config file.")
  172. } else {
  173. kubeConfig = fs.String("kube-config", "", "the absolute path of the k8s config file.")
  174. }
  175. }
  176. // Explicitly initialize klog to enable stderr logging,
  177. // and parse our own flags.
  178. klogFlags := flag.NewFlagSet("klog", flag.ExitOnError)
  179. klog.InitFlags(klogFlags)
  180. err := fs.Parse(os.Args[1:])
  181. switch {
  182. case err == pflag.ErrHelp:
  183. os.Exit(0)
  184. case err != nil:
  185. fmt.Fprintf(os.Stderr, "Error: %s\n\n", err.Error())
  186. fs.Usage()
  187. os.Exit(2)
  188. case *versionFlag:
  189. fmt.Println(version)
  190. os.Exit(0)
  191. }
  192. // set klog verbosity level
  193. if *k8sVerbosity > 0 {
  194. verbosity := klogFlags.Lookup("v")
  195. verbosity.Value.Set(strconv.Itoa(*k8sVerbosity))
  196. klog.V(4).Infof("Kubernetes client verbosity level set to %v", klogFlags.Lookup("v").Value)
  197. }
  198. // Logger component.
  199. var logger log.Logger
  200. {
  201. switch *logFormat {
  202. case "json":
  203. logger = log.NewJSONLogger(log.NewSyncWriter(os.Stderr))
  204. case "fmt":
  205. logger = log.NewLogfmtLogger(log.NewSyncWriter(os.Stderr))
  206. default:
  207. logger = log.NewLogfmtLogger(log.NewSyncWriter(os.Stderr))
  208. }
  209. logger = log.With(logger, "ts", log.DefaultTimestampUTC)
  210. logger = log.With(logger, "caller", log.DefaultCaller)
  211. }
  212. logger.Log("version", version)
  213. // Silence access errors logged internally by client-go
  214. k8slog := log.With(logger,
  215. "type", "internal kubernetes error",
  216. "kubernetes_caller", log.Valuer(func() interface{} {
  217. _, file, line, _ := runtime.Caller(5) // we want to log one level deeper than k8sruntime.HandleError
  218. idx := strings.Index(file, "/k8s.io/")
  219. return file[idx+1:] + ":" + strconv.Itoa(line)
  220. }))
  221. logErrorUnlessAccessRelated := func(err error) {
  222. errLower := strings.ToLower(err.Error())
  223. if k8serrors.IsForbidden(err) || k8serrors.IsNotFound(err) ||
  224. strings.Contains(errLower, "forbidden") ||
  225. strings.Contains(errLower, "not found") {
  226. return
  227. }
  228. k8slog.Log("err", err)
  229. }
  230. k8sruntime.ErrorHandlers = []func(error){logErrorUnlessAccessRelated}
  231. // Argument validation
  232. if *gitReadonly {
  233. if *syncState == fluxsync.GitTagStateMode {
  234. logger.Log("warning", fmt.Sprintf("--git-readonly prevents use of --sync-state=%s. Forcing to --sync-state=%s", fluxsync.GitTagStateMode, fluxsync.NativeStateMode))
  235. *syncState = fluxsync.NativeStateMode
  236. }
  237. gitRelatedFlags := []string{
  238. "git-user",
  239. "git-email",
  240. "git-sync-tag",
  241. "git-set-author",
  242. "git-ci-skip",
  243. "git-ci-skip-message",
  244. }
  245. var changedGitRelatedFlags []string
  246. for _, gitRelatedFlag := range gitRelatedFlags {
  247. if fs.Changed(gitRelatedFlag) {
  248. changedGitRelatedFlags = append(changedGitRelatedFlags, gitRelatedFlag)
  249. }
  250. }
  251. if len(changedGitRelatedFlags) > 0 {
  252. logger.Log("warning", fmt.Sprintf("configuring any of {%s} has no effect when --git-readonly is set", strings.Join(changedGitRelatedFlags, ", ")))
  253. }
  254. }
  255. // Maintain backwards compatibility with the --registry-poll-interval
  256. // flag, but only if the --automation-interval is not set to a custom
  257. // (non default) value.
  258. if fs.Changed("registry-poll-interval") && !fs.Changed("automation-interval") {
  259. *automationInterval = *registryPollInterval
  260. }
  261. // Sort out values for the git tag and notes ref. There are
  262. // running deployments that assume the defaults as given, so don't
  263. // mess with those unless explicitly told.
  264. if fs.Changed("git-label") {
  265. *gitSyncTag = *gitLabel
  266. *gitNotesRef = *gitLabel
  267. for _, f := range []string{"git-sync-tag", "git-notes-ref"} {
  268. if fs.Changed(f) {
  269. logger.Log("overridden", f, "value", *gitLabel)
  270. }
  271. }
  272. }
  273. if *gitSkipMessage == "" && *gitSkip {
  274. *gitSkipMessage = defaultGitSkipMessage
  275. }
  276. for _, path := range *gitPath {
  277. if len(path) > 0 && path[0] == '/' {
  278. logger.Log("err", "subdirectory given as --git-path should not have leading forward slash")
  279. os.Exit(1)
  280. }
  281. }
  282. if *sshKeygenDir == "" {
  283. logger.Log("info", fmt.Sprintf("SSH keygen dir (--ssh-keygen-dir) not provided, so using the deploy key volume (--k8s-secret-volume-mount-path=%s); this may cause problems if the deploy key volume is mounted read-only", *k8sSecretVolumeMountPath))
  284. *sshKeygenDir = *k8sSecretVolumeMountPath
  285. }
  286. // Import GPG keys, if we've been told where to look for them
  287. for _, p := range *gitImportGPG {
  288. keyfiles, err := gpg.ImportKeys(p, *gitVerifySignatures)
  289. if err != nil {
  290. logger.Log("error", fmt.Sprintf("failed to import GPG key(s) from %s", p), "err", err.Error())
  291. }
  292. if keyfiles != nil {
  293. logger.Log("info", fmt.Sprintf("imported GPG key(s) from %s", p), "files", fmt.Sprintf("%v", keyfiles))
  294. }
  295. }
  296. possiblyRequired := stringset(RequireValues)
  297. for _, r := range *registryRequire {
  298. if !possiblyRequired.has(r) {
  299. logger.Log("err", fmt.Sprintf("--registry-require value %q is not in possible values {%s}", r, strings.Join(RequireValues, ",")))
  300. os.Exit(1)
  301. }
  302. }
  303. mandatoryRegistry := stringset(*registryRequire)
  304. if *gitSecret && len(*gitImportGPG) == 0 {
  305. logger.Log("warning", fmt.Sprintf("--git-secret is enabled but there is no GPG key(s) provided using --git-gpg-key-import, we assume you mounted the keyring directly and continue"))
  306. }
  307. // Mechanical components.
  308. // When we can receive from this channel, it indicates that we
  309. // are ready to shut down.
  310. errc := make(chan error)
  311. // This signals other routines to shut down;
  312. shutdown := make(chan struct{})
  313. // .. and this is to wait for other routines to shut down cleanly.
  314. shutdownWg := &sync.WaitGroup{}
  315. go func() {
  316. c := make(chan os.Signal, 1)
  317. signal.Notify(c, syscall.SIGINT, syscall.SIGTERM)
  318. errc <- fmt.Errorf("%s", <-c)
  319. }()
  320. // Cluster component.
  321. var restClientConfig *rest.Config
  322. {
  323. if *k8sInCluster {
  324. logger.Log("msg", "using in cluster config to connect to the cluster")
  325. restClientConfig, err = rest.InClusterConfig()
  326. if err != nil {
  327. logger.Log("err", err)
  328. os.Exit(1)
  329. }
  330. } else {
  331. logger.Log("msg", fmt.Sprintf("using kube config: %q to connect to the cluster", *kubeConfig))
  332. restClientConfig, err = clientcmd.BuildConfigFromFlags("", *kubeConfig)
  333. if err != nil {
  334. logger.Log("err", err)
  335. os.Exit(1)
  336. }
  337. }
  338. restClientConfig.QPS = 50.0
  339. restClientConfig.Burst = 100
  340. }
  341. var clusterVersion string
  342. var sshKeyRing ssh.KeyRing
  343. var k8s cluster.Cluster
  344. var k8sManifests manifests.Manifests
  345. var imageCreds func() registry.ImageCreds
  346. {
  347. clientset, err := k8sclient.NewForConfig(restClientConfig)
  348. if err != nil {
  349. logger.Log("err", err)
  350. os.Exit(1)
  351. }
  352. dynamicClientset, err := k8sclientdynamic.NewForConfig(restClientConfig)
  353. if err != nil {
  354. logger.Log("err", err)
  355. os.Exit(1)
  356. }
  357. fhrClientset, err := hrclient.NewForConfig(restClientConfig)
  358. if err != nil {
  359. logger.Log("error", fmt.Sprintf("Error building hrclient clientset: %v", err))
  360. os.Exit(1)
  361. }
  362. hrClientset, err := helmopclient.NewForConfig(restClientConfig)
  363. if err != nil {
  364. logger.Log("error", fmt.Sprintf("Error building helm operator clientset: %v", err))
  365. os.Exit(1)
  366. }
  367. crdClient, err := crd.NewForConfig(restClientConfig)
  368. if err != nil {
  369. logger.Log("error", fmt.Sprintf("Error building API extensions (CRD) clientset: %v", err))
  370. os.Exit(1)
  371. }
  372. discoClientset := kubernetes.MakeCachedDiscovery(clientset.Discovery(), crdClient, shutdown)
  373. serverVersion, err := clientset.ServerVersion()
  374. if err != nil {
  375. logger.Log("err", err)
  376. os.Exit(1)
  377. }
  378. clusterVersion = "kubernetes-" + serverVersion.GitVersion
  379. if *k8sInCluster {
  380. namespace, err := ioutil.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace")
  381. if err != nil {
  382. logger.Log("err", err)
  383. os.Exit(1)
  384. }
  385. sshKeyRing, err = kubernetes.NewSSHKeyRing(kubernetes.SSHKeyRingConfig{
  386. SecretAPI: clientset.CoreV1().Secrets(string(namespace)),
  387. SecretName: *k8sSecretName,
  388. SecretVolumeMountPath: *k8sSecretVolumeMountPath,
  389. SecretDataKey: *k8sSecretDataKey,
  390. KeyBits: sshKeyBits,
  391. KeyType: sshKeyType,
  392. KeyGenDir: *sshKeygenDir,
  393. })
  394. if err != nil {
  395. logger.Log("err", err)
  396. os.Exit(1)
  397. }
  398. publicKey, privateKeyPath := sshKeyRing.KeyPair()
  399. logger := log.With(logger, "component", "cluster")
  400. logger.Log("identity", privateKeyPath)
  401. logger.Log("identity.pub", strings.TrimSpace(publicKey.Key))
  402. } else {
  403. sshKeyRing = ssh.NewNopSSHKeyRing()
  404. }
  405. logger.Log("host", restClientConfig.Host, "version", clusterVersion)
  406. kubectl := *kubernetesKubectl
  407. if kubectl == "" {
  408. kubectl, err = exec.LookPath("kubectl")
  409. } else {
  410. _, err = os.Stat(kubectl)
  411. }
  412. if err != nil {
  413. logger.Log("err", err)
  414. os.Exit(1)
  415. }
  416. logger.Log("kubectl", kubectl)
  417. client := kubernetes.MakeClusterClientset(clientset, dynamicClientset, fhrClientset, hrClientset, discoClientset)
  418. kubectlApplier := kubernetes.NewKubectl(kubectl, restClientConfig)
  419. allowedNamespaces := append(*k8sNamespaceWhitelist, *k8sAllowNamespace...)
  420. k8sInst := kubernetes.NewCluster(client, kubectlApplier, sshKeyRing, logger, allowedNamespaces, *registryExcludeImage)
  421. k8sInst.GC = *syncGC
  422. k8sInst.DryGC = *dryGC
  423. if err := k8sInst.Ping(); err != nil {
  424. logger.Log("ping", err)
  425. } else {
  426. logger.Log("ping", true)
  427. }
  428. k8s = k8sInst
  429. imageCreds = k8sInst.ImagesToFetch
  430. // There is only one way we currently interpret a repo of
  431. // files as manifests, and that's as Kubernetes yamels.
  432. namespacer, err := kubernetes.NewNamespacer(discoClientset)
  433. if err != nil {
  434. logger.Log("err", err)
  435. os.Exit(1)
  436. }
  437. k8sManifests = kubernetes.NewManifests(namespacer, logger)
  438. }
  439. // Wrap the procedure for collecting images to scan
  440. {
  441. awsConf := registry.AWSRegistryConfig{
  442. Regions: *registryAWSRegions,
  443. AccountIDs: *registryAWSAccountIDs,
  444. BlockIDs: *registryAWSBlockAccountIDs,
  445. }
  446. awsPreflight, credsWithAWSAuth := registry.ImageCredsWithAWSAuth(imageCreds, log.With(logger, "component", "aws"), awsConf)
  447. if mandatoryRegistry.has(RequireECR) {
  448. if err := awsPreflight(); err != nil {
  449. logger.Log("error", "AWS API required (due to --registry-require=ecr), but not available", "err", err)
  450. os.Exit(1)
  451. }
  452. }
  453. imageCreds = credsWithAWSAuth
  454. if *dockerConfig != "" {
  455. credsWithDefaults, err := registry.ImageCredsWithDefaults(imageCreds, *dockerConfig)
  456. if err != nil {
  457. logger.Log("warning", "--docker-config not used; pre-flight check failed", "err", err)
  458. } else {
  459. imageCreds = credsWithDefaults
  460. }
  461. }
  462. }
  463. // Registry components
  464. var cacheRegistry registry.Registry
  465. var cacheWarmer *cache.Warmer
  466. {
  467. // Cache client, for use by registry and cache warmer
  468. var cacheClient cache.Client
  469. var memcacheClient *registryMemcache.MemcacheClient
  470. memcacheConfig := registryMemcache.MemcacheConfig{
  471. Host: *memcachedHostname,
  472. Service: *memcachedService,
  473. Timeout: *memcachedTimeout,
  474. UpdateInterval: 1 * time.Minute,
  475. Logger: log.With(logger, "component", "memcached"),
  476. MaxIdleConns: *registryBurst,
  477. }
  478. // if no memcached service is specified use the ClusterIP name instead of SRV records
  479. if *memcachedService == "" {
  480. memcacheClient = registryMemcache.NewFixedServerMemcacheClient(memcacheConfig,
  481. fmt.Sprintf("%s:%d", *memcachedHostname, *memcachedPort))
  482. } else {
  483. memcacheClient = registryMemcache.NewMemcacheClient(memcacheConfig)
  484. }
  485. defer memcacheClient.Stop()
  486. cacheClient = cache.InstrumentClient(memcacheClient)
  487. cacheRegistry = &cache.Cache{
  488. Reader: cacheClient,
  489. Decorators: []cache.Decorator{
  490. cache.TimestampLabelWhitelist(*registryUseLabels),
  491. },
  492. }
  493. cacheRegistry = registry.NewInstrumentedRegistry(cacheRegistry)
  494. // Remote client, for warmer to refresh entries
  495. registryLogger := log.With(logger, "component", "registry")
  496. registryLimits := &registryMiddleware.RateLimiters{
  497. RPS: *registryRPS,
  498. Burst: *registryBurst,
  499. Logger: log.With(logger, "component", "ratelimiter"),
  500. }
  501. remoteFactory := &registry.RemoteClientFactory{
  502. Logger: registryLogger,
  503. Limiters: registryLimits,
  504. Trace: *registryTrace,
  505. InsecureHosts: *registryInsecure,
  506. }
  507. // Warmer
  508. var err error
  509. cacheWarmer, err = cache.NewWarmer(remoteFactory, cacheClient, *registryBurst)
  510. if err != nil {
  511. logger.Log("err", err)
  512. os.Exit(1)
  513. }
  514. }
  515. // Checkpoint: we want to include the fact of whether the daemon
  516. // was given a Git repo it could clone; but the expected scenario
  517. // is that it will have been set up already, and we don't want to
  518. // report anything before seeing if it works. So, don't start
  519. // until we have failed or succeeded.
  520. updateCheckLogger := log.With(logger, "component", "checkpoint")
  521. checkpointFlags := map[string]string{
  522. "cluster-version": clusterVersion,
  523. "git-configured": strconv.FormatBool(*gitURL != ""),
  524. }
  525. checkpoint.CheckForUpdates(product, version, checkpointFlags, updateCheckLogger)
  526. gitRemote := git.Remote{URL: *gitURL}
  527. gitConfig := git.Config{
  528. Paths: *gitPath,
  529. Branch: *gitBranch,
  530. NotesRef: *gitNotesRef,
  531. UserName: *gitUser,
  532. UserEmail: *gitEmail,
  533. SigningKey: *gitSigningKey,
  534. SetAuthor: *gitSetAuthor,
  535. SkipMessage: *gitSkipMessage,
  536. GitSecret: *gitSecret,
  537. }
  538. repo := git.NewRepo(gitRemote, git.PollInterval(*gitPollInterval), git.Timeout(*gitTimeout), git.Branch(*gitBranch), git.IsReadOnly(*gitReadonly))
  539. {
  540. shutdownWg.Add(1)
  541. go func() {
  542. err := repo.Start(shutdown, shutdownWg)
  543. if err != nil {
  544. errc <- err
  545. }
  546. }()
  547. }
  548. logger.Log(
  549. "url", *gitURL,
  550. "user", *gitUser,
  551. "email", *gitEmail,
  552. "signing-key", *gitSigningKey,
  553. "verify-signatures", *gitVerifySignatures,
  554. "sync-tag", *gitSyncTag,
  555. "state", *syncState,
  556. "readonly", *gitReadonly,
  557. "notes-ref", *gitNotesRef,
  558. "set-author", *gitSetAuthor,
  559. "git-secret", *gitSecret,
  560. )
  561. var jobs *job.Queue
  562. {
  563. jobs = job.NewQueue(shutdown, shutdownWg)
  564. }
  565. var syncProvider fluxsync.State
  566. switch *syncState {
  567. case fluxsync.NativeStateMode:
  568. namespace, err := ioutil.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace")
  569. if err != nil {
  570. logger.Log("err", err)
  571. os.Exit(1)
  572. }
  573. syncProvider, err = fluxsync.NewNativeSyncProvider(
  574. string(namespace),
  575. *k8sSecretName,
  576. )
  577. if err != nil {
  578. logger.Log("err", err)
  579. os.Exit(1)
  580. }
  581. case fluxsync.GitTagStateMode:
  582. syncProvider, err = fluxsync.NewGitTagSyncProvider(
  583. repo,
  584. *gitSyncTag,
  585. *gitSigningKey,
  586. *gitVerifySignatures,
  587. gitConfig,
  588. )
  589. if err != nil {
  590. logger.Log("err", err)
  591. os.Exit(1)
  592. }
  593. default:
  594. logger.Log("error", "unknown sync state mode", "mode", *syncState)
  595. os.Exit(1)
  596. }
  597. daemon := &daemon.Daemon{
  598. V: version,
  599. Cluster: k8s,
  600. Manifests: k8sManifests,
  601. Registry: cacheRegistry,
  602. ImageRefresh: make(chan image.Name, 100), // size chosen by fair dice roll
  603. Repo: repo,
  604. GitConfig: gitConfig,
  605. Jobs: jobs,
  606. JobStatusCache: &job.StatusCache{Size: 100},
  607. Logger: log.With(logger, "component", "daemon"),
  608. ManifestGenerationEnabled: *manifestGeneration,
  609. LoopVars: &daemon.LoopVars{
  610. SyncInterval: *syncInterval,
  611. SyncState: syncProvider,
  612. AutomationInterval: *automationInterval,
  613. GitTimeout: *gitTimeout,
  614. GitVerifySignatures: *gitVerifySignatures,
  615. },
  616. }
  617. {
  618. // Connect to fluxsvc if given an upstream address
  619. if *upstreamURL != "" {
  620. upstreamLogger := log.With(logger, "component", "upstream")
  621. upstreamLogger.Log("URL", *upstreamURL)
  622. upstream, err := daemonhttp.NewUpstream(
  623. &http.Client{Timeout: 10 * time.Second},
  624. fmt.Sprintf("fluxd/%v", version),
  625. client.Token(*token),
  626. transport.NewUpstreamRouter(),
  627. *upstreamURL,
  628. remote.NewErrorLoggingServer(daemon, upstreamLogger),
  629. *rpcTimeout,
  630. upstreamLogger,
  631. )
  632. if err != nil {
  633. logger.Log("err", err)
  634. os.Exit(1)
  635. }
  636. daemon.EventWriter = upstream
  637. go func() {
  638. <-shutdown
  639. upstream.Close()
  640. }()
  641. } else {
  642. logger.Log("upstream", "no upstream URL given")
  643. }
  644. }
  645. shutdownWg.Add(1)
  646. go daemon.Loop(shutdown, shutdownWg, log.With(logger, "component", "sync-loop"))
  647. cacheWarmer.Notify = daemon.AskForAutomatedWorkloadImageUpdates
  648. cacheWarmer.Priority = daemon.ImageRefresh
  649. cacheWarmer.Trace = *registryTrace
  650. shutdownWg.Add(1)
  651. go cacheWarmer.Loop(log.With(logger, "component", "warmer"), shutdown, shutdownWg, imageCreds)
  652. go func() {
  653. mux := http.DefaultServeMux
  654. // Serve /metrics alongside API
  655. if *listenMetricsAddr == "" {
  656. mux.Handle("/metrics", promhttp.Handler())
  657. }
  658. handler := daemonhttp.NewHandler(daemon, daemonhttp.NewRouter())
  659. mux.Handle("/api/flux/", http.StripPrefix("/api/flux", handler))
  660. logger.Log("addr", *listenAddr)
  661. errc <- http.ListenAndServe(*listenAddr, mux)
  662. }()
  663. if *listenMetricsAddr != "" {
  664. go func() {
  665. mux := http.NewServeMux()
  666. mux.Handle("/metrics", promhttp.Handler())
  667. logger.Log("metrics-addr", *listenMetricsAddr)
  668. errc <- http.ListenAndServe(*listenMetricsAddr, mux)
  669. }()
  670. }
  671. // wait here until stopping.
  672. logger.Log("exiting", <-errc)
  673. close(shutdown)
  674. shutdownWg.Wait()
  675. }
  676. func homeDir() string {
  677. // nix
  678. if h := os.Getenv("HOME"); h != "" {
  679. return h
  680. }
  681. // windows
  682. if h := os.Getenv("USERPROFILE"); h != "" {
  683. return h
  684. }
  685. return ""
  686. }