Browse Source

Generate keys in a separate tmpfs volume

In Kubernetes >= 1.10, secrets (and config-maps) will be mounted
read-only. This means we cannot use the tmpfs volume used for the
deploy secret as a workspace for generating new keys (and that we have
to mount the secret with the right mode, since we won't be able to
`chmod` it).

Instead, require _another_ tmpfs to be mounted, and use that. The new,
mandatory flag `--ssh-keygen-dir` is for providing the path. It's
mandatory so that it's harder to accidentally just use a "regular" bit
of the filesystem to generate keys and thereby put them on disk.

So we can still use stable paths, both possible stable locations of
the private key are mentioned in ~/.ssh/config (in the Docker image).
Michael Bridgen 1 year ago
parent
commit
24e3e6061c
5 changed files with 56 additions and 25 deletions
  1. 26
    20
      cluster/kubernetes/sshkeyring.go
  2. 15
    2
      cmd/fluxd/main.go
  3. 11
    1
      deploy/flux-deployment.yaml
  4. 1
    0
      docker/ssh_config
  5. 3
    2
      ssh/keygen.go

+ 26
- 20
cluster/kubernetes/sshkeyring.go View File

@@ -7,6 +7,7 @@ import (
7 7
 	"path/filepath"
8 8
 	"sync"
9 9
 
10
+	"github.com/pkg/errors"
10 11
 	"k8s.io/apimachinery/pkg/types"
11 12
 	"k8s.io/client-go/kubernetes/typed/core/v1"
12 13
 
@@ -30,14 +31,14 @@ type SSHKeyRingConfig struct {
30 31
 	SecretDataKey         string // e.g. "identity"
31 32
 	KeyBits               ssh.OptionalValue
32 33
 	KeyType               ssh.OptionalValue
34
+	KeyGenDir             string // a tmpfs mount; e.g., /var/fluxd/ssh
33 35
 }
34 36
 
35 37
 type sshKeyRing struct {
36 38
 	sync.RWMutex
37 39
 	SSHKeyRingConfig
38
-	publicKey              ssh.PublicKey
39
-	expectedPrivateKeyPath string
40
-	realPrivateKeyPath     string
40
+	publicKey      ssh.PublicKey
41
+	privateKeyPath string
41 42
 }
42 43
 
43 44
 // NewSSHKeyRing constructs an sshKeyRing backed by a kubernetes secret
@@ -46,28 +47,32 @@ type sshKeyRing struct {
46 47
 // generated key if none was found.
47 48
 func NewSSHKeyRing(config SSHKeyRingConfig) (*sshKeyRing, error) {
48 49
 	skr := &sshKeyRing{SSHKeyRingConfig: config}
49
-	skr.expectedPrivateKeyPath = filepath.Join(skr.SecretVolumeMountPath, skr.SecretDataKey)
50
+	mountedPrivateKeyPath := filepath.Join(skr.SecretVolumeMountPath, skr.SecretDataKey)
50 51
 
51
-	fileInfo, err := os.Stat(skr.expectedPrivateKeyPath)
52
+	fileInfo, err := os.Stat(mountedPrivateKeyPath)
52 53
 	switch {
53 54
 	case os.IsNotExist(err):
55
+		// The key is not mounted from the secret, so generate one.
54 56
 		if err := skr.Regenerate(); err != nil {
55 57
 			return nil, err
56 58
 		}
57
-		skr.publicKey, skr.realPrivateKeyPath = skr.KeyPair()
58 59
 	case err != nil:
59
-		return nil, err
60
+		// There's some other problem with that bit of filesystem
61
+		return nil, errors.Wrap(err, "checking for mounted secret")
60 62
 	case fileInfo.Mode() != privateKeyFileMode:
61
-		if err := os.Chmod(skr.expectedPrivateKeyPath, privateKeyFileMode); err != nil {
62
-			return nil, err
63
+		// The key is mounted, but not the right permissions; since
64
+		// it's likely to be read-only, we may not be able to rectify
65
+		// this, but let's try.
66
+		if err := os.Chmod(mountedPrivateKeyPath, privateKeyFileMode); err != nil {
67
+			return nil, errors.Wrap(err, "failed to chmod identity file")
63 68
 		}
64 69
 		fallthrough
65 70
 	default:
66
-		publicKey, err := ssh.ExtractPublicKey(skr.expectedPrivateKeyPath)
71
+		skr.privateKeyPath = mountedPrivateKeyPath
72
+		publicKey, err := ssh.ExtractPublicKey(skr.privateKeyPath)
67 73
 		if err != nil {
68
-			return nil, err
74
+			return nil, errors.Wrap(err, "extracting public key")
69 75
 		}
70
-		skr.realPrivateKeyPath = skr.expectedPrivateKeyPath
71 76
 		skr.publicKey = publicKey
72 77
 	}
73 78
 
@@ -77,16 +82,16 @@ func NewSSHKeyRing(config SSHKeyRingConfig) (*sshKeyRing, error) {
77 82
 // KeyPair returns the current public key and the path to its corresponding
78 83
 // private key. The private key file is guaranteed to exist for the lifetime of
79 84
 // the process, however as the returned pair can be discarded from the keyring
80
-// at any time by use of the regenerate() method it is inadvisable to cache the
85
+// at any time by use of the Regenerate() method it is inadvisable to cache the
81 86
 // results for long periods; instead request the key pair from the ring
82 87
 // immediately prior to each use.
83 88
 func (skr *sshKeyRing) KeyPair() (publicKey ssh.PublicKey, privateKeyPath string) {
84 89
 	skr.RLock()
85 90
 	defer skr.RUnlock()
86
-	return skr.publicKey, skr.expectedPrivateKeyPath
91
+	return skr.publicKey, skr.privateKeyPath
87 92
 }
88 93
 
89
-// regenerate creates a new keypair in the configured SecretVolumeMountPath and
94
+// Regenerate creates a new keypair in the configured SecretVolumeMountPath and
90 95
 // updates the kubernetes secret resource with the private key so that it will
91 96
 // be available to the keyring after restart. If this operation is successful
92 97
 // the keyPair() method will return the new pair; if it fails for any reason,
@@ -97,14 +102,14 @@ func (skr *sshKeyRing) KeyPair() (publicKey ssh.PublicKey, privateKeyPath string
97 102
 // syscall.Mlockall(MCL_FUTURE) in conjunction with an appropriate ulimit to
98 103
 // ensure the private key isn't unintentionally written to persistent storage.
99 104
 func (skr *sshKeyRing) Regenerate() error {
100
-	privateKeyPath, privateKey, publicKey, err := ssh.KeyGen(skr.KeyBits, skr.KeyType, skr.SecretVolumeMountPath)
105
+	tmpPrivateKeyPath, privateKey, publicKey, err := ssh.KeyGen(skr.KeyBits, skr.KeyType, skr.KeyGenDir)
101 106
 	if err != nil {
102 107
 		return err
103 108
 	}
104 109
 
105 110
 	// Prepare a symlink pointing at the new key, to be moved later.
106
-	tmpSymlinkPath := filepath.Join(filepath.Dir(privateKeyPath), "tmp-identity")
107
-	if err = os.Symlink(privateKeyPath, tmpSymlinkPath); err != nil {
111
+	tmpSymlinkPath := filepath.Join(filepath.Dir(tmpPrivateKeyPath), "tmp-identity")
112
+	if err = os.Symlink(tmpPrivateKeyPath, tmpSymlinkPath); err != nil {
108 113
 		return err
109 114
 	}
110 115
 	if err = os.Chmod(tmpSymlinkPath, privateKeyFileMode); err != nil {
@@ -130,13 +135,14 @@ func (skr *sshKeyRing) Regenerate() error {
130 135
 	// The secret is updated, and Kubernetes will eventually make sure
131 136
 	// it's mounted and that `identity` points at it. In the meantime,
132 137
 	// change the symlink to point to our copy of it.
133
-	if err = os.Rename(tmpSymlinkPath, skr.expectedPrivateKeyPath); err != nil {
138
+	generatedPrivateKeyPath := filepath.Join(skr.KeyGenDir, skr.SecretDataKey)
139
+	if err = os.Rename(tmpSymlinkPath, generatedPrivateKeyPath); err != nil {
134 140
 		os.Remove(tmpSymlinkPath)
135 141
 		return err
136 142
 	}
137 143
 
138 144
 	skr.Lock()
139
-	skr.realPrivateKeyPath = privateKeyPath
145
+	skr.privateKeyPath = generatedPrivateKeyPath
140 146
 	skr.publicKey = publicKey
141 147
 	skr.Unlock()
142 148
 

+ 15
- 2
cmd/fluxd/main.go View File

@@ -101,8 +101,9 @@ func main() {
101 101
 		k8sSecretVolumeMountPath = fs.String("k8s-secret-volume-mount-path", "/etc/fluxd/ssh", "Mount location of the k8s secret storing the private SSH key")
102 102
 		k8sSecretDataKey         = fs.String("k8s-secret-data-key", "identity", "Data key holding the private SSH key within the k8s secret")
103 103
 		// SSH key generation
104
-		sshKeyBits = optionalVar(fs, &ssh.KeyBitsValue{}, "ssh-keygen-bits", "-b argument to ssh-keygen (default unspecified)")
105
-		sshKeyType = optionalVar(fs, &ssh.KeyTypeValue{}, "ssh-keygen-type", "-t argument to ssh-keygen (default unspecified)")
104
+		sshKeyBits   = optionalVar(fs, &ssh.KeyBitsValue{}, "ssh-keygen-bits", "-b argument to ssh-keygen (default unspecified)")
105
+		sshKeyType   = optionalVar(fs, &ssh.KeyTypeValue{}, "ssh-keygen-type", "-t argument to ssh-keygen (default unspecified)")
106
+		sshKeygenDir = fs.String("ssh-keygen-dir", "", "directory, ideally on a tmpfs volume, in which to generate new SSH keys when necessary")
106 107
 
107 108
 		upstreamURL = fs.String("connect", "", "Connect to an upstream service e.g., Weave Cloud, at this base address")
108 109
 		token       = fs.String("token", "", "Authentication token for upstream service")
@@ -132,6 +133,8 @@ func main() {
132 133
 	}
133 134
 	logger.Log("started", true)
134 135
 
136
+	// Argument validation
137
+
135 138
 	// Sort out values for the git tag and notes ref. There are
136 139
 	// running deployments that assume the defaults as given, so don't
137 140
 	// mess with those unless explicitly told.
@@ -150,6 +153,15 @@ func main() {
150 153
 		os.Exit(1)
151 154
 	}
152 155
 
156
+	if *sshKeygenDir == "" {
157
+		logger.Log("err", "SSH keygen dir (--ssh-keygen-dir) missing; this is a directory in which to generate SSH keys (ideally mounted from a tmpfs volume)")
158
+		os.Exit(1)
159
+	}
160
+	if _, err := os.Stat(*sshKeygenDir); err != nil && os.IsNotExist(err) {
161
+		logger.Log("err", "SSH keygen dir (supplied as argument --ssh-keygen-dir) does not exist. This should be a directory mounted from a tmpfs volume.")
162
+		os.Exit(1)
163
+	}
164
+
153 165
 	// Cluster component.
154 166
 	var clusterVersion string
155 167
 	var sshKeyRing ssh.KeyRing
@@ -192,6 +204,7 @@ func main() {
192 204
 			SecretDataKey:         *k8sSecretDataKey,
193 205
 			KeyBits:               sshKeyBits,
194 206
 			KeyType:               sshKeyType,
207
+			KeyGenDir:             *sshKeygenDir,
195 208
 		})
196 209
 		if err != nil {
197 210
 			logger.Log("err", err)

+ 11
- 1
deploy/flux-deployment.yaml View File

@@ -15,8 +15,12 @@ spec:
15 15
       serviceAccount: flux
16 16
       volumes:
17 17
       - name: git-key
18
+        defaultMode: 0400 # when mounted read-only, we won't be able to chmod
18 19
         secret:
19 20
           secretName: flux-git-deploy
21
+      - name: git-keygen
22
+        emptyDir:
23
+          medium: Memory
20 24
       containers:
21 25
       - name: flux
22 26
         # There are no ":latest" images for flux. Find the most recent
@@ -28,7 +32,10 @@ spec:
28 32
         - containerPort: 3030 # informational
29 33
         volumeMounts:
30 34
         - name: git-key
31
-          mountPath: /etc/fluxd/ssh
35
+          mountPath: /etc/fluxd/ssh # to match image's ~/.ssh/config
36
+          readOnly: true # this will be the case perforce in K8s >=1.10
37
+        - name: git-keygen
38
+          mountPath: /var/fluxd/keygen # to match image's ~/.ssh/config
32 39
         args:
33 40
 
34 41
         # if you deployed memcached in a different namespace to flux,
@@ -37,6 +44,9 @@ spec:
37 44
         # - --memcached-hostname=memcached.default.svc.cluster.local
38 45
         # - --memcached-service=memcached
39 46
 
47
+        # this must be supplied, and be in the tmpfs (emptyDir) mounted above
48
+        --ssh-keygen-dir=/var/fluxd/keygen
49
+
40 50
         # replace (at least) the following URL
41 51
         - --git-url=git@github.com:weaveworks/flux-example
42 52
         - --git-branch=master

+ 1
- 0
docker/ssh_config View File

@@ -1,4 +1,5 @@
1 1
 Host *
2 2
 StrictHostKeyChecking yes
3 3
 IdentityFile /etc/fluxd/ssh/identity
4
+IdentityFile /var/fluxd/keygen/identity
4 5
 LogLevel error

+ 3
- 2
ssh/keygen.go View File

@@ -2,6 +2,7 @@ package ssh
2 2
 
3 3
 import (
4 4
 	"bytes"
5
+	"errors"
5 6
 	"fmt"
6 7
 	"io/ioutil"
7 8
 	"os/exec"
@@ -156,9 +157,9 @@ type PublicKey struct {
156 157
 // ExtractPublicKey extracts and returns the public key from the specified
157 158
 // private key, along with its fingerprint hashes.
158 159
 func ExtractPublicKey(privateKeyPath string) (PublicKey, error) {
159
-	keyBytes, err := exec.Command("ssh-keygen", "-y", "-f", privateKeyPath).Output()
160
+	keyBytes, err := exec.Command("ssh-keygen", "-y", "-f", privateKeyPath).CombinedOutput()
160 161
 	if err != nil {
161
-		return PublicKey{}, err
162
+		return PublicKey{}, errors.New(string(keyBytes))
162 163
 	}
163 164
 
164 165
 	md5Print, err := ExtractFingerprint(privateKeyPath, "md5")

Loading…
Cancel
Save