Skip to content

Commit 1faee9d

Browse files
committed
digitalocean: bootstrap nodes through kops-controller.
We start with a simple node verifier.
1 parent 8657e25 commit 1faee9d

File tree

10 files changed

+248
-6
lines changed

10 files changed

+248
-6
lines changed

cmd/kops-controller/main.go

+12-1
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ limitations under the License.
1717
package main
1818

1919
import (
20+
"context"
2021
"flag"
2122
"fmt"
2223
"os"
@@ -41,6 +42,7 @@ import (
4142
nodeidentityos "k8s.io/kops/pkg/nodeidentity/openstack"
4243
nodeidentityscw "k8s.io/kops/pkg/nodeidentity/scaleway"
4344
"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
45+
"k8s.io/kops/upup/pkg/fi/cloudup/do"
4446
"k8s.io/kops/upup/pkg/fi/cloudup/gce/tpm/gcetpmverifier"
4547
"k8s.io/kops/upup/pkg/fi/cloudup/hetzner"
4648
"k8s.io/kops/upup/pkg/fi/cloudup/openstack"
@@ -61,6 +63,8 @@ func init() {
6163
}
6264

6365
func main() {
66+
ctx := context.Background()
67+
6468
klog.InitFlags(nil)
6569

6670
// Disable metrics by default (avoid port conflicts, also risky because we are host network)
@@ -97,7 +101,8 @@ func main() {
97101
os.Exit(1)
98102
}
99103

100-
mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
104+
kubeConfig := ctrl.GetConfigOrDie()
105+
mgr, err := ctrl.NewManager(kubeConfig, ctrl.Options{
101106
Scheme: scheme,
102107
MetricsBindAddress: metricsAddress,
103108
LeaderElection: true,
@@ -135,6 +140,12 @@ func main() {
135140
setupLog.Error(err, "unable to create verifier")
136141
os.Exit(1)
137142
}
143+
} else if opt.Server.Provider.DigitalOcean != nil {
144+
verifier, err = do.NewVerifier(ctx, opt.Server.Provider.DigitalOcean)
145+
if err != nil {
146+
setupLog.Error(err, "unable to create verifier")
147+
os.Exit(1)
148+
}
138149
} else {
139150
klog.Fatalf("server cloud provider config not provided")
140151
}

cmd/kops-controller/pkg/config/options.go

+6-4
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ package config
1818

1919
import (
2020
"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
21+
"k8s.io/kops/upup/pkg/fi/cloudup/do"
2122
gcetpm "k8s.io/kops/upup/pkg/fi/cloudup/gce/tpm"
2223
"k8s.io/kops/upup/pkg/fi/cloudup/hetzner"
2324
"k8s.io/kops/upup/pkg/fi/cloudup/openstack"
@@ -65,10 +66,11 @@ type ServerOptions struct {
6566
}
6667

6768
type ServerProviderOptions struct {
68-
AWS *awsup.AWSVerifierOptions `json:"aws,omitempty"`
69-
GCE *gcetpm.TPMVerifierOptions `json:"gce,omitempty"`
70-
Hetzner *hetzner.HetznerVerifierOptions `json:"hetzner,omitempty"`
71-
OpenStack *openstack.OpenStackVerifierOptions `json:"openstack,omitempty"`
69+
AWS *awsup.AWSVerifierOptions `json:"aws,omitempty"`
70+
GCE *gcetpm.TPMVerifierOptions `json:"gce,omitempty"`
71+
Hetzner *hetzner.HetznerVerifierOptions `json:"hetzner,omitempty"`
72+
OpenStack *openstack.OpenStackVerifierOptions `json:"openstack,omitempty"`
73+
DigitalOcean *do.DigitalOceanVerifierOptions `json:"do,omitempty"`
7274
}
7375

7476
// DiscoveryOptions configures our support for discovery, particularly gossip DNS (i.e. k8s.local)

nodeup/pkg/model/bootstrap_client.go

+8
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ import (
2929
"k8s.io/kops/pkg/wellknownports"
3030
"k8s.io/kops/upup/pkg/fi"
3131
"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
32+
"k8s.io/kops/upup/pkg/fi/cloudup/do"
3233
"k8s.io/kops/upup/pkg/fi/cloudup/gce/gcediscovery"
3334
"k8s.io/kops/upup/pkg/fi/cloudup/gce/tpm/gcetpmsigner"
3435
"k8s.io/kops/upup/pkg/fi/cloudup/hetzner"
@@ -80,6 +81,13 @@ func (b BootstrapClientBuilder) Build(c *fi.NodeupModelBuilderContext) error {
8081
}
8182
authenticator = a
8283

84+
case kops.CloudProviderDO:
85+
a, err := do.NewAuthenticator()
86+
if err != nil {
87+
return err
88+
}
89+
authenticator = a
90+
8391
default:
8492
return fmt.Errorf("unsupported cloud provider for authenticator %q", b.CloudProvider())
8593
}

pkg/apis/kops/model/features.go

+4
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,8 @@ func UseKopsControllerForNodeBootstrap(cluster *kops.Cluster) bool {
3131
return true
3232
case kops.CloudProviderOpenstack:
3333
return true
34+
case kops.CloudProviderDO:
35+
return true
3436
default:
3537
return false
3638
}
@@ -41,6 +43,8 @@ func UseChallengeCallback(cloudProvider kops.CloudProviderID) bool {
4143
switch cloudProvider {
4244
case kops.CloudProviderHetzner:
4345
return true
46+
case kops.CloudProviderDO:
47+
return true
4448
default:
4549
return false
4650
}
+71
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
/*
2+
Copyright 2023 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package do
18+
19+
import (
20+
"fmt"
21+
"io"
22+
"net/http"
23+
24+
"k8s.io/kops/pkg/bootstrap"
25+
)
26+
27+
const DOAuthenticationTokenPrefix = "x-digitalocean-droplet-id "
28+
29+
type doAuthenticator struct {
30+
}
31+
32+
var _ bootstrap.Authenticator = &doAuthenticator{}
33+
34+
func NewAuthenticator() (bootstrap.Authenticator, error) {
35+
return &doAuthenticator{}, nil
36+
}
37+
38+
func (o *doAuthenticator) CreateToken(body []byte) (string, error) {
39+
dropletID, err := getMetadataDropletID()
40+
if err != nil {
41+
return "", fmt.Errorf("unable to fetch droplet id: %w", err)
42+
}
43+
return DOAuthenticationTokenPrefix + dropletID, nil
44+
}
45+
46+
const (
47+
dropletIDMetadataURL = "http://169.254.169.254/metadata/v1/id"
48+
)
49+
50+
func getMetadataDropletID() (string, error) {
51+
return getMetadata(dropletIDMetadataURL)
52+
}
53+
54+
func getMetadata(url string) (string, error) {
55+
resp, err := http.Get(url)
56+
if err != nil {
57+
return "", fmt.Errorf("error querying droplet metadata: %w", err)
58+
}
59+
defer resp.Body.Close()
60+
61+
if resp.StatusCode != http.StatusOK {
62+
return "", fmt.Errorf("droplet metadata returned non-200 status code: %d", resp.StatusCode)
63+
}
64+
65+
bodyBytes, err := io.ReadAll(resp.Body)
66+
if err != nil {
67+
return "", fmt.Errorf("error reading droplet metadata: %w", err)
68+
}
69+
70+
return string(bodyBytes), nil
71+
}

upup/pkg/fi/cloudup/do/verifier.go

+125
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
/*
2+
Copyright 2023 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package do
18+
19+
import (
20+
"context"
21+
"errors"
22+
"fmt"
23+
"net"
24+
"net/http"
25+
"os"
26+
"strconv"
27+
"strings"
28+
29+
"github.com/digitalocean/godo"
30+
"golang.org/x/oauth2"
31+
"k8s.io/kops/pkg/bootstrap"
32+
"k8s.io/kops/pkg/wellknownports"
33+
)
34+
35+
type DigitalOceanVerifierOptions struct {
36+
}
37+
38+
type digitalOceanVerifier struct {
39+
doClient *godo.Client
40+
}
41+
42+
var _ bootstrap.Verifier = &digitalOceanVerifier{}
43+
44+
func NewVerifier(ctx context.Context, opt *DigitalOceanVerifierOptions) (bootstrap.Verifier, error) {
45+
accessToken := os.Getenv("DIGITALOCEAN_ACCESS_TOKEN")
46+
if accessToken == "" {
47+
return nil, errors.New("DIGITALOCEAN_ACCESS_TOKEN is required")
48+
}
49+
50+
tokenSource := &TokenSource{
51+
AccessToken: accessToken,
52+
}
53+
54+
oauthClient := oauth2.NewClient(ctx, tokenSource)
55+
doClient := godo.NewClient(oauthClient)
56+
57+
return &digitalOceanVerifier{
58+
doClient: doClient,
59+
}, nil
60+
}
61+
62+
func (o digitalOceanVerifier) VerifyToken(ctx context.Context, rawRequest *http.Request, token string, body []byte, useInstanceIDForNodeName bool) (*bootstrap.VerifyResult, error) {
63+
if !strings.HasPrefix(token, DOAuthenticationTokenPrefix) {
64+
return nil, fmt.Errorf("incorrect authorization type")
65+
}
66+
serverIDString := strings.TrimPrefix(token, DOAuthenticationTokenPrefix)
67+
68+
serverID, err := strconv.Atoi(serverIDString)
69+
if err != nil {
70+
return nil, fmt.Errorf("invalid authorization token")
71+
}
72+
73+
droplet, _, err := o.doClient.Droplets.Get(ctx, serverID)
74+
if err != nil {
75+
return nil, fmt.Errorf("failed to get info for server %v: %w", token, err)
76+
}
77+
78+
var addresses []string
79+
var challengeEndpoints []string
80+
if droplet.Networks != nil {
81+
for _, nic := range droplet.Networks.V4 {
82+
if nic.Type == "private" {
83+
addresses = append(addresses, nic.IPAddress)
84+
challengeEndpoints = append(challengeEndpoints, net.JoinHostPort(nic.IPAddress, strconv.Itoa(wellknownports.NodeupChallenge)))
85+
}
86+
}
87+
for _, nic := range droplet.Networks.V6 {
88+
if nic.Type == "private" {
89+
addresses = append(addresses, nic.IPAddress)
90+
challengeEndpoints = append(challengeEndpoints, net.JoinHostPort(nic.IPAddress, strconv.Itoa(wellknownports.NodeupChallenge)))
91+
}
92+
}
93+
}
94+
95+
// Note: we use TLS passthrough, so we lose the client IP address.
96+
// We therefore don't have a great way to verify the request.
97+
// We do at least prevent duplicate node registrations, preventing some attacks here.
98+
99+
// The node challenge is important here though, verifying the caller has control of the IP address.
100+
101+
nodeName := ""
102+
if len(addresses) == 0 {
103+
// Name seems a better default than the first IP, but we have to match what other components are expecting
104+
nodeName = droplet.Name
105+
} else {
106+
nodeName = addresses[0]
107+
}
108+
109+
if len(challengeEndpoints) == 0 {
110+
return nil, fmt.Errorf("cannot determine challenge endpoint for server %q", serverID)
111+
}
112+
113+
result := &bootstrap.VerifyResult{
114+
NodeName: nodeName,
115+
CertificateNames: addresses,
116+
ChallengeEndpoint: challengeEndpoints[0],
117+
}
118+
119+
for _, tag := range droplet.Tags {
120+
if strings.HasPrefix(tag, TagKubernetesInstanceGroup+":") {
121+
result.InstanceGroupName = strings.TrimPrefix(tag, TagKubernetesInstanceGroup+":")
122+
}
123+
}
124+
return result, nil
125+
}

upup/pkg/fi/cloudup/dotasks/loadbalancer.go

+8
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ import (
2727

2828
"k8s.io/apimachinery/pkg/util/wait"
2929
"k8s.io/klog/v2"
30+
"k8s.io/kops/pkg/wellknownports"
3031
"k8s.io/kops/upup/pkg/fi"
3132
"k8s.io/kops/upup/pkg/fi/cloudup/do"
3233
"k8s.io/kops/util/pkg/vfs"
@@ -130,6 +131,13 @@ func (_ *LoadBalancer) RenderDO(t *do.DOAPITarget, a, e, changes *LoadBalancer)
130131
TargetProtocol: "http",
131132
TargetPort: 80,
132133
},
134+
{
135+
EntryProtocol: "https",
136+
EntryPort: wellknownports.KopsControllerPort,
137+
TargetProtocol: "https",
138+
TargetPort: wellknownports.KopsControllerPort,
139+
TlsPassthrough: true,
140+
},
133141
}
134142

135143
HealthCheck := &godo.HealthCheck{

upup/pkg/fi/cloudup/openstack/verifier.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,7 @@ func NewOpenstackVerifier(opt *OpenStackVerifierOptions) (bootstrap.Verifier, er
8787

8888
kubeClient, err := newClientSet()
8989
if err != nil {
90-
return nil, fmt.Errorf("error building kubernetes client: %v", err)
90+
return nil, fmt.Errorf("error building kubernetes client: %w", err)
9191
}
9292

9393
return &openstackVerifier{

upup/pkg/fi/cloudup/template_functions.go

+4
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ import (
6262
"k8s.io/kops/pkg/wellknownports"
6363
"k8s.io/kops/upup/pkg/fi"
6464
"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
65+
"k8s.io/kops/upup/pkg/fi/cloudup/do"
6566
"k8s.io/kops/upup/pkg/fi/cloudup/gce"
6667
gcetpm "k8s.io/kops/upup/pkg/fi/cloudup/gce/tpm"
6768
"k8s.io/kops/upup/pkg/fi/cloudup/hetzner"
@@ -731,6 +732,9 @@ func (tf *TemplateFunctions) KopsControllerConfig() (string, error) {
731732
case kops.CloudProviderOpenstack:
732733
config.Server.Provider.OpenStack = &openstack.OpenStackVerifierOptions{}
733734

735+
case kops.CloudProviderDO:
736+
config.Server.Provider.DigitalOcean = &do.DigitalOceanVerifierOptions{}
737+
734738
default:
735739
return "", fmt.Errorf("unsupported cloud provider %s", cluster.Spec.GetCloudProvider())
736740
}

upup/pkg/fi/nodeup/command.go

+9
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ import (
5454
"k8s.io/kops/pkg/wellknownports"
5555
"k8s.io/kops/upup/pkg/fi"
5656
"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
57+
"k8s.io/kops/upup/pkg/fi/cloudup/do"
5758
"k8s.io/kops/upup/pkg/fi/cloudup/gce/gcediscovery"
5859
"k8s.io/kops/upup/pkg/fi/cloudup/gce/tpm/gcetpmsigner"
5960
"k8s.io/kops/upup/pkg/fi/cloudup/hetzner"
@@ -761,6 +762,14 @@ func getNodeConfigFromServers(ctx context.Context, bootConfig *nodeup.BootConfig
761762
return nil, err
762763
}
763764
authenticator = a
765+
766+
case api.CloudProviderDO:
767+
a, err := do.NewAuthenticator()
768+
if err != nil {
769+
return nil, err
770+
}
771+
authenticator = a
772+
764773
default:
765774
return nil, fmt.Errorf("unsupported cloud provider for node configuration %s", bootConfig.CloudProvider)
766775
}

0 commit comments

Comments
 (0)