Skip to content

Commit 28e98c0

Browse files
tvonhacht-appletvonhacht-apple
committed
RFC Implementation Supporting ODCR
1 parent 79fbbac commit 28e98c0

File tree

30 files changed

+1253
-178
lines changed

30 files changed

+1253
-178
lines changed

‎charts/karpenter-crd/templates/karpenter.k8s.aws_ec2nodeclasses.yaml

+93
Original file line numberDiff line numberDiff line change
@@ -236,6 +236,40 @@ spec:
236236
x-kubernetes-validations:
237237
- message: must have only one blockDeviceMappings with rootVolume
238238
rule: self.filter(x, has(x.rootVolume)?x.rootVolume==true:false).size() <= 1
239+
capacityReservationSelectorTerms:
240+
description: CapacityReservationSelectorTerms is a list of or Capacity Reservation selector terms. The terms are ORed.
241+
items:
242+
description: |-
243+
CapacityReservationSelectorTerms specify selectors which are ORed together to generate
244+
a list of filters against the EC2 DescribeCapacityReservation API
245+
ID cannot be specified with any other field within a single selector
246+
All other fields are not mutually exclusive and can be combined
247+
properties:
248+
id:
249+
description: |-
250+
The id for the Capacity Reservation
251+
Specifying '*' for this field selects all ids
252+
type: string
253+
ownerId:
254+
description: |-
255+
The id of the AWS account that owns the Capacity Reservation
256+
If no ownerID is specified, only ODCRs owned by the current account will be used
257+
Specifying '*' for this field selects all ownerIDs
258+
type: string
259+
tags:
260+
additionalProperties:
261+
type: string
262+
description: |-
263+
Tags is a map of key/value tags used to select capacity reservations
264+
Specifying '*' for a value selects all values for a given tag key.
265+
maxProperties: 20
266+
type: object
267+
x-kubernetes-validations:
268+
- message: empty tag keys or values aren't supported
269+
rule: self.all(k, k != '' && self[k] != '')
270+
type: object
271+
maxItems: 30
272+
type: array
239273
context:
240274
description: |-
241275
Context is a Reserved field in EC2 APIs
@@ -634,6 +668,65 @@ spec:
634668
- requirements
635669
type: object
636670
type: array
671+
capacityReservations:
672+
description: |-
673+
CapacityReservations contains the current Capacity Reservations values that are available to the
674+
cluster under the CapacityReservations selectors.
675+
items:
676+
description: CapacityReservation contains resolved Capacity Reservation selector values utilized for node launch
677+
properties:
678+
availabilityZone:
679+
description: AvailabilityZone of the Capacity Reservation
680+
type: string
681+
availableInstanceCount:
682+
description: Available Instance Count of the Capacity Reservation
683+
type: integer
684+
endTime:
685+
description: |-
686+
The date and time at which the Capacity Reservation expires. When a Capacity
687+
Reservation expires, the reserved capacity is released and you can no longer
688+
launch instances into it. The Capacity Reservation's state changes to expired
689+
when it reaches its end date and time.
690+
format: date-time
691+
type: string
692+
id:
693+
description: ID of the Capacity Reservation
694+
type: string
695+
instanceMatchCriteria:
696+
description: |-
697+
Indicates the type of instance launches that the Capacity Reservation accepts. The options include:
698+
- open:
699+
The Capacity Reservation accepts all instances that have
700+
matching attributes (instance type, platform, and Availability
701+
Zone). Instances that have matching attributes launch into the
702+
Capacity Reservation automatically without specifying any
703+
additional parameters.
704+
- targeted:
705+
The Capacity Reservation only accepts instances that
706+
have matching attributes (instance type, platform, and
707+
Availability Zone), and explicitly target the Capacity
708+
Reservation. This ensures that only permitted instances can use
709+
the reserved capacity.
710+
type: string
711+
instanceType:
712+
description: Instance Type of the Capacity Reservation
713+
type: string
714+
ownerId:
715+
description: The id of the AWS account that owns the Capacity Reservation
716+
type: string
717+
totalInstanceCount:
718+
description: Total Instance Count of the Capacity Reservation
719+
type: integer
720+
required:
721+
- availabilityZone
722+
- availableInstanceCount
723+
- id
724+
- instanceMatchCriteria
725+
- instanceType
726+
- ownerId
727+
- totalInstanceCount
728+
type: object
729+
type: array
637730
conditions:
638731
description: Conditions contains signals for health and readiness
639732
items:

‎cmd/controller/main.go

+1
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@ func main() {
6565
op.AMIProvider,
6666
op.LaunchTemplateProvider,
6767
op.InstanceTypesProvider,
68+
op.CapacityReservationProvider,
6869
)...).
6970
Start(ctx)
7071
}

‎go.mod

+2
Original file line numberDiff line numberDiff line change
@@ -100,3 +100,5 @@ require (
100100
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
101101
sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
102102
)
103+
104+
replace sigs.k8s.io/karpenter v1.0.1-0.20240921204958-04a921c00ad8 => github.com/tvonhacht-apple/karpenter v0.0.0-20240930185400-e68328e4ea8a

‎go.sum

+2-2
Original file line numberDiff line numberDiff line change
@@ -157,6 +157,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
157157
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
158158
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
159159
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
160+
github.com/tvonhacht-apple/karpenter v0.0.0-20240930185400-e68328e4ea8a h1:+axMcRNEb1rkAd1BHY/YccpWpFbQy9QXVoBLP0pktVY=
161+
github.com/tvonhacht-apple/karpenter v0.0.0-20240930185400-e68328e4ea8a/go.mod h1:odwr/cPdG0y6oTMHsItGMazdAMGUJzz8Kd1+SfrqjKs=
160162
github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
161163
github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
162164
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -276,8 +278,6 @@ sigs.k8s.io/controller-runtime v0.19.0 h1:nWVM7aq+Il2ABxwiCizrVDSlmDcshi9llbaFbC
276278
sigs.k8s.io/controller-runtime v0.19.0/go.mod h1:iRmWllt8IlaLjvTTDLhRBXIEtkCK6hwVBJJsYS9Ajf4=
277279
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=
278280
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
279-
sigs.k8s.io/karpenter v1.0.1-0.20240921204958-04a921c00ad8 h1:F0eGjPSRLd6emk614DNjxgNc97AkVqRSvpnSxxxm0hM=
280-
sigs.k8s.io/karpenter v1.0.1-0.20240921204958-04a921c00ad8/go.mod h1:U70Yuu2wiH1nBC5/nDYYpqYNadYnMgetmVQ5s2NouyU=
281281
sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4=
282282
sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08=
283283
sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E=

‎pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml

+93
Original file line numberDiff line numberDiff line change
@@ -236,6 +236,40 @@ spec:
236236
x-kubernetes-validations:
237237
- message: must have only one blockDeviceMappings with rootVolume
238238
rule: self.filter(x, has(x.rootVolume)?x.rootVolume==true:false).size() <= 1
239+
capacityReservationSelectorTerms:
240+
description: CapacityReservationSelectorTerms is a list of or Capacity Reservation selector terms. The terms are ORed.
241+
items:
242+
description: |-
243+
CapacityReservationSelectorTerms specify selectors which are ORed together to generate
244+
a list of filters against the EC2 DescribeCapacityReservation API
245+
ID cannot be specified with any other field within a single selector
246+
All other fields are not mutually exclusive and can be combined
247+
properties:
248+
id:
249+
description: |-
250+
The id for the Capacity Reservation
251+
Specifying '*' for this field selects all ids
252+
type: string
253+
ownerId:
254+
description: |-
255+
The id of the AWS account that owns the Capacity Reservation
256+
If no ownerID is specified, only ODCRs owned by the current account will be used
257+
Specifying '*' for this field selects all ownerIDs
258+
type: string
259+
tags:
260+
additionalProperties:
261+
type: string
262+
description: |-
263+
Tags is a map of key/value tags used to select capacity reservations
264+
Specifying '*' for a value selects all values for a given tag key.
265+
maxProperties: 20
266+
type: object
267+
x-kubernetes-validations:
268+
- message: empty tag keys or values aren't supported
269+
rule: self.all(k, k != '' && self[k] != '')
270+
type: object
271+
maxItems: 30
272+
type: array
239273
context:
240274
description: |-
241275
Context is a Reserved field in EC2 APIs
@@ -634,6 +668,65 @@ spec:
634668
- requirements
635669
type: object
636670
type: array
671+
capacityReservations:
672+
description: |-
673+
CapacityReservations contains the current Capacity Reservations values that are available to the
674+
cluster under the CapacityReservations selectors.
675+
items:
676+
description: CapacityReservation contains resolved Capacity Reservation selector values utilized for node launch
677+
properties:
678+
availabilityZone:
679+
description: AvailabilityZone of the Capacity Reservation
680+
type: string
681+
availableInstanceCount:
682+
description: Available Instance Count of the Capacity Reservation
683+
type: integer
684+
endTime:
685+
description: |-
686+
The date and time at which the Capacity Reservation expires. When a Capacity
687+
Reservation expires, the reserved capacity is released and you can no longer
688+
launch instances into it. The Capacity Reservation's state changes to expired
689+
when it reaches its end date and time.
690+
format: date-time
691+
type: string
692+
id:
693+
description: ID of the Capacity Reservation
694+
type: string
695+
instanceMatchCriteria:
696+
description: |-
697+
Indicates the type of instance launches that the Capacity Reservation accepts. The options include:
698+
- open:
699+
The Capacity Reservation accepts all instances that have
700+
matching attributes (instance type, platform, and Availability
701+
Zone). Instances that have matching attributes launch into the
702+
Capacity Reservation automatically without specifying any
703+
additional parameters.
704+
- targeted:
705+
The Capacity Reservation only accepts instances that
706+
have matching attributes (instance type, platform, and
707+
Availability Zone), and explicitly target the Capacity
708+
Reservation. This ensures that only permitted instances can use
709+
the reserved capacity.
710+
type: string
711+
instanceType:
712+
description: Instance Type of the Capacity Reservation
713+
type: string
714+
ownerId:
715+
description: The id of the AWS account that owns the Capacity Reservation
716+
type: string
717+
totalInstanceCount:
718+
description: Total Instance Count of the Capacity Reservation
719+
type: integer
720+
required:
721+
- availabilityZone
722+
- availableInstanceCount
723+
- id
724+
- instanceMatchCriteria
725+
- instanceType
726+
- ownerId
727+
- totalInstanceCount
728+
type: object
729+
type: array
637730
conditions:
638731
description: Conditions contains signals for health and readiness
639732
items:

‎pkg/apis/v1/ec2nodeclass.go

+26
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,10 @@ type EC2NodeClassSpec struct {
6464
// +kubebuilder:validation:Enum:={AL2,AL2023,Bottlerocket,Custom,Windows2019,Windows2022}
6565
// +optional
6666
AMIFamily *string `json:"amiFamily,omitempty" hash:"ignore"`
67+
// CapacityReservationSelectorTerms is a list of or Capacity Reservation selector terms. The terms are ORed.
68+
// +kubebuilder:validation:MaxItems:=30
69+
// +optional
70+
CapacityReservationSelectorTerms []CapacityReservationSelectorTerm `json:"capacityReservationSelectorTerms,omitempty" hash:"ignore"`
6771
// UserData to be applied to the provisioned nodes.
6872
// It must be in the appropriate format based on the AMIFamily in use. Karpenter will merge certain fields into
6973
// this UserData to ensure nodes are being provisioned with the correct configuration.
@@ -273,6 +277,28 @@ type KubeletConfiguration struct {
273277
CPUCFSQuota *bool `json:"cpuCFSQuota,omitempty"`
274278
}
275279

280+
// CapacityReservationSelectorTerms specify selectors which are ORed together to generate
281+
// a list of filters against the EC2 DescribeCapacityReservation API
282+
// ID cannot be specified with any other field within a single selector
283+
// All other fields are not mutually exclusive and can be combined
284+
type CapacityReservationSelectorTerm struct {
285+
// The id for the Capacity Reservation
286+
// Specifying '*' for this field selects all ids
287+
// +optional
288+
ID string `json:"id,omitempty"`
289+
// Tags is a map of key/value tags used to select capacity reservations
290+
// Specifying '*' for a value selects all values for a given tag key.
291+
// +kubebuilder:validation:XValidation:message="empty tag keys or values aren't supported",rule="self.all(k, k != '' && self[k] != '')"
292+
// +kubebuilder:validation:MaxProperties:=20
293+
// +optional
294+
Tags map[string]string `json:"tags,omitempty"`
295+
// The id of the AWS account that owns the Capacity Reservation
296+
// If no ownerID is specified, only ODCRs owned by the current account will be used
297+
// Specifying '*' for this field selects all ownerIDs
298+
// +optional
299+
OwnerID string `json:"ownerId,omitempty"`
300+
}
301+
276302
// MetadataOptions contains parameters for specifying the exposure of the
277303
// Instance Metadata Service to provisioned EC2 nodes.
278304
type MetadataOptions struct {

‎pkg/apis/v1/ec2nodeclass_status.go

+48
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ package v1
1717
import (
1818
"github.com/awslabs/operatorpkg/status"
1919
corev1 "k8s.io/api/core/v1"
20+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2021
)
2122

2223
const (
@@ -62,8 +63,55 @@ type AMI struct {
6263
Requirements []corev1.NodeSelectorRequirement `json:"requirements"`
6364
}
6465

66+
// CapacityReservation contains resolved Capacity Reservation selector values utilized for node launch
67+
type CapacityReservation struct {
68+
// ID of the Capacity Reservation
69+
// +required
70+
ID string `json:"id"`
71+
// AvailabilityZone of the Capacity Reservation
72+
// +required
73+
AvailabilityZone string `json:"availabilityZone"`
74+
// Available Instance Count of the Capacity Reservation
75+
// +required
76+
AvailableInstanceCount int `json:"availableInstanceCount"`
77+
// The date and time at which the Capacity Reservation expires. When a Capacity
78+
// Reservation expires, the reserved capacity is released and you can no longer
79+
// launch instances into it. The Capacity Reservation's state changes to expired
80+
// when it reaches its end date and time.
81+
// +optional
82+
EndTime *metav1.Time `json:"endTime,omitempty"`
83+
// Indicates the type of instance launches that the Capacity Reservation accepts. The options include:
84+
// - open:
85+
// The Capacity Reservation accepts all instances that have
86+
// matching attributes (instance type, platform, and Availability
87+
// Zone). Instances that have matching attributes launch into the
88+
// Capacity Reservation automatically without specifying any
89+
// additional parameters.
90+
// - targeted:
91+
// The Capacity Reservation only accepts instances that
92+
// have matching attributes (instance type, platform, and
93+
// Availability Zone), and explicitly target the Capacity
94+
// Reservation. This ensures that only permitted instances can use
95+
// the reserved capacity.
96+
// +required
97+
InstanceMatchCriteria string `json:"instanceMatchCriteria"`
98+
// Instance Type of the Capacity Reservation
99+
// +required
100+
InstanceType string `json:"instanceType"`
101+
// The id of the AWS account that owns the Capacity Reservation
102+
// +required
103+
OwnerID string `json:"ownerId"`
104+
// Total Instance Count of the Capacity Reservation
105+
// +required
106+
TotalInstanceCount int `json:"totalInstanceCount"`
107+
}
108+
65109
// EC2NodeClassStatus contains the resolved state of the EC2NodeClass
66110
type EC2NodeClassStatus struct {
111+
// CapacityReservations contains the current Capacity Reservations values that are available to the
112+
// cluster under the CapacityReservations selectors.
113+
// +optional
114+
CapacityReservations []CapacityReservation `json:"capacityReservations,omitempty"`
67115
// Subnets contains the current Subnet values that are available to the
68116
// cluster under the subnet selectors.
69117
// +optional

‎pkg/apis/v1/labels.go

+2
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ import (
2929
func init() {
3030
karpv1.RestrictedLabelDomains = karpv1.RestrictedLabelDomains.Insert(RestrictedLabelDomains...)
3131
karpv1.WellKnownLabels = karpv1.WellKnownLabels.Insert(
32+
LabelCapactiyReservationID,
3233
LabelInstanceHypervisor,
3334
LabelInstanceEncryptionInTransitSupported,
3435
LabelInstanceCategory,
@@ -124,6 +125,7 @@ var (
124125
AnnotationClusterNameTaggedCompatability = apis.CompatibilityGroup + "/cluster-name-tagged"
125126
AnnotationEC2NodeClassHashVersion = apis.Group + "/ec2nodeclass-hash-version"
126127
AnnotationInstanceTagged = apis.Group + "/tagged"
128+
LabelCapactiyReservationID = apis.Group + "/capacity-reservation-id"
127129

128130
TagNodeClaim = coreapis.Group + "/nodeclaim"
129131
TagManagedLaunchTemplate = apis.Group + "/cluster"

0 commit comments

Comments
 (0)