diff --git a/cmd/routed-eni-cni-plugin/cni.go b/cmd/routed-eni-cni-plugin/cni.go
index bfecb979c3..25ac9332b0 100644
--- a/cmd/routed-eni-cni-plugin/cni.go
+++ b/cmd/routed-eni-cni-plugin/cni.go
@@ -53,6 +53,10 @@ const dummyInterfacePrefix = "dummy"
const npAgentConnTimeout = 2
+const multiNICPodAnnotation = "k8s.amazonaws.com/nicConfig"
+const multiNICAttachment = "multi-nic-attachment"
+const containerVethNamePrefix = "mNicIf"
+
var version string
// NetConf stores the common network config for the CNI plugin
@@ -72,7 +76,12 @@ type NetConf struct {
PluginLogFile string `json:"pluginLogFile"`
- PluginLogLevel string `json:"pluginLogLevel"`
+ PluginLogLevel string `json:"pluginLogLevel"`
+ RuntimeConfig RuntimeConfig `json:"runtimeConfig"`
+}
+
+type RuntimeConfig struct {
+ PodAnnotations map[string]string `json:"io.kubernetes.cri.pod-annotations"`
}
// K8sArgs is the valid CNI_ARGS used for Kubernetes
@@ -153,6 +162,20 @@ func add(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap
mtu := networkutils.GetPodMTU(conf.MTU)
log.Debugf("MTU value set is %d:", mtu)
+ // Derive if Pod requires multiple NIC connected
+ requiresMultiNICAttachment := false
+
+ if value, ok := conf.RuntimeConfig.PodAnnotations[multiNICPodAnnotation]; ok {
+ if value == multiNICAttachment {
+ requiresMultiNICAttachment = true
+ } else {
+ log.Debugf("Multi-NIC annotation mismatch: found=%q, required=%q. Falling back to default configuration.",
+ multiNICAttachment, conf.RuntimeConfig.PodAnnotations[multiNICPodAnnotation])
+ }
+ }
+
+ log.Debugf("pod requires Multi-NIC attachment: %t", requiresMultiNICAttachment)
+
// Set up a connection to the ipamD server.
conn, err := grpcClient.Dial(ipamdAddress, grpc.WithTransportCredentials(insecure.NewCredentials()))
if err != nil {
@@ -174,6 +197,7 @@ func add(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap
ContainerID: args.ContainerID,
NetworkName: conf.Name,
IfName: args.IfName,
+ RequiresMultiNICAttachment: requiresMultiNICAttachment,
})
if err != nil {
@@ -191,47 +215,70 @@ func add(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap
args.ContainerID, args.IfName, r)
// We will let the values in result struct guide us in terms of IP Address Family configured.
- var v4Addr, v6Addr, addr *net.IPNet
- // We don't support dual stack mode currently so it has to be either v4 or v6 mode.
- if r.IPv4Addr != "" {
- v4Addr = &net.IPNet{
- IP: net.ParseIP(r.IPv4Addr),
- Mask: net.CIDRMask(32, 32),
- }
- addr = v4Addr
- } else if r.IPv6Addr != "" {
- v6Addr = &net.IPNet{
- IP: net.ParseIP(r.IPv6Addr),
- Mask: net.CIDRMask(128, 128),
- }
- addr = v6Addr
- }
// AddNetwork guarantees that Gateway string is a valid IPNet
gw := net.ParseIP(r.PodENISubnetGW)
- var hostVethName string
+ var vethMetadata []driver.VirtualInterfaceMetadata
+ var podIPs []*current.IPConfig
+ var podInterfaces []*current.Interface
var dummyInterface *current.Interface
+ for index, ip := range r.IPAddress {
+
+ var hostVethName string
+
+ addr := parseIPAddress(ip)
+ if r.PodVlanId != 0 {
+ hostVethNamePrefix := sgpp.BuildHostVethNamePrefix(conf.VethPrefix, conf.PodSGEnforcingMode)
+ hostVethName = networkutils.GeneratePodHostVethName(hostVethNamePrefix, string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME), index)
+ } else {
+ hostVethName = networkutils.GeneratePodHostVethName(conf.VethPrefix, string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME), index)
+ }
+
+ containerVethName := networkutils.GenerateContainerVethName(args.IfName, containerVethNamePrefix, index)
+
+ vethMetadata = append(vethMetadata, driver.VirtualInterfaceMetadata{
+ IPAddress: addr,
+ DeviceNumber: int(ip.DeviceNumber),
+ RouteTable: int(ip.RouteTableId),
+ HostVethName: hostVethName,
+ ContainerVethName: containerVethName,
+ })
+
+ // Check if we can use PCIID to store RT-ID ?
+ // The RT here is the host route table Id not the container
+ podInterfaces = append(podInterfaces,
+ ¤t.Interface{Name: hostVethName},
+ ¤t.Interface{Name: containerVethName, Sandbox: args.Netns, PciID: fmt.Sprint(ip.RouteTableId)},
+ )
+
+ // This index always points to the container interface so that we get the IP address corresponding to the interface
+ containerInterfaceIndex := len(podInterfaces) - 1
+
+ podIPs = append(podIPs, ¤t.IPConfig{
+ Interface: &containerInterfaceIndex,
+ Address: *addr,
+ Gateway: gw,
+ })
+ }
+
// The dummy interface is purely virtual and is stored in the prevResult struct to assist in cleanup during the DEL command.
- dummyInterfaceName := networkutils.GeneratePodHostVethName(dummyInterfacePrefix, string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME))
+ dummyInterfaceName := networkutils.GeneratePodHostVethName(dummyInterfacePrefix, string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME), 0)
// Non-zero value means pods are using branch ENI
if r.PodVlanId != 0 {
- hostVethNamePrefix := sgpp.BuildHostVethNamePrefix(conf.VethPrefix, conf.PodSGEnforcingMode)
- hostVethName = networkutils.GeneratePodHostVethName(hostVethNamePrefix, string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME))
- err = driverClient.SetupBranchENIPodNetwork(hostVethName, args.IfName, args.Netns, v4Addr, v6Addr, int(r.PodVlanId), r.PodENIMAC,
+ // Since we always return 1 IP, it should be okay to directly address index 0
+ err = driverClient.SetupBranchENIPodNetwork(vethMetadata[0], args.Netns, int(r.PodVlanId), r.PodENIMAC,
r.PodENISubnetGW, int(r.ParentIfIndex), mtu, conf.PodSGEnforcingMode, log)
// For branch ENI mode, the pod VLAN ID is packed in Interface.Mac
dummyInterface = ¤t.Interface{Name: dummyInterfaceName, Mac: fmt.Sprint(r.PodVlanId)}
} else {
- // build hostVethName
- // Note: the maximum length for linux interface name is 15
- hostVethName = networkutils.GeneratePodHostVethName(conf.VethPrefix, string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME))
- err = driverClient.SetupPodNetwork(hostVethName, args.IfName, args.Netns, v4Addr, v6Addr, int(r.DeviceNumber), mtu, log)
+ err = driverClient.SetupPodNetwork(vethMetadata, args.Netns, mtu, log)
// For non-branch ENI, the pod VLAN ID value of 0 is packed in Interface.Mac, while the interface device number is packed in Interface.Sandbox
- dummyInterface = ¤t.Interface{Name: dummyInterfaceName, Mac: fmt.Sprint(0), Sandbox: fmt.Sprint(r.DeviceNumber)}
+ dummyInterface = ¤t.Interface{Name: dummyInterfaceName, Mac: fmt.Sprint(0), Sandbox: fmt.Sprint(vethMetadata[0].DeviceNumber), SocketPath: fmt.Sprint(len(r.IPAddress))}
}
+
log.Debugf("Using dummy interface: %v", dummyInterface)
if err != nil {
@@ -258,24 +305,9 @@ func add(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap
return errors.Wrap(err, "add command: failed to setup network")
}
- containerInterfaceIndex := 1
- ips := []*current.IPConfig{
- {
- Interface: &containerInterfaceIndex,
- Address: *addr,
- Gateway: gw,
- },
- }
-
- hostInterface := ¤t.Interface{Name: hostVethName}
- containerInterface := ¤t.Interface{Name: args.IfName, Sandbox: args.Netns}
-
result := ¤t.Result{
- IPs: ips,
- Interfaces: []*current.Interface{
- hostInterface,
- containerInterface,
- },
+ IPs: podIPs,
+ Interfaces: podInterfaces,
}
// dummy interface is appended to PrevResult for use during cleanup
@@ -302,6 +334,7 @@ func add(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap
K8S_POD_NAME: string(k8sArgs.K8S_POD_NAME),
K8S_POD_NAMESPACE: string(k8sArgs.K8S_POD_NAMESPACE),
NETWORK_POLICY_MODE: r.NetworkPolicyMode,
+ InterfaceCount: int32(len(r.IPAddress)),
})
// No need to cleanup IP and network, kubelet will send delete.
@@ -338,6 +371,10 @@ func del(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap
log.Errorf("Failed to load k8s config from args: %v", err)
return errors.Wrap(err, "del cmd: failed to load k8s config from args")
}
+ // During del we cannot depend on the annotation as annotation change doesn't restart a POD
+ // We can add it to the dummy interface and use that during the deletion flow
+ // Or Pass the number of interfaces so that it is returned by prev result.
+ // OR store this information in ipamd datastore so that it always returns what is assigned.
// For pods using branch ENI, try to delete using previous result
handled, err := tryDelWithPrevResult(driverClient, conf, k8sArgs, args.IfName, args.Netns, log)
@@ -415,21 +452,21 @@ func del(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap
log.Infof("Received del network response from ipamd for pod %s namespace %s sandbox %s: %+v", string(k8sArgs.K8S_POD_NAME),
string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_INFRA_CONTAINER_ID), r)
- var deletedPodIP net.IP
- var maskLen int
- if r.IPv4Addr != "" {
- deletedPodIP = net.ParseIP(r.IPv4Addr)
- maskLen = 32
- } else if r.IPv6Addr != "" {
- deletedPodIP = net.ParseIP(r.IPv6Addr)
- maskLen = 128
- }
+ var vethMetadata []driver.VirtualInterfaceMetadata
+
+ for _, ip := range r.IPAddress {
+ addr := parseIPAddress(ip)
- if deletedPodIP != nil {
- addr := &net.IPNet{
- IP: deletedPodIP,
- Mask: net.CIDRMask(maskLen, maskLen),
+ if addr != nil {
+ vethMetadata = append(vethMetadata, driver.VirtualInterfaceMetadata{
+ IPAddress: addr,
+ DeviceNumber: int(ip.DeviceNumber),
+ RouteTable: int(ip.RouteTableId),
+ })
}
+ }
+
+ if len(vethMetadata) > 0 {
// vlanID != 0 means pod using security group
if r.PodVlanId != 0 {
@@ -437,9 +474,9 @@ func del(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap
log.Infof("Ignoring TeardownPodENI as Netns is empty for SG pod:%s namespace: %s containerID:%s", k8sArgs.K8S_POD_NAME, k8sArgs.K8S_POD_NAMESPACE, k8sArgs.K8S_POD_INFRA_CONTAINER_ID)
return nil
}
- err = driverClient.TeardownBranchENIPodNetwork(addr, int(r.PodVlanId), conf.PodSGEnforcingMode, log)
+ err = driverClient.TeardownBranchENIPodNetwork(vethMetadata[0], int(r.PodVlanId), conf.PodSGEnforcingMode, log)
} else {
- err = driverClient.TeardownPodNetwork(addr, int(r.DeviceNumber), log)
+ err = driverClient.TeardownPodNetwork(vethMetadata, log)
}
if err != nil {
@@ -448,7 +485,7 @@ func del(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap
return errors.Wrap(err, "del cmd: failed on tear down pod network")
}
} else {
- log.Warnf("Container %s did not have a valid IP %s", args.ContainerID, r.IPv4Addr)
+ log.Warnf("Container %s did not have a valid IP %+v", args.ContainerID, r.IPAddress)
}
// Set up a connection to the network policy agent
@@ -479,16 +516,16 @@ func del(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap
return nil
}
-func getContainerIP(prevResult *current.Result, contVethName string) (net.IPNet, error) {
- containerIfaceIndex, _, found := cniutils.FindInterfaceByName(prevResult.Interfaces, contVethName)
+func getContainerNetworkMetadata(prevResult *current.Result, contVethName string) (net.IPNet, *current.Interface, error) {
+ containerIfaceIndex, containerInterface, found := cniutils.FindInterfaceByName(prevResult.Interfaces, contVethName)
if !found {
- return net.IPNet{}, errors.Errorf("cannot find contVethName %s in prevResult", contVethName)
+ return net.IPNet{}, nil, errors.Errorf("cannot find contVethName %s in prevResult", contVethName)
}
containerIPs := cniutils.FindIPConfigsByIfaceIndex(prevResult.IPs, containerIfaceIndex)
if len(containerIPs) != 1 {
- return net.IPNet{}, errors.Errorf("found %d containerIPs for %v in prevResult", len(containerIPs), contVethName)
+ return net.IPNet{}, nil, errors.Errorf("found %d containerIPs for %v in prevResult", len(containerIPs), contVethName)
}
- return containerIPs[0].Address, nil
+ return containerIPs[0].Address, containerInterface, nil
}
// tryDelWithPrevResult will try to process CNI delete request without IPAMD.
@@ -500,7 +537,7 @@ func tryDelWithPrevResult(driverClient driver.NetworkAPIs, conf *NetConf, k8sArg
return false, nil
}
- dummyIfaceName := networkutils.GeneratePodHostVethName(dummyInterfacePrefix, string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME))
+ dummyIfaceName := networkutils.GeneratePodHostVethName(dummyInterfacePrefix, string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME), 0)
_, dummyIface, found := cniutils.FindInterfaceByName(prevResult.Interfaces, dummyIfaceName)
if !found {
return false, nil
@@ -518,12 +555,16 @@ func tryDelWithPrevResult(driverClient driver.NetworkAPIs, conf *NetConf, k8sArg
return true, nil
}
- containerIP, err := getContainerIP(prevResult, contVethName)
+ containerIP, _, err := getContainerNetworkMetadata(prevResult, contVethName)
if err != nil {
return false, err
}
- if err := driverClient.TeardownBranchENIPodNetwork(&containerIP, podVlanID, conf.PodSGEnforcingMode, log); err != nil {
+ vethMetadata := driver.VirtualInterfaceMetadata{
+ IPAddress: &containerIP,
+ }
+
+ if err := driverClient.TeardownBranchENIPodNetwork(vethMetadata, podVlanID, conf.PodSGEnforcingMode, log); err != nil {
return true, err
}
return true, nil
@@ -538,7 +579,7 @@ func teardownPodNetworkWithPrevResult(driverClient driver.NetworkAPIs, conf *Net
log.Infof("PrevResult not available for pod. Pod may have already been deleted.")
return false
}
- dummyIfaceName := networkutils.GeneratePodHostVethName(dummyInterfacePrefix, string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME))
+ dummyIfaceName := networkutils.GeneratePodHostVethName(dummyInterfacePrefix, string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME), 0)
_, dummyIface, found := cniutils.FindInterfaceByName(prevResult.Interfaces, dummyIfaceName)
if !found {
return false
@@ -554,13 +595,48 @@ func teardownPodNetworkWithPrevResult(driverClient driver.NetworkAPIs, conf *Net
log.Errorf("Invalid device number for pod: %s", dummyIface.Sandbox)
return false
}
- containerIP, err := getContainerIP(prevResult, contVethName)
- if err != nil {
- log.Errorf("Failed to get container IP: %v", err)
- return false
+
+ // Since we are always using dummy interface to store the device number of network card 0 IP
+ // RT ID for NC-0 is also stored in the container interface entry. So we have a path for migration where
+ // getting the device number from dummy interface can be deprecated entirely. This is currently done to keep it backwards compatible
+ routeTableId := deviceNumber + 1
+
+ var interfacesAttached = 1
+ if dummyIface.SocketPath != "" {
+ interfacesAttached, err = strconv.Atoi(dummyIface.SocketPath)
+ if err != nil {
+ log.Errorf("error getting number of interfaces attached to the pod: %s", dummyIface.SocketPath)
+ return false
+ }
+ }
+
+ var vethMetadata []driver.VirtualInterfaceMetadata
+ for v := range interfacesAttached {
+ containerInterfaceName := networkutils.GenerateContainerVethName(contVethName, containerVethNamePrefix, v)
+ containerIP, containerInterface, err := getContainerNetworkMetadata(prevResult, containerInterfaceName)
+ if err != nil {
+ log.Errorf("Failed to get container IP: %v", err)
+ return false
+ }
+
+ // If this property is set, that means the container metadata has the route table ID which we can use
+ // If this is not set, it is a pod launched before this change was introduced.
+ // So it is only managing network card 0 at that time and device number + 1 is the route table ID which we have above
+ if dummyIface.SocketPath != "" {
+ routeTableId, err = strconv.Atoi(containerInterface.PciID)
+ if err != nil {
+ log.Errorf("error getting route table number of the interface %s", containerInterface.PciID)
+ return false
+ }
+ }
+
+ vethMetadata = append(vethMetadata, driver.VirtualInterfaceMetadata{
+ IPAddress: &containerIP,
+ RouteTable: routeTableId,
+ })
}
- if err := driverClient.TeardownPodNetwork(&containerIP, deviceNumber, log); err != nil {
+ if err := driverClient.TeardownPodNetwork(vethMetadata, log); err != nil {
log.Errorf("Failed to teardown pod network: %v", err)
return false
}
@@ -587,3 +663,20 @@ func main() {
}
os.Exit(exitCode)
}
+
+func parseIPAddress(v *pb.IPAddress) *net.IPNet {
+
+ if v.IPv4Addr != "" {
+ return &net.IPNet{
+ IP: net.ParseIP(v.IPv4Addr),
+ Mask: net.CIDRMask(32, 32),
+ }
+ } else if v.IPv6Addr != "" {
+ return &net.IPNet{
+ IP: net.ParseIP(v.IPv6Addr),
+ Mask: net.CIDRMask(128, 128),
+ }
+ }
+
+ return nil
+}
diff --git a/cmd/routed-eni-cni-plugin/cni_test.go b/cmd/routed-eni-cni-plugin/cni_test.go
index 8954489fb4..dfc376b024 100644
--- a/cmd/routed-eni-cni-plugin/cni_test.go
+++ b/cmd/routed-eni-cni-plugin/cni_test.go
@@ -30,6 +30,7 @@ import (
"github.com/stretchr/testify/assert"
"google.golang.org/grpc"
+ "github.com/aws/amazon-vpc-cni-k8s/cmd/routed-eni-cni-plugin/driver"
mock_driver "github.com/aws/amazon-vpc-cni-k8s/cmd/routed-eni-cni-plugin/driver/mocks"
mock_grpcwrapper "github.com/aws/amazon-vpc-cni-k8s/pkg/grpcwrapper/mocks"
mock_rpcwrapper "github.com/aws/amazon-vpc-cni-k8s/pkg/rpcwrapper/mocks"
@@ -103,15 +104,16 @@ func TestCmdAdd(t *testing.T) {
enforceNpReply := &rpc.EnforceNpReply{Success: true}
mockNP.EXPECT().EnforceNpToPod(gomock.Any(), gomock.Any()).Return(enforceNpReply, nil).Times(1)
- addNetworkReply := &rpc.AddNetworkReply{Success: true, IPv4Addr: ipAddr, DeviceNumber: devNum, NetworkPolicyMode: "none"}
+ addrs := []*rpc.IPAddress{&rpc.IPAddress{
+ IPv4Addr: ipAddr,
+ DeviceNumber: devNum,
+ RouteTableId: devNum + 1,
+ }}
+
+ addNetworkReply := &rpc.AddNetworkReply{Success: true, IPAddress: addrs, NetworkPolicyMode: "none"}
mockC.EXPECT().AddNetwork(gomock.Any(), gomock.Any()).Return(addNetworkReply, nil)
- v4Addr := &net.IPNet{
- IP: net.ParseIP(addNetworkReply.IPv4Addr),
- Mask: net.IPv4Mask(255, 255, 255, 255),
- }
- mocksNetwork.EXPECT().SetupPodNetwork(gomock.Any(), cmdArgs.IfName, cmdArgs.Netns,
- v4Addr, nil, int(addNetworkReply.DeviceNumber), gomock.Any(), gomock.Any()).Return(nil)
+ mocksNetwork.EXPECT().SetupPodNetwork(gomock.Any(), cmdArgs.Netns, gomock.Any(), gomock.Any()).Return(nil)
mocksTypes.EXPECT().PrintResult(gomock.Any(), gomock.Any()).Return(nil).Times(1)
@@ -144,18 +146,19 @@ func TestCmdAddWithNPenabled(t *testing.T) {
mockNP := mock_rpc.NewMockNPBackendClient(ctrl)
mocksRPC.EXPECT().NewNPBackendClient(npConn).Return(mockNP)
- addNetworkReply := &rpc.AddNetworkReply{Success: true, IPv4Addr: ipAddr, DeviceNumber: devNum, NetworkPolicyMode: "strict"}
+ addrs := []*rpc.IPAddress{&rpc.IPAddress{
+ IPv4Addr: ipAddr,
+ DeviceNumber: devNum,
+ RouteTableId: devNum + 1,
+ }}
+
+ addNetworkReply := &rpc.AddNetworkReply{Success: true, IPAddress: addrs, NetworkPolicyMode: "strict"}
mockC.EXPECT().AddNetwork(gomock.Any(), gomock.Any()).Return(addNetworkReply, nil)
enforceNpReply := &rpc.EnforceNpReply{Success: true}
mockNP.EXPECT().EnforceNpToPod(gomock.Any(), gomock.Any()).Return(enforceNpReply, nil)
- v4Addr := &net.IPNet{
- IP: net.ParseIP(addNetworkReply.IPv4Addr),
- Mask: net.IPv4Mask(255, 255, 255, 255),
- }
- mocksNetwork.EXPECT().SetupPodNetwork(gomock.Any(), cmdArgs.IfName, cmdArgs.Netns,
- v4Addr, nil, int(addNetworkReply.DeviceNumber), gomock.Any(), gomock.Any()).Return(nil)
+ mocksNetwork.EXPECT().SetupPodNetwork(gomock.Any(), cmdArgs.Netns, gomock.Any(), gomock.Any()).Return(nil)
mocksTypes.EXPECT().PrintResult(gomock.Any(), gomock.Any()).Return(nil)
@@ -188,18 +191,19 @@ func TestCmdAddWithNPenabledWithErr(t *testing.T) {
mockNP := mock_rpc.NewMockNPBackendClient(ctrl)
mocksRPC.EXPECT().NewNPBackendClient(npConn).Return(mockNP)
- addNetworkReply := &rpc.AddNetworkReply{Success: true, IPv4Addr: ipAddr, DeviceNumber: devNum, NetworkPolicyMode: "strict"}
+ addrs := []*rpc.IPAddress{&rpc.IPAddress{
+ IPv4Addr: ipAddr,
+ DeviceNumber: devNum,
+ RouteTableId: devNum + 1,
+ }}
+
+ addNetworkReply := &rpc.AddNetworkReply{Success: true, IPAddress: addrs, NetworkPolicyMode: "strict"}
mockC.EXPECT().AddNetwork(gomock.Any(), gomock.Any()).Return(addNetworkReply, nil)
enforceNpReply := &rpc.EnforceNpReply{Success: false}
mockNP.EXPECT().EnforceNpToPod(gomock.Any(), gomock.Any()).Return(enforceNpReply, errors.New("Error on EnforceNpReply"))
- v4Addr := &net.IPNet{
- IP: net.ParseIP(addNetworkReply.IPv4Addr),
- Mask: net.IPv4Mask(255, 255, 255, 255),
- }
- mocksNetwork.EXPECT().SetupPodNetwork(gomock.Any(), cmdArgs.IfName, cmdArgs.Netns,
- v4Addr, nil, int(addNetworkReply.DeviceNumber), gomock.Any(), gomock.Any()).Return(nil)
+ mocksNetwork.EXPECT().SetupPodNetwork(gomock.Any(), cmdArgs.Netns, gomock.Any(), gomock.Any()).Return(nil)
err := add(cmdArgs, mocksTypes, mocksGRPC, mocksRPC, mocksNetwork)
assert.Error(t, err)
@@ -224,7 +228,13 @@ func TestCmdAddNetworkErr(t *testing.T) {
mockC := mock_rpc.NewMockCNIBackendClient(ctrl)
mocksRPC.EXPECT().NewCNIBackendClient(conn).Return(mockC)
- addNetworkReply := &rpc.AddNetworkReply{Success: false, IPv4Addr: ipAddr, DeviceNumber: devNum, NetworkPolicyMode: "none"}
+ addrs := []*rpc.IPAddress{&rpc.IPAddress{
+ IPv4Addr: ipAddr,
+ DeviceNumber: devNum,
+ RouteTableId: devNum + 1,
+ }}
+
+ addNetworkReply := &rpc.AddNetworkReply{Success: false, IPAddress: addrs, NetworkPolicyMode: "none"}
mockC.EXPECT().AddNetwork(gomock.Any(), gomock.Any()).Return(addNetworkReply, errors.New("Error on AddNetworkReply"))
err := add(cmdArgs, mocksTypes, mocksGRPC, mocksRPC, mocksNetwork)
@@ -251,19 +261,18 @@ func TestCmdAddErrSetupPodNetwork(t *testing.T) {
mockC := mock_rpc.NewMockCNIBackendClient(ctrl)
mocksRPC.EXPECT().NewCNIBackendClient(conn).Return(mockC)
- addNetworkReply := &rpc.AddNetworkReply{Success: true, IPv4Addr: ipAddr, DeviceNumber: devNum, NetworkPolicyMode: "none"}
+ addrs := []*rpc.IPAddress{&rpc.IPAddress{
+ IPv4Addr: ipAddr,
+ DeviceNumber: devNum,
+ RouteTableId: devNum + 1,
+ }}
+ addNetworkReply := &rpc.AddNetworkReply{Success: true, IPAddress: addrs, NetworkPolicyMode: "none"}
mockC.EXPECT().AddNetwork(gomock.Any(), gomock.Any()).Return(addNetworkReply, nil)
- addr := &net.IPNet{
- IP: net.ParseIP(addNetworkReply.IPv4Addr),
- Mask: net.IPv4Mask(255, 255, 255, 255),
- }
-
- mocksNetwork.EXPECT().SetupPodNetwork(gomock.Any(), cmdArgs.IfName, cmdArgs.Netns,
- addr, nil, int(addNetworkReply.DeviceNumber), gomock.Any(), gomock.Any()).Return(errors.New("error on SetupPodNetwork"))
+ mocksNetwork.EXPECT().SetupPodNetwork(gomock.Any(), cmdArgs.Netns, gomock.Any(), gomock.Any()).Return(errors.New("error on SetupPodNetwork"))
// when SetupPodNetwork fails, expect to return IP back to datastore
- delNetworkReply := &rpc.DelNetworkReply{Success: true, IPv4Addr: ipAddr, DeviceNumber: devNum}
+ delNetworkReply := &rpc.DelNetworkReply{Success: true, IPAddress: addrs}
mockC.EXPECT().DelNetwork(gomock.Any(), gomock.Any()).Return(delNetworkReply, nil)
err := add(cmdArgs, mocksTypes, mocksGRPC, mocksRPC, mocksNetwork)
@@ -296,18 +305,19 @@ func TestCmdDel(t *testing.T) {
mockNP := mock_rpc.NewMockNPBackendClient(ctrl)
mocksRPC.EXPECT().NewNPBackendClient(npConn).Return(mockNP)
- delNetworkReply := &rpc.DelNetworkReply{Success: true, IPv4Addr: ipAddr, DeviceNumber: devNum}
+ addrs := []*rpc.IPAddress{&rpc.IPAddress{
+ IPv4Addr: ipAddr,
+ DeviceNumber: devNum,
+ RouteTableId: devNum + 1,
+ }}
+
+ delNetworkReply := &rpc.DelNetworkReply{Success: true, IPAddress: addrs}
mockC.EXPECT().DelNetwork(gomock.Any(), gomock.Any()).Return(delNetworkReply, nil)
deleteNpReply := &rpc.DeleteNpReply{Success: true}
mockNP.EXPECT().DeletePodNp(gomock.Any(), gomock.Any()).Return(deleteNpReply, nil)
- addr := &net.IPNet{
- IP: net.ParseIP(delNetworkReply.IPv4Addr),
- Mask: net.IPv4Mask(255, 255, 255, 255),
- }
-
- mocksNetwork.EXPECT().TeardownPodNetwork(addr, int(delNetworkReply.DeviceNumber), gomock.Any()).Return(nil)
+ mocksNetwork.EXPECT().TeardownPodNetwork(gomock.Any(), gomock.Any()).Return(nil)
err := del(cmdArgs, mocksTypes, mocksGRPC, mocksRPC, mocksNetwork)
assert.Nil(t, err)
@@ -332,7 +342,13 @@ func TestCmdDelErrDelNetwork(t *testing.T) {
mockC := mock_rpc.NewMockCNIBackendClient(ctrl)
mocksRPC.EXPECT().NewCNIBackendClient(conn).Return(mockC)
- delNetworkReply := &rpc.DelNetworkReply{Success: false, IPv4Addr: ipAddr, DeviceNumber: devNum}
+ addrs := []*rpc.IPAddress{&rpc.IPAddress{
+ IPv4Addr: ipAddr,
+ DeviceNumber: devNum,
+ RouteTableId: devNum + 1,
+ }}
+
+ delNetworkReply := &rpc.DelNetworkReply{Success: false, IPAddress: addrs}
mockC.EXPECT().DelNetwork(gomock.Any(), gomock.Any()).Return(delNetworkReply, errors.New("error on DelNetwork"))
@@ -359,17 +375,16 @@ func TestCmdDelErrTeardown(t *testing.T) {
mocksGRPC.EXPECT().Dial(gomock.Any(), gomock.Any()).Return(conn, nil)
mockC := mock_rpc.NewMockCNIBackendClient(ctrl)
mocksRPC.EXPECT().NewCNIBackendClient(conn).Return(mockC)
-
- delNetworkReply := &rpc.DelNetworkReply{Success: true, IPv4Addr: ipAddr, DeviceNumber: devNum}
+ addrs := []*rpc.IPAddress{&rpc.IPAddress{
+ IPv4Addr: ipAddr,
+ DeviceNumber: devNum,
+ RouteTableId: devNum + 1,
+ }}
+ delNetworkReply := &rpc.DelNetworkReply{Success: true, IPAddress: addrs}
mockC.EXPECT().DelNetwork(gomock.Any(), gomock.Any()).Return(delNetworkReply, nil)
- addr := &net.IPNet{
- IP: net.ParseIP(delNetworkReply.IPv4Addr),
- Mask: net.IPv4Mask(255, 255, 255, 255),
- }
-
- mocksNetwork.EXPECT().TeardownPodNetwork(addr, int(delNetworkReply.DeviceNumber), gomock.Any()).Return(errors.New("error on teardown"))
+ mocksNetwork.EXPECT().TeardownPodNetwork(gomock.Any(), gomock.Any()).Return(errors.New("error on teardown"))
err := del(cmdArgs, mocksTypes, mocksGRPC, mocksRPC, mocksNetwork)
assert.Error(t, err)
@@ -400,18 +415,17 @@ func TestCmdAddForPodENINetwork(t *testing.T) {
mockNP := mock_rpc.NewMockNPBackendClient(ctrl)
mocksRPC.EXPECT().NewNPBackendClient(npConn).Return(mockNP)
- addNetworkReply := &rpc.AddNetworkReply{Success: true, IPv4Addr: ipAddr, PodENISubnetGW: "10.0.0.1", PodVlanId: 1,
+ addrs := []*rpc.IPAddress{&rpc.IPAddress{
+ IPv4Addr: ipAddr}}
+
+ addNetworkReply := &rpc.AddNetworkReply{Success: true, IPAddress: addrs, PodENISubnetGW: "10.0.0.1", PodVlanId: 1,
PodENIMAC: "eniHardwareAddr", ParentIfIndex: 2, NetworkPolicyMode: "none"}
mockC.EXPECT().AddNetwork(gomock.Any(), gomock.Any()).Return(addNetworkReply, nil)
enforceNpReply := &rpc.EnforceNpReply{Success: true}
mockNP.EXPECT().EnforceNpToPod(gomock.Any(), gomock.Any()).Return(enforceNpReply, nil)
- addr := &net.IPNet{
- IP: net.ParseIP(addNetworkReply.IPv4Addr),
- Mask: net.IPv4Mask(255, 255, 255, 255),
- }
- mocksNetwork.EXPECT().SetupBranchENIPodNetwork(gomock.Any(), cmdArgs.IfName, cmdArgs.Netns, addr, nil, 1, "eniHardwareAddr",
+ mocksNetwork.EXPECT().SetupBranchENIPodNetwork(gomock.Any(), cmdArgs.Netns, 1, "eniHardwareAddr",
"10.0.0.1", 2, gomock.Any(), sgpp.EnforcingModeStrict, gomock.Any()).Return(nil)
mocksTypes.EXPECT().PrintResult(gomock.Any(), gomock.Any()).Return(nil)
@@ -446,17 +460,19 @@ func TestCmdDelForPodENINetwork(t *testing.T) {
mockNP := mock_rpc.NewMockNPBackendClient(ctrl)
mocksRPC.EXPECT().NewNPBackendClient(npConn).Return(mockNP)
- delNetworkReply := &rpc.DelNetworkReply{Success: true, IPv4Addr: ipAddr, PodVlanId: 1}
+ addrs := []*rpc.IPAddress{&rpc.IPAddress{
+ IPv4Addr: ipAddr,
+ DeviceNumber: devNum,
+ RouteTableId: devNum + 1,
+ }}
+
+ delNetworkReply := &rpc.DelNetworkReply{Success: true, IPAddress: addrs, PodVlanId: 1}
mockC.EXPECT().DelNetwork(gomock.Any(), gomock.Any()).Return(delNetworkReply, nil)
deleteNpReply := &rpc.DeleteNpReply{Success: true}
mockNP.EXPECT().DeletePodNp(gomock.Any(), gomock.Any()).Return(deleteNpReply, nil)
- addr := &net.IPNet{
- IP: net.ParseIP(delNetworkReply.IPv4Addr),
- Mask: net.IPv4Mask(255, 255, 255, 255),
- }
- mocksNetwork.EXPECT().TeardownBranchENIPodNetwork(addr, 1, sgpp.EnforcingModeStrict, gomock.Any()).Return(nil)
+ mocksNetwork.EXPECT().TeardownBranchENIPodNetwork(gomock.Any(), 1, sgpp.EnforcingModeStrict, gomock.Any()).Return(nil)
err := del(cmdArgs, mocksTypes, mocksGRPC, mocksRPC, mocksNetwork)
assert.Nil(t, err)
@@ -843,7 +859,11 @@ func Test_tryDelWithPrevResult(t *testing.T) {
driverClient := mock_driver.NewMockNetworkAPIs(ctrl)
for _, call := range tt.fields.teardownBranchENIPodNetworkCalls {
- driverClient.EXPECT().TeardownBranchENIPodNetwork(call.containerAddr, call.vlanID, call.podSGEnforcingMode, gomock.Any()).Return(call.err)
+ vethMetadata := driver.VirtualInterfaceMetadata{
+ IPAddress: call.containerAddr,
+ }
+
+ driverClient.EXPECT().TeardownBranchENIPodNetwork(vethMetadata, call.vlanID, call.podSGEnforcingMode, gomock.Any()).Return(call.err)
}
got, err := tryDelWithPrevResult(driverClient, tt.args.conf, tt.args.k8sArgs, tt.args.contVethName, "/proc/1/ns", testLogger)
@@ -1152,7 +1172,7 @@ func Test_teardownPodNetworkWithPrevResult(t *testing.T) {
driverClient := mock_driver.NewMockNetworkAPIs(ctrl)
for _, call := range tt.fields.teardownPodNetworkCalls {
- driverClient.EXPECT().TeardownPodNetwork(call.containerAddr, call.deviceNumber, gomock.Any()).Return(call.err)
+ driverClient.EXPECT().TeardownPodNetwork(gomock.Any(), gomock.Any()).Return(call.err)
}
handled := teardownPodNetworkWithPrevResult(driverClient, tt.args.conf, tt.args.k8sArgs, tt.args.contVethName, testLogger)
diff --git a/cmd/routed-eni-cni-plugin/driver/driver.go b/cmd/routed-eni-cni-plugin/driver/driver.go
index 2c650894e6..b57f3b6a1c 100644
--- a/cmd/routed-eni-cni-plugin/driver/driver.go
+++ b/cmd/routed-eni-cni-plugin/driver/driver.go
@@ -45,18 +45,25 @@ const (
v6DADTimeout = 10 * time.Second
)
+type VirtualInterfaceMetadata struct {
+ IPAddress *net.IPNet
+ DeviceNumber int
+ RouteTable int
+ HostVethName string
+ ContainerVethName string
+}
+
// NetworkAPIs defines network API calls
type NetworkAPIs interface {
// SetupPodNetwork sets up pod network for normal ENI based pods
- SetupPodNetwork(hostVethName string, contVethName string, netnsPath string, v4Addr *net.IPNet, v6Addr *net.IPNet, deviceNumber int, mtu int, log logger.Logger) error
+ SetupPodNetwork(vethMetadata []VirtualInterfaceMetadata, netnsPath string, mtu int, log logger.Logger) error
// TeardownPodNetwork clean up pod network for normal ENI based pods
- TeardownPodNetwork(containerAddr *net.IPNet, deviceNumber int, log logger.Logger) error
-
+ TeardownPodNetwork(vethMetadata []VirtualInterfaceMetadata, log logger.Logger) error
// SetupBranchENIPodNetwork sets up pod network for branch ENI based pods
- SetupBranchENIPodNetwork(hostVethName string, contVethName string, netnsPath string, v4Addr *net.IPNet, v6Addr *net.IPNet, vlanID int, eniMAC string,
+ SetupBranchENIPodNetwork(vethMetadata VirtualInterfaceMetadata, netnsPath string, vlanID int, eniMAC string,
subnetGW string, parentIfIndex int, mtu int, podSGEnforcingMode sgpp.EnforcingMode, log logger.Logger) error
// TeardownBranchENIPodNetwork cleans up pod network for branch ENI based pods
- TeardownBranchENIPodNetwork(containerAddr *net.IPNet, vlanID int, podSGEnforcingMode sgpp.EnforcingMode, log logger.Logger) error
+ TeardownBranchENIPodNetwork(vethMetadata VirtualInterfaceMetadata, vlanID int, podSGEnforcingMode sgpp.EnforcingMode, log logger.Logger) error
}
type linuxNetwork struct {
@@ -79,24 +86,24 @@ func New() NetworkAPIs {
type createVethPairContext struct {
contVethName string
hostVethName string
- v4Addr *net.IPNet
- v6Addr *net.IPNet
+ ipAddr *net.IPNet
netLink netlinkwrapper.NetLink
ip ipwrapper.IP
mtu int
procSys procsyswrapper.ProcSys
+ index int
}
-func newCreateVethPairContext(contVethName string, hostVethName string, v4Addr *net.IPNet, v6Addr *net.IPNet, mtu int) *createVethPairContext {
+func newCreateVethPairContext(contVethName string, hostVethName string, ipAddr *net.IPNet, mtu int, index int) *createVethPairContext {
return &createVethPairContext{
contVethName: contVethName,
hostVethName: hostVethName,
- v4Addr: v4Addr,
- v6Addr: v6Addr,
+ ipAddr: ipAddr,
netLink: netlinkwrapper.NewNetLink(),
ip: ipwrapper.NewIP(),
mtu: mtu,
procSys: procsyswrapper.NewProcSys(),
+ index: index,
}
}
@@ -137,7 +144,8 @@ func (createVethContext *createVethPairContext) run(hostNS ns.NetNS) error {
return errors.Wrapf(err, "setup NS network: failed to set link %q up", createVethContext.contVethName)
}
- if createVethContext.v6Addr != nil && createVethContext.v6Addr.IP.To16() != nil {
+ // this means it's a V6 IP address
+ if createVethContext.ipAddr.IP.To4() == nil {
// Enable v6 support on Container's veth interface.
if err = createVethContext.procSys.Set(fmt.Sprintf("net/ipv6/conf/%s/disable_ipv6", createVethContext.contVethName), "0"); err != nil {
if !os.IsNotExist(err) {
@@ -146,7 +154,7 @@ func (createVethContext *createVethPairContext) run(hostNS ns.NetNS) error {
}
// Enable v6 support on Container's lo interface inside the Pod networking namespace.
- if err = createVethContext.procSys.Set(fmt.Sprintf("net/ipv6/conf/lo/disable_ipv6"), "0"); err != nil {
+ if err = createVethContext.procSys.Set("net/ipv6/conf/lo/disable_ipv6", "0"); err != nil {
if !os.IsNotExist(err) {
return errors.Wrapf(err, "setupVeth network: failed to enable IPv6 on container's lo interface")
}
@@ -163,27 +171,45 @@ func (createVethContext *createVethPairContext) run(hostNS ns.NetNS) error {
var addr *netlink.Addr
var defNet *net.IPNet
- if createVethContext.v4Addr != nil {
- gw = net.IPv4(169, 254, 1, 1)
+ if createVethContext.ipAddr.IP.To4() != nil {
+ gw = net.IPv4(169, 254, 1, byte(createVethContext.index)+1)
maskLen = 32
- addr = &netlink.Addr{IPNet: createVethContext.v4Addr}
+ addr = &netlink.Addr{IPNet: createVethContext.ipAddr}
defNet = &net.IPNet{IP: net.IPv4zero, Mask: net.CIDRMask(0, maskLen)}
- } else if createVethContext.v6Addr != nil {
- gw = net.IP{0xfe, 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}
+ } else {
+ gw = net.IP{0xfe, 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, byte(createVethContext.index) + 1}
maskLen = 128
- addr = &netlink.Addr{IPNet: createVethContext.v6Addr}
+ addr = &netlink.Addr{IPNet: createVethContext.ipAddr}
defNet = &net.IPNet{IP: net.IPv6zero, Mask: net.CIDRMask(0, maskLen)}
}
gwNet := &net.IPNet{IP: gw, Mask: net.CIDRMask(maskLen, maskLen)}
+ // If Index > 0 that means it has multiple IPs. Add IP rule + add default route to
+ rtTable := unix.RT_TABLE_MAIN
+ if createVethContext.index > 0 {
+ rtTable = createVethContext.index
+ }
+
if err = createVethContext.netLink.RouteReplace(&netlink.Route{
LinkIndex: contVeth.Attrs().Index,
Scope: netlink.SCOPE_LINK,
- Dst: gwNet}); err != nil {
+ Dst: gwNet,
+ Table: rtTable}); err != nil {
return errors.Wrap(err, "setup NS network: failed to add default gateway")
}
+ if createVethContext.index > 0 {
+ // Add a from interface rule
+ fromInterfaceRule := createVethContext.netLink.NewRule()
+ fromInterfaceRule.Src = createVethContext.ipAddr
+ fromInterfaceRule.Priority = networkutils.FromInterfaceRulePriority
+ fromInterfaceRule.Table = rtTable
+ if err := createVethContext.netLink.RuleAdd(fromInterfaceRule); err != nil && !networkutils.IsRuleExistsError(err) {
+ return errors.Wrapf(err, "failed to setup fromInterface rule, containerAddr=%s, rtTable=%v", createVethContext.ipAddr.String(), createVethContext.index)
+ }
+ }
+
// Add a default route via dummy next hop(169.254.1.1 or fe80::1). Then all outgoing traffic will be routed by this
// default route via dummy next hop (169.254.1.1 or fe80::1)
if err = createVethContext.netLink.RouteAdd(&netlink.Route{
@@ -191,6 +217,7 @@ func (createVethContext *createVethPairContext) run(hostNS ns.NetNS) error {
Scope: netlink.SCOPE_UNIVERSE,
Dst: defNet,
Gw: gw,
+ Table: rtTable,
}); err != nil {
return errors.Wrap(err, "setup NS network: failed to add default route")
}
@@ -212,8 +239,8 @@ func (createVethContext *createVethPairContext) run(hostNS ns.NetNS) error {
if err = createVethContext.netLink.NeighAdd(neigh); err != nil {
return errors.Wrap(err, "setup NS network: failed to add static ARP")
}
-
- if createVethContext.v6Addr != nil && createVethContext.v6Addr.IP.To16() != nil {
+ // if IP is not IPv4 or a v4 in v6 address, it return nil
+ if createVethContext.ipAddr.IP.To4() == nil {
if err := cniutils.WaitForAddressesToBeStable(createVethContext.netLink, createVethContext.contVethName, v6DADTimeout, WAIT_INTERVAL); err != nil {
return errors.Wrap(err, "setup NS network: failed while waiting for v6 addresses to be stable")
}
@@ -229,55 +256,60 @@ func (createVethContext *createVethPairContext) run(hostNS ns.NetNS) error {
// SetupPodNetwork wires up linux networking for a pod's network
// we expect v4Addr and v6Addr to have correct IPAddress Family.
-func (n *linuxNetwork) SetupPodNetwork(hostVethName string, contVethName string, netnsPath string, v4Addr *net.IPNet, v6Addr *net.IPNet,
- deviceNumber int, mtu int, log logger.Logger) error {
- log.Debugf("SetupPodNetwork: hostVethName=%s, contVethName=%s, netnsPath=%s, v4Addr=%v, v6Addr=%v, deviceNumber=%d, mtu=%d",
- hostVethName, contVethName, netnsPath, v4Addr, v6Addr, deviceNumber, mtu)
+func (n *linuxNetwork) SetupPodNetwork(vethMetadata []VirtualInterfaceMetadata, netnsPath string, mtu int, log logger.Logger) error {
+ for index, vethData := range vethMetadata {
- hostVeth, err := n.setupVeth(hostVethName, contVethName, netnsPath, v4Addr, v6Addr, mtu, log)
- if err != nil {
- return errors.Wrapf(err, "SetupPodNetwork: failed to setup veth pair")
- }
+ log.Debugf("SetupPodNetwork: hostVethName=%s, contVethName=%s, netnsPath=%s, ipAddr=%v, routeTableNumber=%d, mtu=%d",
+ vethData.HostVethName, vethData.ContainerVethName, netnsPath, vethData.IPAddress, vethData.RouteTable, mtu)
- var containerAddr *net.IPNet
- if v4Addr != nil {
- containerAddr = v4Addr
- } else if v6Addr != nil {
- containerAddr = v6Addr
- }
+ hostVeth, err := n.setupVeth(vethData.HostVethName, vethData.ContainerVethName, netnsPath, vethData.IPAddress, mtu, log, index)
+ if err != nil {
+ return errors.Wrapf(err, "SetupPodNetwork: failed to setup veth pair")
+ }
- rtTable := unix.RT_TABLE_MAIN
- if deviceNumber > 0 {
- rtTable = deviceNumber + 1
- }
- if err := n.setupIPBasedContainerRouteRules(hostVeth, containerAddr, rtTable, log); err != nil {
- return errors.Wrapf(err, "SetupPodNetwork: unable to setup IP based container routes and rules")
+ rtTable := unix.RT_TABLE_MAIN
+ if vethData.RouteTable > 1 {
+ rtTable = vethData.RouteTable
+ }
+
+ if err := n.setupIPBasedContainerRouteRules(hostVeth, vethData.IPAddress, rtTable, log); err != nil {
+ return errors.Wrapf(err, "SetupPodNetwork: unable to setup IP based container routes and rules")
+ }
}
return nil
}
// TeardownPodNetwork cleanup ip rules
-func (n *linuxNetwork) TeardownPodNetwork(containerAddr *net.IPNet, deviceNumber int, log logger.Logger) error {
- log.Debugf("TeardownPodNetwork: containerAddr=%s, deviceNumber=%d", containerAddr.String(), deviceNumber)
+func (n *linuxNetwork) TeardownPodNetwork(vethMetadata []VirtualInterfaceMetadata, log logger.Logger) error {
- rtTable := unix.RT_TABLE_MAIN
- if deviceNumber > 0 {
- rtTable = deviceNumber + 1
- }
- if err := n.teardownIPBasedContainerRouteRules(containerAddr, rtTable, log); err != nil {
- return errors.Wrapf(err, "TeardownPodNetwork: unable to teardown IP based container routes and rules")
+ for _, vethData := range vethMetadata {
+
+ log.Debugf("TeardownPodNetwork: containerAddr=%s, routeTable=%d", vethData.IPAddress.String(), vethData.RouteTable)
+
+ // Route table ID for primary ENI (Network 0, Device 0) => (0* MaxENI + 0 + 1)
+ // which is why we only update if the RT > 1
+ rtTable := unix.RT_TABLE_MAIN
+ if vethData.RouteTable > 1 {
+ rtTable = vethData.RouteTable
+ }
+
+ if err := n.teardownIPBasedContainerRouteRules(vethData.IPAddress, rtTable, log); err != nil {
+ return errors.Wrapf(err, "TeardownPodNetwork: unable to teardown IP based container routes and rules")
+ }
}
+
return nil
}
// SetupBranchENIPodNetwork sets up the network ns for pods requesting its own security group
// we expect v4Addr and v6Addr to have correct IPAddress Family.
-func (n *linuxNetwork) SetupBranchENIPodNetwork(hostVethName string, contVethName string, netnsPath string, v4Addr *net.IPNet, v6Addr *net.IPNet,
+func (n *linuxNetwork) SetupBranchENIPodNetwork(vethMetadata VirtualInterfaceMetadata, netnsPath string,
vlanID int, eniMAC string, subnetGW string, parentIfIndex int, mtu int, podSGEnforcingMode sgpp.EnforcingMode, log logger.Logger) error {
- log.Debugf("SetupBranchENIPodNetwork: hostVethName=%s, contVethName=%s, netnsPath=%s, v4Addr=%v, v6Addr=%v, vlanID=%d, eniMAC=%s, subnetGW=%s, parentIfIndex=%d, mtu=%d, podSGEnforcingMode=%v",
- hostVethName, contVethName, netnsPath, v4Addr, v6Addr, vlanID, eniMAC, subnetGW, parentIfIndex, mtu, podSGEnforcingMode)
- hostVeth, err := n.setupVeth(hostVethName, contVethName, netnsPath, v4Addr, v6Addr, mtu, log)
+ log.Debugf("SetupBranchENIPodNetwork: hostVethName=%s, contVethName=%s, netnsPath=%s, ipAddr=%v, vlanID=%d, eniMAC=%s, subnetGW=%s, parentIfIndex=%d, mtu=%d, podSGEnforcingMode=%v",
+ vethMetadata.HostVethName, vethMetadata.ContainerVethName, netnsPath, vethMetadata.IPAddress, vlanID, eniMAC, subnetGW, parentIfIndex, mtu, podSGEnforcingMode)
+
+ hostVeth, err := n.setupVeth(vethMetadata.HostVethName, vethMetadata.ContainerVethName, netnsPath, vethMetadata.IPAddress, mtu, log, 0)
if err != nil {
return errors.Wrapf(err, "SetupBranchENIPodNetwork: failed to setup veth pair")
}
@@ -288,13 +320,15 @@ func (n *linuxNetwork) SetupBranchENIPodNetwork(hostVethName string, contVethNam
// now since we obtain vlanID from prevResult during pod deletion, we should be able to correctly purge hostVeth during pod deletion and thus don't need this logic.
// this logic is kept here for safety purpose.
oldFromHostVethRule := n.netLink.NewRule()
- oldFromHostVethRule.IifName = hostVethName
+ oldFromHostVethRule.IifName = vethMetadata.HostVethName
oldFromHostVethRule.Priority = networkutils.VlanRulePriority
- if v6Addr != nil {
+
+ // If IPv4 it returns the IP address back
+ if vethMetadata.IPAddress.IP.To4() == nil {
oldFromHostVethRule.Family = unix.AF_INET6
}
if err := networkutils.NetLinkRuleDelAll(n.netLink, oldFromHostVethRule); err != nil {
- return errors.Wrapf(err, "SetupBranchENIPodNetwork: failed to delete hostVeth rule for %s", hostVethName)
+ return errors.Wrapf(err, "SetupBranchENIPodNetwork: failed to delete hostVeth rule for %s", vethMetadata.HostVethName)
}
rtTable := vlanID + 100
@@ -303,20 +337,13 @@ func (n *linuxNetwork) SetupBranchENIPodNetwork(hostVethName string, contVethNam
return errors.Wrapf(err, "SetupBranchENIPodNetwork: failed to setup vlan")
}
- var containerAddr *net.IPNet
- if v4Addr != nil {
- containerAddr = v4Addr
- } else if v6Addr != nil {
- containerAddr = v6Addr
- }
-
switch podSGEnforcingMode {
case sgpp.EnforcingModeStrict:
- if err := n.setupIIFBasedContainerRouteRules(hostVeth, containerAddr, vlanLink, rtTable, log); err != nil {
+ if err := n.setupIIFBasedContainerRouteRules(hostVeth, vethMetadata.IPAddress, vlanLink, rtTable, log); err != nil {
return errors.Wrapf(err, "SetupBranchENIPodNetwork: unable to setup IIF based container routes and rules")
}
case sgpp.EnforcingModeStandard:
- if err := n.setupIPBasedContainerRouteRules(hostVeth, containerAddr, rtTable, log); err != nil {
+ if err := n.setupIPBasedContainerRouteRules(hostVeth, vethMetadata.IPAddress, rtTable, log); err != nil {
return errors.Wrapf(err, "SetupBranchENIPodNetwork: unable to setup IP based container routes and rules")
}
}
@@ -324,15 +351,15 @@ func (n *linuxNetwork) SetupBranchENIPodNetwork(hostVethName string, contVethNam
}
// TeardownBranchENIPodNetwork tears down the vlan and corresponding ip rules.
-func (n *linuxNetwork) TeardownBranchENIPodNetwork(containerAddr *net.IPNet, vlanID int, _ sgpp.EnforcingMode, log logger.Logger) error {
- log.Debugf("TeardownBranchENIPodNetwork: containerAddr=%s, vlanID=%d", containerAddr.String(), vlanID)
+func (n *linuxNetwork) TeardownBranchENIPodNetwork(vethMetadata VirtualInterfaceMetadata, vlanID int, _ sgpp.EnforcingMode, log logger.Logger) error {
+ log.Debugf("TeardownBranchENIPodNetwork: containerAddr=%s, vlanID=%d", vethMetadata.IPAddress.String(), vlanID)
if err := n.teardownVlan(vlanID, log); err != nil {
return errors.Wrapf(err, "TeardownBranchENIPodNetwork: failed to teardown vlan")
}
ipFamily := unix.AF_INET
- if containerAddr.IP.To4() == nil {
+ if vethMetadata.IPAddress.IP.To4() == nil {
ipFamily = unix.AF_INET6
}
// to handle the migration between different enforcingMode, we try to clean up rules under both mode since the pod might be setup with a different mode.
@@ -340,7 +367,7 @@ func (n *linuxNetwork) TeardownBranchENIPodNetwork(containerAddr *net.IPNet, vla
if err := n.teardownIIFBasedContainerRouteRules(rtTable, ipFamily, log); err != nil {
return errors.Wrapf(err, "TeardownBranchENIPodNetwork: unable to teardown IIF based container routes and rules")
}
- if err := n.teardownIPBasedContainerRouteRules(containerAddr, rtTable, log); err != nil {
+ if err := n.teardownIPBasedContainerRouteRules(vethMetadata.IPAddress, rtTable, log); err != nil {
return errors.Wrapf(err, "TeardownBranchENIPodNetwork: unable to teardown IP based container routes and rules")
}
@@ -348,7 +375,7 @@ func (n *linuxNetwork) TeardownBranchENIPodNetwork(containerAddr *net.IPNet, vla
}
// setupVeth sets up veth for the pod.
-func (n *linuxNetwork) setupVeth(hostVethName string, contVethName string, netnsPath string, v4Addr *net.IPNet, v6Addr *net.IPNet, mtu int, log logger.Logger) (netlink.Link, error) {
+func (n *linuxNetwork) setupVeth(hostVethName string, contVethName string, netnsPath string, ipAddr *net.IPNet, mtu int, log logger.Logger, index int) (netlink.Link, error) {
// Clean up if hostVeth exists.
if oldHostVeth, err := n.netLink.LinkByName(hostVethName); err == nil {
if err = n.netLink.LinkDel(oldHostVeth); err != nil {
@@ -357,7 +384,7 @@ func (n *linuxNetwork) setupVeth(hostVethName string, contVethName string, netns
log.Debugf("Successfully deleted old hostVeth %s", hostVethName)
}
- createVethContext := newCreateVethPairContext(contVethName, hostVethName, v4Addr, v6Addr, mtu)
+ createVethContext := newCreateVethPairContext(contVethName, hostVethName, ipAddr, mtu, index)
if err := n.ns.WithNetNSPath(netnsPath, createVethContext.run); err != nil {
return nil, errors.Wrap(err, "failed to setup veth network")
}
diff --git a/cmd/routed-eni-cni-plugin/driver/driver_test.go b/cmd/routed-eni-cni-plugin/driver/driver_test.go
index 7af04ebef5..2f4f5b63ce 100644
--- a/cmd/routed-eni-cni-plugin/driver/driver_test.go
+++ b/cmd/routed-eni-cni-plugin/driver/driver_test.go
@@ -106,10 +106,10 @@ func Test_linuxNetwork_SetupPodNetwork(t *testing.T) {
hostVethName string
contVethName string
netnsPath string
- v4Addr *net.IPNet
- v6Addr *net.IPNet
+ ipAddr *net.IPNet
deviceNumber int
mtu int
+ routeTableId int
}
tests := []struct {
name string
@@ -174,9 +174,9 @@ func Test_linuxNetwork_SetupPodNetwork(t *testing.T) {
hostVethName: "eni8ea2c11fe35",
contVethName: "eth0",
netnsPath: "/proc/42/ns/net",
- v4Addr: containerAddr,
- v6Addr: nil,
+ ipAddr: containerAddr, // v4 address
deviceNumber: 0,
+ routeTableId: 1,
mtu: 9001,
},
},
@@ -240,9 +240,9 @@ func Test_linuxNetwork_SetupPodNetwork(t *testing.T) {
hostVethName: "eni8ea2c11fe35",
contVethName: "eth0",
netnsPath: "/proc/42/ns/net",
- v4Addr: containerAddr,
- v6Addr: nil,
+ ipAddr: containerAddr, // v4 address
deviceNumber: 3,
+ routeTableId: 4,
mtu: 9001,
},
},
@@ -266,9 +266,9 @@ func Test_linuxNetwork_SetupPodNetwork(t *testing.T) {
hostVethName: "eni8ea2c11fe35",
contVethName: "eth0",
netnsPath: "/proc/42/ns/net",
- v4Addr: containerAddr,
- v6Addr: nil,
+ ipAddr: containerAddr, // v4 address
deviceNumber: 3,
+ routeTableId: 4,
mtu: 9001,
},
wantErr: errors.New("SetupPodNetwork: failed to setup veth pair: failed to setup veth network: some error"),
@@ -326,9 +326,9 @@ func Test_linuxNetwork_SetupPodNetwork(t *testing.T) {
hostVethName: "eni8ea2c11fe35",
contVethName: "eth0",
netnsPath: "/proc/42/ns/net",
- v4Addr: containerAddr,
- v6Addr: nil,
+ ipAddr: containerAddr, // v4 Address
deviceNumber: 3,
+ routeTableId: 4,
mtu: 9001,
},
wantErr: errors.New("SetupPodNetwork: unable to setup IP based container routes and rules: failed to setup container route, containerAddr=192.168.100.42/32, hostVeth=eni8ea2c11fe35, rtTable=main: some error"),
@@ -374,7 +374,17 @@ func Test_linuxNetwork_SetupPodNetwork(t *testing.T) {
ns: ns,
procSys: procSys,
}
- err := n.SetupPodNetwork(tt.args.hostVethName, tt.args.contVethName, tt.args.netnsPath, tt.args.v4Addr, tt.args.v6Addr, tt.args.deviceNumber, tt.args.mtu, testLogger)
+ vIfMetadata := []VirtualInterfaceMetadata{
+ {
+ IPAddress: tt.args.ipAddr,
+ DeviceNumber: tt.args.deviceNumber,
+ RouteTable: tt.args.routeTableId,
+ HostVethName: tt.args.hostVethName,
+ ContainerVethName: tt.args.contVethName,
+ },
+ }
+
+ err := n.SetupPodNetwork(vIfMetadata, tt.args.netnsPath, tt.args.mtu, testLogger)
if tt.wantErr != nil {
assert.EqualError(t, err, tt.wantErr.Error())
} else {
@@ -420,6 +430,7 @@ func Test_linuxNetwork_TeardownPodNetwork(t *testing.T) {
type args struct {
containerAddr *net.IPNet
deviceNumber int
+ routeTableId int
}
tests := []struct {
name string
@@ -444,6 +455,7 @@ func Test_linuxNetwork_TeardownPodNetwork(t *testing.T) {
args: args{
containerAddr: containerAddr,
deviceNumber: 0,
+ routeTableId: 1,
},
},
{
@@ -470,6 +482,7 @@ func Test_linuxNetwork_TeardownPodNetwork(t *testing.T) {
args: args{
containerAddr: containerAddr,
deviceNumber: 3,
+ routeTableId: 4,
},
},
{
@@ -485,6 +498,7 @@ func Test_linuxNetwork_TeardownPodNetwork(t *testing.T) {
args: args{
containerAddr: containerAddr,
deviceNumber: 3,
+ routeTableId: 4,
},
wantErr: errors.New("TeardownPodNetwork: unable to teardown IP based container routes and rules: failed to delete toContainer rule, containerAddr=192.168.100.42/32, rtTable=main: some error"),
},
@@ -506,7 +520,14 @@ func Test_linuxNetwork_TeardownPodNetwork(t *testing.T) {
n := &linuxNetwork{
netLink: netLink,
}
- err := n.TeardownPodNetwork(tt.args.containerAddr, tt.args.deviceNumber, testLogger)
+ vIfMetadata := []VirtualInterfaceMetadata{
+ {
+ IPAddress: tt.args.containerAddr,
+ DeviceNumber: tt.args.deviceNumber,
+ RouteTable: tt.args.routeTableId,
+ },
+ }
+ err := n.TeardownPodNetwork(vIfMetadata, testLogger)
if tt.wantErr != nil {
assert.EqualError(t, err, tt.wantErr.Error())
} else {
@@ -647,8 +668,7 @@ func Test_linuxNetwork_SetupBranchENIPodNetwork(t *testing.T) {
hostVethName string
contVethName string
netnsPath string
- v4Addr *net.IPNet
- v6Addr *net.IPNet
+ ipAddr *net.IPNet
vlanID int
eniMAC string
subnetGW string
@@ -770,8 +790,7 @@ func Test_linuxNetwork_SetupBranchENIPodNetwork(t *testing.T) {
hostVethName: "eni8ea2c11fe35",
contVethName: "eth0",
netnsPath: "/proc/42/ns/net",
- v4Addr: containerAddr,
- v6Addr: nil,
+ ipAddr: containerAddr,
vlanID: vlanID,
eniMAC: eniMac,
subnetGW: subnetGW,
@@ -888,8 +907,7 @@ func Test_linuxNetwork_SetupBranchENIPodNetwork(t *testing.T) {
hostVethName: "eni8ea2c11fe35",
contVethName: "eth0",
netnsPath: "/proc/42/ns/net",
- v4Addr: nil,
- v6Addr: containerV6Addr,
+ ipAddr: containerV6Addr, // v6Address
vlanID: vlanID,
eniMAC: eniMac,
subnetGW: subnetV6GW,
@@ -1006,8 +1024,7 @@ func Test_linuxNetwork_SetupBranchENIPodNetwork(t *testing.T) {
hostVethName: "eni8ea2c11fe35",
contVethName: "eth0",
netnsPath: "/proc/42/ns/net",
- v4Addr: containerAddr,
- v6Addr: nil,
+ ipAddr: containerAddr, //v4 Address
vlanID: vlanID,
eniMAC: eniMac,
subnetGW: subnetGW,
@@ -1124,8 +1141,7 @@ func Test_linuxNetwork_SetupBranchENIPodNetwork(t *testing.T) {
hostVethName: "eni8ea2c11fe35",
contVethName: "eth0",
netnsPath: "/proc/42/ns/net",
- v4Addr: nil,
- v6Addr: containerV6Addr,
+ ipAddr: containerV6Addr, // v6
vlanID: vlanID,
eniMAC: eniMac,
subnetGW: subnetV6GW,
@@ -1154,8 +1170,7 @@ func Test_linuxNetwork_SetupBranchENIPodNetwork(t *testing.T) {
hostVethName: "eni8ea2c11fe35",
contVethName: "eth0",
netnsPath: "/proc/42/ns/net",
- v4Addr: containerAddr,
- v6Addr: nil,
+ ipAddr: containerAddr,
vlanID: vlanID,
eniMAC: eniMac,
subnetGW: subnetGW,
@@ -1213,8 +1228,7 @@ func Test_linuxNetwork_SetupBranchENIPodNetwork(t *testing.T) {
hostVethName: "eni8ea2c11fe35",
contVethName: "eth0",
netnsPath: "/proc/42/ns/net",
- v4Addr: containerAddr,
- v6Addr: nil,
+ ipAddr: containerAddr,
vlanID: vlanID,
eniMAC: eniMac,
subnetGW: subnetGW,
@@ -1282,8 +1296,7 @@ func Test_linuxNetwork_SetupBranchENIPodNetwork(t *testing.T) {
hostVethName: "eni8ea2c11fe35",
contVethName: "eth0",
netnsPath: "/proc/42/ns/net",
- v4Addr: containerAddr,
- v6Addr: nil,
+ ipAddr: containerAddr,
vlanID: vlanID,
eniMAC: eniMac,
subnetGW: subnetGW,
@@ -1394,8 +1407,7 @@ func Test_linuxNetwork_SetupBranchENIPodNetwork(t *testing.T) {
hostVethName: "eni8ea2c11fe35",
contVethName: "eth0",
netnsPath: "/proc/42/ns/net",
- v4Addr: containerAddr,
- v6Addr: nil,
+ ipAddr: containerAddr,
vlanID: vlanID,
eniMAC: eniMac,
subnetGW: subnetGW,
@@ -1506,8 +1518,7 @@ func Test_linuxNetwork_SetupBranchENIPodNetwork(t *testing.T) {
hostVethName: "eni8ea2c11fe35",
contVethName: "eth0",
netnsPath: "/proc/42/ns/net",
- v4Addr: containerAddr,
- v6Addr: nil,
+ ipAddr: containerAddr,
vlanID: vlanID,
eniMAC: eniMac,
subnetGW: subnetGW,
@@ -1571,7 +1582,14 @@ func Test_linuxNetwork_SetupBranchENIPodNetwork(t *testing.T) {
ns: ns,
procSys: procSys,
}
- err := n.SetupBranchENIPodNetwork(tt.args.hostVethName, tt.args.contVethName, tt.args.netnsPath, tt.args.v4Addr, tt.args.v6Addr, tt.args.vlanID, tt.args.eniMAC, tt.args.subnetGW, tt.args.parentIfIndex, tt.args.mtu, tt.args.podSGEnforcingMode, testLogger)
+
+ vIfMetadata := VirtualInterfaceMetadata{
+ IPAddress: tt.args.ipAddr,
+ HostVethName: tt.args.hostVethName,
+ ContainerVethName: tt.args.contVethName,
+ }
+
+ err := n.SetupBranchENIPodNetwork(vIfMetadata, tt.args.netnsPath, tt.args.vlanID, tt.args.eniMAC, tt.args.subnetGW, tt.args.parentIfIndex, tt.args.mtu, tt.args.podSGEnforcingMode, testLogger)
if tt.wantErr != nil {
assert.EqualError(t, err, tt.wantErr.Error())
} else {
@@ -1952,7 +1970,12 @@ func Test_linuxNetwork_TeardownBranchENIPodNetwork(t *testing.T) {
n := &linuxNetwork{
netLink: netLink,
}
- err := n.TeardownBranchENIPodNetwork(tt.args.containerAddr, tt.args.vlanID, tt.args.podSGEnforcingMode, testLogger)
+
+ vIfMetadata := VirtualInterfaceMetadata{
+ IPAddress: tt.args.containerAddr,
+ }
+
+ err := n.TeardownBranchENIPodNetwork(vIfMetadata, tt.args.vlanID, tt.args.podSGEnforcingMode, testLogger)
if tt.wantErr != nil {
assert.EqualError(t, err, tt.wantErr.Error())
} else {
@@ -2043,8 +2066,7 @@ func Test_createVethPairContext_run(t *testing.T) {
type args struct {
contVethName string
hostVethName string
- v4Addr *net.IPNet
- v6Addr *net.IPNet
+ ipAddr *net.IPNet
mtu int
}
tests := []struct {
@@ -2091,6 +2113,7 @@ func Test_createVethPairContext_run(t *testing.T) {
route: &netlink.Route{
LinkIndex: contVethWithIndex1.Attrs().Index,
Scope: netlink.SCOPE_LINK,
+ Table: 254,
Dst: &net.IPNet{
IP: net.IPv4(169, 254, 1, 1),
Mask: net.CIDRMask(32, 32),
@@ -2103,6 +2126,7 @@ func Test_createVethPairContext_run(t *testing.T) {
route: &netlink.Route{
LinkIndex: contVethWithIndex1.Attrs().Index,
Scope: netlink.SCOPE_UNIVERSE,
+ Table: 254,
Dst: &net.IPNet{
IP: net.IPv4zero,
Mask: net.CIDRMask(0, 32),
@@ -2148,12 +2172,11 @@ func Test_createVethPairContext_run(t *testing.T) {
args: args{
contVethName: "eth0",
hostVethName: "eni8ea2c11fe35",
- v4Addr: &net.IPNet{
+ ipAddr: &net.IPNet{
IP: net.ParseIP("192.168.120.1"),
Mask: net.CIDRMask(32, 32),
},
- v6Addr: nil,
- mtu: 9001,
+ mtu: 9001,
},
},
{
@@ -2198,6 +2221,7 @@ func Test_createVethPairContext_run(t *testing.T) {
route: &netlink.Route{
LinkIndex: contVethWithIndex1.Attrs().Index,
Scope: netlink.SCOPE_LINK,
+ Table: 254,
Dst: &net.IPNet{
IP: net.IP{0xfe, 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1},
Mask: net.CIDRMask(128, 128),
@@ -2210,6 +2234,7 @@ func Test_createVethPairContext_run(t *testing.T) {
route: &netlink.Route{
LinkIndex: contVethWithIndex1.Attrs().Index,
Scope: netlink.SCOPE_UNIVERSE,
+ Table: 254,
Dst: &net.IPNet{
IP: net.IPv6zero,
Mask: net.CIDRMask(0, 128),
@@ -2278,8 +2303,7 @@ func Test_createVethPairContext_run(t *testing.T) {
args: args{
contVethName: "eth0",
hostVethName: "eni8ea2c11fe35",
- v4Addr: nil,
- v6Addr: &net.IPNet{
+ ipAddr: &net.IPNet{
IP: net.ParseIP("2001:db8:3333:4444:5555:6666:7777:8888"),
Mask: net.CIDRMask(128, 128),
},
@@ -2306,12 +2330,11 @@ func Test_createVethPairContext_run(t *testing.T) {
args: args{
contVethName: "eth0",
hostVethName: "eni8ea2c11fe35",
- v4Addr: &net.IPNet{
+ ipAddr: &net.IPNet{
IP: net.ParseIP("192.168.120.1"),
Mask: net.CIDRMask(32, 32),
},
- v6Addr: nil,
- mtu: 9001,
+ mtu: 9001,
},
wantErr: errors.New("some error"),
},
@@ -2340,12 +2363,11 @@ func Test_createVethPairContext_run(t *testing.T) {
args: args{
contVethName: "eth0",
hostVethName: "eni8ea2c11fe35",
- v4Addr: &net.IPNet{
+ ipAddr: &net.IPNet{
IP: net.ParseIP("192.168.120.1"),
Mask: net.CIDRMask(32, 32),
},
- v6Addr: nil,
- mtu: 9001,
+ mtu: 9001,
},
wantErr: errors.New("setup NS network: failed to find link \"eni8ea2c11fe35\": some error"),
},
@@ -2380,12 +2402,11 @@ func Test_createVethPairContext_run(t *testing.T) {
args: args{
contVethName: "eth0",
hostVethName: "eni8ea2c11fe35",
- v4Addr: &net.IPNet{
+ ipAddr: &net.IPNet{
IP: net.ParseIP("192.168.120.1"),
Mask: net.CIDRMask(32, 32),
},
- v6Addr: nil,
- mtu: 9001,
+ mtu: 9001,
},
wantErr: errors.New("setup NS network: failed to set link \"eni8ea2c11fe35\" up: some error"),
},
@@ -2423,12 +2444,11 @@ func Test_createVethPairContext_run(t *testing.T) {
args: args{
contVethName: "eth0",
hostVethName: "eni8ea2c11fe35",
- v4Addr: &net.IPNet{
+ ipAddr: &net.IPNet{
IP: net.ParseIP("192.168.120.1"),
Mask: net.CIDRMask(32, 32),
},
- v6Addr: nil,
- mtu: 9001,
+ mtu: 9001,
},
wantErr: errors.New("setup NS network: failed to find link \"eth0\": some error"),
},
@@ -2470,12 +2490,11 @@ func Test_createVethPairContext_run(t *testing.T) {
args: args{
contVethName: "eth0",
hostVethName: "eni8ea2c11fe35",
- v4Addr: &net.IPNet{
+ ipAddr: &net.IPNet{
IP: net.ParseIP("192.168.120.1"),
Mask: net.CIDRMask(32, 32),
},
- v6Addr: nil,
- mtu: 9001,
+ mtu: 9001,
},
wantErr: errors.New("setup NS network: failed to set link \"eth0\" up: some error"),
},
@@ -2517,6 +2536,7 @@ func Test_createVethPairContext_run(t *testing.T) {
route: &netlink.Route{
LinkIndex: contVethWithIndex1.Attrs().Index,
Scope: netlink.SCOPE_LINK,
+ Table: 254,
Dst: &net.IPNet{
IP: net.IPv4(169, 254, 1, 1),
Mask: net.CIDRMask(32, 32),
@@ -2529,12 +2549,11 @@ func Test_createVethPairContext_run(t *testing.T) {
args: args{
contVethName: "eth0",
hostVethName: "eni8ea2c11fe35",
- v4Addr: &net.IPNet{
+ ipAddr: &net.IPNet{
IP: net.ParseIP("192.168.120.1"),
Mask: net.CIDRMask(32, 32),
},
- v6Addr: nil,
- mtu: 9001,
+ mtu: 9001,
},
wantErr: errors.New("setup NS network: failed to add default gateway: some error"),
},
@@ -2576,6 +2595,7 @@ func Test_createVethPairContext_run(t *testing.T) {
route: &netlink.Route{
LinkIndex: contVethWithIndex1.Attrs().Index,
Scope: netlink.SCOPE_LINK,
+ Table: 254,
Dst: &net.IPNet{
IP: net.IPv4(169, 254, 1, 1),
Mask: net.CIDRMask(32, 32),
@@ -2588,6 +2608,7 @@ func Test_createVethPairContext_run(t *testing.T) {
route: &netlink.Route{
LinkIndex: contVethWithIndex1.Attrs().Index,
Scope: netlink.SCOPE_UNIVERSE,
+ Table: 254,
Dst: &net.IPNet{
IP: net.IPv4zero,
Mask: net.CIDRMask(0, 32),
@@ -2601,12 +2622,11 @@ func Test_createVethPairContext_run(t *testing.T) {
args: args{
contVethName: "eth0",
hostVethName: "eni8ea2c11fe35",
- v4Addr: &net.IPNet{
+ ipAddr: &net.IPNet{
IP: net.ParseIP("192.168.120.1"),
Mask: net.CIDRMask(32, 32),
},
- v6Addr: nil,
- mtu: 9001,
+ mtu: 9001,
},
wantErr: errors.New("setup NS network: failed to add default route: some error"),
},
@@ -2648,6 +2668,7 @@ func Test_createVethPairContext_run(t *testing.T) {
route: &netlink.Route{
LinkIndex: contVethWithIndex1.Attrs().Index,
Scope: netlink.SCOPE_LINK,
+ Table: 254,
Dst: &net.IPNet{
IP: net.IPv4(169, 254, 1, 1),
Mask: net.CIDRMask(32, 32),
@@ -2660,6 +2681,7 @@ func Test_createVethPairContext_run(t *testing.T) {
route: &netlink.Route{
LinkIndex: contVethWithIndex1.Attrs().Index,
Scope: netlink.SCOPE_UNIVERSE,
+ Table: 254,
Dst: &net.IPNet{
IP: net.IPv4zero,
Mask: net.CIDRMask(0, 32),
@@ -2684,12 +2706,11 @@ func Test_createVethPairContext_run(t *testing.T) {
args: args{
contVethName: "eth0",
hostVethName: "eni8ea2c11fe35",
- v4Addr: &net.IPNet{
+ ipAddr: &net.IPNet{
IP: net.ParseIP("192.168.120.1"),
Mask: net.CIDRMask(32, 32),
},
- v6Addr: nil,
- mtu: 9001,
+ mtu: 9001,
},
wantErr: errors.New("setup NS network: failed to add IP addr to \"eth0\": some error"),
},
@@ -2731,6 +2752,7 @@ func Test_createVethPairContext_run(t *testing.T) {
route: &netlink.Route{
LinkIndex: contVethWithIndex1.Attrs().Index,
Scope: netlink.SCOPE_LINK,
+ Table: 254,
Dst: &net.IPNet{
IP: net.IPv4(169, 254, 1, 1),
Mask: net.CIDRMask(32, 32),
@@ -2743,6 +2765,7 @@ func Test_createVethPairContext_run(t *testing.T) {
route: &netlink.Route{
LinkIndex: contVethWithIndex1.Attrs().Index,
Scope: netlink.SCOPE_UNIVERSE,
+ Table: 254,
Dst: &net.IPNet{
IP: net.IPv4zero,
Mask: net.CIDRMask(0, 32),
@@ -2777,12 +2800,11 @@ func Test_createVethPairContext_run(t *testing.T) {
args: args{
contVethName: "eth0",
hostVethName: "eni8ea2c11fe35",
- v4Addr: &net.IPNet{
+ ipAddr: &net.IPNet{
IP: net.ParseIP("192.168.120.1"),
Mask: net.CIDRMask(32, 32),
},
- v6Addr: nil,
- mtu: 9001,
+ mtu: 9001,
},
wantErr: errors.New("setup NS network: failed to add static ARP: some error"),
},
@@ -2824,6 +2846,7 @@ func Test_createVethPairContext_run(t *testing.T) {
route: &netlink.Route{
LinkIndex: contVethWithIndex1.Attrs().Index,
Scope: netlink.SCOPE_LINK,
+ Table: 254,
Dst: &net.IPNet{
IP: net.IPv4(169, 254, 1, 1),
Mask: net.CIDRMask(32, 32),
@@ -2836,6 +2859,7 @@ func Test_createVethPairContext_run(t *testing.T) {
route: &netlink.Route{
LinkIndex: contVethWithIndex1.Attrs().Index,
Scope: netlink.SCOPE_UNIVERSE,
+ Table: 254,
Dst: &net.IPNet{
IP: net.IPv4zero,
Mask: net.CIDRMask(0, 32),
@@ -2881,12 +2905,11 @@ func Test_createVethPairContext_run(t *testing.T) {
args: args{
contVethName: "eth0",
hostVethName: "eni8ea2c11fe35",
- v4Addr: &net.IPNet{
+ ipAddr: &net.IPNet{
IP: net.ParseIP("192.168.120.1"),
Mask: net.CIDRMask(32, 32),
},
- v6Addr: nil,
- mtu: 9001,
+ mtu: 9001,
},
wantErr: errors.New("setup NS network: failed to move veth to host netns: some error"),
},
@@ -2934,8 +2957,7 @@ func Test_createVethPairContext_run(t *testing.T) {
args: args{
contVethName: "eth0",
hostVethName: "eni8ea2c11fe35",
- v4Addr: nil,
- v6Addr: &net.IPNet{
+ ipAddr: &net.IPNet{
IP: net.ParseIP("2001:db8:3333:4444:5555:6666:7777:8888"),
Mask: net.CIDRMask(128, 128),
},
@@ -2991,8 +3013,7 @@ func Test_createVethPairContext_run(t *testing.T) {
args: args{
contVethName: "eth0",
hostVethName: "eni8ea2c11fe35",
- v4Addr: nil,
- v6Addr: &net.IPNet{
+ ipAddr: &net.IPNet{
IP: net.ParseIP("2001:db8:3333:4444:5555:6666:7777:8888"),
Mask: net.CIDRMask(128, 128),
},
@@ -3042,6 +3063,7 @@ func Test_createVethPairContext_run(t *testing.T) {
route: &netlink.Route{
LinkIndex: contVethWithIndex1.Attrs().Index,
Scope: netlink.SCOPE_LINK,
+ Table: 254,
Dst: &net.IPNet{
IP: net.IP{0xfe, 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1},
Mask: net.CIDRMask(128, 128),
@@ -3054,6 +3076,7 @@ func Test_createVethPairContext_run(t *testing.T) {
route: &netlink.Route{
LinkIndex: contVethWithIndex1.Attrs().Index,
Scope: netlink.SCOPE_UNIVERSE,
+ Table: 254,
Dst: &net.IPNet{
IP: net.IPv6zero,
Mask: net.CIDRMask(0, 128),
@@ -3104,8 +3127,7 @@ func Test_createVethPairContext_run(t *testing.T) {
args: args{
contVethName: "eth0",
hostVethName: "eni8ea2c11fe35",
- v4Addr: nil,
- v6Addr: &net.IPNet{
+ ipAddr: &net.IPNet{
IP: net.ParseIP("2001:db8:3333:4444:5555:6666:7777:8888"),
Mask: net.CIDRMask(128, 128),
},
@@ -3161,8 +3183,7 @@ func Test_createVethPairContext_run(t *testing.T) {
createVethContext := &createVethPairContext{
contVethName: tt.args.contVethName,
hostVethName: tt.args.hostVethName,
- v4Addr: tt.args.v4Addr,
- v6Addr: tt.args.v6Addr,
+ ipAddr: tt.args.ipAddr,
mtu: tt.args.mtu,
netLink: netLink,
procSys: procSys,
@@ -3592,7 +3613,7 @@ func Test_linuxNetwork_setupVeth(t *testing.T) {
ns: ns,
procSys: procSys,
}
- got, err := n.setupVeth(tt.args.hostVethName, tt.args.contVethName, tt.args.netnsPath, nil, nil, tt.args.mtu, testLogger)
+ got, err := n.setupVeth(tt.args.hostVethName, tt.args.contVethName, tt.args.netnsPath, nil, tt.args.mtu, testLogger, 0)
if tt.wantErr != nil {
assert.EqualError(t, err, tt.wantErr.Error())
} else {
diff --git a/cmd/routed-eni-cni-plugin/driver/mocks/driver_mocks.go b/cmd/routed-eni-cni-plugin/driver/mocks/driver_mocks.go
index 7de6603fb6..81da89c7ac 100644
--- a/cmd/routed-eni-cni-plugin/driver/mocks/driver_mocks.go
+++ b/cmd/routed-eni-cni-plugin/driver/mocks/driver_mocks.go
@@ -1,27 +1,13 @@
-// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License"). You may
-// not use this file except in compliance with the License. A copy of the
-// License is located at
-//
-// http://aws.amazon.com/apache2.0/
-//
-// or in the "license" file accompanying this file. This file is distributed
-// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
-// express or implied. See the License for the specific language governing
-// permissions and limitations under the License.
-//
-
// Code generated by MockGen. DO NOT EDIT.
-// Source: github.com/aws/amazon-vpc-cni-k8s/cmd/routed-eni-cni-plugin/driver (interfaces: NetworkAPIs)
+// Source: driver.go
-// Package mock_driver is a generated GoMock package.
-package mock_driver
+// Package mocks is a generated GoMock package.
+package mocks
import (
- net "net"
reflect "reflect"
+ driver "github.com/aws/amazon-vpc-cni-k8s/cmd/routed-eni-cni-plugin/driver"
sgpp "github.com/aws/amazon-vpc-cni-k8s/pkg/sgpp"
logger "github.com/aws/amazon-vpc-cni-k8s/pkg/utils/logger"
gomock "github.com/golang/mock/gomock"
@@ -51,57 +37,57 @@ func (m *MockNetworkAPIs) EXPECT() *MockNetworkAPIsMockRecorder {
}
// SetupBranchENIPodNetwork mocks base method.
-func (m *MockNetworkAPIs) SetupBranchENIPodNetwork(arg0, arg1, arg2 string, arg3, arg4 *net.IPNet, arg5 int, arg6, arg7 string, arg8, arg9 int, arg10 sgpp.EnforcingMode, arg11 logger.Logger) error {
+func (m *MockNetworkAPIs) SetupBranchENIPodNetwork(vethMetadata driver.VirtualInterfaceMetadata, netnsPath string, vlanID int, eniMAC, subnetGW string, parentIfIndex, mtu int, podSGEnforcingMode sgpp.EnforcingMode, log logger.Logger) error {
m.ctrl.T.Helper()
- ret := m.ctrl.Call(m, "SetupBranchENIPodNetwork", arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9, arg10, arg11)
+ ret := m.ctrl.Call(m, "SetupBranchENIPodNetwork", vethMetadata, netnsPath, vlanID, eniMAC, subnetGW, parentIfIndex, mtu, podSGEnforcingMode, log)
ret0, _ := ret[0].(error)
return ret0
}
// SetupBranchENIPodNetwork indicates an expected call of SetupBranchENIPodNetwork.
-func (mr *MockNetworkAPIsMockRecorder) SetupBranchENIPodNetwork(arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9, arg10, arg11 interface{}) *gomock.Call {
+func (mr *MockNetworkAPIsMockRecorder) SetupBranchENIPodNetwork(vethMetadata, netnsPath, vlanID, eniMAC, subnetGW, parentIfIndex, mtu, podSGEnforcingMode, log interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetupBranchENIPodNetwork", reflect.TypeOf((*MockNetworkAPIs)(nil).SetupBranchENIPodNetwork), arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9, arg10, arg11)
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetupBranchENIPodNetwork", reflect.TypeOf((*MockNetworkAPIs)(nil).SetupBranchENIPodNetwork), vethMetadata, netnsPath, vlanID, eniMAC, subnetGW, parentIfIndex, mtu, podSGEnforcingMode, log)
}
// SetupPodNetwork mocks base method.
-func (m *MockNetworkAPIs) SetupPodNetwork(arg0, arg1, arg2 string, arg3, arg4 *net.IPNet, arg5, arg6 int, arg7 logger.Logger) error {
+func (m *MockNetworkAPIs) SetupPodNetwork(vethMetadata []driver.VirtualInterfaceMetadata, netnsPath string, mtu int, log logger.Logger) error {
m.ctrl.T.Helper()
- ret := m.ctrl.Call(m, "SetupPodNetwork", arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7)
+ ret := m.ctrl.Call(m, "SetupPodNetwork", vethMetadata, netnsPath, mtu, log)
ret0, _ := ret[0].(error)
return ret0
}
// SetupPodNetwork indicates an expected call of SetupPodNetwork.
-func (mr *MockNetworkAPIsMockRecorder) SetupPodNetwork(arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7 interface{}) *gomock.Call {
+func (mr *MockNetworkAPIsMockRecorder) SetupPodNetwork(vethMetadata, netnsPath, mtu, log interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetupPodNetwork", reflect.TypeOf((*MockNetworkAPIs)(nil).SetupPodNetwork), arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7)
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetupPodNetwork", reflect.TypeOf((*MockNetworkAPIs)(nil).SetupPodNetwork), vethMetadata, netnsPath, mtu, log)
}
// TeardownBranchENIPodNetwork mocks base method.
-func (m *MockNetworkAPIs) TeardownBranchENIPodNetwork(arg0 *net.IPNet, arg1 int, arg2 sgpp.EnforcingMode, arg3 logger.Logger) error {
+func (m *MockNetworkAPIs) TeardownBranchENIPodNetwork(vethMetadata driver.VirtualInterfaceMetadata, vlanID int, podSGEnforcingMode sgpp.EnforcingMode, log logger.Logger) error {
m.ctrl.T.Helper()
- ret := m.ctrl.Call(m, "TeardownBranchENIPodNetwork", arg0, arg1, arg2, arg3)
+ ret := m.ctrl.Call(m, "TeardownBranchENIPodNetwork", vethMetadata, vlanID, podSGEnforcingMode, log)
ret0, _ := ret[0].(error)
return ret0
}
// TeardownBranchENIPodNetwork indicates an expected call of TeardownBranchENIPodNetwork.
-func (mr *MockNetworkAPIsMockRecorder) TeardownBranchENIPodNetwork(arg0, arg1, arg2, arg3 interface{}) *gomock.Call {
+func (mr *MockNetworkAPIsMockRecorder) TeardownBranchENIPodNetwork(vethMetadata, vlanID, podSGEnforcingMode, log interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TeardownBranchENIPodNetwork", reflect.TypeOf((*MockNetworkAPIs)(nil).TeardownBranchENIPodNetwork), arg0, arg1, arg2, arg3)
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TeardownBranchENIPodNetwork", reflect.TypeOf((*MockNetworkAPIs)(nil).TeardownBranchENIPodNetwork), vethMetadata, vlanID, podSGEnforcingMode, log)
}
// TeardownPodNetwork mocks base method.
-func (m *MockNetworkAPIs) TeardownPodNetwork(arg0 *net.IPNet, arg1 int, arg2 logger.Logger) error {
+func (m *MockNetworkAPIs) TeardownPodNetwork(vethMetadata []driver.VirtualInterfaceMetadata, log logger.Logger) error {
m.ctrl.T.Helper()
- ret := m.ctrl.Call(m, "TeardownPodNetwork", arg0, arg1, arg2)
+ ret := m.ctrl.Call(m, "TeardownPodNetwork", vethMetadata, log)
ret0, _ := ret[0].(error)
return ret0
}
// TeardownPodNetwork indicates an expected call of TeardownPodNetwork.
-func (mr *MockNetworkAPIsMockRecorder) TeardownPodNetwork(arg0, arg1, arg2 interface{}) *gomock.Call {
+func (mr *MockNetworkAPIsMockRecorder) TeardownPodNetwork(vethMetadata, log interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TeardownPodNetwork", reflect.TypeOf((*MockNetworkAPIs)(nil).TeardownPodNetwork), arg0, arg1, arg2)
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TeardownPodNetwork", reflect.TypeOf((*MockNetworkAPIs)(nil).TeardownPodNetwork), vethMetadata, log)
}
diff --git a/misc/10-aws.conflist b/misc/10-aws.conflist
index dde6b53631..82617a10a6 100644
--- a/misc/10-aws.conflist
+++ b/misc/10-aws.conflist
@@ -10,7 +10,8 @@
"mtu": "__MTU__",
"podSGEnforcingMode": "__PODSGENFORCINGMODE__",
"pluginLogFile": "__PLUGINLOGFILE__",
- "pluginLogLevel": "__PLUGINLOGLEVEL__"
+ "pluginLogLevel": "__PLUGINLOGLEVEL__",
+ "capabilities": {"io.kubernetes.cri.pod-annotations": true}
},
{
"name": "egress-cni",
diff --git a/pkg/awsutils/awsutils.go b/pkg/awsutils/awsutils.go
index 0f539cf6d8..7e5972a663 100644
--- a/pkg/awsutils/awsutils.go
+++ b/pkg/awsutils/awsutils.go
@@ -99,7 +99,7 @@ var log = logger.Get()
// APIs defines interfaces calls for adding/getting/deleting ENIs/secondary IPs. The APIs are not thread-safe.
type APIs interface {
// AllocENI creates an ENI and attaches it to the instance
- AllocENI(useCustomCfg bool, sg []*string, eniCfgSubnet string, numIPs int) (eni string, err error)
+ AllocENI(sg []*string, eniCfgSubnet string, numIPs int, networkCard int) (eni string, err error)
// FreeENI detaches ENI interface and deletes it
FreeENI(eniName string) error
@@ -143,6 +143,9 @@ type APIs interface {
// GetLocalIPv4 returns the primary IPv4 address on the primary ENI interface
GetLocalIPv4() net.IP
+ // GetLocalIPv6 returns the primary IPv6 address on the primary ENI interface
+ GetLocalIPv6() net.IP
+
// GetVPCIPv6CIDRs returns VPC's IPv6 CIDRs from instance metadata
GetVPCIPv6CIDRs() ([]string, error)
@@ -180,7 +183,7 @@ type APIs interface {
IsPrimaryENI(eniID string) bool
//RefreshSGIDs
- RefreshSGIDs(mac string, store *datastore.DataStore) error
+ RefreshSGIDs(mac string, ds *datastore.DataStoreAccess) error
//GetInstanceHypervisorFamily returns the hypervisor family for the instance
GetInstanceHypervisorFamily() string
@@ -206,6 +209,7 @@ type EC2InstanceMetadataCache struct {
securityGroups StringSet
subnetID string
localIPv4 net.IP
+ localIPv6 net.IP
v4Enabled bool
v6Enabled bool
instanceID string
@@ -257,6 +261,9 @@ type ENIMetadata struct {
// IPv6 Prefixes allocated for the network interface
IPv6Prefixes []ec2types.Ipv6PrefixSpecification
+
+ // Network card the ENI is attached on
+ NetworkCard int
}
// PrimaryIPv4Address returns the primary IPv4 address of this node
@@ -284,11 +291,12 @@ type TagMap map[string]string
// DescribeAllENIsResult contains the fully
type DescribeAllENIsResult struct {
- ENIMetadata []ENIMetadata
- TagMap map[string]TagMap
- TrunkENI string
- EFAENIs map[string]bool
- MultiCardENIIDs []string
+ ENIMetadata []ENIMetadata
+ TagMap map[string]TagMap
+ TrunkENI string
+ EFAENIs map[string]bool
+ MultiCardENIIDs []string
+ NetworkCardENIMap map[int][]string
}
// msSince returns milliseconds since start.
@@ -497,7 +505,7 @@ func (cache *EC2InstanceMetadataCache) initWithEC2Metadata(ctx context.Context)
}
// RefreshSGIDs retrieves security groups
-func (cache *EC2InstanceMetadataCache) RefreshSGIDs(mac string, store *datastore.DataStore) error {
+func (cache *EC2InstanceMetadataCache) RefreshSGIDs(mac string, dsAccess *datastore.DataStoreAccess) error {
ctx := context.TODO()
sgIDs, err := cache.imds.GetSecurityGroupIDs(ctx, mac)
@@ -524,19 +532,19 @@ func (cache *EC2InstanceMetadataCache) RefreshSGIDs(mac string, store *datastore
cache.securityGroups.Set(sgIDs)
if !cache.useCustomNetworking && (addedSGsCount != 0 || deletedSGsCount != 0) {
- eniInfos := store.GetENIInfos()
-
var eniIDs []string
- for eniID := range eniInfos.ENIs {
- eniIDs = append(eniIDs, eniID)
+ for _, ds := range dsAccess.DataStores {
+ eniInfos := ds.GetENIInfos()
+ for eniID := range eniInfos.ENIs {
+ eniIDs = append(eniIDs, eniID)
+ }
}
newENIs := StringSet{}
newENIs.Set(eniIDs)
- tempfilteredENIs := newENIs.Difference(&cache.multiCardENIs)
- filteredENIs := tempfilteredENIs.Difference(&cache.unmanagedENIs)
+ filteredENIs := newENIs.Difference(&cache.unmanagedENIs)
// This will update SG for managed ENIs created by EKS.
for _, eniID := range filteredENIs.SortedList() {
@@ -633,6 +641,7 @@ func (cache *EC2InstanceMetadataCache) getENIMetadata(eniMAC string) (ENIMetadat
}
ipv4Available := false
ipv6Available := false
+ networkCard := 0
// Efa-only interfaces do not have any ipv4s or ipv6s associated with it. If we don't find any local-ipv4 or ipv6 info in imds we assume it to be efa-only interface and validate this later via ec2 call
for _, field := range macImdsFields {
if field == "local-ipv4s" {
@@ -655,6 +664,14 @@ func (cache *EC2InstanceMetadataCache) getENIMetadata(eniMAC string) (ENIMetadat
log.Debugf("Found IPv6 addresses associated with interface. This is not efa-only interface")
}
}
+ if field == "network-card" {
+ networkCard, err = cache.imds.GetNetworkCard(ctx, eniMAC)
+ if err != nil {
+ awsAPIErrInc("GetNetworkCard", err)
+ log.Errorf("Network Card data not found from", networkCard)
+ return ENIMetadata{}, err
+ }
+ }
}
if !ipv4Available && !ipv6Available {
@@ -668,6 +685,7 @@ func (cache *EC2InstanceMetadataCache) getENIMetadata(eniMAC string) (ENIMetadat
SubnetIPv6CIDR: "",
IPv6Addresses: make([]ec2types.NetworkInterfaceIpv6Address, 0),
IPv6Prefixes: make([]ec2types.Ipv6PrefixSpecification, 0),
+ NetworkCard: networkCard,
}, nil
}
@@ -701,10 +719,11 @@ func (cache *EC2InstanceMetadataCache) getENIMetadata(eniMAC string) (ENIMetadat
var ec2ip6s []ec2types.NetworkInterfaceIpv6Address
var subnetV6Cidr string
if cache.v6Enabled {
- // For IPv6 ENIs, do not error on missing IPv6 information
+ // For IPv6 ENIs, we have to return the error if Subnet is not discovered
v6cidr, err := cache.imds.GetSubnetIPv6CIDRBlocks(ctx, eniMAC)
if err != nil {
awsAPIErrInc("GetSubnetIPv6CIDRBlocks", err)
+ return ENIMetadata{}, err
} else {
subnetV6Cidr = v6cidr.String()
}
@@ -764,11 +783,12 @@ func (cache *EC2InstanceMetadataCache) getENIMetadata(eniMAC string) (ENIMetadat
SubnetIPv6CIDR: subnetV6Cidr,
IPv6Addresses: ec2ip6s,
IPv6Prefixes: ec2ipv6Prefixes,
+ NetworkCard: networkCard,
}, nil
}
// awsGetFreeDeviceNumber calls EC2 API DescribeInstances to get the next free device index
-func (cache *EC2InstanceMetadataCache) awsGetFreeDeviceNumber() (int, error) {
+func (cache *EC2InstanceMetadataCache) awsGetFreeDeviceNumber(networkCard int) (int, error) {
input := &ec2.DescribeInstancesInput{
InstanceIds: []string{cache.instanceID},
}
@@ -793,44 +813,44 @@ func (cache *EC2InstanceMetadataCache) awsGetFreeDeviceNumber() (int, error) {
inst := result.Reservations[0].Instances[0]
var device [maxENIs]bool
for _, eni := range inst.NetworkInterfaces {
- // We don't support multi-card yet, so only account for network card zero
- if eni.Attachment != nil && aws.ToInt32(eni.Attachment.NetworkCardIndex) == 0 {
+ if eni.Attachment != nil && aws.ToInt32(eni.Attachment.NetworkCardIndex) == int32(networkCard) {
if aws.ToInt32(eni.Attachment.DeviceIndex) > maxENIs {
- log.Warnf("The Device Index %d of the attached ENI %s > instance max slot %d",
+ log.Warnf("The Device Index %d of the attached ENI %s > instance max slot %d for network card",
aws.ToInt32(eni.Attachment.DeviceIndex), aws.ToString(eni.NetworkInterfaceId),
- maxENIs)
+ maxENIs, networkCard)
} else {
- log.Debugf("Discovered device number is used: %d", aws.ToInt32(eni.Attachment.DeviceIndex))
+ log.Debugf("Discovered device number is used for network card %d: %d", networkCard, aws.ToInt32(eni.Attachment.DeviceIndex))
device[aws.ToInt32(eni.Attachment.DeviceIndex)] = true
}
}
}
- for freeDeviceIndex := 0; freeDeviceIndex < maxENIs; freeDeviceIndex++ {
+ for freeDeviceIndex := range maxENIs {
if !device[freeDeviceIndex] {
- log.Debugf("Found a free device number: %d", freeDeviceIndex)
+ log.Debugf("Found a free device number for network card %d : %d", networkCard, freeDeviceIndex)
return freeDeviceIndex, nil
}
}
- return 0, errors.New("awsGetFreeDeviceNumber: no available device number")
+ return 0, errors.New(fmt.Sprintf("awsGetFreeDeviceNumber: no available device number for network card %d", networkCard))
}
// AllocENI creates an ENI and attaches it to the instance
// returns: newly created ENI ID
-func (cache *EC2InstanceMetadataCache) AllocENI(useCustomCfg bool, sg []*string, eniCfgSubnet string, numIPs int) (string, error) {
- eniID, err := cache.createENI(useCustomCfg, sg, eniCfgSubnet, numIPs)
+func (cache *EC2InstanceMetadataCache) AllocENI(sg []*string, eniCfgSubnet string, numIPs int, networkCard int) (string, error) {
+
+ eniID, err := cache.createENI(sg, eniCfgSubnet, numIPs)
if err != nil {
return "", errors.Wrap(err, "AllocENI: failed to create ENI")
}
- attachmentID, err := cache.attachENI(eniID)
+ attachmentID, err := cache.attachENI(eniID, networkCard)
if err != nil {
derr := cache.deleteENI(eniID, maxENIBackoffDelay)
if derr != nil {
awsUtilsErrInc("AllocENIDeleteErr", err)
log.Errorf("Failed to delete newly created untagged ENI! %v", err)
}
- return "", errors.Wrap(err, "AllocENI: error attaching ENI")
+ return "", errors.Wrap(err, fmt.Sprintf("AllocENI: error attaching ENI for network card %d", networkCard))
}
// Also change the ENI's attribute so that the ENI will be deleted when the instance is deleted.
@@ -857,14 +877,14 @@ func (cache *EC2InstanceMetadataCache) AllocENI(useCustomCfg bool, sg []*string,
return "", errors.Wrap(err, "AllocENI: unable to change the ENI's attribute")
}
- log.Infof("Successfully created and attached a new ENI %s to instance", eniID)
+ log.Infof("Successfully created and attached a new ENI %s to instance on network card %d", eniID, networkCard)
return eniID, nil
}
// attachENI calls EC2 API to attach the ENI and returns the attachment id
-func (cache *EC2InstanceMetadataCache) attachENI(eniID string) (string, error) {
+func (cache *EC2InstanceMetadataCache) attachENI(eniID string, networkCard int) (string, error) {
// attach to instance
- freeDevice, err := cache.awsGetFreeDeviceNumber()
+ freeDevice, err := cache.awsGetFreeDeviceNumber(networkCard)
if err != nil {
return "", errors.Wrap(err, "attachENI: failed to get a free device number")
}
@@ -873,7 +893,7 @@ func (cache *EC2InstanceMetadataCache) attachENI(eniID string) (string, error) {
DeviceIndex: aws.Int32(int32(freeDevice)),
InstanceId: aws.String(cache.instanceID),
NetworkInterfaceId: aws.String(eniID),
- NetworkCardIndex: aws.Int32(0),
+ NetworkCardIndex: aws.Int32(int32(networkCard)),
}
start := time.Now()
attachOutput, err := cache.ec2SVC.AttachNetworkInterface(context.Background(), attachInput)
@@ -884,58 +904,80 @@ func (cache *EC2InstanceMetadataCache) attachENI(eniID string) (string, error) {
awsAPIErrInc("AttachNetworkInterface", err)
prometheusmetrics.Ec2ApiErr.WithLabelValues("AttachNetworkInterface").Inc()
log.Errorf("Failed to attach ENI %s: %v", eniID, err)
- return "", errors.Wrap(err, "attachENI: failed to attach ENI")
+ return "", errors.Wrap(err, fmt.Sprintf("attachENI: failed to attach ENI for network card %d", networkCard))
}
return aws.ToString(attachOutput.AttachmentId), err
}
-// return ENI id, error
-func (cache *EC2InstanceMetadataCache) createENI(useCustomCfg bool, sg []*string, eniCfgSubnet string, numIPs int) (string, error) {
- eniDescription := eniDescriptionPrefix + cache.instanceID
+// createENITags creates all the tags required to be added to the ENI
+//
+// Parameters:
+// - eniDescription (string):
+//
+// Returns:
+// - []ec2types.TagSpecification: Returns the tags by converting it into AWS SDK class
+func (cache *EC2InstanceMetadataCache) createENITags(eniDescription string) []ec2types.TagSpecification {
+
tags := map[string]string{
eniCreatedAtTagKey: time.Now().Format(time.RFC3339),
}
for key, value := range cache.buildENITags() {
tags[key] = value
}
- tagSpec := []ec2types.TagSpecification{
+ return []ec2types.TagSpecification{
{
ResourceType: ec2types.ResourceTypeNetworkInterface,
Tags: convertTagsToSDKTags(tags),
},
}
- var needIPs = numIPs
+}
- ipLimit := cache.GetENIIPv4Limit()
- if ipLimit < needIPs {
- needIPs = ipLimit
+func (cache *EC2InstanceMetadataCache) createENIInput(eniDescription string, tags []ec2types.TagSpecification, needIPs int) *ec2.CreateNetworkInterfaceInput {
+ input := &ec2.CreateNetworkInterfaceInput{
+ Description: aws.String(eniDescription),
+ Groups: cache.securityGroups.SortedList(),
+ SubnetId: aws.String(cache.subnetID),
+ TagSpecifications: tags,
+ }
+ // Even though IPv6 PD is enabled, we require a Primary IP for the ENI.
+ // This always creates an ENI which has 1 Primary IPv6 address
+ // We use assignIPv6Prefix to assign a prefix during setupENI
+ if cache.v6Enabled {
+ input.Ipv6AddressCount = aws.Int32(int32(needIPs))
+ return input
}
-
- log.Infof("Trying to allocate %d IP addresses on new ENI", needIPs)
- log.Debugf("PD enabled - %t", cache.enablePrefixDelegation)
-
- input := &ec2.CreateNetworkInterfaceInput{}
if cache.enablePrefixDelegation {
- input = &ec2.CreateNetworkInterfaceInput{
- Description: aws.String(eniDescription),
- Groups: cache.securityGroups.SortedList(),
- SubnetId: aws.String(cache.subnetID),
- TagSpecifications: tagSpec,
- Ipv4PrefixCount: aws.Int32(int32(needIPs)),
- }
+ input.Ipv4PrefixCount = aws.Int32(int32(needIPs))
} else {
- input = &ec2.CreateNetworkInterfaceInput{
- Description: aws.String(eniDescription),
- Groups: cache.securityGroups.SortedList(),
- SubnetId: aws.String(cache.subnetID),
- TagSpecifications: tagSpec,
- SecondaryPrivateIpAddressCount: aws.Int32(int32(needIPs)),
+ input.SecondaryPrivateIpAddressCount = aws.Int32(int32(needIPs))
+ }
+
+ return input
+}
+
+// return ENI id, error
+func (cache *EC2InstanceMetadataCache) createENI(sg []*string, eniCfgSubnet string, numIPs int) (string, error) {
+ eniDescription := eniDescriptionPrefix + cache.instanceID
+ tags := cache.createENITags(eniDescription)
+
+ var needIPs = numIPs
+
+ if !cache.v6Enabled {
+ ipLimit := cache.GetENIIPv4Limit()
+ if ipLimit < needIPs {
+ needIPs = ipLimit
}
}
+ log.Infof("Trying to allocate %d IP addresses on new ENI", needIPs)
+ log.Debugf("PD enabled - %t", cache.enablePrefixDelegation)
+
var err error
var networkInterfaceID string
+
+ input := cache.createENIInput(eniDescription, tags, needIPs)
+
if cache.useCustomNetworking {
input = createENIUsingCustomCfg(sg, eniCfgSubnet, input)
log.Infof("Creating ENI with security groups: %v in subnet: %s", input.Groups, aws.ToString(input.SubnetId))
@@ -945,7 +987,7 @@ func (cache *EC2InstanceMetadataCache) createENI(useCustomCfg bool, sg []*string
return networkInterfaceID, nil
}
} else {
- if cache.useSubnetDiscovery {
+ if cache.useSubnetDiscovery && !cache.v6Enabled {
subnetResult, vpcErr := cache.getVpcSubnets()
if vpcErr != nil {
log.Warnf("Failed to call ec2:DescribeSubnets: %v", vpcErr)
@@ -1346,6 +1388,9 @@ func (cache *EC2InstanceMetadataCache) DescribeAllENIs() (DescribeAllENIsResult,
return DescribeAllENIsResult{}, errors.Wrap(err, "DescribeAllENIs: failed to get local ENI metadata")
}
+ // Get number of Network Cards supported by the instance
+ networkCards := len(cache.GetNetworkCards())
+
eniMap := make(map[string]ENIMetadata, len(allENIs))
var eniIDs []string
for _, eni := range allENIs {
@@ -1405,20 +1450,24 @@ func (cache *EC2InstanceMetadataCache) DescribeAllENIs() (DescribeAllENIsResult,
// Collect ENI response into ENI metadata and tags.
var trunkENI string
var multiCardENIIDs []string
+ networkCardENIMap := make(map[int][]string, networkCards)
efaENIs := make(map[string]bool, 0)
tagMap := make(map[string]TagMap, len(ec2Response.NetworkInterfaces))
+
for _, ec2res := range ec2Response.NetworkInterfaces {
eniID := aws.ToString(ec2res.NetworkInterfaceId)
attachment := ec2res.Attachment
// Validate that Attachment is populated by EC2 response before logging
if attachment != nil {
log.Infof("Got network card index %v for ENI %v", aws.ToInt32(attachment.NetworkCardIndex), eniID)
- if aws.ToInt32(attachment.DeviceIndex) == 0 && !aws.ToBool(attachment.DeleteOnTermination) {
+ if aws.ToInt32(attachment.DeviceIndex) == 0 && aws.ToInt32(attachment.NetworkCardIndex) == 0 && !aws.ToBool(attachment.DeleteOnTermination) {
log.Warn("Primary ENI will not get deleted when node terminates because 'delete_on_termination' is set to false")
}
- if aws.ToInt32(attachment.NetworkCardIndex) > 0 {
- multiCardENIIDs = append(multiCardENIIDs, eniID)
+ // Maintain a mapping of NIC to ENIs
+ if attachment != nil {
+ networkCardENIMap[int(aws.ToInt32(attachment.NetworkCardIndex))] = append(networkCardENIMap[int(aws.ToInt32(attachment.NetworkCardIndex))], eniID)
}
+
} else {
log.Infof("Got empty attachment for ENI %v", eniID)
}
@@ -1434,6 +1483,7 @@ func (cache *EC2InstanceMetadataCache) DescribeAllENIs() (DescribeAllENIsResult,
if interfaceType == "efa" || interfaceType == "efa-only" {
efaENIs[eniID] = true
}
+
if interfaceType != "efa-only" {
if len(eniMetadata.IPv4Addresses) == 0 && len(eniMetadata.IPv6Addresses) == 0 {
log.Errorf("Missing IP addresses from IMDS. Non efa-only interface should have IP address associated with it %s", eniID)
@@ -1449,11 +1499,12 @@ func (cache *EC2InstanceMetadataCache) DescribeAllENIs() (DescribeAllENIsResult,
tagMap[eniMetadata.ENIID] = convertSDKTagsToTags(ec2res.TagSet)
}
return DescribeAllENIsResult{
- ENIMetadata: verifiedENIs,
- TagMap: tagMap,
- TrunkENI: trunkENI,
- EFAENIs: efaENIs,
- MultiCardENIIDs: multiCardENIIDs,
+ ENIMetadata: verifiedENIs,
+ TagMap: tagMap,
+ TrunkENI: trunkENI,
+ EFAENIs: efaENIs,
+ MultiCardENIIDs: multiCardENIIDs,
+ NetworkCardENIMap: networkCardENIMap,
}, nil
}
@@ -1786,12 +1837,13 @@ func (cache *EC2InstanceMetadataCache) waitForENIAndIPsAttached(eni string, want
if eni == returnedENI.ENIID {
// Check how many Secondary IPs or Prefixes have been attached
var eniIPCount int
- log.Debugf("ENI ID: %v IP Addr: %s, IPv4Prefixes:- %v, IPv6Prefixes:- %v", returnedENI.ENIID,
- returnedENI.IPv4Addresses, returnedENI.IPv4Prefixes, returnedENI.IPv6Prefixes)
+ log.Debugf("ENI ID: %v IP Addr: %d, IPv4Prefixes:- %d, IPv6Prefixes:- %d", returnedENI.ENIID,
+ len(returnedENI.IPv4Addresses), len(returnedENI.IPv4Prefixes), len(returnedENI.IPv6Prefixes))
if cache.enablePrefixDelegation {
eniIPCount = len(returnedENI.IPv4Prefixes)
+ // We look for IPv6Address instead if prefix here
if cache.v6Enabled {
- eniIPCount = len(returnedENI.IPv6Prefixes)
+ eniIPCount = len(returnedENI.IPv6Addresses)
}
} else {
//Ignore primary IP of the ENI
@@ -1800,7 +1852,7 @@ func (cache *EC2InstanceMetadataCache) waitForENIAndIPsAttached(eni string, want
}
if eniIPCount < 1 {
- log.Debugf("No secondary IPv4 addresses/prefixes available yet on ENI %s", returnedENI.ENIID)
+ log.Debugf("No secondary IPv4/IPv6 addresses/prefixes available yet on ENI %s", returnedENI.ENIID)
return ErrNoSecondaryIPsFound
}
@@ -1816,6 +1868,7 @@ func (cache *EC2InstanceMetadataCache) waitForENIAndIPsAttached(eni string, want
log.Debugf("Not able to find the right ENI yet (attempt %d/%d)", attempt, maxENIEC2APIRetries)
return ErrENINotFound
})
+
prometheusmetrics.AwsAPILatency.WithLabelValues("waitForENIAndIPsAttached", fmt.Sprint(err != nil), awsReqStatus(err)).Observe(msSince(start))
if err != nil {
// If we have at least 1 Secondary IP, by now return what we have without an error
@@ -2057,6 +2110,18 @@ func (cache *EC2InstanceMetadataCache) GetLocalIPv4() net.IP {
return cache.localIPv4
}
+// GetLocalIPv4 returns the primary IP address on the primary interface
+func (cache *EC2InstanceMetadataCache) GetLocalIPv6() net.IP {
+ ctx := context.TODO()
+
+ localIPv6, err := cache.imds.GetLocalIPv6(ctx)
+ if err != nil {
+ awsAPIErrInc("GetIPv6", err)
+ }
+
+ return localIPv6
+}
+
// GetVPCIPv6CIDRs returns VPC CIDRs
func (cache *EC2InstanceMetadataCache) GetVPCIPv6CIDRs() ([]string, error) {
ctx := context.TODO()
diff --git a/pkg/awsutils/awsutils_test.go b/pkg/awsutils/awsutils_test.go
index 59da59ee6f..3ad27d6637 100644
--- a/pkg/awsutils/awsutils_test.go
+++ b/pkg/awsutils/awsutils_test.go
@@ -59,6 +59,7 @@ const (
metadataIPv4Prefixes = "/ipv4-prefix"
metadataIPv6s = "/ipv6s"
metadataIPv6Prefixes = "/ipv6-prefix"
+ metadataSubnetV6CIDR = "/subnet-ipv6-cidr-blocks"
az = "us-east-1a"
localIP = "10.0.0.10"
@@ -71,6 +72,7 @@ const (
sgs = sg1 + " " + sg2
subnetID = "subnet-6b245523"
subnetCIDR = "10.0.1.0/24"
+ subnetCIDRv6 = "2001:db8::/56"
vpcID = "vpc-3c133421"
primaryeniID = "eni-00000000"
eniID = primaryeniID
@@ -83,13 +85,13 @@ const (
eni2PrivateIP = "10.0.0.2"
eni2Prefix = "10.0.2.0/28"
eni2v6IP = "2001:db8:8:4::2"
- eni2v6Prefix = "2001:db8::/64"
+ eni2v6Prefix = "2001:db8::/80"
eni2ID = "eni-12341234"
metadataVPCIPv4CIDRs = "192.168.0.0/16 100.66.0.0/1"
myNodeName = "testNodeName"
imdsMACFields = "security-group-ids subnet-id vpc-id vpc-ipv4-cidr-blocks device-number interface-id subnet-ipv4-cidr-block local-ipv4s ipv4-prefix ipv6-prefix"
imdsMACFieldsEfaOnly = "security-group-ids subnet-id vpc-id vpc-ipv4-cidr-blocks device-number interface-id subnet-ipv4-cidr-block ipv4-prefix ipv6-prefix"
- imdsMACFieldsV6Only = "security-group-ids subnet-id vpc-id vpc-ipv4-cidr-blocks device-number interface-id subnet-ipv6-cidr-blocks ipv6s ipv6-prefix"
+ imdsMACFieldsV6Only = "security-group-ids subnet-id vpc-id vpc-ipv4-cidr-blocks vpc-ipv6-cidr-blocks device-number interface-id subnet-ipv6-cidr-blocks ipv6s ipv6-prefix"
imdsMACFieldsV4AndV6 = "security-group-ids subnet-id vpc-id vpc-ipv4-cidr-blocks device-number interface-id subnet-ipv4-cidr-block ipv6s local-ipv4s"
)
@@ -102,14 +104,15 @@ func testMetadata(overrides map[string]interface{}) FakeIMDS {
metadataMAC: primaryMAC,
metadataMACPath: primaryMAC,
metadataMACPath + primaryMAC: imdsMACFields,
- metadataMACPath + primaryMAC + metadataDeviceNum: eni1Device,
- metadataMACPath + primaryMAC + metadataInterface: primaryeniID,
- metadataMACPath + primaryMAC + metadataSGs: sgs,
- metadataMACPath + primaryMAC + metadataIPv4s: eni1PrivateIP,
- metadataMACPath + primaryMAC + metadataSubnetID: subnetID,
- metadataMACPath + primaryMAC + metadataVpcID: vpcID,
- metadataMACPath + primaryMAC + metadataSubnetCIDR: subnetCIDR,
- metadataMACPath + primaryMAC + metadataVPCcidrs: metadataVPCIPv4CIDRs,
+ metadataMACPath + primaryMAC + metadataDeviceNum: eni1Device,
+ metadataMACPath + primaryMAC + metadataInterface: primaryeniID,
+ metadataMACPath + primaryMAC + metadataSGs: sgs,
+ metadataMACPath + primaryMAC + metadataIPv4s: eni1PrivateIP,
+ metadataMACPath + primaryMAC + metadataSubnetID: subnetID,
+ metadataMACPath + primaryMAC + metadataVpcID: vpcID,
+ metadataMACPath + primaryMAC + metadataSubnetCIDR: subnetCIDR,
+ metadataMACPath + primaryMAC + metadataSubnetV6CIDR: subnetCIDRv6,
+ metadataMACPath + primaryMAC + metadataVPCcidrs: metadataVPCIPv4CIDRs,
}
for k, v := range overrides {
@@ -253,6 +256,7 @@ func TestGetAttachedENIsWithIPv6Only(t *testing.T) {
metadataMACPath + eni2MAC + metadataDeviceNum: eni2Device,
metadataMACPath + eni2MAC + metadataInterface: eni2ID,
metadataMACPath + eni2MAC + metadataIPv6s: eni2v6IP,
+ metadataMACPath + eni2MAC + metadataSubnetV6CIDR: subnetCIDRv6,
metadataMACPath + eni2MAC + metadataIPv6Prefixes: eni2v6Prefix,
})
@@ -313,7 +317,7 @@ func TestAWSGetFreeDeviceNumberOnErr(t *testing.T) {
mockEC2.EXPECT().DescribeInstances(gomock.Any(), gomock.Any()).Return(nil, errors.New("error on DescribeInstances"))
cache := &EC2InstanceMetadataCache{ec2SVC: mockEC2}
- _, err := cache.awsGetFreeDeviceNumber()
+ _, err := cache.awsGetFreeDeviceNumber(0)
assert.Error(t, err)
}
@@ -338,7 +342,7 @@ func TestAWSGetFreeDeviceNumberNoDevice(t *testing.T) {
mockEC2.EXPECT().DescribeInstances(gomock.Any(), gomock.Any(), gomock.Any()).Return(result, nil)
cache := &EC2InstanceMetadataCache{ec2SVC: mockEC2}
- _, err := cache.awsGetFreeDeviceNumber()
+ _, err := cache.awsGetFreeDeviceNumber(0)
assert.Error(t, err)
}
@@ -515,7 +519,7 @@ func TestAllocENI(t *testing.T) {
useSubnetDiscovery: true,
}
- _, err := cache.AllocENI(false, nil, "", 5)
+ _, err := cache.AllocENI(nil, "", 5, 0)
assert.NoError(t, err)
}
@@ -565,7 +569,7 @@ func TestAllocENINoFreeDevice(t *testing.T) {
useSubnetDiscovery: true,
}
- _, err := cache.AllocENI(false, nil, "", 5)
+ _, err := cache.AllocENI(nil, "", 5, 0)
assert.Error(t, err)
}
@@ -618,7 +622,7 @@ func TestAllocENIMaxReached(t *testing.T) {
useSubnetDiscovery: true,
}
- _, err := cache.AllocENI(false, nil, "", 5)
+ _, err := cache.AllocENI(nil, "", 5, 0)
assert.Error(t, err)
}
@@ -665,7 +669,7 @@ func TestAllocENIWithIPAddresses(t *testing.T) {
mockEC2.EXPECT().ModifyNetworkInterfaceAttribute(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil)
cache := &EC2InstanceMetadataCache{ec2SVC: mockEC2, instanceType: "c5n.18xlarge", useSubnetDiscovery: true}
- _, err := cache.AllocENI(false, nil, subnetID, 5)
+ _, err := cache.AllocENI(nil, subnetID, 5, 0)
assert.NoError(t, err)
// when required IP numbers(50) is higher than ENI's limit(49)
@@ -675,7 +679,7 @@ func TestAllocENIWithIPAddresses(t *testing.T) {
mockEC2.EXPECT().AttachNetworkInterface(gomock.Any(), gomock.Any(), gomock.Any()).Return(attachResult, nil)
mockEC2.EXPECT().ModifyNetworkInterfaceAttribute(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil)
cache = &EC2InstanceMetadataCache{ec2SVC: mockEC2, instanceType: "c5n.18xlarge", useSubnetDiscovery: true}
- _, err = cache.AllocENI(false, nil, subnetID, 49)
+ _, err = cache.AllocENI(nil, subnetID, 49, 0)
assert.NoError(t, err)
}
@@ -709,7 +713,7 @@ func TestAllocENIWithIPAddressesAlreadyFull(t *testing.T) {
instanceType: "t3.xlarge",
useSubnetDiscovery: true,
}
- _, err := cache.AllocENI(true, nil, "", 14)
+ _, err := cache.AllocENI(nil, "", 14, 0)
assert.Error(t, err)
}
@@ -763,7 +767,7 @@ func TestAllocENIWithPrefixAddresses(t *testing.T) {
enablePrefixDelegation: true,
useSubnetDiscovery: true,
}
- _, err := cache.AllocENI(false, nil, subnetID, 1)
+ _, err := cache.AllocENI(nil, subnetID, 1, 0)
assert.NoError(t, err)
}
@@ -798,7 +802,7 @@ func TestAllocENIWithPrefixesAlreadyFull(t *testing.T) {
enablePrefixDelegation: true,
useSubnetDiscovery: true,
}
- _, err := cache.AllocENI(true, nil, "", 1)
+ _, err := cache.AllocENI(nil, "", 1, 0)
assert.Error(t, err)
}
@@ -1159,6 +1163,8 @@ func TestEC2InstanceMetadataCache_waitForENIAndPrefixesAttached(t *testing.T) {
isPrimary := true
primaryIP := eni2PrivateIP
prefixIP := eni2Prefix
+ primaryIPv6 := eni1v6IP
+
eni1Metadata := ENIMetadata{
ENIID: eni2ID,
MAC: eni2MAC,
@@ -1176,24 +1182,23 @@ func TestEC2InstanceMetadataCache_waitForENIAndPrefixesAttached(t *testing.T) {
},
},
}
- v6PrefixIP := eni2v6Prefix
+ // v6PrefixIP := eni2v6Prefix
eni2Metadata := ENIMetadata{
ENIID: eni2ID,
MAC: eni2MAC,
DeviceNumber: 1,
SubnetIPv4CIDR: subnetCIDR,
+ SubnetIPv6CIDR: subnetCIDRv6,
IPv4Addresses: []ec2types.NetworkInterfacePrivateIpAddress{
{
Primary: &isPrimary,
PrivateIpAddress: &primaryIP,
},
},
- IPv6Prefixes: []ec2types.Ipv6PrefixSpecification{
- {
- Ipv6Prefix: &v6PrefixIP,
- },
+ IPv6Prefixes: []ec2types.Ipv6PrefixSpecification{},
+ IPv6Addresses: []ec2types.NetworkInterfaceIpv6Address{
+ {Ipv6Address: &primaryIPv6},
},
- IPv6Addresses: []ec2types.NetworkInterfaceIpv6Address{},
}
tests := []struct {
name string
@@ -1210,30 +1215,40 @@ func TestEC2InstanceMetadataCache_waitForENIAndPrefixesAttached(t *testing.T) {
t.Run(tt.name, func(t *testing.T) {
ctrl, mockEC2 := setup(t)
defer ctrl.Finish()
+
eniIPs := eni2PrivateIP
eniPrefixes := eni2Prefix
metaDataPrefixPath := metadataIPv4Prefixes
+ metaDataSubnetCIDRPath := metadataSubnetCIDR
+ subnetCIDRValue := subnetCIDR
+ macFields := imdsMACFields
+
+ // We create ENI with single v6 IP and then attach the IPv6 Prefixes later
if tt.args.v6Enabled {
- eniPrefixes = eni2v6Prefix
- metaDataPrefixPath = metadataIPv6Prefixes
+ eniPrefixes = eni1v6IP
+ metaDataPrefixPath = metadataIPv6s
+ metaDataSubnetCIDRPath = metadataSubnetV6CIDR
+ subnetCIDRValue = subnetCIDRv6
+ macFields = imdsMACFieldsV6Only
}
if tt.args.foundPrefixes == 0 {
eniPrefixes = ""
}
mockMetadata := testMetadata(map[string]interface{}{
- metadataMACPath: primaryMAC + " " + eni2MAC,
- metadataMACPath + eni2MAC: imdsMACFields,
- metadataMACPath + eni2MAC + metadataDeviceNum: eni2Device,
- metadataMACPath + eni2MAC + metadataInterface: eni2ID,
- metadataMACPath + eni2MAC + metadataSubnetCIDR: subnetCIDR,
- metadataMACPath + eni2MAC + metadataIPv4s: eniIPs,
- metadataMACPath + eni2MAC + metaDataPrefixPath: eniPrefixes,
+ metadataMACPath: primaryMAC + " " + eni2MAC,
+ metadataMACPath + eni2MAC: macFields,
+ metadataMACPath + eni2MAC + metadataDeviceNum: eni2Device,
+ metadataMACPath + eni2MAC + metadataInterface: eni2ID,
+ metadataMACPath + eni2MAC + metaDataSubnetCIDRPath: subnetCIDRValue,
+ metadataMACPath + eni2MAC + metadataIPv4s: eniIPs,
+ metadataMACPath + eni2MAC + metaDataPrefixPath: eniPrefixes,
})
+
cache := &EC2InstanceMetadataCache{imds: TypedIMDS{mockMetadata}, ec2SVC: mockEC2,
enablePrefixDelegation: true, v4Enabled: tt.args.v4Enabled, v6Enabled: tt.args.v6Enabled}
gotEniMetadata, err := cache.waitForENIAndIPsAttached(tt.args.eni, tt.args.wantedSecondaryIPs, tt.args.maxBackoffDelay)
if (err != nil) != tt.wantErr {
- t.Errorf("waitForENIAndIPsAttached() error = %v, wantErr %v", err, tt.wantErr)
+ t.Errorf("waitForENIAndIPsAttached() error = %+v, wantErr %+v", err, tt.wantErr)
return
}
if !reflect.DeepEqual(gotEniMetadata, tt.wantEniMetadata) {
diff --git a/pkg/awsutils/imds.go b/pkg/awsutils/imds.go
index 0c3152442c..80d388db7c 100644
--- a/pkg/awsutils/imds.go
+++ b/pkg/awsutils/imds.go
@@ -463,6 +463,30 @@ func (typedimds TypedIMDS) GetLocalIPv4s(ctx context.Context, mac string) ([]net
imdsErr := new(imdsRequestError)
oe := new(smithy.OperationError)
if errors.As(err, &imdsErr) || errors.As(err, &oe) {
+ if IsNotFound(err) {
+ // No IPv4 address on the interface, not an error
+ return nil, nil
+ }
+ log.Warnf("%v", err)
+ return nil, newIMDSRequestError(err.Error(), err)
+ }
+ return nil, err
+ }
+ return ips, err
+}
+
+// GetLocalIPv4s returns the private IPv6 addresses associated with the interface. First returned address is the primary address.
+func (typedimds TypedIMDS) GetLocalIPv6s(ctx context.Context, mac string) ([]net.IP, error) {
+ key := fmt.Sprintf("network/interfaces/macs/%s/local-ipv6s", mac)
+ ips, err := typedimds.getIPs(ctx, key)
+ if err != nil {
+ imdsErr := new(imdsRequestError)
+ oe := new(smithy.OperationError)
+ if errors.As(err, &imdsErr) || errors.As(err, &oe) {
+ if IsNotFound(err) {
+ // No IPv6 address on the interface, not an error
+ return nil, nil
+ }
log.Warnf("%v", err)
return nil, newIMDSRequestError(err.Error(), err)
}
@@ -510,6 +534,26 @@ func (typedimds TypedIMDS) GetIPv6Prefixes(ctx context.Context, mac string) ([]n
return prefixes, err
}
+// GetLocalIPv6 returns the IPv6 addresses associated with the primary interface.
+func (typedimds TypedIMDS) GetLocalIPv6(ctx context.Context) (net.IP, error) {
+ key := fmt.Sprintf("ipv6")
+ ips, err := typedimds.getIP(ctx, key)
+ if err != nil {
+ imdsErr := new(imdsRequestError)
+ oe := new(smithy.OperationError)
+ if errors.As(err, &imdsErr) || errors.As(err, &oe) {
+ if IsNotFound(err) {
+ // No IPv6. Not an error, just a disappointment :(
+ return nil, nil
+ }
+ log.Warnf("%v", err)
+ return nil, newIMDSRequestError(err.Error(), err)
+ }
+ return nil, err
+ }
+ return ips, err
+}
+
// GetIPv6s returns the IPv6 addresses associated with the interface.
func (typedimds TypedIMDS) GetIPv6s(ctx context.Context, mac string) ([]net.IP, error) {
key := fmt.Sprintf("network/interfaces/macs/%s/ipv6s", mac)
@@ -572,6 +616,25 @@ func (typedimds TypedIMDS) GetVPCIPv6CIDRBlocks(ctx context.Context, mac string)
return ipnets, err
}
+// GetNetworkCard returns the Network card the interface is attached on
+func (typedimds TypedIMDS) GetNetworkCard(ctx context.Context, mac string) (int, error) {
+ key := fmt.Sprintf("network/interfaces/macs/%s/network-card", mac)
+ networkCard, err := typedimds.getInt(ctx, key)
+ if err != nil {
+ imdsErr := new(imdsRequestError)
+ oe := new(smithy.OperationError)
+ if errors.As(err, &imdsErr) || errors.As(err, &oe) {
+ if IsNotFound(err) {
+ // If no network card field, it is connected to Network card 0
+ return networkCard, nil
+ }
+ log.Warnf("%v", err)
+ return -1, newIMDSRequestError(err.Error(), err)
+ }
+ }
+ return networkCard, err
+}
+
// GetSubnetIPv6CIDRBlocks returns the IPv6 CIDR block for the subnet in which the interface resides.
func (typedimds TypedIMDS) GetSubnetIPv6CIDRBlocks(ctx context.Context, mac string) (net.IPNet, error) {
key := fmt.Sprintf("network/interfaces/macs/%s/subnet-ipv6-cidr-blocks", mac)
diff --git a/pkg/awsutils/imds_test.go b/pkg/awsutils/imds_test.go
index 5663b64e5a..6db3b30685 100644
--- a/pkg/awsutils/imds_test.go
+++ b/pkg/awsutils/imds_test.go
@@ -159,10 +159,13 @@ func TestGetLocalIPv4s(t *testing.T) {
assert.Equal(t, ips, []net.IP{net.IPv4(10, 0, 114, 236), net.IPv4(10, 0, 120, 181)})
}
- _, err = f.GetLocalIPv4s(context.TODO(), "00:00:de:ad:be:ef")
- if assert.Error(t, err) {
- assert.True(t, IsNotFound(err))
- }
+ // This can be remove as we check if "local-ipv4s" field is present in the mac fields. The field not found is verified in the function itself
+ // The function itself will return nil
+
+ // _, err = f.GetLocalIPv4s(context.TODO(), "00:00:de:ad:be:ef")
+ // if assert.Error(t, err) {
+ // assert.True(t, IsNotFound(err))
+ // }
}
func TestGetIPv6s(t *testing.T) {
@@ -217,10 +220,11 @@ func TestGetVPCIPv4CIDRBlocks(t *testing.T) {
assert.Equal(t, ips, []net.IPNet{{IP: net.IPv4(10, 0, 0, 0), Mask: net.CIDRMask(16, 32)}})
}
- _, err = f.GetLocalIPv4s(context.TODO(), "00:00:de:ad:be:ef")
- if assert.Error(t, err) {
- assert.True(t, IsNotFound(err))
- }
+ // This check can be removed as we check if the field is present in mac supported fields before returning as error not found.
+ // _, err = f.GetLocalIPv4s(context.TODO(), "00:00:de:ad:be:ef")
+ // if assert.Error(t, err) {
+ // assert.True(t, IsNotFound(err))
+ // }
}
func TestGetSubnetIPv6CIDRBlocks(t *testing.T) {
diff --git a/pkg/awsutils/mocks/awsutils_mocks.go b/pkg/awsutils/mocks/awsutils_mocks.go
index 7cefc58316..15ede4adfa 100644
--- a/pkg/awsutils/mocks/awsutils_mocks.go
+++ b/pkg/awsutils/mocks/awsutils_mocks.go
@@ -1,22 +1,8 @@
-// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License"). You may
-// not use this file except in compliance with the License. A copy of the
-// License is located at
-//
-// http://aws.amazon.com/apache2.0/
-//
-// or in the "license" file accompanying this file. This file is distributed
-// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
-// express or implied. See the License for the specific language governing
-// permissions and limitations under the License.
-//
-
// Code generated by MockGen. DO NOT EDIT.
-// Source: github.com/aws/amazon-vpc-cni-k8s/pkg/awsutils (interfaces: APIs)
+// Source: awsutils.go
-// Package mock_awsutils is a generated GoMock package.
-package mock_awsutils
+// Package mocks is a generated GoMock package.
+package mocks
import (
net "net"
@@ -54,90 +40,90 @@ func (m *MockAPIs) EXPECT() *MockAPIsMockRecorder {
}
// AllocENI mocks base method.
-func (m *MockAPIs) AllocENI(arg0 bool, arg1 []*string, arg2 string, arg3 int) (string, error) {
+func (m *MockAPIs) AllocENI(sg []*string, eniCfgSubnet string, numIPs, networkCard int) (string, error) {
m.ctrl.T.Helper()
- ret := m.ctrl.Call(m, "AllocENI", arg0, arg1, arg2, arg3)
+ ret := m.ctrl.Call(m, "AllocENI", sg, eniCfgSubnet, numIPs, networkCard)
ret0, _ := ret[0].(string)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// AllocENI indicates an expected call of AllocENI.
-func (mr *MockAPIsMockRecorder) AllocENI(arg0, arg1, arg2, arg3 interface{}) *gomock.Call {
+func (mr *MockAPIsMockRecorder) AllocENI(sg, eniCfgSubnet, numIPs, networkCard interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AllocENI", reflect.TypeOf((*MockAPIs)(nil).AllocENI), arg0, arg1, arg2, arg3)
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AllocENI", reflect.TypeOf((*MockAPIs)(nil).AllocENI), sg, eniCfgSubnet, numIPs, networkCard)
}
// AllocIPAddress mocks base method.
-func (m *MockAPIs) AllocIPAddress(arg0 string) error {
+func (m *MockAPIs) AllocIPAddress(eniID string) error {
m.ctrl.T.Helper()
- ret := m.ctrl.Call(m, "AllocIPAddress", arg0)
+ ret := m.ctrl.Call(m, "AllocIPAddress", eniID)
ret0, _ := ret[0].(error)
return ret0
}
// AllocIPAddress indicates an expected call of AllocIPAddress.
-func (mr *MockAPIsMockRecorder) AllocIPAddress(arg0 interface{}) *gomock.Call {
+func (mr *MockAPIsMockRecorder) AllocIPAddress(eniID interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AllocIPAddress", reflect.TypeOf((*MockAPIs)(nil).AllocIPAddress), arg0)
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AllocIPAddress", reflect.TypeOf((*MockAPIs)(nil).AllocIPAddress), eniID)
}
// AllocIPAddresses mocks base method.
-func (m *MockAPIs) AllocIPAddresses(arg0 string, arg1 int) (*ec2.AssignPrivateIpAddressesOutput, error) {
+func (m *MockAPIs) AllocIPAddresses(eniID string, numIPs int) (*ec2.AssignPrivateIpAddressesOutput, error) {
m.ctrl.T.Helper()
- ret := m.ctrl.Call(m, "AllocIPAddresses", arg0, arg1)
+ ret := m.ctrl.Call(m, "AllocIPAddresses", eniID, numIPs)
ret0, _ := ret[0].(*ec2.AssignPrivateIpAddressesOutput)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// AllocIPAddresses indicates an expected call of AllocIPAddresses.
-func (mr *MockAPIsMockRecorder) AllocIPAddresses(arg0, arg1 interface{}) *gomock.Call {
+func (mr *MockAPIsMockRecorder) AllocIPAddresses(eniID, numIPs interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AllocIPAddresses", reflect.TypeOf((*MockAPIs)(nil).AllocIPAddresses), arg0, arg1)
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AllocIPAddresses", reflect.TypeOf((*MockAPIs)(nil).AllocIPAddresses), eniID, numIPs)
}
// AllocIPv6Prefixes mocks base method.
-func (m *MockAPIs) AllocIPv6Prefixes(arg0 string) ([]*string, error) {
+func (m *MockAPIs) AllocIPv6Prefixes(eniID string) ([]*string, error) {
m.ctrl.T.Helper()
- ret := m.ctrl.Call(m, "AllocIPv6Prefixes", arg0)
+ ret := m.ctrl.Call(m, "AllocIPv6Prefixes", eniID)
ret0, _ := ret[0].([]*string)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// AllocIPv6Prefixes indicates an expected call of AllocIPv6Prefixes.
-func (mr *MockAPIsMockRecorder) AllocIPv6Prefixes(arg0 interface{}) *gomock.Call {
+func (mr *MockAPIsMockRecorder) AllocIPv6Prefixes(eniID interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AllocIPv6Prefixes", reflect.TypeOf((*MockAPIs)(nil).AllocIPv6Prefixes), arg0)
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AllocIPv6Prefixes", reflect.TypeOf((*MockAPIs)(nil).AllocIPv6Prefixes), eniID)
}
// DeallocIPAddresses mocks base method.
-func (m *MockAPIs) DeallocIPAddresses(arg0 string, arg1 []string) error {
+func (m *MockAPIs) DeallocIPAddresses(eniID string, ips []string) error {
m.ctrl.T.Helper()
- ret := m.ctrl.Call(m, "DeallocIPAddresses", arg0, arg1)
+ ret := m.ctrl.Call(m, "DeallocIPAddresses", eniID, ips)
ret0, _ := ret[0].(error)
return ret0
}
// DeallocIPAddresses indicates an expected call of DeallocIPAddresses.
-func (mr *MockAPIsMockRecorder) DeallocIPAddresses(arg0, arg1 interface{}) *gomock.Call {
+func (mr *MockAPIsMockRecorder) DeallocIPAddresses(eniID, ips interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeallocIPAddresses", reflect.TypeOf((*MockAPIs)(nil).DeallocIPAddresses), arg0, arg1)
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeallocIPAddresses", reflect.TypeOf((*MockAPIs)(nil).DeallocIPAddresses), eniID, ips)
}
// DeallocPrefixAddresses mocks base method.
-func (m *MockAPIs) DeallocPrefixAddresses(arg0 string, arg1 []string) error {
+func (m *MockAPIs) DeallocPrefixAddresses(eniID string, ips []string) error {
m.ctrl.T.Helper()
- ret := m.ctrl.Call(m, "DeallocPrefixAddresses", arg0, arg1)
+ ret := m.ctrl.Call(m, "DeallocPrefixAddresses", eniID, ips)
ret0, _ := ret[0].(error)
return ret0
}
// DeallocPrefixAddresses indicates an expected call of DeallocPrefixAddresses.
-func (mr *MockAPIsMockRecorder) DeallocPrefixAddresses(arg0, arg1 interface{}) *gomock.Call {
+func (mr *MockAPIsMockRecorder) DeallocPrefixAddresses(eniID, ips interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeallocPrefixAddresses", reflect.TypeOf((*MockAPIs)(nil).DeallocPrefixAddresses), arg0, arg1)
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeallocPrefixAddresses", reflect.TypeOf((*MockAPIs)(nil).DeallocPrefixAddresses), eniID, ips)
}
// DescribeAllENIs mocks base method.
@@ -170,17 +156,17 @@ func (mr *MockAPIsMockRecorder) FetchInstanceTypeLimits() *gomock.Call {
}
// FreeENI mocks base method.
-func (m *MockAPIs) FreeENI(arg0 string) error {
+func (m *MockAPIs) FreeENI(eniName string) error {
m.ctrl.T.Helper()
- ret := m.ctrl.Call(m, "FreeENI", arg0)
+ ret := m.ctrl.Call(m, "FreeENI", eniName)
ret0, _ := ret[0].(error)
return ret0
}
// FreeENI indicates an expected call of FreeENI.
-func (mr *MockAPIsMockRecorder) FreeENI(arg0 interface{}) *gomock.Call {
+func (mr *MockAPIsMockRecorder) FreeENI(eniName interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FreeENI", reflect.TypeOf((*MockAPIs)(nil).FreeENI), arg0)
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FreeENI", reflect.TypeOf((*MockAPIs)(nil).FreeENI), eniName)
}
// GetAttachedENIs mocks base method.
@@ -227,48 +213,48 @@ func (mr *MockAPIsMockRecorder) GetENILimit() *gomock.Call {
}
// GetIPv4PrefixesFromEC2 mocks base method.
-func (m *MockAPIs) GetIPv4PrefixesFromEC2(arg0 string) ([]types.Ipv4PrefixSpecification, error) {
+func (m *MockAPIs) GetIPv4PrefixesFromEC2(eniID string) ([]types.Ipv4PrefixSpecification, error) {
m.ctrl.T.Helper()
- ret := m.ctrl.Call(m, "GetIPv4PrefixesFromEC2", arg0)
+ ret := m.ctrl.Call(m, "GetIPv4PrefixesFromEC2", eniID)
ret0, _ := ret[0].([]types.Ipv4PrefixSpecification)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// GetIPv4PrefixesFromEC2 indicates an expected call of GetIPv4PrefixesFromEC2.
-func (mr *MockAPIsMockRecorder) GetIPv4PrefixesFromEC2(arg0 interface{}) *gomock.Call {
+func (mr *MockAPIsMockRecorder) GetIPv4PrefixesFromEC2(eniID interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetIPv4PrefixesFromEC2", reflect.TypeOf((*MockAPIs)(nil).GetIPv4PrefixesFromEC2), arg0)
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetIPv4PrefixesFromEC2", reflect.TypeOf((*MockAPIs)(nil).GetIPv4PrefixesFromEC2), eniID)
}
// GetIPv4sFromEC2 mocks base method.
-func (m *MockAPIs) GetIPv4sFromEC2(arg0 string) ([]types.NetworkInterfacePrivateIpAddress, error) {
+func (m *MockAPIs) GetIPv4sFromEC2(eniID string) ([]types.NetworkInterfacePrivateIpAddress, error) {
m.ctrl.T.Helper()
- ret := m.ctrl.Call(m, "GetIPv4sFromEC2", arg0)
+ ret := m.ctrl.Call(m, "GetIPv4sFromEC2", eniID)
ret0, _ := ret[0].([]types.NetworkInterfacePrivateIpAddress)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// GetIPv4sFromEC2 indicates an expected call of GetIPv4sFromEC2.
-func (mr *MockAPIsMockRecorder) GetIPv4sFromEC2(arg0 interface{}) *gomock.Call {
+func (mr *MockAPIsMockRecorder) GetIPv4sFromEC2(eniID interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetIPv4sFromEC2", reflect.TypeOf((*MockAPIs)(nil).GetIPv4sFromEC2), arg0)
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetIPv4sFromEC2", reflect.TypeOf((*MockAPIs)(nil).GetIPv4sFromEC2), eniID)
}
// GetIPv6PrefixesFromEC2 mocks base method.
-func (m *MockAPIs) GetIPv6PrefixesFromEC2(arg0 string) ([]types.Ipv6PrefixSpecification, error) {
+func (m *MockAPIs) GetIPv6PrefixesFromEC2(eniID string) ([]types.Ipv6PrefixSpecification, error) {
m.ctrl.T.Helper()
- ret := m.ctrl.Call(m, "GetIPv6PrefixesFromEC2", arg0)
+ ret := m.ctrl.Call(m, "GetIPv6PrefixesFromEC2", eniID)
ret0, _ := ret[0].([]types.Ipv6PrefixSpecification)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// GetIPv6PrefixesFromEC2 indicates an expected call of GetIPv6PrefixesFromEC2.
-func (mr *MockAPIsMockRecorder) GetIPv6PrefixesFromEC2(arg0 interface{}) *gomock.Call {
+func (mr *MockAPIsMockRecorder) GetIPv6PrefixesFromEC2(eniID interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetIPv6PrefixesFromEC2", reflect.TypeOf((*MockAPIs)(nil).GetIPv6PrefixesFromEC2), arg0)
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetIPv6PrefixesFromEC2", reflect.TypeOf((*MockAPIs)(nil).GetIPv6PrefixesFromEC2), eniID)
}
// GetInstanceHypervisorFamily mocks base method.
@@ -327,6 +313,20 @@ func (mr *MockAPIsMockRecorder) GetLocalIPv4() *gomock.Call {
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLocalIPv4", reflect.TypeOf((*MockAPIs)(nil).GetLocalIPv4))
}
+// GetLocalIPv6 mocks base method.
+func (m *MockAPIs) GetLocalIPv6() net.IP {
+ m.ctrl.T.Helper()
+ ret := m.ctrl.Call(m, "GetLocalIPv6")
+ ret0, _ := ret[0].(net.IP)
+ return ret0
+}
+
+// GetLocalIPv6 indicates an expected call of GetLocalIPv6.
+func (mr *MockAPIsMockRecorder) GetLocalIPv6() *gomock.Call {
+ mr.mock.ctrl.T.Helper()
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLocalIPv6", reflect.TypeOf((*MockAPIs)(nil).GetLocalIPv6))
+}
+
// GetNetworkCards mocks base method.
func (m *MockAPIs) GetNetworkCards() []vpc.NetworkCard {
m.ctrl.T.Helper()
@@ -412,17 +412,17 @@ func (mr *MockAPIsMockRecorder) InitCachedPrefixDelegation(arg0 interface{}) *go
}
// IsMultiCardENI mocks base method.
-func (m *MockAPIs) IsMultiCardENI(arg0 string) bool {
+func (m *MockAPIs) IsMultiCardENI(eniID string) bool {
m.ctrl.T.Helper()
- ret := m.ctrl.Call(m, "IsMultiCardENI", arg0)
+ ret := m.ctrl.Call(m, "IsMultiCardENI", eniID)
ret0, _ := ret[0].(bool)
return ret0
}
// IsMultiCardENI indicates an expected call of IsMultiCardENI.
-func (mr *MockAPIsMockRecorder) IsMultiCardENI(arg0 interface{}) *gomock.Call {
+func (mr *MockAPIsMockRecorder) IsMultiCardENI(eniID interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsMultiCardENI", reflect.TypeOf((*MockAPIs)(nil).IsMultiCardENI), arg0)
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsMultiCardENI", reflect.TypeOf((*MockAPIs)(nil).IsMultiCardENI), eniID)
}
// IsPrefixDelegationSupported mocks base method.
@@ -440,98 +440,98 @@ func (mr *MockAPIsMockRecorder) IsPrefixDelegationSupported() *gomock.Call {
}
// IsPrimaryENI mocks base method.
-func (m *MockAPIs) IsPrimaryENI(arg0 string) bool {
+func (m *MockAPIs) IsPrimaryENI(eniID string) bool {
m.ctrl.T.Helper()
- ret := m.ctrl.Call(m, "IsPrimaryENI", arg0)
+ ret := m.ctrl.Call(m, "IsPrimaryENI", eniID)
ret0, _ := ret[0].(bool)
return ret0
}
// IsPrimaryENI indicates an expected call of IsPrimaryENI.
-func (mr *MockAPIsMockRecorder) IsPrimaryENI(arg0 interface{}) *gomock.Call {
+func (mr *MockAPIsMockRecorder) IsPrimaryENI(eniID interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsPrimaryENI", reflect.TypeOf((*MockAPIs)(nil).IsPrimaryENI), arg0)
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsPrimaryENI", reflect.TypeOf((*MockAPIs)(nil).IsPrimaryENI), eniID)
}
// IsUnmanagedENI mocks base method.
-func (m *MockAPIs) IsUnmanagedENI(arg0 string) bool {
+func (m *MockAPIs) IsUnmanagedENI(eniID string) bool {
m.ctrl.T.Helper()
- ret := m.ctrl.Call(m, "IsUnmanagedENI", arg0)
+ ret := m.ctrl.Call(m, "IsUnmanagedENI", eniID)
ret0, _ := ret[0].(bool)
return ret0
}
// IsUnmanagedENI indicates an expected call of IsUnmanagedENI.
-func (mr *MockAPIsMockRecorder) IsUnmanagedENI(arg0 interface{}) *gomock.Call {
+func (mr *MockAPIsMockRecorder) IsUnmanagedENI(eniID interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsUnmanagedENI", reflect.TypeOf((*MockAPIs)(nil).IsUnmanagedENI), arg0)
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsUnmanagedENI", reflect.TypeOf((*MockAPIs)(nil).IsUnmanagedENI), eniID)
}
// RefreshSGIDs mocks base method.
-func (m *MockAPIs) RefreshSGIDs(arg0 string, arg1 *datastore.DataStore) error {
+func (m *MockAPIs) RefreshSGIDs(mac string, ds *datastore.DataStoreAccess) error {
m.ctrl.T.Helper()
- ret := m.ctrl.Call(m, "RefreshSGIDs", arg0, arg1)
+ ret := m.ctrl.Call(m, "RefreshSGIDs", mac, ds)
ret0, _ := ret[0].(error)
return ret0
}
// RefreshSGIDs indicates an expected call of RefreshSGIDs.
-func (mr *MockAPIsMockRecorder) RefreshSGIDs(arg0, arg1 interface{}) *gomock.Call {
+func (mr *MockAPIsMockRecorder) RefreshSGIDs(mac, ds interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RefreshSGIDs", reflect.TypeOf((*MockAPIs)(nil).RefreshSGIDs), arg0, arg1)
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RefreshSGIDs", reflect.TypeOf((*MockAPIs)(nil).RefreshSGIDs), mac, ds)
}
// SetMultiCardENIs mocks base method.
-func (m *MockAPIs) SetMultiCardENIs(arg0 []string) error {
+func (m *MockAPIs) SetMultiCardENIs(eniID []string) error {
m.ctrl.T.Helper()
- ret := m.ctrl.Call(m, "SetMultiCardENIs", arg0)
+ ret := m.ctrl.Call(m, "SetMultiCardENIs", eniID)
ret0, _ := ret[0].(error)
return ret0
}
// SetMultiCardENIs indicates an expected call of SetMultiCardENIs.
-func (mr *MockAPIsMockRecorder) SetMultiCardENIs(arg0 interface{}) *gomock.Call {
+func (mr *MockAPIsMockRecorder) SetMultiCardENIs(eniID interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetMultiCardENIs", reflect.TypeOf((*MockAPIs)(nil).SetMultiCardENIs), arg0)
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetMultiCardENIs", reflect.TypeOf((*MockAPIs)(nil).SetMultiCardENIs), eniID)
}
// SetUnmanagedENIs mocks base method.
-func (m *MockAPIs) SetUnmanagedENIs(arg0 []string) {
+func (m *MockAPIs) SetUnmanagedENIs(eniIDs []string) {
m.ctrl.T.Helper()
- m.ctrl.Call(m, "SetUnmanagedENIs", arg0)
+ m.ctrl.Call(m, "SetUnmanagedENIs", eniIDs)
}
// SetUnmanagedENIs indicates an expected call of SetUnmanagedENIs.
-func (mr *MockAPIsMockRecorder) SetUnmanagedENIs(arg0 interface{}) *gomock.Call {
+func (mr *MockAPIsMockRecorder) SetUnmanagedENIs(eniIDs interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetUnmanagedENIs", reflect.TypeOf((*MockAPIs)(nil).SetUnmanagedENIs), arg0)
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetUnmanagedENIs", reflect.TypeOf((*MockAPIs)(nil).SetUnmanagedENIs), eniIDs)
}
// TagENI mocks base method.
-func (m *MockAPIs) TagENI(arg0 string, arg1 map[string]string) error {
+func (m *MockAPIs) TagENI(eniID string, currentTags map[string]string) error {
m.ctrl.T.Helper()
- ret := m.ctrl.Call(m, "TagENI", arg0, arg1)
+ ret := m.ctrl.Call(m, "TagENI", eniID, currentTags)
ret0, _ := ret[0].(error)
return ret0
}
// TagENI indicates an expected call of TagENI.
-func (mr *MockAPIsMockRecorder) TagENI(arg0, arg1 interface{}) *gomock.Call {
+func (mr *MockAPIsMockRecorder) TagENI(eniID, currentTags interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TagENI", reflect.TypeOf((*MockAPIs)(nil).TagENI), arg0, arg1)
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TagENI", reflect.TypeOf((*MockAPIs)(nil).TagENI), eniID, currentTags)
}
// WaitForENIAndIPsAttached mocks base method.
-func (m *MockAPIs) WaitForENIAndIPsAttached(arg0 string, arg1 int) (awsutils.ENIMetadata, error) {
+func (m *MockAPIs) WaitForENIAndIPsAttached(eni string, wantedSecondaryIPs int) (awsutils.ENIMetadata, error) {
m.ctrl.T.Helper()
- ret := m.ctrl.Call(m, "WaitForENIAndIPsAttached", arg0, arg1)
+ ret := m.ctrl.Call(m, "WaitForENIAndIPsAttached", eni, wantedSecondaryIPs)
ret0, _ := ret[0].(awsutils.ENIMetadata)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// WaitForENIAndIPsAttached indicates an expected call of WaitForENIAndIPsAttached.
-func (mr *MockAPIsMockRecorder) WaitForENIAndIPsAttached(arg0, arg1 interface{}) *gomock.Call {
+func (mr *MockAPIsMockRecorder) WaitForENIAndIPsAttached(eni, wantedSecondaryIPs interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WaitForENIAndIPsAttached", reflect.TypeOf((*MockAPIs)(nil).WaitForENIAndIPsAttached), arg0, arg1)
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WaitForENIAndIPsAttached", reflect.TypeOf((*MockAPIs)(nil).WaitForENIAndIPsAttached), eni, wantedSecondaryIPs)
}
diff --git a/pkg/ipamd/datastore/data_store.go b/pkg/ipamd/datastore/data_store.go
index 02d5cd21f0..8d03b126a2 100644
--- a/pkg/ipamd/datastore/data_store.go
+++ b/pkg/ipamd/datastore/data_store.go
@@ -79,6 +79,9 @@ const backfillNetworkIface = "unknown"
// ErrUnknownPod is an error when there is no pod in data store matching pod name, namespace, sandbox id
var ErrUnknownPod = errors.New("datastore: unknown pod")
+// ErrNoAvailableIPInDataStore is an error when IPAM cannot assign an IP address from the datastore
+var ErrNoAvailableIPInDataStore = errors.New("AssignPodIPAddress: no available IP/Prefix addresses")
+
// IPAMKey is the IPAM primary key. Quoting CNI spec:
//
// Plugins that store state should do so using a primary key of
@@ -103,6 +106,7 @@ func (k IPAMKey) String() string {
type IPAMMetadata struct {
K8SPodNamespace string `json:"k8sPodNamespace,omitempty"`
K8SPodName string `json:"k8sPodName,omitempty"`
+ InterfacesCount int `json:"interfacesCount,omitempty"`
}
// ENI represents a single ENI. Exported fields will be marshaled for introspection.
@@ -461,7 +465,9 @@ func (ds *DataStore) AddENI(eniID string, deviceNumber int, isPrimary, isTrunk,
IsEFA: isEFA,
ID: eniID,
DeviceNumber: deviceNumber,
- AvailableIPv4Cidrs: make(map[string]*CidrInfo)}
+ AvailableIPv4Cidrs: make(map[string]*CidrInfo),
+ IPv6Cidrs: make(map[string]*CidrInfo),
+ }
prometheusmetrics.Enis.Set(float64(len(ds.eniPool)))
// Initialize ENI IPs In Use to 0 when an ENI is created
@@ -574,7 +580,7 @@ func (ds *DataStore) AddIPv6CidrToStore(eniID string, ipv6Cidr net.IPNet, isPref
}
// Check if already present in datastore.
_, ok = curENI.IPv6Cidrs[strIPv6Cidr]
- ds.log.Debugf("IP not in DS")
+ ds.log.Debugf("IP not in DS")
if ok {
ds.log.Debugf("IPv6 prefix %s already in DS", strIPv6Cidr)
return errors.New(IPAlreadyInStoreError)
@@ -661,7 +667,7 @@ func (ds *DataStore) AssignPodIPv6Address(ipamKey IPAMKey, ipamMetadata IPAMMeta
}
}
prometheusmetrics.NoAvailableIPAddrs.Inc()
- return "", -1, errors.New("AssignPodIPv6Address: no available IP addresses")
+ return "", -1, ErrNoAvailableIPInDataStore
}
// AssignPodIPv4Address assigns an IPv4 address to pod
@@ -732,13 +738,13 @@ func (ds *DataStore) AssignPodIPv4Address(ipamKey IPAMKey, ipamMetadata IPAMMeta
prometheusmetrics.NoAvailableIPAddrs.Inc()
ds.log.Errorf("DataStore has no available IP/Prefix addresses")
- return "", -1, errors.New("AssignPodIPv4Address: no available IP/Prefix addresses")
+ return "", -1, ErrNoAvailableIPInDataStore
}
// assignPodIPAddressUnsafe mark Address as assigned.
func (ds *DataStore) assignPodIPAddressUnsafe(addr *AddressInfo, ipamKey IPAMKey, ipamMetadata IPAMMetadata, assignedTime time.Time) {
- ds.log.Infof("assignPodIPAddressUnsafe: Assign IP %v to sandbox %s",
- addr.Address, ipamKey)
+ ds.log.Infof("assignPodIPAddressUnsafe: Assign IP %v to sandbox %s with metadata %",
+ addr.Address, ipamKey, ipamMetadata)
if addr.Assigned() {
panic("addr already assigned")
@@ -1082,7 +1088,7 @@ func (ds *DataStore) RemoveENIFromDataStore(eniID string, force bool) error {
// UnassignPodIPAddress a) find out the IP address based on PodName and PodNameSpace
// b) mark IP address as unassigned c) returns IP address, ENI's device number, error
-func (ds *DataStore) UnassignPodIPAddress(ipamKey IPAMKey) (e *ENI, ip string, deviceNumber int, err error) {
+func (ds *DataStore) UnassignPodIPAddress(ipamKey IPAMKey) (e *ENI, ip string, deviceNumber int, interfaces int, err error) {
ds.lock.Lock()
defer ds.lock.Unlock()
ds.log.Debugf("UnassignPodIPAddress: IP address pool stats: total %d, assigned %d, sandbox %s", ds.total, ds.assigned, ipamKey)
@@ -1099,7 +1105,7 @@ func (ds *DataStore) UnassignPodIPAddress(ipamKey IPAMKey) (e *ENI, ip string, d
// If entry is still not found, IPAMD has no knowledge of this pod, so there is nothing to do.
if addr == nil {
ds.log.Warnf("UnassignPodIPAddress: Failed to find sandbox %s", ipamKey)
- return nil, "", 0, ErrUnknownPod
+ return nil, "", 0, -1, ErrUnknownPod
}
}
@@ -1109,17 +1115,22 @@ func (ds *DataStore) UnassignPodIPAddress(ipamKey IPAMKey) (e *ENI, ip string, d
if err := ds.writeBackingStoreUnsafe(); err != nil {
// Unwind un-assignment
ds.assignPodIPAddressUnsafe(addr, ipamKey, originalIPAMMetadata, originalAssignedTime)
- return nil, "", 0, err
+ return nil, "", 0, -1, err
}
addr.UnassignedTime = time.Now()
+ // Interfaces Count 0 means this property did not exist in the datastore when we restored. A Pod entry always have atleast one interface
+ if originalIPAMMetadata.InterfacesCount == 0 {
+ originalIPAMMetadata.InterfacesCount += 1
+ }
+
//Update prometheus for ips per cidr
prometheusmetrics.IpsPerCidr.With(prometheus.Labels{"cidr": availableCidr.Cidr.String()}).Dec()
ds.log.Infof("UnassignPodIPAddress: sandbox %s's ipAddr %s, DeviceNumber %d",
ipamKey, addr.Address, eni.DeviceNumber)
// Decrement ENI IP usage when a pod is deallocated
prometheusmetrics.EniIPsInUse.WithLabelValues(eni.ID).Dec()
- return eni, addr.Address, eni.DeviceNumber, nil
+ return eni, addr.Address, eni.DeviceNumber, originalIPAMMetadata.InterfacesCount, nil
}
// AllocatedIPs returns a recent snapshot of allocated sandbox<->IPs.
@@ -1534,3 +1545,39 @@ func (ds *DataStore) DeleteFromContainerRule(entry *CheckpointEntry) {
ds.log.Errorf("failed to delete fromPod rule, addr=%s", addr.String())
}
}
+
+type DataStoreAccess struct {
+ DataStores []*DataStore
+}
+
+func InitializeDataStores(networkCards int, defaultDataStorePath string, enablePD bool, log logger.Logger) *DataStoreAccess {
+ var dslist []*DataStore
+ log.Infof("Initializing %d datastores for managed network cards", networkCards)
+ for i := range networkCards {
+ dsBackingStorePath := defaultDataStorePath
+ if i > 0 {
+ baseName := strings.TrimSuffix(defaultDataStorePath, ".json")
+ dsBackingStorePath = fmt.Sprintf("%s-nic-%d.json", baseName, i)
+ }
+ checkpointer := NewJSONFile(dsBackingStorePath)
+ dslist = append(dslist, NewDataStore(log, checkpointer, enablePD))
+ }
+
+ return &DataStoreAccess{
+ DataStores: dslist,
+ }
+}
+
+func (ds *DataStoreAccess) GetDataStore(index int) *DataStore {
+ return ds.DataStores[index]
+}
+
+func (ds *DataStoreAccess) ReadAllDataStores(enableIPv6 bool) error {
+
+ for i := range ds.DataStores {
+ if err := ds.GetDataStore(i).ReadBackingStore(enableIPv6); err != nil {
+ return errors.Wrap(err, "Error while reading datastore")
+ }
+ }
+ return nil
+}
diff --git a/pkg/ipamd/datastore/data_store_test.go b/pkg/ipamd/datastore/data_store_test.go
index 3efd872dbd..e7b6d90285 100644
--- a/pkg/ipamd/datastore/data_store_test.go
+++ b/pkg/ipamd/datastore/data_store_test.go
@@ -670,14 +670,15 @@ func TestPodIPv4Address(t *testing.T) {
_, _, err = ds.AssignPodIPv4Address(key4, IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-4"})
assert.Error(t, err)
// Unassign unknown Pod
- _, _, _, err = ds.UnassignPodIPAddress(key4)
+ _, _, _, _, err = ds.UnassignPodIPAddress(key4)
assert.Error(t, err)
- _, _, deviceNum, err := ds.UnassignPodIPAddress(key2)
+ _, _, deviceNum, interfaces, err := ds.UnassignPodIPAddress(key2)
assert.NoError(t, err)
assert.Equal(t, ds.total, 3)
assert.Equal(t, ds.assigned, 2)
assert.Equal(t, deviceNum, pod1Ns2Device)
+ assert.Equal(t, interfaces, 1)
assert.Equal(t, len(ds.eniPool["eni-2"].AvailableIPv4Cidrs), 1)
assert.Equal(t, ds.eniPool["eni-2"].AssignedIPv4Addresses(), 0)
expectedCheckpointData = &CheckpointData{
@@ -860,10 +861,11 @@ func TestPodIPv4AddressWithPDEnabled(t *testing.T) {
cmp.Diff(checkpoint.Data, expectedCheckpointData, checkpointDataCmpOpts),
)
- _, _, deviceNum, err := ds.UnassignPodIPAddress(key2)
+ _, _, deviceNum, interfaces, err := ds.UnassignPodIPAddress(key2)
assert.NoError(t, err)
assert.Equal(t, ds.total, 16)
assert.Equal(t, ds.assigned, 2)
+ assert.Equal(t, interfaces, 1)
assert.Equal(t, deviceNum, pod1Ns2Device)
assert.Equal(t, len(ds.eniPool["eni-1"].AvailableIPv4Cidrs), 1)
assert.Equal(t, ds.eniPool["eni-1"].AssignedIPv4Addresses(), 2)
@@ -919,7 +921,7 @@ func TestGetIPStatsV4(t *testing.T) {
*ds.GetIPStats("4"),
)
- _, _, _, err = ds.UnassignPodIPAddress(key2)
+ _, _, _, _, err = ds.UnassignPodIPAddress(key2)
assert.NoError(t, err)
assert.Equal(t,
@@ -972,7 +974,7 @@ func TestGetIPStatsV4WithPD(t *testing.T) {
*ds.GetIPStats("4"),
)
- _, _, _, err = ds.UnassignPodIPAddress(key2)
+ _, _, _, _, err = ds.UnassignPodIPAddress(key2)
assert.NoError(t, err)
assert.Equal(t,
diff --git a/pkg/ipamd/introspect.go b/pkg/ipamd/introspect.go
index 7db9bd0f8d..8406be8571 100644
--- a/pkg/ipamd/introspect.go
+++ b/pkg/ipamd/introspect.go
@@ -123,7 +123,7 @@ func (c *IPAMContext) setupIntrospectionServer() *http.Server {
func eniV1RequestHandler(ipam *IPAMContext) func(http.ResponseWriter, *http.Request) {
return func(w http.ResponseWriter, r *http.Request) {
- responseJSON, err := json.Marshal(ipam.dataStore.GetENIInfos())
+ responseJSON, err := json.Marshal(ipam.dataStore[0].GetENIInfos())
if err != nil {
log.Errorf("Failed to marshal ENI data: %v", err)
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
diff --git a/pkg/ipamd/ipamd.go b/pkg/ipamd/ipamd.go
index 9b9b07ccdb..30d62e60ac 100644
--- a/pkg/ipamd/ipamd.go
+++ b/pkg/ipamd/ipamd.go
@@ -185,6 +185,8 @@ const (
// envEnableNetworkPolicy is used to enable IPAMD/CNI to send pod create events to network policy agent.
envNetworkPolicyMode = "NETWORK_POLICY_ENFORCING_MODE"
defaultNetworkPolicyMode = "standard"
+
+ DefaultNetworkCard = 0
)
var log = logger.Get()
@@ -196,7 +198,8 @@ var (
// IPAMContext contains node level control information
type IPAMContext struct {
awsClient awsutils.APIs
- dataStore *datastore.DataStore
+ dataStore []*datastore.DataStore
+ dataStoreAccess *datastore.DataStoreAccess
k8sClient client.Client
enableIPv4 bool
enableIPv6 bool
@@ -207,7 +210,7 @@ type IPAMContext struct {
maxIPsPerENI int
maxENI int
maxPrefixesPerENI int
- unmanagedENI int
+ unmanagedENI []int
numNetworkCards int
warmENITarget int
@@ -364,6 +367,7 @@ func New(k8sClient client.Client) (*IPAMContext, error) {
c.enableManageUntaggedMode = enableManageUntaggedMode()
c.enablePodIPAnnotation = enablePodIPAnnotation()
c.numNetworkCards = len(c.awsClient.GetNetworkCards())
+ c.unmanagedENI = make([]int, c.numNetworkCards)
c.networkPolicyMode, err = getNetworkPolicyMode()
if err != nil {
@@ -383,8 +387,6 @@ func New(k8sClient client.Client) (*IPAMContext, error) {
c.awsClient.InitCachedPrefixDelegation(c.enablePrefixDelegation)
c.myNodeName = os.Getenv(envNodeName)
- checkpointer := datastore.NewJSONFile(dsBackingStorePath())
- c.dataStore = datastore.NewDataStore(log, checkpointer, c.enablePrefixDelegation)
if err := c.nodeInit(); err != nil {
return nil, err
@@ -396,44 +398,59 @@ func (c *IPAMContext) nodeInit() error {
prometheusmetrics.IpamdActionsInprogress.WithLabelValues("nodeInit").Add(float64(1))
defer prometheusmetrics.IpamdActionsInprogress.WithLabelValues("nodeInit").Sub(float64(1))
var err error
- var vpcV4CIDRs []string
+ var vpcCIDRs []string
+ var primaryIP net.IP
ctx := context.TODO()
log.Debugf("Start node init")
- primaryV4IP := c.awsClient.GetLocalIPv4()
+
if err = c.initENIAndIPLimits(); err != nil {
return err
}
if c.enableIPv4 {
- // Subnets currently will have both v4 and v6 CIDRs. Once EC2 launches v6 only Subnets, that will no longer
- // be true and so it is safe (and only required) to get the v4 CIDR info only when IPv4 mode is enabled.
- vpcV4CIDRs, err = c.awsClient.GetVPCIPv4CIDRs()
+ primaryIP = c.awsClient.GetLocalIPv4()
+ vpcCIDRs, err = c.awsClient.GetVPCIPv4CIDRs()
+ if err != nil {
+ return err
+ }
+ } else {
+ primaryIP = c.awsClient.GetLocalIPv6()
+ vpcCIDRs, err = c.awsClient.GetVPCIPv6CIDRs()
if err != nil {
return err
}
}
primaryENIMac := c.awsClient.GetPrimaryENImac()
- err = c.networkClient.SetupHostNetwork(vpcV4CIDRs, primaryENIMac, &primaryV4IP, c.enablePodENI, c.enableIPv4, c.enableIPv6)
+
+ // For multi-card instance, non-VPC traffic is still routed out of Primary ENI
+ err = c.networkClient.SetupHostNetwork(vpcCIDRs, primaryENIMac, &primaryIP, c.enablePodENI, c.enableIPv4, c.enableIPv6)
if err != nil {
return errors.Wrap(err, "ipamd init: failed to set up host network")
}
+
err = c.networkClient.CleanUpStaleAWSChains(c.enableIPv4, c.enableIPv6)
if err != nil {
// We should not error if clean up fails since these chains don't affect the rules
log.Debugf("Failed to clean up stale AWS chains: %v", err)
}
+ // Queries IMDS for all attached ENIs and then compares it against EC2.
+ // Groups the ENIs into different types
metadataResult, err := c.awsClient.DescribeAllENIs()
if err != nil {
return errors.Wrap(err, "ipamd init: failed to retrieve attached ENIs info")
}
log.Debugf("DescribeAllENIs success: ENIs: %d, tagged: %d", len(metadataResult.ENIMetadata), len(metadataResult.TagMap))
- c.awsClient.SetMultiCardENIs(metadataResult.MultiCardENIIDs)
c.setUnmanagedENIs(metadataResult.TagMap)
enis := c.filterUnmanagedENIs(metadataResult.ENIMetadata)
+ fmt.Printf("datastore %+v", c.dataStoreAccess)
+
+ if c.dataStoreAccess == nil {
+ c.dataStoreAccess = datastore.InitializeDataStores(c.numNetworkCards, dsBackingStorePath(), c.enablePrefixDelegation, log)
+ }
for _, eni := range enis {
log.Debugf("Discovered ENI %s, trying to set it up", eni.ENIID)
@@ -471,33 +488,21 @@ func (c *IPAMContext) nodeInit() error {
}
}
- if err := c.dataStore.ReadBackingStore(c.enableIPv6); err != nil {
+ // Read from all backing datastores corresponding to the NICs
+ if err := c.dataStoreAccess.ReadAllDataStores(c.enableIPv6); err != nil {
return err
}
- if c.enableIPv6 {
- // Security Groups for Pods cannot be enabled for IPv4 at this point, as Custom Networking must be enabled first.
- if c.enablePodENI {
- // Try to patch CNINode with Security Groups for Pods feature.
- c.tryEnableSecurityGroupsForPods(ctx)
- }
- // We will not support upgrading/converting an existing IPv4 cluster to operate in IPv6 mode. So, we will always
- // start with a clean slate in IPv6 mode. We also do not have to deal with dynamic update of Prefix Delegation
- // feature in IPv6 mode as we do not support (yet) a non-PD v6 option. In addition, we do not support custom
- // networking in IPv6 mode yet, so we will skip the corresponding setup. This will save us from checking
- // if IPv6 is enabled at multiple places. Once we start supporting these features in IPv6 mode, we can do away
- // with this check and not change anything else in the below setup.
- return nil
- }
-
- if c.enablePrefixDelegation {
- // During upgrade or if prefix delgation knob is disabled to enabled then we
- // might have secondary IPs attached to ENIs so doing a cleanup if not used before moving on
- c.tryUnassignIPsFromENIs()
- } else {
- // When prefix delegation knob is enabled to disabled then we might
- // have unused prefixes attached to the ENIs so need to cleanup
- c.tryUnassignPrefixesFromENIs()
+ if c.enableIPv4 {
+ if c.enablePrefixDelegation {
+ // During upgrade or if prefix delgation knob is disabled to enabled then we
+ // might have secondary IPs attached to ENIs so doing a cleanup if not used before moving on
+ c.tryUnassignIPsFromENIs()
+ } else {
+ // When prefix delegation knob is enabled to disabled then we might
+ // have unused prefixes attached to the ENIs so need to cleanup
+ c.tryUnassignPrefixesFromENIs()
+ }
}
if err = c.configureIPRulesForPods(); err != nil {
@@ -505,21 +510,21 @@ func (c *IPAMContext) nodeInit() error {
}
// Spawning updateCIDRsRulesOnChange go-routine
go wait.Forever(func() {
- vpcV4CIDRs = c.updateCIDRsRulesOnChange(vpcV4CIDRs)
+ vpcCIDRs = c.updateCIDRsRulesOnChange(vpcCIDRs)
}, 30*time.Second)
// RefreshSGIDs populates the ENI cache with ENI -> security group ID mappings, and so it must be called:
// 1. after managed/unmanaged ENIs have been determined
// 2. before any new ENIs are attached
- if c.enableIPv4 && !c.disableENIProvisioning {
- if err := c.awsClient.RefreshSGIDs(primaryENIMac, c.dataStore); err != nil {
+ if !c.disableENIProvisioning {
+ if err := c.awsClient.RefreshSGIDs(primaryENIMac, c.dataStoreAccess); err != nil {
return err
}
// Refresh security groups and VPC CIDR blocks in the background
// Ignoring errors since we will retry in 30s
go wait.Forever(func() {
- c.awsClient.RefreshSGIDs(primaryENIMac, c.dataStore)
+ c.awsClient.RefreshSGIDs(primaryENIMac, c.dataStoreAccess)
}, 30*time.Second)
}
@@ -539,6 +544,8 @@ func (c *IPAMContext) nodeInit() error {
}
c.maxPods = int(maxPods)
+ // Custom Networking is not supported with IPv6 and we do not need to have to add a condition here for IPv6 as we validate
+ // the configs during initialization
if c.useCustomNetworking {
// When custom networking is enabled and a valid ENIConfig is found, IPAMD patches the CNINode
// resource for this instance. The operation is safe as enabling/disabling custom networking
@@ -565,19 +572,15 @@ func (c *IPAMContext) nodeInit() error {
c.tryEnableSecurityGroupsForPods(ctx)
}
- // On node init, check if datastore pool needs to be increased. If so, attach CIDRs from existing ENIs and attach new ENIs.
- datastorePoolTooLow, _ := c.isDatastorePoolTooLow()
- if !c.disableENIProvisioning && datastorePoolTooLow {
- if err := c.increaseDatastorePool(ctx); err != nil {
- // Note that the only error currently returned by increaseDatastorePool is an error attaching CIDRs (other than insufficient IPs)
- podENIErrInc("nodeInit")
- return errors.New("error while trying to increase datastore pool")
- }
- // If custom networking is enabled and the pool is empty, return an error, as there is a misconfiguration and
- // the node should not become ready.
- if c.useCustomNetworking && c.isDatastorePoolEmpty() {
- podENIErrInc("nodeInit")
- return errors.New("Failed to attach any ENIs for custom networking")
+ if !c.disableENIProvisioning {
+ if c.enableIPv6 {
+ // We do not return an error here as create ENI failure shouldn not cause the node to go in not ready state.
+ // However that does restrict multi-nic pods to get failed
+ c.createSecondaryIPv6ENIs(ctx)
+ } else {
+ if err = c.handlePreScaling(ctx); err != nil {
+ return err
+ }
}
}
@@ -585,6 +588,28 @@ func (c *IPAMContext) nodeInit() error {
return nil
}
+func (c *IPAMContext) handlePreScaling(ctx context.Context) error {
+
+ // On node init, check if datastore pool needs to be increased. If so, attach CIDRs from existing ENIs and attach new ENIs.
+ for networkCard, decisions := range c.isDatastorePoolTooLow() {
+ log.Debugf("is datastore pool low for networkCard %d, decision %t", networkCard, decisions.IsLow)
+ if decisions.IsLow {
+ if err := c.increaseDatastorePool(ctx, networkCard); err != nil {
+ // Note that the only error currently returned by increaseDatastorePool is an error attaching CIDRs (other than insufficient IPs)
+ podENIErrInc("nodeInit")
+ return errors.New("error while trying to increase datastore pool")
+ }
+ // If custom networking is enabled and the pool is empty, return an error, as there is a misconfiguration and
+ // the node should not become ready.
+ if c.useCustomNetworking && c.isDatastorePoolEmpty(networkCard) {
+ podENIErrInc("nodeInit")
+ return errors.New("Failed to attach any ENIs for custom networking")
+ }
+ }
+ }
+ return nil
+}
+
func (c *IPAMContext) configureIPRulesForPods() error {
rules, err := c.networkClient.GetRuleList()
if err != nil {
@@ -592,15 +617,17 @@ func (c *IPAMContext) configureIPRulesForPods() error {
return nil
}
- for _, info := range c.dataStore.AllocatedIPs() {
- // TODO(gus): This should really be done via CNI CHECK calls, rather than in ipam (requires upstream k8s changes).
+ for _, ds := range c.dataStoreAccess.DataStores {
+ for _, info := range ds.AllocatedIPs() {
+ // TODO(gus): This should really be done via CNI CHECK calls, rather than in ipam (requires upstream k8s changes).
- // Update ip rules in case there is a change in VPC CIDRs, AWS_VPC_K8S_CNI_EXTERNALSNAT setting
- srcIPNet := net.IPNet{IP: net.ParseIP(info.IP), Mask: net.IPv4Mask(255, 255, 255, 255)}
+ // Update ip rules in case there is a change in VPC CIDRs, AWS_VPC_K8S_CNI_EXTERNALSNAT setting
+ srcIPNet := net.IPNet{IP: net.ParseIP(info.IP), Mask: net.IPv4Mask(255, 255, 255, 255)}
- err = c.networkClient.UpdateRuleListBySrc(rules, srcIPNet)
- if err != nil {
- log.Warnf("UpdateRuleListBySrc in nodeInit() failed for IP %s: %v", info.IP, err)
+ err = c.networkClient.UpdateRuleListBySrc(rules, srcIPNet)
+ if err != nil {
+ log.Warnf("UpdateRuleListBySrc in nodeInit() failed for IP %s: %v", info.IP, err)
+ }
}
}
@@ -645,7 +672,7 @@ func (c *IPAMContext) updateIPStats(unmanaged int) {
func (c *IPAMContext) StartNodeIPPoolManager() {
// For IPv6, if Security Groups for Pods is enabled, wait until trunk ENI is attached and add it to the datastore.
if c.enableIPv6 {
- if c.enablePodENI && c.dataStore.GetTrunkENI() == "" {
+ if c.enablePodENI && c.dataStoreAccess.GetDataStore(DefaultNetworkCard).GetTrunkENI() == "" {
for !c.checkForTrunkENI() {
time.Sleep(ipPoolMonitorInterval)
}
@@ -671,48 +698,59 @@ func (c *IPAMContext) StartNodeIPPoolManager() {
func (c *IPAMContext) updateIPPoolIfRequired(ctx context.Context) {
// When IPv4 Security Groups for Pods is configured, do not write to CNINode until there is room for a trunk ENI
- if c.enablePodENI && c.enableIPv4 && c.dataStore.GetTrunkENI() == "" {
+ if c.enablePodENI && c.enableIPv4 && c.dataStoreAccess.GetDataStore(DefaultNetworkCard).GetTrunkENI() == "" {
c.tryEnableSecurityGroupsForPods(ctx)
}
- datastorePoolTooLow, stats := c.isDatastorePoolTooLow()
- // Each iteration, log the current datastore IP stats
- log.Debugf("IP stats - total IPs: %d, assigned IPs: %d, cooldown IPs: %d", stats.TotalIPs, stats.AssignedIPs, stats.CooldownIPs)
+ isScaleDownExecuted := false
+ for networkCard, decisions := range c.isDatastorePoolTooLow() {
+ // Each iteration, log the current datastore IP stats
+ log.Debugf("IP stats for Network Card %d - total IPs: %d, assigned IPs: %d, cooldown IPs: %d", networkCard, decisions.Stats.TotalIPs, decisions.Stats.AssignedIPs, decisions.Stats.CooldownIPs)
+ if decisions.IsLow {
+ c.increaseDatastorePool(ctx, networkCard)
+ } else if c.isDatastorePoolTooHigh(decisions.Stats, networkCard) && !c.shouldSkipDataStorePoolDecrease() {
+ isScaleDownExecuted = true
+ c.decreaseDatastorePool(networkCard)
+ }
+ if c.shouldRemoveExtraENIs(decisions.Stats, networkCard) {
+ c.tryFreeENI(networkCard)
+ }
+ }
- if datastorePoolTooLow {
- c.increaseDatastorePool(ctx)
- } else if c.isDatastorePoolTooHigh(stats) {
- c.decreaseDatastorePool(decreaseIPPoolInterval)
+ // Store the last update time so that we only scale down every
+ if isScaleDownExecuted {
+ now := time.Now()
+ c.lastDecreaseIPPool = now
+ c.lastNodeIPPoolAction = now
}
- if c.shouldRemoveExtraENIs() {
- c.tryFreeENI()
+}
+
+func (c *IPAMContext) shouldSkipDataStorePoolDecrease() bool {
+
+ timeSinceLastPoolDecrease := time.Since(c.lastDecreaseIPPool)
+
+ if timeSinceLastPoolDecrease <= decreaseIPPoolInterval {
+ log.Debugf("Skipping decrease Datastore pool because time since last %v <= %v", timeSinceLastPoolDecrease, decreaseIPPoolInterval)
+ return true
}
+
+ return false
}
// decreaseDatastorePool runs every `interval` and attempts to return unused ENIs and IPs
-func (c *IPAMContext) decreaseDatastorePool(interval time.Duration) {
- log.Debug("Starting to decrease pool size")
+func (c *IPAMContext) decreaseDatastorePool(networkCard int) {
prometheusmetrics.IpamdActionsInprogress.WithLabelValues("decreaseDatastorePool").Add(float64(1))
defer prometheusmetrics.IpamdActionsInprogress.WithLabelValues("decreaseDatastorePool").Sub(float64(1))
- now := time.Now()
- timeSinceLast := now.Sub(c.lastDecreaseIPPool)
- if timeSinceLast <= interval {
- log.Debugf("Skipping decrease Datastore pool because time since last %v <= %v", timeSinceLast, interval)
- return
- }
-
- log.Debugf("Starting to decrease Datastore pool")
- c.tryUnassignCidrsFromAll()
- c.lastDecreaseIPPool = now
- c.lastNodeIPPoolAction = now
+ log.Debugf("Starting to decrease pool size for network card %d", networkCard)
+ c.tryUnassignCidrsFromAll(networkCard)
log.Debugf("Successfully decreased IP pool")
- c.logPoolStats(c.dataStore.GetIPStats(ipV4AddrFamily))
+ c.logPoolStats(c.dataStoreAccess.GetDataStore(networkCard).GetIPStats(ipV4AddrFamily), networkCard)
}
// tryFreeENI always tries to free one ENI
-func (c *IPAMContext) tryFreeENI() {
+func (c *IPAMContext) tryFreeENI(networkCard int) {
if c.isTerminating() {
log.Debug("AWS CNI is terminating, not detaching any ENIs")
return
@@ -723,12 +761,12 @@ func (c *IPAMContext) tryFreeENI() {
return
}
- eni := c.dataStore.RemoveUnusedENIFromStore(c.warmIPTarget, c.minimumIPTarget, c.warmPrefixTarget)
+ eni := c.dataStoreAccess.GetDataStore(networkCard).RemoveUnusedENIFromStore(c.warmIPTarget, c.minimumIPTarget, c.warmPrefixTarget)
if eni == "" {
return
}
- log.Debugf("Start freeing ENI %s", eni)
+ log.Debugf("Start freeing ENI %s from network card %d", eni, networkCard)
err := c.awsClient.FreeENI(eni)
if err != nil {
ipamdErrInc("decreaseIPPoolFreeENIFailed")
@@ -739,18 +777,18 @@ func (c *IPAMContext) tryFreeENI() {
}
// When warm IP/prefix targets are defined, free extra IPs
-func (c *IPAMContext) tryUnassignCidrsFromAll() {
- _, over, warmIPTargetsDefined := c.datastoreTargetState(nil)
+func (c *IPAMContext) tryUnassignCidrsFromAll(networkCard int) {
+ _, over, warmIPTargetsDefined := c.datastoreTargetState(nil, networkCard)
// If WARM IP targets are not defined, check if WARM_PREFIX_TARGET is defined.
if !warmIPTargetsDefined {
- over = c.computeExtraPrefixesOverWarmTarget()
+ over = c.computeExtraPrefixesOverWarmTarget(networkCard)
}
if over > 0 {
- eniInfos := c.dataStore.GetENIInfos()
+ eniInfos := c.dataStoreAccess.GetDataStore(networkCard).GetENIInfos()
for eniID := range eniInfos.ENIs {
// Either returns prefixes or IPs [Cidrs]
- cidrs := c.dataStore.FindFreeableCidrs(eniID)
+ cidrs := c.dataStoreAccess.GetDataStore(networkCard).FindFreeableCidrs(eniID)
if cidrs == nil {
log.Errorf("Error finding unassigned IPs for ENI %s", eniID)
continue
@@ -769,9 +807,9 @@ func (c *IPAMContext) tryUnassignCidrsFromAll() {
for _, toDelete := range cidrs {
// Do not force the delete, since a freeable Cidr might have been assigned to a pod
// before we get around to deleting it.
- err := c.dataStore.DelIPv4CidrFromStore(eniID, toDelete.Cidr, false /* force */)
+ err := c.dataStoreAccess.GetDataStore(networkCard).DelIPv4CidrFromStore(eniID, toDelete.Cidr, false /* force */)
if err != nil {
- log.Warnf("Failed to delete Cidr %s on ENI %s from datastore: %s", toDelete, eniID, err)
+ log.Warnf("Failed to delete Cidr %s on ENI %s from datastore for network card %d: %s", toDelete, eniID, networkCard, err)
ipamdErrInc("decreaseIPPool")
continue
} else {
@@ -790,18 +828,25 @@ func (c *IPAMContext) tryUnassignCidrsFromAll() {
}
}
-// PRECONDITION: isDatastorePoolTooLow returned true
-func (c *IPAMContext) increaseDatastorePool(ctx context.Context) error {
- log.Debug("Starting to increase pool size")
- prometheusmetrics.IpamdActionsInprogress.WithLabelValues("increaseDatastorePool").Add(float64(1))
- defer prometheusmetrics.IpamdActionsInprogress.WithLabelValues("increaseDatastorePool").Sub(float64(1))
-
+func (c *IPAMContext) isENIAttachmentAllowed() bool {
if c.isTerminating() {
log.Debug("AWS CNI is terminating, will not try to attach any new IPs or ENIs right now")
- return nil
+ return false
}
if !c.manageENIsNonScheduleable && c.isNodeNonSchedulable() {
log.Debug("AWS CNI is on a non schedulable node, will not try to attach any new IPs or ENIs right now")
+ return false
+ }
+ return true
+}
+
+// PRECONDITION: isDatastorePoolTooLow returned true
+func (c *IPAMContext) increaseDatastorePool(ctx context.Context, networkCard int) error {
+ log.Debugf("Starting to increase pool size for network card %d", networkCard)
+ prometheusmetrics.IpamdActionsInprogress.WithLabelValues("increaseDatastorePool").Add(float64(1))
+ defer prometheusmetrics.IpamdActionsInprogress.WithLabelValues("increaseDatastorePool").Sub(float64(1))
+
+ if !c.isENIAttachmentAllowed() {
return nil
}
@@ -811,7 +856,7 @@ func (c *IPAMContext) increaseDatastorePool(ctx context.Context) error {
return nil
}
- increasedPool, err := c.tryAssignCidrs()
+ increasedPool, err := c.tryAssignCidrs(networkCard)
if err != nil {
if containsInsufficientCIDRsOrSubnetIPs(err) {
log.Errorf("Unable to attach IPs/Prefixes for the ENI, subnet doesn't seem to have enough IPs/Prefixes. Consider using new subnet or carve a reserved range using create-subnet-cidr-reservation")
@@ -822,12 +867,12 @@ func (c *IPAMContext) increaseDatastorePool(ctx context.Context) error {
return err
}
if increasedPool {
- c.updateLastNodeIPPoolAction()
+ c.updateLastNodeIPPoolAction(networkCard)
} else {
// If we did not add any IPs, try to allocate an ENI.
- if c.hasRoomForEni() {
- if err = c.tryAllocateENI(ctx); err == nil {
- c.updateLastNodeIPPoolAction()
+ if c.hasRoomForEni(networkCard) {
+ if err = c.tryAllocateENI(ctx, networkCard); err == nil {
+ c.updateLastNodeIPPoolAction(networkCard)
} else {
// Note that no error is returned if ENI allocation fails. This is because ENI allocation failure should not cause node to be "NotReady".
log.Debugf("Error trying to allocate ENI: %v", err)
@@ -839,13 +884,46 @@ func (c *IPAMContext) increaseDatastorePool(ctx context.Context) error {
return nil
}
-func (c *IPAMContext) updateLastNodeIPPoolAction() {
+func (c *IPAMContext) createSecondaryIPv6ENIs(ctx context.Context) error {
+
+ log.Info("Trying to see if new ENIs need to be created")
+ if !c.isENIAttachmentAllowed() {
+ return nil
+ }
+
+ log.Info("Attaching ENIs allowed")
+
+ for networkCard, ds := range c.dataStoreAccess.DataStores {
+ if networkCard == 0 {
+ continue
+ }
+
+ log.Infof("Checking for DS %d", networkCard)
+ if ds.GetENIs() == 0 {
+ // This means we have not added an ENI before as it would have been detected ENI detection
+ log.Infof("Attaching ENI to Network Card %d, MaxENI :%d", networkCard, c.maxENI)
+ if c.hasRoomForEni(networkCard) {
+ if err := c.tryAllocateENI(ctx, networkCard); err == nil {
+ log.Infof("Allocated ENI successfully to Network Card %d", networkCard)
+ } else {
+ // Note that no error is returned if ENI allocation fails. This is because ENI allocation failure should not cause node to be "NotReady".
+ log.Debugf("Error trying to allocate ENI: %v", err)
+ }
+ } else {
+ log.Debugf("Skipping ENI allocation as the max ENI limit is already reached for the instance")
+ }
+ }
+ }
+ return nil
+}
+
+func (c *IPAMContext) updateLastNodeIPPoolAction(networkCard int) {
c.lastNodeIPPoolAction = time.Now()
- stats := c.dataStore.GetIPStats(ipV4AddrFamily)
- c.logPoolStats(stats)
+ stats := c.dataStoreAccess.GetDataStore(networkCard).GetIPStats(ipV4AddrFamily)
+ c.logPoolStats(stats, networkCard)
}
-func (c *IPAMContext) tryAllocateENI(ctx context.Context) error {
+func (c *IPAMContext) tryAllocateENI(ctx context.Context, networkCard int) error {
var securityGroups []*string
var eniCfgSubnet string
@@ -864,9 +942,9 @@ func (c *IPAMContext) tryAllocateENI(ctx context.Context) error {
eniCfgSubnet = eniCfg.Subnet
}
- resourcesToAllocate := c.GetENIResourcesToAllocate()
+ resourcesToAllocate := c.GetENIResourcesToAllocate(networkCard)
if resourcesToAllocate > 0 {
- eni, err := c.awsClient.AllocENI(c.useCustomNetworking, securityGroups, eniCfgSubnet, resourcesToAllocate)
+ eni, err := c.awsClient.AllocENI(securityGroups, eniCfgSubnet, resourcesToAllocate, networkCard)
if err != nil {
log.Errorf("Failed to increase pool size due to not able to allocate ENI %v", err)
ipamdErrInc("increaseIPPoolAllocENI")
@@ -882,15 +960,14 @@ func (c *IPAMContext) tryAllocateENI(ctx context.Context) error {
eniMetadata, err := c.awsClient.WaitForENIAndIPsAttached(eni, resourcesToAllocate)
if err != nil {
ipamdErrInc("increaseIPPoolwaitENIAttachedFailed")
- log.Errorf("Failed to increase pool size: Unable to discover attached ENI from metadata service %v", err)
+ log.Errorf("Failed to increase pool size for Network card %d: Unable to discover attached ENI from metadata service %v", networkCard, err)
return err
}
-
// The CNI does not create trunk or EFA ENIs, so they will always be false here
err = c.setupENI(eni, eniMetadata, false, false)
if err != nil {
ipamdErrInc("increaseIPPoolsetupENIFailed")
- log.Errorf("Failed to increase pool size: %v", err)
+ log.Errorf("Failed to increase pool size for network card %d: %v", networkCard, err)
return err
}
} else {
@@ -901,20 +978,20 @@ func (c *IPAMContext) tryAllocateENI(ctx context.Context) error {
// For an ENI, fill in missing IPs or prefixes.
// PRECONDITION: isDatastorePoolTooLow returned true
-func (c *IPAMContext) tryAssignCidrs() (increasedPool bool, err error) {
+func (c *IPAMContext) tryAssignCidrs(networkCard int) (increasedPool bool, err error) {
if c.enablePrefixDelegation {
- return c.tryAssignPrefixes()
+ return c.tryAssignPrefixes(networkCard)
} else {
- return c.tryAssignIPs()
+ return c.tryAssignIPs(networkCard)
}
}
// For an ENI, try to fill in missing IPs on an existing ENI.
// PRECONDITION: isDatastorePoolTooLow returned true
-func (c *IPAMContext) tryAssignIPs() (increasedPool bool, err error) {
+func (c *IPAMContext) tryAssignIPs(networkCard int) (increasedPool bool, err error) {
// If WARM_IP_TARGET is set, only proceed if we are short of target
- short, _, warmIPTargetsDefined := c.datastoreTargetState(nil)
+ short, _, warmIPTargetsDefined := c.datastoreTargetState(nil, networkCard)
if warmIPTargetsDefined && short == 0 {
return false, nil
}
@@ -926,7 +1003,7 @@ func (c *IPAMContext) tryAssignIPs() (increasedPool bool, err error) {
}
// Find an ENI where we can add more IPs
- enis := c.dataStore.GetAllocatableENIs(c.maxIPsPerENI, c.useCustomNetworking)
+ enis := c.dataStoreAccess.GetDataStore(networkCard).GetAllocatableENIs(c.maxIPsPerENI, c.useCustomNetworking)
for _, eni := range enis {
if len(eni.AvailableIPv4Cidrs) < c.maxIPsPerENI {
currentNumberOfAllocatedIPs := len(eni.AvailableIPv4Cidrs)
@@ -967,14 +1044,14 @@ func (c *IPAMContext) tryAssignIPs() (increasedPool bool, err error) {
ec2ip4s = append(ec2ip4s, ec2types.NetworkInterfacePrivateIpAddress{PrivateIpAddress: ec2Addr.PrivateIpAddress})
}
}
- c.addENIsecondaryIPsToDataStore(ec2ip4s, eni.ID)
+ c.addENIsecondaryIPsToDataStore(ec2ip4s, eni.ID, networkCard)
return true, nil
}
}
return false, nil
}
-func (c *IPAMContext) assignIPv6Prefix(eniID string) (err error) {
+func (c *IPAMContext) assignIPv6Prefix(eniID string, networkCard int) (err error) {
log.Debugf("Assigning an IPv6Prefix for ENI: %s", eniID)
//Let's make an EC2 API call to get a list of IPv6 prefixes (if any) that are already attached to the
//current ENI. We will make this call only once during boot up/init and doing so will shield us from any
@@ -1009,16 +1086,16 @@ func (c *IPAMContext) assignIPv6Prefix(eniID string) (err error) {
//Will use the first IPv6 Prefix attached for IP address allocation.
ec2v6Prefixes = []ec2types.Ipv6PrefixSpecification{ec2v6Prefixes[0]}
}
- c.addENIv6prefixesToDataStore(ec2v6Prefixes, eniID)
+ c.addENIv6prefixesToDataStore(ec2v6Prefixes, eniID, networkCard)
return nil
}
// PRECONDITION: isDatastorePoolTooLow returned true
-func (c *IPAMContext) tryAssignPrefixes() (increasedPool bool, err error) {
- toAllocate := c.getPrefixesNeeded()
+func (c *IPAMContext) tryAssignPrefixes(networkCard int) (increasedPool bool, err error) {
+ toAllocate := c.getPrefixesNeeded(networkCard)
// Returns an ENI which has space for more prefixes to be attached, but this
// ENI might not suffice the WARM_IP_TARGET/WARM_PREFIX_TARGET
- enis := c.dataStore.GetAllocatableENIs(c.maxPrefixesPerENI, c.useCustomNetworking)
+ enis := c.dataStoreAccess.GetDataStore(networkCard).GetAllocatableENIs(c.maxPrefixesPerENI, c.useCustomNetworking)
for _, eni := range enis {
currentNumberOfAllocatedPrefixes := len(eni.AvailableIPv4Cidrs)
resourcesToAllocate := min((c.maxPrefixesPerENI - currentNumberOfAllocatedPrefixes), toAllocate)
@@ -1053,7 +1130,7 @@ func (c *IPAMContext) tryAssignPrefixes() (increasedPool bool, err error) {
}
ec2Prefixes = output.AssignedIpv4Prefixes
}
- c.addENIv4prefixesToDataStore(ec2Prefixes, eni.ID)
+ c.addENIv4prefixesToDataStore(ec2Prefixes, eni.ID, networkCard)
return true, nil
}
return false, nil
@@ -1063,76 +1140,82 @@ func (c *IPAMContext) tryAssignPrefixes() (increasedPool bool, err error) {
// 1) add ENI to datastore
// 2) set up linux ENI related networking stack.
// 3) add all ENI's secondary IP addresses to datastore
+
+// TODO: Will primary ENI so that we can skip Network Setup ?
func (c *IPAMContext) setupENI(eni string, eniMetadata awsutils.ENIMetadata, isTrunkENI, isEFAENI bool) error {
primaryENI := c.awsClient.GetPrimaryENI()
// Add the ENI to the datastore
- err := c.dataStore.AddENI(eni, eniMetadata.DeviceNumber, eni == primaryENI, isTrunkENI, isEFAENI)
+ err := c.dataStoreAccess.GetDataStore(eniMetadata.NetworkCard).AddENI(eni, eniMetadata.DeviceNumber, eni == primaryENI, isTrunkENI, isEFAENI)
if err != nil && err.Error() != datastore.DuplicatedENIError {
return errors.Wrapf(err, "failed to add ENI %s to data store", eni)
}
- // Store the addressable IP for the ENI
+
+ // Store the addressable IP for the ENI.
if c.enableIPv6 {
c.primaryIP[eni] = eniMetadata.PrimaryIPv6Address()
} else {
c.primaryIP[eni] = eniMetadata.PrimaryIPv4Address()
}
- if c.enableIPv6 && eni == primaryENI {
- // In v6 PD mode, VPC CNI will only manage the primary ENI and trunk ENI. Once we start supporting secondary
- // IP and custom networking modes for IPv6, this restriction can be relaxed.
- err := c.assignIPv6Prefix(eni)
- if err != nil {
- return errors.Wrapf(err, "Failed to allocate IPv6 Prefixes to Primary ENI")
- }
- } else {
- // For other ENIs, set up the network
- if eni != primaryENI {
- subnetCidr := eniMetadata.SubnetIPv4CIDR
- if c.enableIPv6 {
- subnetCidr = eniMetadata.SubnetIPv6CIDR
- }
- err = c.networkClient.SetupENINetwork(c.primaryIP[eni], eniMetadata.MAC, eniMetadata.DeviceNumber, subnetCidr)
+ if c.enableIPv6 {
+ if !isTrunkENI {
+ // In v6 PD mode, VPC CNI will manage the primary ENI, ENIs on NC > 0 and trunk ENI. Once we start supporting secondary
+ // IP and custom networking modes for IPv6, this restriction can be relaxed.
+ err := c.assignIPv6Prefix(eni, eniMetadata.NetworkCard)
if err != nil {
- // Failed to set up the ENI
- errRemove := c.dataStore.RemoveENIFromDataStore(eni, true)
- if errRemove != nil {
- log.Warnf("failed to remove ENI %s: %v", eni, errRemove)
- }
- delete(c.primaryIP, eni)
- return errors.Wrapf(err, "failed to set up ENI %s network", eni)
+ return errors.Wrapf(err, "Failed to allocate IPv6 Prefixes to ENI")
}
}
- if !c.enableIPv6 {
- log.Infof("Found ENIs having %d secondary IPs and %d Prefixes", len(eniMetadata.IPv4Addresses), len(eniMetadata.IPv4Prefixes))
- // Either case add the IPs and prefixes to datastore.
- c.addENIsecondaryIPsToDataStore(eniMetadata.IPv4Addresses, eni)
- c.addENIv4prefixesToDataStore(eniMetadata.IPv4Prefixes, eni)
- } else {
- // This is a trunk ENI in IPv6 PD mode, so do not add IPs or prefixes to datastore
- log.Infof("Found IPv6 trunk ENI having %d secondary IPs and %d Prefixes", len(eniMetadata.IPv6Addresses), len(eniMetadata.IPv6Prefixes))
+ }
+
+ // For other ENIs, set up the network
+ if eni != primaryENI {
+ subnetCidr := eniMetadata.SubnetIPv4CIDR
+ if c.enableIPv6 {
+ subnetCidr = eniMetadata.SubnetIPv6CIDR
+ }
+ err = c.networkClient.SetupENINetwork(c.primaryIP[eni], eniMetadata.MAC, eniMetadata.DeviceNumber, eniMetadata.NetworkCard, subnetCidr, c.maxENI)
+ if err != nil {
+ // Failed to set up the ENI
+ errRemove := c.dataStoreAccess.GetDataStore(eniMetadata.NetworkCard).RemoveENIFromDataStore(eni, true)
+ if errRemove != nil {
+ log.Warnf("failed to remove ENI %s: %v", eni, errRemove)
+ }
+ delete(c.primaryIP, eni)
+ return errors.Wrapf(err, "failed to set up ENI %s network", eni)
}
}
+
+ if !c.enableIPv6 {
+ log.Infof("Found ENIs having %d secondary IPs and %d Prefixes", len(eniMetadata.IPv4Addresses), len(eniMetadata.IPv4Prefixes))
+ // Either case add the IPs and prefixes to datastore.
+ c.addENIsecondaryIPsToDataStore(eniMetadata.IPv4Addresses, eni, eniMetadata.NetworkCard)
+ c.addENIv4prefixesToDataStore(eniMetadata.IPv4Prefixes, eni, eniMetadata.NetworkCard)
+ } else {
+ log.Infof("Found IPv6 ENI %s having %d secondary IPs and %d Prefixes and isTrunk: %t", eni, len(eniMetadata.IPv6Addresses), len(eniMetadata.IPv6Prefixes), isTrunkENI)
+ }
+
return nil
}
-func (c *IPAMContext) addENIsecondaryIPsToDataStore(ec2PrivateIpAddrs []ec2types.NetworkInterfacePrivateIpAddress, eni string) {
+func (c *IPAMContext) addENIsecondaryIPsToDataStore(ec2PrivateIpAddrs []ec2types.NetworkInterfacePrivateIpAddress, eni string, networkCard int) {
// Add all the secondary IPs
for _, ec2PrivateIpAddr := range ec2PrivateIpAddrs {
if aws.ToBool(ec2PrivateIpAddr.Primary) {
continue
}
cidr := net.IPNet{IP: net.ParseIP(aws.ToString(ec2PrivateIpAddr.PrivateIpAddress)), Mask: net.IPv4Mask(255, 255, 255, 255)}
- err := c.dataStore.AddIPv4CidrToStore(eni, cidr, false)
+ err := c.dataStoreAccess.GetDataStore(networkCard).AddIPv4CidrToStore(eni, cidr, false)
if err != nil && err.Error() != datastore.IPAlreadyInStoreError {
- log.Warnf("Failed to increase IP pool, failed to add IP %s to data store", ec2PrivateIpAddr.PrivateIpAddress)
+ log.Warnf("Failed to increase IP pool, failed to add IP %s to data store for network card %d", ec2PrivateIpAddr.PrivateIpAddress, networkCard)
// continue to add next address
ipamdErrInc("addENIsecondaryIPsToDataStoreFailed")
}
}
- c.logPoolStats(c.dataStore.GetIPStats(ipV4AddrFamily))
+ c.logPoolStats(c.dataStoreAccess.GetDataStore(networkCard).GetIPStats(ipV4AddrFamily), networkCard)
}
-func (c *IPAMContext) addENIv4prefixesToDataStore(ec2PrefixAddrs []ec2types.Ipv4PrefixSpecification, eni string) {
+func (c *IPAMContext) addENIv4prefixesToDataStore(ec2PrefixAddrs []ec2types.Ipv4PrefixSpecification, eni string, networkCard int) {
// Walk thru all prefixes
for _, ec2PrefixAddr := range ec2PrefixAddrs {
strIpv4Prefix := aws.ToString(ec2PrefixAddr.Ipv4Prefix)
@@ -1143,17 +1226,17 @@ func (c *IPAMContext) addENIv4prefixesToDataStore(ec2PrefixAddrs []ec2types.Ipv4
continue
}
cidr := *ipnet
- err = c.dataStore.AddIPv4CidrToStore(eni, cidr, true)
+ err = c.dataStoreAccess.GetDataStore(networkCard).AddIPv4CidrToStore(eni, cidr, true)
if err != nil && err.Error() != datastore.IPAlreadyInStoreError {
- log.Warnf("Failed to increase Prefix pool, failed to add Prefix %s to data store", ec2PrefixAddr.Ipv4Prefix)
+ log.Warnf("Failed to increase Prefix pool, failed to add Prefix %s to data store for network card %d", ec2PrefixAddr.Ipv4Prefix, networkCard)
// continue to add next address
ipamdErrInc("addENIv4prefixesToDataStoreFailed")
}
}
- c.logPoolStats(c.dataStore.GetIPStats(ipV4AddrFamily))
+ c.logPoolStats(c.dataStoreAccess.GetDataStore(networkCard).GetIPStats(ipV4AddrFamily), networkCard)
}
-func (c *IPAMContext) addENIv6prefixesToDataStore(ec2PrefixAddrs []ec2types.Ipv6PrefixSpecification, eni string) {
+func (c *IPAMContext) addENIv6prefixesToDataStore(ec2PrefixAddrs []ec2types.Ipv6PrefixSpecification, eni string, networkCard int) {
log.Debugf("Updating datastore with IPv6Prefix(es) for ENI: %v, count: %v", eni, len(ec2PrefixAddrs))
// Walk through all prefixes
for _, ec2PrefixAddr := range ec2PrefixAddrs {
@@ -1165,14 +1248,15 @@ func (c *IPAMContext) addENIv6prefixesToDataStore(ec2PrefixAddrs []ec2types.Ipv6
continue
}
cidr := *ipnet
- err = c.dataStore.AddIPv6CidrToStore(eni, cidr, true)
+ err = c.dataStoreAccess.GetDataStore(networkCard).AddIPv6CidrToStore(eni, cidr, true)
if err != nil && err.Error() != datastore.IPAlreadyInStoreError {
- log.Warnf("Failed to increase Prefix pool, failed to add Prefix %s to data store", ec2PrefixAddr.Ipv6Prefix)
+ log.Warnf("Failed to increase Prefix pool, failed to add Prefix %s to data store for network card %d", ec2PrefixAddr.Ipv6Prefix, networkCard)
// continue to add next address
ipamdErrInc("addENIv6prefixesToDataStoreFailed")
}
+ c.logPoolStats(c.dataStoreAccess.GetDataStore(networkCard).GetIPStats(ipV6AddrFamily), networkCard)
}
- c.logPoolStats(c.dataStore.GetIPStats(ipV6AddrFamily))
+
}
// getMaxENI returns the maximum number of ENIs to attach to this instance. This is calculated as the lesser of
@@ -1231,17 +1315,18 @@ func getWarmPrefixTarget() int {
}
// logPoolStats logs usage information for allocated addresses/prefixes.
-func (c *IPAMContext) logPoolStats(dataStoreStats *datastore.DataStoreStats) {
- prefix := "IP pool stats"
+func (c *IPAMContext) logPoolStats(dataStoreStats *datastore.DataStoreStats, networkCard int) {
+ prefix := "IP pool stats for network card"
if c.enablePrefixDelegation {
- prefix = "Prefix pool stats"
+ prefix = "Prefix pool stats for network card"
}
- log.Debugf("%s: %s, c.maxIPsPerENI = %d", prefix, dataStoreStats, c.maxIPsPerENI)
+ log.Debugf("%s %d: %s, c.maxIPsPerENI = %d", prefix, networkCard, dataStoreStats, c.maxIPsPerENI)
}
func (c *IPAMContext) tryEnableSecurityGroupsForPods(ctx context.Context) {
- // For IPv4, check that there is room for a trunk ENI before patching CNINode CRD
- if c.enableIPv4 && (c.dataStore.GetENIs() >= (c.maxENI - c.unmanagedENI)) {
+
+ // For IPv4, check that there is room for a trunk ENI before patching CNINode CRD. We only check on the Default Network Card
+ if c.enableIPv4 && (c.dataStoreAccess.GetDataStore(DefaultNetworkCard).GetENIs() >= (c.maxENI - c.unmanagedENI[DefaultNetworkCard])) {
log.Error("No slot available for a trunk ENI to be attached.")
return
}
@@ -1260,13 +1345,12 @@ func (c *IPAMContext) tryEnableSecurityGroupsForPods(ctx context.Context) {
// PD enabled: If the WARM_PREFIX_TARGET is spread across ENIs and we have more than needed, this function will return true.
// If the number of prefixes are on just one ENI, and there are more than available, it returns true so getDeletableENI will
// recheck if we need the ENI for prefix target.
-func (c *IPAMContext) shouldRemoveExtraENIs() bool {
+func (c *IPAMContext) shouldRemoveExtraENIs(stats *datastore.DataStoreStats, networkCard int) bool {
// When WARM_IP_TARGET is set, return true as verification is always done in getDeletableENI()
if c.warmIPTargetsDefined() {
return true
}
- stats := c.dataStore.GetIPStats(ipV4AddrFamily)
available := stats.AvailableAddresses()
var shouldRemoveExtra bool
@@ -1279,27 +1363,27 @@ func (c *IPAMContext) shouldRemoveExtraENIs() bool {
shouldRemoveExtra = available >= (warmTarget)*c.maxIPsPerENI
if shouldRemoveExtra {
- c.logPoolStats(stats)
+ c.logPoolStats(stats, networkCard)
log.Debugf("It might be possible to remove extra ENIs because available (%d) >= (ENI/Prefix target + 1 (%d) + 1) * addrsPerENI (%d)", available, warmTarget, c.maxIPsPerENI)
} else if c.enablePrefixDelegation {
// When prefix target count is reduced, datastore would have deleted extra prefixes over the warm prefix target.
// Hence available will be less than (warmTarget)*c.maxIPsPerENI, but there can be some extra ENIs which are not used hence see if we can clean it up.
- shouldRemoveExtra = c.dataStore.CheckFreeableENIexists()
+ shouldRemoveExtra = c.dataStoreAccess.GetDataStore(networkCard).CheckFreeableENIexists()
}
return shouldRemoveExtra
}
-func (c *IPAMContext) computeExtraPrefixesOverWarmTarget() int {
+func (c *IPAMContext) computeExtraPrefixesOverWarmTarget(networkCard int) int {
if !c.warmPrefixTargetDefined() {
return 0
}
- freePrefixes := c.dataStore.GetFreePrefixes()
+ freePrefixes := c.dataStoreAccess.GetDataStore(networkCard).GetFreePrefixes()
over := max(freePrefixes-c.warmPrefixTarget, 0)
- stats := c.dataStore.GetIPStats(ipV4AddrFamily)
+ stats := c.dataStoreAccess.GetDataStore(networkCard).GetIPStats(ipV4AddrFamily)
log.Debugf("computeExtraPrefixesOverWarmTarget - available: %d, over: %d, warm_prefix_target: %d", stats.AvailableAddresses(), over, c.warmPrefixTarget)
- c.logPoolStats(stats)
+ c.logPoolStats(stats, networkCard)
return over
}
@@ -1334,12 +1418,20 @@ func (c *IPAMContext) checkForTrunkENI() bool {
return false
}
+func (c *IPAMContext) getENIsByNetworkCard(allENIs []awsutils.ENIMetadata) map[int][]awsutils.ENIMetadata {
+ var eniNetworkCardMap = make(map[int][]awsutils.ENIMetadata, c.numNetworkCards)
+ for _, eni := range allENIs {
+ eniNetworkCardMap[eni.NetworkCard] = append(eniNetworkCardMap[eni.NetworkCard], eni)
+ }
+ return eniNetworkCardMap
+}
+
// nodeIPPoolReconcile reconcile ENI and IP info from metadata service and IP addresses in datastore
func (c *IPAMContext) nodeIPPoolReconcile(ctx context.Context, interval time.Duration) {
// To reduce the number of EC2 API calls, skip reconciliation if IPs were recently added to the datastore.
timeSinceLast := time.Since(c.lastNodeIPPoolAction)
// Make an exception if node needs a trunk ENI and one is not currently attached.
- needsTrunkEni := c.enablePodENI && c.dataStore.GetTrunkENI() == ""
+ needsTrunkEni := c.enablePodENI && c.dataStoreAccess.GetDataStore(0).GetTrunkENI() == ""
if timeSinceLast <= interval && !needsTrunkEni {
return
}
@@ -1360,103 +1452,109 @@ func (c *IPAMContext) nodeIPPoolReconcile(ctx context.Context, interval time.Dur
ipamdErrInc("reconcileFailedGetENIs")
return
}
- attachedENIs := c.filterUnmanagedENIs(allENIs)
- currentENIs := c.dataStore.GetENIInfos().ENIs
- trunkENI := c.dataStore.GetTrunkENI()
- // Initialize the set with the known EFA interfaces
- efaENIs := c.dataStore.GetEFAENIs()
- // Check if a new ENI was added, if so we need to update the tags.
- needToUpdateTags := false
- for _, attachedENI := range attachedENIs {
- if _, ok := currentENIs[attachedENI.ENIID]; !ok {
- needToUpdateTags = true
- break
+ attachedENIs := c.filterUnmanagedENIs(allENIs)
+ attachedENIsByNetworkCard := c.getENIsByNetworkCard(allENIs)
+
+ for networkCard, ds := range c.dataStoreAccess.DataStores {
+ currentENIs := ds.GetENIInfos().ENIs
+ trunkENI := ds.GetTrunkENI()
+ // Initialize the set with the known EFA interfaces
+ efaENIs := ds.GetEFAENIs()
+
+ // Check if a new ENI was added, if so we need to update the tags.
+ needToUpdateTags := false
+ for _, attachedENI := range attachedENIsByNetworkCard[networkCard] {
+ if _, ok := currentENIs[attachedENI.ENIID]; !ok {
+ needToUpdateTags = true
+ break
+ }
}
- }
- var eniTagMap map[string]awsutils.TagMap
- if needToUpdateTags {
- log.Debugf("A new ENI added but not by ipamd, updating tags by calling EC2")
- metadataResult, err := c.awsClient.DescribeAllENIs()
- if err != nil {
- log.Warnf("Failed to call EC2 to describe ENIs, aborting reconcile: %v", err)
- return
- }
+ var eniTagMap map[string]awsutils.TagMap
+ if needToUpdateTags {
+ log.Debugf("A new ENI added but not by ipamd, updating tags by calling EC2")
+ metadataResult, err := c.awsClient.DescribeAllENIs()
+ if err != nil {
+ log.Warnf("Failed to call EC2 to describe ENIs, aborting reconcile: %v", err)
+ return
+ }
- if c.enablePodENI && metadataResult.TrunkENI != "" {
- log.Debugf("Trunk interface (%s) has been added to the node already.", metadataResult.TrunkENI)
- }
- // Update trunk ENI
- trunkENI = metadataResult.TrunkENI
- // Just copy values of the EFA set
- efaENIs = metadataResult.EFAENIs
- eniTagMap = metadataResult.TagMap
- c.setUnmanagedENIs(metadataResult.TagMap)
- c.awsClient.SetMultiCardENIs(metadataResult.MultiCardENIIDs)
- attachedENIs = c.filterUnmanagedENIs(metadataResult.ENIMetadata)
- }
+ if c.enablePodENI && metadataResult.TrunkENI != "" {
+ log.Debugf("Trunk interface (%s) has been added to the node already.", metadataResult.TrunkENI)
+ }
+ // Update trunk ENI
+ trunkENI = metadataResult.TrunkENI
+ // Just copy values of the EFA set
+ efaENIs = metadataResult.EFAENIs
+ eniTagMap = metadataResult.TagMap
+ c.setUnmanagedENIs(metadataResult.TagMap)
+ // c.awsClient.SetMultiCardENIs(metadataResult.MultiCardENIIDs)
+ attachedENIs = c.filterUnmanagedENIs(metadataResult.ENIMetadata)
+ }
+
+ // Mark phase
+ for _, attachedENI := range attachedENIs {
+ eniIPPool, eniPrefixPool, err := ds.GetENICIDRs(attachedENI.ENIID)
+ if err == nil {
+ // If the attached ENI is in the data store
+ log.Debugf("Reconcile existing ENI %s IP pool", attachedENI.ENIID)
+ // Reconcile IP pool
+ c.eniIPPoolReconcile(eniIPPool, attachedENI, attachedENI.ENIID, networkCard)
+ // If the attached ENI is in the data store
+ log.Debugf("Reconcile existing ENI %s IP prefixes", attachedENI.ENIID)
+ // Reconcile IP pool
+ c.eniPrefixPoolReconcile(eniPrefixPool, attachedENI, attachedENI.ENIID, networkCard)
+ // Mark action, remove this ENI from currentENIs map
+ delete(currentENIs, attachedENI.ENIID)
+ continue
+ }
- // Mark phase
- for _, attachedENI := range attachedENIs {
- eniIPPool, eniPrefixPool, err := c.dataStore.GetENICIDRs(attachedENI.ENIID)
- if err == nil {
- // If the attached ENI is in the data store
- log.Debugf("Reconcile existing ENI %s IP pool", attachedENI.ENIID)
- // Reconcile IP pool
- c.eniIPPoolReconcile(eniIPPool, attachedENI, attachedENI.ENIID)
- // If the attached ENI is in the data store
- log.Debugf("Reconcile existing ENI %s IP prefixes", attachedENI.ENIID)
- // Reconcile IP pool
- c.eniPrefixPoolReconcile(eniPrefixPool, attachedENI, attachedENI.ENIID)
- // Mark action, remove this ENI from currentENIs map
- delete(currentENIs, attachedENI.ENIID)
- continue
- }
+ isTrunkENI := attachedENI.ENIID == trunkENI
+ isEFAENI := efaENIs[attachedENI.ENIID]
+ if !isTrunkENI && !c.disableENIProvisioning {
+ if err := c.awsClient.TagENI(attachedENI.ENIID, eniTagMap[attachedENI.ENIID]); err != nil {
+ log.Errorf("IP pool reconcile: failed to tag managed ENI %v: %v", attachedENI.ENIID, err)
+ ipamdErrInc("eniReconcileAdd")
+ continue
+ }
+ }
- isTrunkENI := attachedENI.ENIID == trunkENI
- isEFAENI := efaENIs[attachedENI.ENIID]
- if !isTrunkENI && !c.disableENIProvisioning {
- if err := c.awsClient.TagENI(attachedENI.ENIID, eniTagMap[attachedENI.ENIID]); err != nil {
- log.Errorf("IP pool reconcile: failed to tag managed ENI %v: %v", attachedENI.ENIID, err)
+ // Add new ENI
+ log.Debugf("Reconcile and add a new ENI %s", attachedENI.ENIID)
+ err = c.setupENI(attachedENI.ENIID, attachedENI, isTrunkENI, isEFAENI)
+ if err != nil {
+ log.Errorf("IP pool reconcile: Failed to set up ENI %s network: %v", attachedENI.ENIID, err)
ipamdErrInc("eniReconcileAdd")
+ // Continue if having trouble with ONLY 1 ENI, instead of bailout here?
continue
}
+ prometheusmetrics.ReconcileCnt.With(prometheus.Labels{"fn": "eniReconcileAdd"}).Inc()
}
- // Add new ENI
- log.Debugf("Reconcile and add a new ENI %s", attachedENI)
- err = c.setupENI(attachedENI.ENIID, attachedENI, isTrunkENI, isEFAENI)
- if err != nil {
- log.Errorf("IP pool reconcile: Failed to set up ENI %s network: %v", attachedENI.ENIID, err)
- ipamdErrInc("eniReconcileAdd")
- // Continue if having trouble with ONLY 1 ENI, instead of bailout here?
- continue
+ // Sweep phase: since the marked ENI have been removed, the remaining ones needs to be sweeped
+ for eni := range currentENIs {
+ log.Infof("Reconcile and delete detached ENI %s", eni)
+ // Force the delete, since aws local metadata has told us that this ENI is no longer
+ // attached, so any IPs assigned from this ENI will no longer work.
+ err = c.dataStoreAccess.GetDataStore(networkCard).RemoveENIFromDataStore(eni, true /* force */)
+ if err != nil {
+ log.Errorf("IP pool reconcile: Failed to delete ENI during reconcile: %v", err)
+ ipamdErrInc("eniReconcileDel")
+ continue
+ }
+ delete(c.primaryIP, eni)
+ prometheusmetrics.ReconcileCnt.With(prometheus.Labels{"fn": "eniReconcileDel"}).Inc()
}
- prometheusmetrics.ReconcileCnt.With(prometheus.Labels{"fn": "eniReconcileAdd"}).Inc()
- }
- // Sweep phase: since the marked ENI have been removed, the remaining ones needs to be sweeped
- for eni := range currentENIs {
- log.Infof("Reconcile and delete detached ENI %s", eni)
- // Force the delete, since aws local metadata has told us that this ENI is no longer
- // attached, so any IPs assigned from this ENI will no longer work.
- err = c.dataStore.RemoveENIFromDataStore(eni, true /* force */)
- if err != nil {
- log.Errorf("IP pool reconcile: Failed to delete ENI during reconcile: %v", err)
- ipamdErrInc("eniReconcileDel")
- continue
- }
- delete(c.primaryIP, eni)
- prometheusmetrics.ReconcileCnt.With(prometheus.Labels{"fn": "eniReconcileDel"}).Inc()
+ log.Debug("Successfully Reconciled ENI/IP pool")
+ c.logPoolStats(c.dataStoreAccess.GetDataStore(networkCard).GetIPStats(ipV4AddrFamily), networkCard)
}
c.lastNodeIPPoolAction = time.Now()
- log.Debug("Successfully Reconciled ENI/IP pool")
- c.logPoolStats(c.dataStore.GetIPStats(ipV4AddrFamily))
}
-func (c *IPAMContext) eniIPPoolReconcile(ipPool []string, attachedENI awsutils.ENIMetadata, eni string) {
+func (c *IPAMContext) eniIPPoolReconcile(ipPool []string, attachedENI awsutils.ENIMetadata, eni string, networkCard int) {
attachedENIIPs := attachedENI.IPv4Addresses
needEC2Reconcile := true
// Here we can't trust attachedENI since the IMDS metadata can be stale. We need to check with EC2 API.
@@ -1475,7 +1573,7 @@ func (c *IPAMContext) eniIPPoolReconcile(ipPool []string, attachedENI awsutils.E
}
// Add all known attached IPs to the datastore
- seenIPs := c.verifyAndAddIPsToDatastore(eni, attachedENIIPs, needEC2Reconcile)
+ seenIPs := c.verifyAndAddIPsToDatastore(eni, attachedENIIPs, needEC2Reconcile, networkCard)
// Sweep phase, delete remaining IPs since they should not remain in the datastore
for _, existingIP := range ipPool {
@@ -1486,7 +1584,7 @@ func (c *IPAMContext) eniIPPoolReconcile(ipPool []string, attachedENI awsutils.E
log.Debugf("Reconcile and delete IP %s on ENI %s", existingIP, eni)
// Force the delete, since we have verified with EC2 that these secondary IPs are no longer assigned to this ENI
ipv4Addr := net.IPNet{IP: net.ParseIP(existingIP), Mask: net.IPv4Mask(255, 255, 255, 255)}
- err := c.dataStore.DelIPv4CidrFromStore(eni, ipv4Addr, true /* force */)
+ err := c.dataStoreAccess.GetDataStore(networkCard).DelIPv4CidrFromStore(eni, ipv4Addr, true /* force */)
if err != nil {
log.Errorf("Failed to reconcile and delete IP %s on ENI %s, %v", existingIP, eni, err)
ipamdErrInc("ipReconcileDel")
@@ -1497,7 +1595,7 @@ func (c *IPAMContext) eniIPPoolReconcile(ipPool []string, attachedENI awsutils.E
}
}
-func (c *IPAMContext) eniPrefixPoolReconcile(prefixPool []string, attachedENI awsutils.ENIMetadata, eni string) {
+func (c *IPAMContext) eniPrefixPoolReconcile(prefixPool []string, attachedENI awsutils.ENIMetadata, eni string, networkCard int) {
attachedENIIPs := attachedENI.IPv4Prefixes
needEC2Reconcile := true
// Here we can't trust attachedENI since the IMDS metadata can be stale. We need to check with EC2 API.
@@ -1517,7 +1615,7 @@ func (c *IPAMContext) eniPrefixPoolReconcile(prefixPool []string, attachedENI aw
}
// Add all known attached IPs to the datastore
- seenIPs := c.verifyAndAddPrefixesToDatastore(eni, attachedENIIPs, needEC2Reconcile)
+ seenIPs := c.verifyAndAddPrefixesToDatastore(eni, attachedENIIPs, needEC2Reconcile, networkCard)
// Sweep phase, delete remaining Prefixes since they should not remain in the datastore
for _, existingIP := range prefixPool {
@@ -1532,7 +1630,7 @@ func (c *IPAMContext) eniPrefixPoolReconcile(prefixPool []string, attachedENI aw
log.Debugf("Failed to parse so continuing with next prefix")
continue
}
- err = c.dataStore.DelIPv4CidrFromStore(eni, *ipv4Cidr, true /* force */)
+ err = c.dataStoreAccess.GetDataStore(networkCard).DelIPv4CidrFromStore(eni, *ipv4Cidr, true /* force */)
if err != nil {
log.Errorf("Failed to reconcile and delete IP %s on ENI %s, %v", existingIP, eni, err)
ipamdErrInc("ipReconcileDel")
@@ -1545,7 +1643,7 @@ func (c *IPAMContext) eniPrefixPoolReconcile(prefixPool []string, attachedENI aw
// verifyAndAddIPsToDatastore updates the datastore with the known secondary IPs. IPs who are out of cooldown gets added
// back to the datastore after being verified against EC2.
-func (c *IPAMContext) verifyAndAddIPsToDatastore(eni string, attachedENIIPs []ec2types.NetworkInterfacePrivateIpAddress, needEC2Reconcile bool) map[string]bool {
+func (c *IPAMContext) verifyAndAddIPsToDatastore(eni string, attachedENIIPs []ec2types.NetworkInterfacePrivateIpAddress, needEC2Reconcile bool, networkCard int) map[string]bool {
var ec2VerifiedAddresses []ec2types.NetworkInterfacePrivateIpAddress
seenIPs := make(map[string]bool)
for _, privateIPv4 := range attachedENIIPs {
@@ -1599,7 +1697,7 @@ func (c *IPAMContext) verifyAndAddIPsToDatastore(eni string, attachedENIIPs []ec
}
log.Infof("Trying to add %s", strPrivateIPv4)
// Try to add the IP
- err := c.dataStore.AddIPv4CidrToStore(eni, ipv4Addr, false)
+ err := c.dataStoreAccess.GetDataStore(networkCard).AddIPv4CidrToStore(eni, ipv4Addr, false)
if err != nil && err.Error() != datastore.IPAlreadyInStoreError {
log.Errorf("Failed to reconcile IP %s on ENI %s", strPrivateIPv4, eni)
ipamdErrInc("ipReconcileAdd")
@@ -1616,7 +1714,7 @@ func (c *IPAMContext) verifyAndAddIPsToDatastore(eni string, attachedENIIPs []ec
// verifyAndAddPrefixesToDatastore updates the datastore with the known Prefixes. Prefixes who are out of cooldown gets added
// back to the datastore after being verified against EC2.
-func (c *IPAMContext) verifyAndAddPrefixesToDatastore(eni string, attachedENIPrefixes []ec2types.Ipv4PrefixSpecification, needEC2Reconcile bool) map[string]bool {
+func (c *IPAMContext) verifyAndAddPrefixesToDatastore(eni string, attachedENIPrefixes []ec2types.Ipv4PrefixSpecification, needEC2Reconcile bool, networkCard int) map[string]bool {
var ec2VerifiedAddresses []ec2types.Ipv4PrefixSpecification
seenIPs := make(map[string]bool)
for _, privateIPv4Cidr := range attachedENIPrefixes {
@@ -1671,7 +1769,7 @@ func (c *IPAMContext) verifyAndAddPrefixesToDatastore(eni string, attachedENIPre
}
}
- err = c.dataStore.AddIPv4CidrToStore(eni, *ipv4CidrPtr, true)
+ err = c.dataStoreAccess.GetDataStore(networkCard).AddIPv4CidrToStore(eni, *ipv4CidrPtr, true)
if err != nil && err.Error() != datastore.IPAlreadyInStoreError {
log.Errorf("Failed to reconcile Prefix %s on ENI %s", strPrivateIPv4Cidr, eni)
ipamdErrInc("prefixReconcileAdd")
@@ -1801,36 +1899,39 @@ func enablePodIPAnnotation() bool {
}
// filterUnmanagedENIs filters out ENIs marked with the "node.k8s.amazonaws.com/no_manage" tag
+// TODO: We have to mark some ENIs as primary on multlicard instance for IPv6 and the numfiltered count
func (c *IPAMContext) filterUnmanagedENIs(enis []awsutils.ENIMetadata) []awsutils.ENIMetadata {
numFiltered := 0
ret := make([]awsutils.ENIMetadata, 0, len(enis))
+
for _, eni := range enis {
//Filter out any Unmanaged ENIs. VPC CNI will only work with Primary ENI in IPv6 Prefix Delegation mode until
//we open up IPv6 support in Secondary IP and Custom networking modes. Filtering out the ENIs here will
//help us avoid myriad of if/else loops elsewhere in the code.
- if c.enableIPv6 && !c.awsClient.IsPrimaryENI(eni.ENIID) {
- log.Debugf("Skipping ENI %s: IPv6 Mode is enabled and VPC CNI will only manage Primary ENI in v6 PD mode",
- eni.ENIID)
- numFiltered++
- continue
+ // We shouldn't need the IsPrimaryENI check as ENIs not created by vpc-cni will be marked unmanaged (including trunk ENI)
+ if c.enableIPv6 {
+ if !c.awsClient.IsPrimaryENI(eni.ENIID) && c.awsClient.IsUnmanagedENI(eni.ENIID) {
+ log.Debugf("Skipping ENI %s: IPv6 Mode is enabled and VPC CNI will only ENIs created by it in v6 PD mode",
+ eni.ENIID)
+ numFiltered++
+ c.unmanagedENI[eni.NetworkCard] += 1
+ continue
+ }
} else if c.awsClient.IsUnmanagedENI(eni.ENIID) {
log.Debugf("Skipping ENI %s: since it is unmanaged", eni.ENIID)
numFiltered++
- continue
- } else if c.awsClient.IsMultiCardENI(eni.ENIID) {
- log.Debugf("Skipping ENI %s: since on non-zero network card", eni.ENIID)
+ c.unmanagedENI[eni.NetworkCard] += 1
continue
}
ret = append(ret, eni)
}
- c.unmanagedENI = numFiltered
c.updateIPStats(numFiltered)
return ret
}
// datastoreTargetState determines the number of IPs `short` or `over` our WARM_IP_TARGET, accounting for the MINIMUM_IP_TARGET.
// With prefix delegation, this function determines the number of Prefixes `short` or `over`
-func (c *IPAMContext) datastoreTargetState(stats *datastore.DataStoreStats) (short int, over int, enabled bool) {
+func (c *IPAMContext) datastoreTargetState(stats *datastore.DataStoreStats, networkCard int) (short int, over int, enabled bool) {
if !c.warmIPTargetsDefined() {
// there is no WARM_IP_TARGET defined and no MINIMUM_IP_TARGET, fallback to use all IP addresses on ENI
return 0, 0, false
@@ -1838,7 +1939,7 @@ func (c *IPAMContext) datastoreTargetState(stats *datastore.DataStoreStats) (sho
// Calculating DataStore stats can be expensive, so allow the caller to optionally pass stats it already calculated
if stats == nil {
- stats = c.dataStore.GetIPStats(ipV4AddrFamily)
+ stats = c.dataStoreAccess.GetDataStore(networkCard).GetIPStats(ipV4AddrFamily)
}
available := stats.AvailableAddresses()
@@ -1870,7 +1971,7 @@ func (c *IPAMContext) datastoreTargetState(stats *datastore.DataStoreStats) (sho
// over will be number of prefixes over than needed but could be spread across used prefixes,
// say, after couple of pod churns, 3 prefixes are allocated with 1 IP each assigned and warm ip target is 15
// (J : is this needed? since we have to walk thru the loop of prefixes)
- freePrefixes := c.dataStore.GetFreePrefixes()
+ freePrefixes := c.dataStoreAccess.GetDataStore(networkCard).GetFreePrefixes()
overPrefix := max(min(freePrefixes, stats.TotalPrefixes-prefixNeededForWarmIP), 0)
overPrefix = max(min(overPrefix, stats.TotalPrefixes-prefixNeededForMinIP), 0)
return shortPrefix, overPrefix, true
@@ -1879,12 +1980,12 @@ func (c *IPAMContext) datastoreTargetState(stats *datastore.DataStoreStats) (sho
}
// datastorePrefixTargetState determines the number of prefixes short to reach WARM_PREFIX_TARGET
-func (c *IPAMContext) datastorePrefixTargetState() (short int, enabled bool) {
+func (c *IPAMContext) datastorePrefixTargetState(networkCard int) (short int, enabled bool) {
if !c.warmPrefixTargetDefined() {
return 0, false
}
// /28 will consume 16 IPs so let's not allocate if not needed.
- freePrefixesInStore := c.dataStore.GetFreePrefixes()
+ freePrefixesInStore := c.dataStoreAccess.GetDataStore(networkCard).GetFreePrefixes()
toAllocate := max(c.warmPrefixTarget-freePrefixesInStore, 0)
log.Debugf("Prefix target is %d, short of %d prefixes, free %d prefixes", c.warmPrefixTarget, toAllocate, freePrefixesInStore)
@@ -1954,7 +2055,7 @@ func min(x, y int) int {
}
func (c *IPAMContext) getTrunkLinkIndex() (int, error) {
- trunkENI := c.dataStore.GetTrunkENI()
+ trunkENI := c.dataStoreAccess.GetDataStore(0).GetTrunkENI()
attachedENIs, err := c.awsClient.GetAttachedENIs()
if err != nil {
return -1, err
@@ -2048,14 +2149,20 @@ func (c *IPAMContext) AnnotatePod(podName string, podNamespace string, key strin
func (c *IPAMContext) tryUnassignIPsFromENIs() {
log.Debugf("tryUnassignIPsFromENIs")
- eniInfos := c.dataStore.GetENIInfos()
- for eniID := range eniInfos.ENIs {
- c.tryUnassignIPFromENI(eniID)
+ // From all datastores, get ENIInfos and unassign IPs
+ for networkCard, ds := range c.dataStoreAccess.DataStores {
+ eniInfos := ds.GetENIInfos()
+ for eniID := range eniInfos.ENIs {
+ c.tryUnassignIPFromENI(eniID, networkCard)
+ }
}
+
}
-func (c *IPAMContext) tryUnassignIPFromENI(eniID string) {
- freeableIPs := c.dataStore.FreeableIPs(eniID)
+func (c *IPAMContext) tryUnassignIPFromENI(eniID string, networkCard int) {
+
+ ds := c.dataStoreAccess.GetDataStore(networkCard)
+ freeableIPs := ds.FreeableIPs(eniID)
if len(freeableIPs) == 0 {
log.Debugf("No freeable IPs")
return
@@ -2066,7 +2173,7 @@ func (c *IPAMContext) tryUnassignIPFromENI(eniID string) {
for _, toDelete := range freeableIPs {
// Don't force the delete, since a freeable IP might have been assigned to a pod
// before we get around to deleting it.
- err := c.dataStore.DelIPv4CidrFromStore(eniID, toDelete, false /* force */)
+ err := ds.DelIPv4CidrFromStore(eniID, toDelete, false /* force */)
if err != nil {
log.Warnf("Failed to delete IP %s on ENI %s from datastore: %s", toDelete, eniID, err)
ipamdErrInc("decreaseIPPool")
@@ -2086,14 +2193,19 @@ func (c *IPAMContext) tryUnassignIPFromENI(eniID string) {
func (c *IPAMContext) tryUnassignPrefixesFromENIs() {
log.Debugf("tryUnassignPrefixesFromENIs")
- eniInfos := c.dataStore.GetENIInfos()
- for eniID := range eniInfos.ENIs {
- c.tryUnassignPrefixFromENI(eniID)
+
+ // From all datastores get ENIs and remove prefixes
+ for networkCard, ds := range c.dataStoreAccess.DataStores {
+ eniInfos := ds.GetENIInfos()
+ for eniID := range eniInfos.ENIs {
+ c.tryUnassignPrefixFromENI(eniID, networkCard)
+ }
}
}
-func (c *IPAMContext) tryUnassignPrefixFromENI(eniID string) {
- freeablePrefixes := c.dataStore.FreeablePrefixes(eniID)
+func (c *IPAMContext) tryUnassignPrefixFromENI(eniID string, networkCard int) {
+ ds := c.dataStoreAccess.GetDataStore(networkCard)
+ freeablePrefixes := ds.FreeablePrefixes(eniID)
if len(freeablePrefixes) == 0 {
return
}
@@ -2102,7 +2214,7 @@ func (c *IPAMContext) tryUnassignPrefixFromENI(eniID string) {
for _, toDelete := range freeablePrefixes {
// Don't force the delete, since a freeable Prefix might have been assigned to a pod
// before we get around to deleting it.
- err := c.dataStore.DelIPv4CidrFromStore(eniID, toDelete, false /* force */)
+ err := ds.DelIPv4CidrFromStore(eniID, toDelete, false /* force */)
if err != nil {
log.Warnf("Failed to delete Prefix %s on ENI %s from datastore: %s", toDelete, eniID, err)
ipamdErrInc("decreaseIPPool")
@@ -2120,13 +2232,18 @@ func (c *IPAMContext) tryUnassignPrefixFromENI(eniID string) {
}
}
-func (c *IPAMContext) GetENIResourcesToAllocate() int {
+func (c *IPAMContext) GetENIResourcesToAllocate(networkCard int) int {
var resourcesToAllocate int
if c.enablePrefixDelegation {
- resourcesToAllocate = min(c.getPrefixesNeeded(), c.maxPrefixesPerENI)
+ if c.enableIPv6 {
+ // IPv6 always uses PD, so it should always be 1
+ resourcesToAllocate = 1
+ } else {
+ resourcesToAllocate = min(c.getPrefixesNeeded(networkCard), c.maxPrefixesPerENI)
+ }
} else {
resourcesToAllocate = c.maxIPsPerENI
- short, _, warmTargetDefined := c.datastoreTargetState(nil)
+ short, _, warmTargetDefined := c.datastoreTargetState(nil, networkCard)
if warmTargetDefined {
resourcesToAllocate = min(short, c.maxIPsPerENI)
}
@@ -2150,32 +2267,23 @@ func (c *IPAMContext) GetIPv4Limit() (int, int, error) {
return maxIPsPerENI, maxPrefixesPerENI, nil
}
-func (c *IPAMContext) isDatastorePoolEmpty() bool {
- stats := c.dataStore.GetIPStats(ipV4AddrFamily)
+func (c *IPAMContext) isDatastorePoolEmpty(networkCard int) bool {
+ stats := c.dataStoreAccess.GetDataStore(networkCard).GetIPStats(ipV4AddrFamily)
return stats.TotalIPs == 0
}
// Return whether the maximum number of ENIs that can be attached to the node has already been reached
-func (c *IPAMContext) hasRoomForEni() bool {
+func (c *IPAMContext) hasRoomForEni(networkCard int) bool {
trunkEni := 0
- if c.enablePodENI && c.dataStore.GetTrunkENI() == "" {
+ if c.enablePodENI && networkCard == DefaultNetworkCard && c.dataStoreAccess.GetDataStore(DefaultNetworkCard).GetTrunkENI() == "" {
trunkEni = 1
}
- return c.dataStore.GetENIs() < (c.maxENI - c.unmanagedENI - trunkEni)
+ return c.dataStoreAccess.GetDataStore(networkCard).GetENIs() < (c.maxENI - c.unmanagedENI[networkCard] - trunkEni)
}
-func (c *IPAMContext) isDatastorePoolTooLow() (bool, *datastore.DataStoreStats) {
- stats := c.dataStore.GetIPStats(ipV4AddrFamily)
- // If max pods has been reached, pool is not too low
- if stats.TotalIPs >= c.maxPods {
- return false, stats
- }
-
- short, _, warmTargetDefined := c.datastoreTargetState(stats)
- if warmTargetDefined {
- return short > 0, stats
- }
+func (c *IPAMContext) isDatastorePoolTooLow() []Decisions {
+ decisions := make([]Decisions, 0, len(c.dataStoreAccess.DataStores))
warmTarget := c.warmENITarget
totalIPs := c.maxIPsPerENI
if c.enablePrefixDelegation {
@@ -2184,28 +2292,45 @@ func (c *IPAMContext) isDatastorePoolTooLow() (bool, *datastore.DataStoreStats)
totalIPs = maxIpsPerPrefix
}
- available := stats.AvailableAddresses()
- poolTooLow := available < totalIPs*warmTarget || (warmTarget == 0 && available == 0)
- if poolTooLow {
- log.Debugf("IP pool is too low: available (%d) < ENI target (%d) * addrsPerENI (%d)", available, warmTarget, totalIPs)
- c.logPoolStats(stats)
+ for networkCard, ds := range c.dataStoreAccess.DataStores {
+
+ stats := ds.GetIPStats(ipV4AddrFamily)
+ // If max pods has been reached, pool is not too low
+ if stats.TotalIPs >= c.maxPods {
+ decisions = append(decisions, Decisions{Stats: stats, IsLow: false})
+ continue
+ }
+
+ short, _, warmTargetDefined := c.datastoreTargetState(stats, networkCard)
+ if warmTargetDefined {
+ decisions = append(decisions, Decisions{Stats: stats, IsLow: short > 0})
+ continue
+ }
+
+ available := stats.AvailableAddresses()
+ poolTooLow := available < totalIPs*warmTarget || (warmTarget == 0 && available == 0)
+ if poolTooLow {
+ log.Debugf("IP pool is too low for Network Card %d: available (%d) < ENI target (%d) * addrsPerENI (%d)", networkCard, available, warmTarget, totalIPs)
+ c.logPoolStats(stats, networkCard)
+ }
+ decisions = append(decisions, Decisions{Stats: stats, IsLow: poolTooLow})
}
- return poolTooLow, stats
+ return decisions
}
-func (c *IPAMContext) isDatastorePoolTooHigh(stats *datastore.DataStoreStats) bool {
+func (c *IPAMContext) isDatastorePoolTooHigh(stats *datastore.DataStoreStats, networkCard int) bool {
// NOTE: IPs may be allocated in chunks (full ENIs of prefixes), so the "too-high" condition does not check max pods. The limit is enforced on the allocation side.
- _, over, warmTargetDefined := c.datastoreTargetState(stats)
+ _, over, warmTargetDefined := c.datastoreTargetState(stats, networkCard)
if warmTargetDefined {
return over > 0
}
// For the existing ENIs check if we can cleanup prefixes
if c.warmPrefixTargetDefined() {
- freePrefixes := c.dataStore.GetFreePrefixes()
+ freePrefixes := c.dataStoreAccess.GetDataStore(networkCard).GetFreePrefixes()
poolTooHigh := freePrefixes > c.warmPrefixTarget
if poolTooHigh {
- log.Debugf("Prefix pool is high so might be able to deallocate - free prefixes: %d, warm prefix target: %d", freePrefixes, c.warmPrefixTarget)
+ log.Debugf("Prefix pool is high so might be able to deallocate - free prefixes: %d, warm prefix target: %d from NetworkCard %d", freePrefixes, c.warmPrefixTarget, networkCard)
}
return poolTooHigh
}
@@ -2248,14 +2373,14 @@ func (c *IPAMContext) DeallocCidrs(eniID string, deletableCidrs []datastore.Cidr
}
// getPrefixesNeeded returns the number of prefixes need to be allocated to the ENI
-func (c *IPAMContext) getPrefixesNeeded() int {
+func (c *IPAMContext) getPrefixesNeeded(networkCard int) int {
// By default allocate 1 prefix at a time
toAllocate := 1
// TODO - post GA we can evaluate to see if these two calls can be merged.
// datastoreTargetState already has complex math so adding Prefix target will make it even more complex.
- short, _, warmIPTargetsDefined := c.datastoreTargetState(nil)
- shortPrefixes, warmPrefixTargetDefined := c.datastorePrefixTargetState()
+ short, _, warmIPTargetsDefined := c.datastoreTargetState(nil, networkCard)
+ shortPrefixes, warmPrefixTargetDefined := c.datastorePrefixTargetState(networkCard)
// WARM_IP_TARGET takes precendence over WARM_PREFIX_TARGET
if warmIPTargetsDefined {
@@ -2263,19 +2388,20 @@ func (c *IPAMContext) getPrefixesNeeded() int {
} else if warmPrefixTargetDefined {
toAllocate = max(toAllocate, shortPrefixes)
}
- log.Debugf("ToAllocate: %d", toAllocate)
+ log.Debugf("ToAllocate: %d, Network Card %d", toAllocate, networkCard)
return toAllocate
}
func (c *IPAMContext) initENIAndIPLimits() (err error) {
- if c.enableIPv4 {
- nodeMaxENI, err := c.getMaxENI()
- if err != nil {
- log.Error("Failed to get ENI limit")
- return err
- }
- c.maxENI = nodeMaxENI
+ nodeMaxENI, err := c.getMaxENI()
+ if err != nil {
+ log.Error("Failed to get ENI limit")
+ return err
+ }
+ c.maxENI = nodeMaxENI
+
+ if c.enableIPv4 {
c.maxIPsPerENI, c.maxPrefixesPerENI, err = c.GetIPv4Limit()
if err != nil {
return err
@@ -2355,3 +2481,8 @@ func (c *IPAMContext) AddFeatureToCNINode(ctx context.Context, featureName rcv1a
newCNINode.Spec.Features = append(newCNINode.Spec.Features, newFeature)
return c.k8sClient.Patch(ctx, newCNINode, client.MergeFromWithOptions(cniNode, client.MergeFromWithOptimisticLock{}))
}
+
+type Decisions struct {
+ Stats *datastore.DataStoreStats
+ IsLow bool
+}
diff --git a/pkg/ipamd/ipamd_test.go b/pkg/ipamd/ipamd_test.go
index deaa083f47..8b621eb9c6 100644
--- a/pkg/ipamd/ipamd_test.go
+++ b/pkg/ipamd/ipamd_test.go
@@ -85,6 +85,8 @@ const (
v6prefix01 = "2001:db8::/64"
instanceID = "i-0e1f3b9eb950e4980"
externalEniConfigLabel = "vpc.amazonaws.com/externalEniConfig"
+ defaultNetworkCard = 0
+ maxENIPerNIC = 4
)
type testMocks struct {
@@ -124,19 +126,22 @@ func TestNodeInit(t *testing.T) {
}
mockContext := &IPAMContext{
- awsClient: m.awsutils,
- k8sClient: m.k8sClient,
- maxIPsPerENI: 14,
- maxENI: 4,
- warmENITarget: 1,
- warmIPTarget: 3,
- primaryIP: make(map[string]string),
- terminating: int32(0),
- networkClient: m.network,
- dataStore: datastore.NewDataStore(log, datastore.NewTestCheckpoint(fakeCheckpoint), false),
- myNodeName: myNodeName,
- enableIPv4: true,
- enableIPv6: false,
+ awsClient: m.awsutils,
+ k8sClient: m.k8sClient,
+ maxIPsPerENI: 14,
+ maxENI: 4,
+ warmENITarget: 1,
+ warmIPTarget: 3,
+ numNetworkCards: 1,
+ primaryIP: make(map[string]string),
+ terminating: int32(0),
+ networkClient: m.network,
+ dataStoreAccess: &datastore.DataStoreAccess{
+ DataStores: []*datastore.DataStore{datastore.NewDataStore(log, datastore.NewTestCheckpoint(fakeCheckpoint), false)},
+ },
+ myNodeName: myNodeName,
+ enableIPv4: true,
+ enableIPv6: false,
}
eni1, eni2, _ := getDummyENIMetadata()
@@ -149,8 +154,6 @@ func TestNodeInit(t *testing.T) {
m.awsutils.EXPECT().IsUnmanagedENI(eni1.ENIID).Return(false).AnyTimes()
m.awsutils.EXPECT().IsUnmanagedENI(eni2.ENIID).Return(false).AnyTimes()
m.awsutils.EXPECT().TagENI(gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
- m.awsutils.EXPECT().IsMultiCardENI(eni1.ENIID).Return(false).AnyTimes()
- m.awsutils.EXPECT().IsMultiCardENI(eni2.ENIID).Return(false).AnyTimes()
primaryIP := net.ParseIP(ipaddr01)
m.awsutils.EXPECT().GetVPCIPv4CIDRs().AnyTimes().Return(cidrs, nil)
@@ -169,7 +172,7 @@ func TestNodeInit(t *testing.T) {
MultiCardENIIDs: nil,
}
m.awsutils.EXPECT().DescribeAllENIs().Return(resp, nil)
- m.network.EXPECT().SetupENINetwork(gomock.Any(), secMAC, secDevice, secSubnet)
+ m.network.EXPECT().SetupENINetwork(gomock.Any(), secMAC, secDevice, defaultNetworkCard, secSubnet, maxENIPerNIC)
m.awsutils.EXPECT().SetMultiCardENIs(resp.MultiCardENIIDs).AnyTimes()
m.awsutils.EXPECT().GetLocalIPv4().Return(primaryIP)
@@ -213,17 +216,20 @@ func TestNodeInitwithPDenabledIPv4Mode(t *testing.T) {
}
mockContext := &IPAMContext{
- awsClient: m.awsutils,
- k8sClient: m.k8sClient,
- maxIPsPerENI: 224,
- maxPrefixesPerENI: 14,
- maxENI: 4,
- warmENITarget: 1,
- warmIPTarget: 3,
- primaryIP: make(map[string]string),
- terminating: int32(0),
- networkClient: m.network,
- dataStore: datastore.NewDataStore(log, datastore.NewTestCheckpoint(fakeCheckpoint), true),
+ awsClient: m.awsutils,
+ k8sClient: m.k8sClient,
+ maxIPsPerENI: 224,
+ maxPrefixesPerENI: 14,
+ maxENI: 4,
+ warmENITarget: 1,
+ numNetworkCards: 1,
+ warmIPTarget: 3,
+ primaryIP: make(map[string]string),
+ terminating: int32(0),
+ networkClient: m.network,
+ dataStoreAccess: &datastore.DataStoreAccess{
+ DataStores: []*datastore.DataStore{datastore.NewDataStore(log, datastore.NewTestCheckpoint(fakeCheckpoint), true)},
+ },
myNodeName: myNodeName,
enablePrefixDelegation: true,
enableIPv4: true,
@@ -239,8 +245,6 @@ func TestNodeInitwithPDenabledIPv4Mode(t *testing.T) {
m.awsutils.EXPECT().IsUnmanagedENI(eni1.ENIID).Return(false).AnyTimes()
m.awsutils.EXPECT().IsUnmanagedENI(eni2.ENIID).Return(false).AnyTimes()
m.awsutils.EXPECT().TagENI(gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
- m.awsutils.EXPECT().IsMultiCardENI(eni1.ENIID).Return(false).AnyTimes()
- m.awsutils.EXPECT().IsMultiCardENI(eni2.ENIID).Return(false).AnyTimes()
primaryIP := net.ParseIP(ipaddr01)
m.awsutils.EXPECT().GetVPCIPv4CIDRs().AnyTimes().Return(cidrs, nil)
@@ -258,10 +262,9 @@ func TestNodeInitwithPDenabledIPv4Mode(t *testing.T) {
EFAENIs: make(map[string]bool),
}
m.awsutils.EXPECT().DescribeAllENIs().Return(resp, nil)
- m.network.EXPECT().SetupENINetwork(gomock.Any(), secMAC, secDevice, secSubnet)
+ m.network.EXPECT().SetupENINetwork(gomock.Any(), secMAC, secDevice, defaultNetworkCard, secSubnet, maxENIPerNIC)
m.awsutils.EXPECT().GetLocalIPv4().Return(primaryIP)
- m.awsutils.EXPECT().SetMultiCardENIs(resp.MultiCardENIIDs).AnyTimes()
var rules []netlink.Rule
m.network.EXPECT().GetRuleList().Return(rules, nil)
@@ -300,17 +303,19 @@ func TestNodeInitwithPDenabledIPv6Mode(t *testing.T) {
}
mockContext := &IPAMContext{
- awsClient: m.awsutils,
- k8sClient: m.k8sClient,
- maxIPsPerENI: 224,
- maxPrefixesPerENI: 1,
- maxENI: 1,
- warmENITarget: 1,
- warmIPTarget: 1,
- primaryIP: make(map[string]string),
- terminating: int32(0),
- networkClient: m.network,
- dataStore: datastore.NewDataStore(log, datastore.NewTestCheckpoint(fakeCheckpoint), true),
+ awsClient: m.awsutils,
+ k8sClient: m.k8sClient,
+ maxIPsPerENI: 224,
+ maxPrefixesPerENI: 1,
+ maxENI: 1,
+ warmENITarget: 1,
+ warmIPTarget: 1,
+ primaryIP: make(map[string]string),
+ terminating: int32(0),
+ networkClient: m.network,
+ dataStoreAccess: &datastore.DataStoreAccess{
+ DataStores: []*datastore.DataStore{datastore.NewDataStore(log, datastore.NewTestCheckpoint(fakeCheckpoint), true)},
+ },
myNodeName: myNodeName,
enablePrefixDelegation: true,
enableIPv4: false,
@@ -322,9 +327,9 @@ func TestNodeInitwithPDenabledIPv6Mode(t *testing.T) {
var cidrs []string
m.awsutils.EXPECT().IsUnmanagedENI(eni1.ENIID).Return(false).AnyTimes()
m.awsutils.EXPECT().TagENI(gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
- m.awsutils.EXPECT().IsMultiCardENI(eni1.ENIID).Return(false).AnyTimes()
primaryIP := net.ParseIP(ipaddr01)
+ primaryIPv6 := net.ParseIP(v6ipaddr01)
m.network.EXPECT().SetupHostNetwork(cidrs, eni1.MAC, &primaryIP, false, false, true).Return(nil)
m.network.EXPECT().CleanUpStaleAWSChains(false, true).Return(nil)
m.awsutils.EXPECT().GetIPv6PrefixesFromEC2(eni1.ENIID).AnyTimes().Return(eni1.IPv6Prefixes, nil)
@@ -339,9 +344,10 @@ func TestNodeInitwithPDenabledIPv6Mode(t *testing.T) {
TrunkENI: "",
EFAENIs: make(map[string]bool),
}
+ m.awsutils.EXPECT().GetENILimit().Return(1)
m.awsutils.EXPECT().DescribeAllENIs().Return(resp, nil)
m.awsutils.EXPECT().GetLocalIPv4().Return(primaryIP)
- m.awsutils.EXPECT().SetMultiCardENIs(resp.MultiCardENIIDs).AnyTimes()
+ m.awsutils.EXPECT().GetLocalIPv6().Return(primaryIPv6)
fakeNode := v1.Node{
TypeMeta: metav1.TypeMeta{Kind: "Node"},
@@ -521,9 +527,9 @@ func testIncreaseIPPool(t *testing.T, useENIConfig bool, unschedulabeNode bool,
primaryIP: make(map[string]string),
terminating: int32(0),
}
- mockContext.dataStore = testDatastore()
+ mockContext.dataStoreAccess = testDatastore()
if subnetDiscovery {
- mockContext.dataStore.AddENI(primaryENIid, primaryDevice, true, false, false)
+ mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).AddENI(primaryENIid, primaryDevice, true, false, false)
}
primary := true
@@ -621,7 +627,7 @@ func testIncreaseIPPool(t *testing.T, useENIConfig bool, unschedulabeNode bool,
}
m.k8sClient.Create(ctx, &fakeENIConfig)
}
- mockContext.increaseDatastorePool(ctx)
+ mockContext.increaseDatastorePool(ctx, defaultNetworkCard)
}
func assertAllocationExternalCalls(shouldCall bool, useENIConfig bool, m *testMocks, sg []*string, podENIConfig *eniconfigscheme.ENIConfigSpec, eni2 string, eniMetadata []awsutils.ENIMetadata, subnetDiscovery bool) {
@@ -651,7 +657,7 @@ func assertAllocationExternalCalls(shouldCall bool, useENIConfig bool, m *testMo
}
m.awsutils.EXPECT().GetPrimaryENI().Times(callCount).Return(primaryENIid)
m.awsutils.EXPECT().WaitForENIAndIPsAttached(secENIid, 14).Times(callCount).Return(eniMetadata[1], nil)
- m.network.EXPECT().SetupENINetwork(gomock.Any(), secMAC, secDevice, secSubnet).Times(callCount)
+ m.network.EXPECT().SetupENINetwork(gomock.Any(), secMAC, secDevice, defaultNetworkCard, secSubnet, maxENIPerNIC).Times(callCount)
}
func TestIncreasePrefixPoolDefault(t *testing.T) {
@@ -691,9 +697,9 @@ func testIncreasePrefixPool(t *testing.T, useENIConfig, subnetDiscovery bool) {
enablePrefixDelegation: true,
}
- mockContext.dataStore = testDatastorewithPrefix()
+ mockContext.dataStoreAccess = testDatastorewithPrefix()
if subnetDiscovery {
- mockContext.dataStore.AddENI(primaryENIid, primaryDevice, true, false, false)
+ mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).AddENI(primaryENIid, primaryDevice, true, false, false)
}
primary := true
@@ -770,7 +776,7 @@ func testIncreasePrefixPool(t *testing.T, useENIConfig, subnetDiscovery bool) {
m.awsutils.EXPECT().GetPrimaryENI().Return(primaryENIid)
m.awsutils.EXPECT().WaitForENIAndIPsAttached(secENIid, 1).Return(eniMetadata[1], nil)
- m.network.EXPECT().SetupENINetwork(gomock.Any(), secMAC, secDevice, secSubnet)
+ m.network.EXPECT().SetupENINetwork(gomock.Any(), secMAC, secDevice, defaultNetworkCard, secSubnet, maxENIPerNIC)
if mockContext.useCustomNetworking {
mockContext.myNodeName = myNodeName
@@ -800,7 +806,7 @@ func testIncreasePrefixPool(t *testing.T, useENIConfig, subnetDiscovery bool) {
m.k8sClient.Create(ctx, &fakeENIConfig)
}
- mockContext.increaseDatastorePool(ctx)
+ mockContext.increaseDatastorePool(ctx, defaultNetworkCard)
}
// TestDecreaseIPPool checks that the deallocation honors the warm IP targets when deallocations happens across multiple enis
@@ -821,38 +827,38 @@ func TestDecreaseIPPool(t *testing.T) {
testAddr2 := net.IPNet{IP: net.ParseIP(ipaddr02), Mask: net.IPv4Mask(255, 255, 255, 255)}
testAddr11 := net.IPNet{IP: net.ParseIP(ipaddr11), Mask: net.IPv4Mask(255, 255, 255, 255)}
testAddr12 := net.IPNet{IP: net.ParseIP(ipaddr12), Mask: net.IPv4Mask(255, 255, 255, 255)}
+ // TODO Fix getting datastore
+ mockContext.dataStoreAccess = testDatastore()
- mockContext.dataStore = testDatastore()
-
- mockContext.dataStore.AddENI(primaryENIid, primaryDevice, true, false, false)
- mockContext.dataStore.AddIPv4CidrToStore(primaryENIid, testAddr1, false)
- mockContext.dataStore.AddIPv4CidrToStore(primaryENIid, testAddr2, false)
- mockContext.dataStore.AssignPodIPv4Address(datastore.IPAMKey{ContainerID: "container1"}, datastore.IPAMMetadata{K8SPodName: "pod1"})
+ mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).AddENI(primaryENIid, primaryDevice, true, false, false)
+ mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).AddIPv4CidrToStore(primaryENIid, testAddr1, false)
+ mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).AddIPv4CidrToStore(primaryENIid, testAddr2, false)
+ mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).AssignPodIPv4Address(datastore.IPAMKey{ContainerID: "container1"}, datastore.IPAMMetadata{K8SPodName: "pod1"})
- mockContext.dataStore.AddENI(secENIid, secDevice, true, false, false)
- mockContext.dataStore.AddIPv4CidrToStore(secENIid, testAddr11, false)
- mockContext.dataStore.AddIPv4CidrToStore(secENIid, testAddr12, false)
- mockContext.dataStore.AssignPodIPv4Address(datastore.IPAMKey{ContainerID: "container2"}, datastore.IPAMMetadata{K8SPodName: "pod2"})
+ mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).AddENI(secENIid, secDevice, true, false, false)
+ mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).AddIPv4CidrToStore(secENIid, testAddr11, false)
+ mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).AddIPv4CidrToStore(secENIid, testAddr12, false)
+ mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).AssignPodIPv4Address(datastore.IPAMKey{ContainerID: "container2"}, datastore.IPAMMetadata{K8SPodName: "pod2"})
m.awsutils.EXPECT().DeallocPrefixAddresses(gomock.Any(), gomock.Any()).Times(1)
m.awsutils.EXPECT().DeallocIPAddresses(gomock.Any(), gomock.Any()).Times(1)
- short, over, enabled := mockContext.datastoreTargetState(nil)
+ short, over, enabled := mockContext.datastoreTargetState(nil, defaultNetworkCard)
assert.Equal(t, 0, short) // there would not be any shortage
assert.Equal(t, 1, over) // out of 4 IPs we have 2 IPs assigned, warm IP target is 1, so over is 1
assert.Equal(t, true, enabled) // there is warm ip target enabled with the value of 1
- mockContext.decreaseDatastorePool(10 * time.Second)
+ mockContext.decreaseDatastorePool(defaultNetworkCard)
- short, over, enabled = mockContext.datastoreTargetState(nil)
+ short, over, enabled = mockContext.datastoreTargetState(nil, defaultNetworkCard)
assert.Equal(t, 0, short) // there would not be any shortage
assert.Equal(t, 0, over) // after the above deallocation this should be zero
assert.Equal(t, true, enabled) // there is warm ip target enabled with the value of 1
// make another call just to ensure that more deallocations do not happen
- mockContext.decreaseDatastorePool(10 * time.Second)
+ mockContext.decreaseDatastorePool(defaultNetworkCard)
- short, over, enabled = mockContext.datastoreTargetState(nil)
+ short, over, enabled = mockContext.datastoreTargetState(nil, defaultNetworkCard)
assert.Equal(t, 0, short) // there would not be any shortage
assert.Equal(t, 0, over) // after the above deallocation this should be zero
assert.Equal(t, true, enabled) // there is warm ip target enabled with the value of 1
@@ -884,7 +890,7 @@ func TestTryAddIPToENI(t *testing.T) {
terminating: int32(0),
}
- mockContext.dataStore = testDatastore()
+ mockContext.dataStoreAccess = testDatastore()
m.awsutils.EXPECT().AllocENI(false, nil, "", warmIPTarget).Return(secENIid, nil)
eniMetadata := []awsutils.ENIMetadata{
@@ -919,7 +925,7 @@ func TestTryAddIPToENI(t *testing.T) {
}
m.awsutils.EXPECT().WaitForENIAndIPsAttached(secENIid, 3).Return(eniMetadata[1], nil)
m.awsutils.EXPECT().GetPrimaryENI().Return(primaryENIid)
- m.network.EXPECT().SetupENINetwork(gomock.Any(), secMAC, secDevice, secSubnet)
+ m.network.EXPECT().SetupENINetwork(gomock.Any(), secMAC, secDevice, defaultNetworkCard, secSubnet, maxENIPerNIC)
mockContext.myNodeName = myNodeName
@@ -931,7 +937,7 @@ func TestTryAddIPToENI(t *testing.T) {
Status: v1.NodeStatus{},
}
m.k8sClient.Create(ctx, &fakeNode)
- mockContext.increaseDatastorePool(ctx)
+ mockContext.increaseDatastorePool(ctx, defaultNetworkCard)
}
func TestNodeIPPoolReconcile(t *testing.T) {
@@ -946,7 +952,7 @@ func TestNodeIPPoolReconcile(t *testing.T) {
terminating: int32(0),
}
- mockContext.dataStore = testDatastore()
+ mockContext.dataStoreAccess = testDatastore()
primary := true
primaryENIMetadata := getPrimaryENIMetadata()
@@ -970,7 +976,7 @@ func TestNodeIPPoolReconcile(t *testing.T) {
m.awsutils.EXPECT().SetMultiCardENIs(resp.MultiCardENIIDs).AnyTimes()
mockContext.nodeIPPoolReconcile(ctx, 0)
- curENIs := mockContext.dataStore.GetENIInfos()
+ curENIs := mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).GetENIInfos()
assert.Equal(t, 1, len(curENIs.ENIs))
assert.Equal(t, 2, curENIs.TotalIPs)
@@ -992,7 +998,7 @@ func TestNodeIPPoolReconcile(t *testing.T) {
m.awsutils.EXPECT().GetIPv4sFromEC2(primaryENIid).Return(oneIPUnassigned[0].IPv4Addresses, nil)
mockContext.nodeIPPoolReconcile(ctx, 0)
- curENIs = mockContext.dataStore.GetENIInfos()
+ curENIs = mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).GetENIInfos()
assert.Equal(t, 1, len(curENIs.ENIs))
assert.Equal(t, 0, curENIs.TotalIPs)
@@ -1013,13 +1019,13 @@ func TestNodeIPPoolReconcile(t *testing.T) {
MultiCardENIIDs: nil,
}
m.awsutils.EXPECT().DescribeAllENIs().Return(resp2, nil)
- m.network.EXPECT().SetupENINetwork(gomock.Any(), secMAC, secDevice, primarySubnet)
+ m.network.EXPECT().SetupENINetwork(gomock.Any(), secMAC, secDevice, defaultNetworkCard, primarySubnet, maxENIPerNIC)
m.awsutils.EXPECT().SetMultiCardENIs(resp2.MultiCardENIIDs).AnyTimes()
mockContext.nodeIPPoolReconcile(ctx, 0)
// Verify that we now have 2 ENIs, primary ENI with 0 secondary IPs, and secondary ENI with 1 secondary IP
- curENIs = mockContext.dataStore.GetENIInfos()
+ curENIs = mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).GetENIInfos()
assert.Equal(t, 2, len(curENIs.ENIs))
assert.Equal(t, 1, curENIs.TotalIPs)
@@ -1027,7 +1033,7 @@ func TestNodeIPPoolReconcile(t *testing.T) {
m.awsutils.EXPECT().GetAttachedENIs().Return(oneIPUnassigned, nil)
mockContext.nodeIPPoolReconcile(ctx, 0)
- curENIs = mockContext.dataStore.GetENIInfos()
+ curENIs = mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).GetENIInfos()
assert.Equal(t, 1, len(curENIs.ENIs))
assert.Equal(t, 0, curENIs.TotalIPs)
}
@@ -1045,7 +1051,7 @@ func TestNodePrefixPoolReconcile(t *testing.T) {
enablePrefixDelegation: true,
}
- mockContext.dataStore = testDatastorewithPrefix()
+ mockContext.dataStoreAccess = testDatastorewithPrefix()
primary := true
primaryENIMetadata := getPrimaryENIMetadataPDenabled()
@@ -1069,7 +1075,7 @@ func TestNodePrefixPoolReconcile(t *testing.T) {
m.awsutils.EXPECT().SetMultiCardENIs(resp.MultiCardENIIDs).AnyTimes()
mockContext.nodeIPPoolReconcile(ctx, 0)
- curENIs := mockContext.dataStore.GetENIInfos()
+ curENIs := mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).GetENIInfos()
assert.Equal(t, 1, len(curENIs.ENIs))
assert.Equal(t, 16, curENIs.TotalIPs)
@@ -1094,7 +1100,7 @@ func TestNodePrefixPoolReconcile(t *testing.T) {
// m.awsutils.EXPECT().GetIPv4sFromEC2(primaryENIid).Return(oneIPUnassigned[0].IPv4Addresses, nil)
mockContext.nodeIPPoolReconcile(ctx, 0)
- curENIs = mockContext.dataStore.GetENIInfos()
+ curENIs = mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).GetENIInfos()
assert.Equal(t, 1, len(curENIs.ENIs))
assert.Equal(t, 0, curENIs.TotalIPs)
@@ -1114,13 +1120,13 @@ func TestNodePrefixPoolReconcile(t *testing.T) {
EFAENIs: make(map[string]bool),
}
m.awsutils.EXPECT().DescribeAllENIs().Return(resp2, nil)
- m.network.EXPECT().SetupENINetwork(gomock.Any(), secMAC, secDevice, primarySubnet)
+ m.network.EXPECT().SetupENINetwork(gomock.Any(), secMAC, secDevice, defaultNetworkCard, primarySubnet, maxENIPerNIC)
m.awsutils.EXPECT().SetMultiCardENIs(resp2.MultiCardENIIDs).AnyTimes()
mockContext.nodeIPPoolReconcile(ctx, 0)
// Verify that we now have 2 ENIs, primary ENI with 0 prefixes, and secondary ENI with 1 prefix
- curENIs = mockContext.dataStore.GetENIInfos()
+ curENIs = mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).GetENIInfos()
assert.Equal(t, 2, len(curENIs.ENIs))
assert.Equal(t, 16, curENIs.TotalIPs)
@@ -1128,7 +1134,7 @@ func TestNodePrefixPoolReconcile(t *testing.T) {
m.awsutils.EXPECT().GetAttachedENIs().Return(oneIPUnassigned, nil)
mockContext.nodeIPPoolReconcile(ctx, 0)
- curENIs = mockContext.dataStore.GetENIInfos()
+ curENIs = mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).GetENIInfos()
assert.Equal(t, 1, len(curENIs.ENIs))
assert.Equal(t, 0, curENIs.TotalIPs)
}
@@ -1177,38 +1183,38 @@ func TestGetWarmIPTargetState(t *testing.T) {
primaryIP: make(map[string]string),
terminating: int32(0),
}
- mockContext.dataStore = testDatastore()
+ mockContext.dataStoreAccess = testDatastore()
- _, _, warmIPTargetDefined := mockContext.datastoreTargetState(nil)
+ _, _, warmIPTargetDefined := mockContext.datastoreTargetState(nil, defaultNetworkCard)
assert.False(t, warmIPTargetDefined)
mockContext.warmIPTarget = 5
- short, over, warmIPTargetDefined := mockContext.datastoreTargetState(nil)
+ short, over, warmIPTargetDefined := mockContext.datastoreTargetState(nil, defaultNetworkCard)
assert.True(t, warmIPTargetDefined)
assert.Equal(t, 5, short)
assert.Equal(t, 0, over)
// add 2 addresses to datastore
- _ = mockContext.dataStore.AddENI("eni-1", 1, true, false, false)
+ _ = mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).AddENI("eni-1", 1, true, false, false)
ipv4Addr := net.IPNet{IP: net.ParseIP("1.1.1.1"), Mask: net.IPv4Mask(255, 255, 255, 255)}
- _ = mockContext.dataStore.AddIPv4CidrToStore("eni-1", ipv4Addr, false)
+ _ = mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).AddIPv4CidrToStore("eni-1", ipv4Addr, false)
ipv4Addr = net.IPNet{IP: net.ParseIP("1.1.1.2"), Mask: net.IPv4Mask(255, 255, 255, 255)}
- _ = mockContext.dataStore.AddIPv4CidrToStore("eni-1", ipv4Addr, false)
+ _ = mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).AddIPv4CidrToStore("eni-1", ipv4Addr, false)
- short, over, warmIPTargetDefined = mockContext.datastoreTargetState(nil)
+ short, over, warmIPTargetDefined = mockContext.datastoreTargetState(nil, defaultNetworkCard)
assert.True(t, warmIPTargetDefined)
assert.Equal(t, 3, short)
assert.Equal(t, 0, over)
// add 3 more addresses to datastore
ipv4Addr = net.IPNet{IP: net.ParseIP("1.1.1.3"), Mask: net.IPv4Mask(255, 255, 255, 255)}
- _ = mockContext.dataStore.AddIPv4CidrToStore("eni-1", ipv4Addr, false)
+ _ = mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).AddIPv4CidrToStore("eni-1", ipv4Addr, false)
ipv4Addr = net.IPNet{IP: net.ParseIP("1.1.1.4"), Mask: net.IPv4Mask(255, 255, 255, 255)}
- _ = mockContext.dataStore.AddIPv4CidrToStore("eni-1", ipv4Addr, false)
+ _ = mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).AddIPv4CidrToStore("eni-1", ipv4Addr, false)
ipv4Addr = net.IPNet{IP: net.ParseIP("1.1.1.5"), Mask: net.IPv4Mask(255, 255, 255, 255)}
- _ = mockContext.dataStore.AddIPv4CidrToStore("eni-1", ipv4Addr, false)
+ _ = mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).AddIPv4CidrToStore("eni-1", ipv4Addr, false)
- short, over, warmIPTargetDefined = mockContext.datastoreTargetState(nil)
+ short, over, warmIPTargetDefined = mockContext.datastoreTargetState(nil, defaultNetworkCard)
assert.True(t, warmIPTargetDefined)
assert.Equal(t, 0, short)
assert.Equal(t, 0, over)
@@ -1226,35 +1232,35 @@ func TestGetWarmIPTargetStateWithPDenabled(t *testing.T) {
enablePrefixDelegation: true,
}
- mockContext.dataStore = testDatastorewithPrefix()
+ mockContext.dataStoreAccess = testDatastorewithPrefix()
- _, _, warmIPTargetDefined := mockContext.datastoreTargetState(nil)
+ _, _, warmIPTargetDefined := mockContext.datastoreTargetState(nil, defaultNetworkCard)
assert.False(t, warmIPTargetDefined)
mockContext.warmIPTarget = 5
- short, over, warmIPTargetDefined := mockContext.datastoreTargetState(nil)
+ short, over, warmIPTargetDefined := mockContext.datastoreTargetState(nil, defaultNetworkCard)
assert.True(t, warmIPTargetDefined)
assert.Equal(t, 1, short)
assert.Equal(t, 0, over)
// add 2 addresses to datastore
- _ = mockContext.dataStore.AddENI("eni-1", 1, true, false, false)
+ _ = mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).AddENI("eni-1", 1, true, false, false)
_, ipnet, _ := net.ParseCIDR("10.1.1.0/28")
- _ = mockContext.dataStore.AddIPv4CidrToStore("eni-1", *ipnet, true)
- _ = mockContext.dataStore.AddENI("eni-2", 2, true, false, false)
+ _ = mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).AddIPv4CidrToStore("eni-1", *ipnet, true)
+ _ = mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).AddENI("eni-2", 2, true, false, false)
_, ipnet, _ = net.ParseCIDR("20.1.1.0/28")
- _ = mockContext.dataStore.AddIPv4CidrToStore("eni-1", *ipnet, true)
+ _ = mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).AddIPv4CidrToStore("eni-1", *ipnet, true)
- short, over, warmIPTargetDefined = mockContext.datastoreTargetState(nil)
+ short, over, warmIPTargetDefined = mockContext.datastoreTargetState(nil, defaultNetworkCard)
assert.True(t, warmIPTargetDefined)
assert.Equal(t, 0, short)
assert.Equal(t, 1, over)
// Del 1 address
_, ipnet, _ = net.ParseCIDR("20.1.1.0/28")
- _ = mockContext.dataStore.DelIPv4CidrFromStore("eni-1", *ipnet, true)
+ _ = mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).DelIPv4CidrFromStore("eni-1", *ipnet, true)
- short, over, warmIPTargetDefined = mockContext.datastoreTargetState(nil)
+ short, over, warmIPTargetDefined = mockContext.datastoreTargetState(nil, defaultNetworkCard)
assert.True(t, warmIPTargetDefined)
assert.Equal(t, 0, short)
assert.Equal(t, 0, over)
@@ -1265,12 +1271,12 @@ func TestIPAMContext_nodeIPPoolTooLow(t *testing.T) {
defer m.ctrl.Finish()
type fields struct {
- maxIPsPerENI int
- maxEni int
- warmENITarget int
- warmIPTarget int
- datastore *datastore.DataStore
- maxPods int
+ maxIPsPerENI int
+ maxEni int
+ warmENITarget int
+ warmIPTarget int
+ datastoreAccess *datastore.DataStoreAccess
+ maxPods int
}
tests := []struct {
@@ -1292,7 +1298,7 @@ func TestIPAMContext_nodeIPPoolTooLow(t *testing.T) {
t.Run(tt.name, func(t *testing.T) {
c := &IPAMContext{
awsClient: m.awsutils,
- dataStore: tt.fields.datastore,
+ dataStoreAccess: tt.fields.datastoreAccess,
useCustomNetworking: false,
networkClient: m.network,
maxIPsPerENI: tt.fields.maxIPsPerENI,
@@ -1302,8 +1308,10 @@ func TestIPAMContext_nodeIPPoolTooLow(t *testing.T) {
enablePrefixDelegation: false,
maxPods: tt.fields.maxPods,
}
- if got, _ := c.isDatastorePoolTooLow(); got != tt.want {
- t.Errorf("nodeIPPoolTooLow() = %v, want %v", got, tt.want)
+ if decisions := c.isDatastorePoolTooLow(); decisions != nil {
+ if decisions[defaultNetworkCard].IsLow != tt.want {
+ t.Errorf("nodeIPPoolTooLow() = %v, want %v", decisions[defaultNetworkCard].IsLow, tt.want)
+ }
}
})
}
@@ -1318,7 +1326,7 @@ func TestIPAMContext_nodePrefixPoolTooLow(t *testing.T) {
maxEni int
maxPrefixesPerENI int
warmPrefixTarget int
- datastore *datastore.DataStore
+ datastoreAccess *datastore.DataStoreAccess
maxPods int
}
@@ -1340,7 +1348,7 @@ func TestIPAMContext_nodePrefixPoolTooLow(t *testing.T) {
t.Run(tt.name, func(t *testing.T) {
c := &IPAMContext{
awsClient: m.awsutils,
- dataStore: tt.fields.datastore,
+ dataStoreAccess: tt.fields.datastoreAccess,
useCustomNetworking: false,
networkClient: m.network,
maxPrefixesPerENI: tt.fields.maxPrefixesPerENI,
@@ -1350,37 +1358,44 @@ func TestIPAMContext_nodePrefixPoolTooLow(t *testing.T) {
enablePrefixDelegation: true,
maxPods: tt.fields.maxPods,
}
- if got, _ := c.isDatastorePoolTooLow(); got != tt.want {
- t.Errorf("nodeIPPoolTooLow() = %v, want %v", got, tt.want)
+ if decisions := c.isDatastorePoolTooLow(); decisions != nil {
+ if decisions[defaultNetworkCard].IsLow != tt.want {
+ t.Errorf("nodeIPPoolTooLow() = %v, want %v", decisions[defaultNetworkCard].IsLow, tt.want)
+ }
}
})
}
}
-func testDatastore() *datastore.DataStore {
- return datastore.NewDataStore(log, datastore.NewTestCheckpoint(datastore.CheckpointData{Version: datastore.CheckpointFormatVersion}), false)
+func testDatastore() *datastore.DataStoreAccess {
+ return &datastore.DataStoreAccess{
+ DataStores: []*datastore.DataStore{datastore.NewDataStore(log, datastore.NewTestCheckpoint(datastore.CheckpointData{Version: datastore.CheckpointFormatVersion}), false)},
+ }
}
-func testDatastorewithPrefix() *datastore.DataStore {
- return datastore.NewDataStore(log, datastore.NewTestCheckpoint(datastore.CheckpointData{Version: datastore.CheckpointFormatVersion}), true)
+func testDatastorewithPrefix() *datastore.DataStoreAccess {
+
+ return &datastore.DataStoreAccess{
+ DataStores: []*datastore.DataStore{datastore.NewDataStore(log, datastore.NewTestCheckpoint(datastore.CheckpointData{Version: datastore.CheckpointFormatVersion}), true)},
+ }
}
-func datastoreWith3FreeIPs() *datastore.DataStore {
+func datastoreWith3FreeIPs() *datastore.DataStoreAccess {
datastoreWith3FreeIPs := testDatastore()
- _ = datastoreWith3FreeIPs.AddENI(primaryENIid, 1, true, false, false)
+ _ = datastoreWith3FreeIPs.GetDataStore(defaultNetworkCard).AddENI(primaryENIid, 1, true, false, false)
ipv4Addr := net.IPNet{IP: net.ParseIP(ipaddr01), Mask: net.IPv4Mask(255, 255, 255, 255)}
- _ = datastoreWith3FreeIPs.AddIPv4CidrToStore(primaryENIid, ipv4Addr, false)
+ _ = datastoreWith3FreeIPs.GetDataStore(defaultNetworkCard).AddIPv4CidrToStore(primaryENIid, ipv4Addr, false)
ipv4Addr = net.IPNet{IP: net.ParseIP(ipaddr02), Mask: net.IPv4Mask(255, 255, 255, 255)}
- _ = datastoreWith3FreeIPs.AddIPv4CidrToStore(primaryENIid, ipv4Addr, false)
+ _ = datastoreWith3FreeIPs.GetDataStore(defaultNetworkCard).AddIPv4CidrToStore(primaryENIid, ipv4Addr, false)
ipv4Addr = net.IPNet{IP: net.ParseIP(ipaddr03), Mask: net.IPv4Mask(255, 255, 255, 255)}
- _ = datastoreWith3FreeIPs.AddIPv4CidrToStore(primaryENIid, ipv4Addr, false)
+ _ = datastoreWith3FreeIPs.GetDataStore(defaultNetworkCard).AddIPv4CidrToStore(primaryENIid, ipv4Addr, false)
return datastoreWith3FreeIPs
}
-func datastoreWith1Pod1() *datastore.DataStore {
+func datastoreWith1Pod1() *datastore.DataStoreAccess {
datastoreWith1Pod1 := datastoreWith3FreeIPs()
- _, _, _ = datastoreWith1Pod1.AssignPodIPv4Address(datastore.IPAMKey{
+ _, _, _ = datastoreWith1Pod1.GetDataStore(defaultNetworkCard).AssignPodIPv4Address(datastore.IPAMKey{
NetworkName: "net0",
ContainerID: "sandbox-1",
IfName: "eth0",
@@ -1391,7 +1406,7 @@ func datastoreWith1Pod1() *datastore.DataStore {
return datastoreWith1Pod1
}
-func datastoreWith3Pods() *datastore.DataStore {
+func datastoreWith3Pods() *datastore.DataStoreAccess {
datastoreWith3Pods := datastoreWith3FreeIPs()
for i := 0; i < 3; i++ {
@@ -1400,7 +1415,7 @@ func datastoreWith3Pods() *datastore.DataStore {
ContainerID: fmt.Sprintf("sandbox-%d", i),
IfName: "eth0",
}
- _, _, _ = datastoreWith3Pods.AssignPodIPv4Address(key, datastore.IPAMMetadata{
+ _, _, _ = datastoreWith3Pods.GetDataStore(defaultNetworkCard).AssignPodIPv4Address(key, datastore.IPAMMetadata{
K8SPodNamespace: "default",
K8SPodName: fmt.Sprintf("sample-pod-%d", i),
})
@@ -1408,18 +1423,18 @@ func datastoreWith3Pods() *datastore.DataStore {
return datastoreWith3Pods
}
-func datastoreWithFreeIPsFromPrefix() *datastore.DataStore {
+func datastoreWithFreeIPsFromPrefix() *datastore.DataStoreAccess {
datastoreWithFreeIPs := testDatastorewithPrefix()
- _ = datastoreWithFreeIPs.AddENI(primaryENIid, 1, true, false, false)
+ _ = datastoreWithFreeIPs.GetDataStore(defaultNetworkCard).AddENI(primaryENIid, 1, true, false, false)
_, ipnet, _ := net.ParseCIDR(prefix01)
- _ = datastoreWithFreeIPs.AddIPv4CidrToStore(primaryENIid, *ipnet, true)
+ _ = datastoreWithFreeIPs.GetDataStore(defaultNetworkCard).AddIPv4CidrToStore(primaryENIid, *ipnet, true)
return datastoreWithFreeIPs
}
-func datastoreWith1Pod1FromPrefix() *datastore.DataStore {
+func datastoreWith1Pod1FromPrefix() *datastore.DataStoreAccess {
datastoreWith1Pod1 := datastoreWithFreeIPsFromPrefix()
- _, _, _ = datastoreWith1Pod1.AssignPodIPv4Address(datastore.IPAMKey{
+ _, _, _ = datastoreWith1Pod1.GetDataStore(defaultNetworkCard).AssignPodIPv4Address(datastore.IPAMKey{
NetworkName: "net0",
ContainerID: "sandbox-1",
IfName: "eth0",
@@ -1430,7 +1445,7 @@ func datastoreWith1Pod1FromPrefix() *datastore.DataStore {
return datastoreWith1Pod1
}
-func datastoreWith3PodsFromPrefix() *datastore.DataStore {
+func datastoreWith3PodsFromPrefix() *datastore.DataStoreAccess {
datastoreWith3Pods := datastoreWithFreeIPsFromPrefix()
for i := 0; i < 3; i++ {
@@ -1439,7 +1454,7 @@ func datastoreWith3PodsFromPrefix() *datastore.DataStore {
ContainerID: fmt.Sprintf("sandbox-%d", i),
IfName: "eth0",
}
- _, _, _ = datastoreWith3Pods.AssignPodIPv4Address(key,
+ _, _, _ = datastoreWith3Pods.GetDataStore(defaultNetworkCard).AssignPodIPv4Address(key,
datastore.IPAMMetadata{
K8SPodNamespace: "default",
K8SPodName: fmt.Sprintf("sample-pod-%d", i),
@@ -1669,16 +1684,16 @@ func TestNodeIPPoolReconcileBadIMDSData(t *testing.T) {
terminating: int32(0),
}
- mockContext.dataStore = testDatastore()
+ mockContext.dataStoreAccess = testDatastore()
primaryENIMetadata := getPrimaryENIMetadata()
testAddr1 := *primaryENIMetadata.IPv4Addresses[0].PrivateIpAddress
// Add ENI and IPs to datastore
eniID := primaryENIMetadata.ENIID
- _ = mockContext.dataStore.AddENI(eniID, primaryENIMetadata.DeviceNumber, true, false, false)
+ _ = mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).AddENI(eniID, primaryENIMetadata.DeviceNumber, true, false, false)
mockContext.primaryIP[eniID] = testAddr1
- mockContext.addENIsecondaryIPsToDataStore(primaryENIMetadata.IPv4Addresses, eniID)
- curENIs := mockContext.dataStore.GetENIInfos()
+ mockContext.addENIsecondaryIPsToDataStore(primaryENIMetadata.IPv4Addresses, eniID, defaultNetworkCard)
+ curENIs := mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).GetENIInfos()
assert.Equal(t, 1, len(curENIs.ENIs))
assert.Equal(t, 2, curENIs.TotalIPs)
eniMetadataList := []awsutils.ENIMetadata{primaryENIMetadata}
@@ -1708,7 +1723,7 @@ func TestNodeIPPoolReconcileBadIMDSData(t *testing.T) {
// eniIPPoolReconcile() calls EC2 to get the actual count, but that call fails
m.awsutils.EXPECT().GetIPv4sFromEC2(primaryENIid).Return(nil, errors.New("ec2 API call failed"))
mockContext.nodeIPPoolReconcile(ctx, 0)
- curENIs = mockContext.dataStore.GetENIInfos()
+ curENIs = mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).GetENIInfos()
assert.Equal(t, 1, len(curENIs.ENIs))
assert.Equal(t, 2, curENIs.TotalIPs)
@@ -1730,14 +1745,14 @@ func TestNodeIPPoolReconcileBadIMDSData(t *testing.T) {
// eniIPPoolReconcile() calls EC2 to get the actual count that should still be 2
m.awsutils.EXPECT().GetIPv4sFromEC2(primaryENIid).Return(primaryENIMetadata.IPv4Addresses, nil)
mockContext.nodeIPPoolReconcile(ctx, 0)
- curENIs = mockContext.dataStore.GetENIInfos()
+ curENIs = mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).GetENIInfos()
assert.Equal(t, 1, len(curENIs.ENIs))
assert.Equal(t, 2, curENIs.TotalIPs)
// If no ENI is found, we abort the reconcile
m.awsutils.EXPECT().GetAttachedENIs().Return(nil, nil)
mockContext.nodeIPPoolReconcile(ctx, 0)
- curENIs = mockContext.dataStore.GetENIInfos()
+ curENIs = mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).GetENIInfos()
assert.Equal(t, 1, len(curENIs.ENIs))
assert.Equal(t, 2, curENIs.TotalIPs)
}
@@ -1755,16 +1770,16 @@ func TestNodePrefixPoolReconcileBadIMDSData(t *testing.T) {
enablePrefixDelegation: true,
}
- mockContext.dataStore = testDatastorewithPrefix()
+ mockContext.dataStoreAccess = testDatastorewithPrefix()
primaryENIMetadata := getPrimaryENIMetadataPDenabled()
testAddr1 := *primaryENIMetadata.IPv4Addresses[0].PrivateIpAddress
// Add ENI and IPs to datastore
eniID := primaryENIMetadata.ENIID
- _ = mockContext.dataStore.AddENI(eniID, primaryENIMetadata.DeviceNumber, true, false, false)
+ _ = mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).AddENI(eniID, primaryENIMetadata.DeviceNumber, true, false, false)
mockContext.primaryIP[eniID] = testAddr1
- mockContext.addENIv4prefixesToDataStore(primaryENIMetadata.IPv4Prefixes, eniID)
- curENIs := mockContext.dataStore.GetENIInfos()
+ mockContext.addENIv4prefixesToDataStore(primaryENIMetadata.IPv4Prefixes, eniID, defaultNetworkCard)
+ curENIs := mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).GetENIInfos()
assert.Equal(t, 1, len(curENIs.ENIs))
assert.Equal(t, 16, curENIs.TotalIPs)
eniMetadataList := []awsutils.ENIMetadata{primaryENIMetadata}
@@ -1794,7 +1809,7 @@ func TestNodePrefixPoolReconcileBadIMDSData(t *testing.T) {
// eniIPPoolReconcile() calls EC2 to get the actual count, but that call fails
m.awsutils.EXPECT().GetIPv4PrefixesFromEC2(primaryENIid).Return(nil, errors.New("ec2 API call failed"))
mockContext.nodeIPPoolReconcile(ctx, 0)
- curENIs = mockContext.dataStore.GetENIInfos()
+ curENIs = mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).GetENIInfos()
assert.Equal(t, 1, len(curENIs.ENIs))
assert.Equal(t, 16, curENIs.TotalIPs)
@@ -1816,14 +1831,14 @@ func TestNodePrefixPoolReconcileBadIMDSData(t *testing.T) {
// eniIPPoolReconcile() calls EC2 to get the actual count that should still be 16
m.awsutils.EXPECT().GetIPv4PrefixesFromEC2(primaryENIid).Return(primaryENIMetadata.IPv4Prefixes, nil)
mockContext.nodeIPPoolReconcile(ctx, 0)
- curENIs = mockContext.dataStore.GetENIInfos()
+ curENIs = mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).GetENIInfos()
assert.Equal(t, 1, len(curENIs.ENIs))
assert.Equal(t, 16, curENIs.TotalIPs)
// If no ENI is found, we abort the reconcile
m.awsutils.EXPECT().GetAttachedENIs().Return(nil, nil)
mockContext.nodeIPPoolReconcile(ctx, 0)
- curENIs = mockContext.dataStore.GetENIInfos()
+ curENIs = mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).GetENIInfos()
assert.Equal(t, 1, len(curENIs.ENIs))
assert.Equal(t, 16, curENIs.TotalIPs)
}
@@ -1937,7 +1952,7 @@ func TestIPAMContext_setupENI(t *testing.T) {
}
// mockContext.primaryIP[]
- mockContext.dataStore = testDatastore()
+ mockContext.dataStoreAccess = testDatastore()
primary := true
notPrimary := false
testAddr1 := ipaddr01
@@ -1964,7 +1979,7 @@ func TestIPAMContext_setupENI(t *testing.T) {
newENIMetadata := getSecondaryENIMetadata()
m.awsutils.EXPECT().GetPrimaryENI().Return(primaryENIid)
- m.network.EXPECT().SetupENINetwork(gomock.Any(), secMAC, secDevice, primarySubnet).Return(errors.New("not able to set route 0.0.0.0/0 via 10.10.10.1 table 2"))
+ m.network.EXPECT().SetupENINetwork(gomock.Any(), secMAC, secDevice, defaultNetworkCard, primarySubnet, maxENIPerNIC).Return(errors.New("not able to set route 0.0.0.0/0 via 10.10.10.1 table 2"))
err = mockContext.setupENI(newENIMetadata.ENIID, newENIMetadata, false, false)
assert.Error(t, err)
@@ -1983,7 +1998,7 @@ func TestIPAMContext_setupENIwithPDenabled(t *testing.T) {
}
// mockContext.primaryIP[]
- mockContext.dataStore = testDatastorewithPrefix()
+ mockContext.dataStoreAccess = testDatastorewithPrefix()
primary := true
notPrimary := false
testAddr1 := ipaddr01
@@ -2010,7 +2025,7 @@ func TestIPAMContext_setupENIwithPDenabled(t *testing.T) {
newENIMetadata := getSecondaryENIMetadata()
m.awsutils.EXPECT().GetPrimaryENI().Return(primaryENIid)
- m.network.EXPECT().SetupENINetwork(gomock.Any(), secMAC, secDevice, primarySubnet).Return(errors.New("not able to set route 0.0.0.0/0 via 10.10.10.1 table 2"))
+ m.network.EXPECT().SetupENINetwork(gomock.Any(), secMAC, secDevice, defaultNetworkCard, primarySubnet, maxENIPerNIC).Return(errors.New("not able to set route 0.0.0.0/0 via 10.10.10.1 table 2"))
err = mockContext.setupENI(newENIMetadata.ENIID, newENIMetadata, false, false)
assert.Error(t, err)
@@ -2023,10 +2038,12 @@ func TestIPAMContext_enableSecurityGroupsForPods(t *testing.T) {
ctx := context.Background()
mockContext := &IPAMContext{
- k8sClient: m.k8sClient,
- enableIPv4: true,
- enableIPv6: false,
- dataStore: datastore.NewDataStore(log, datastore.NewTestCheckpoint(datastore.CheckpointData{Version: datastore.CheckpointFormatVersion}), false),
+ k8sClient: m.k8sClient,
+ enableIPv4: true,
+ enableIPv6: false,
+ dataStoreAccess: &datastore.DataStoreAccess{
+ DataStores: []*datastore.DataStore{datastore.NewDataStore(log, datastore.NewTestCheckpoint(datastore.CheckpointData{Version: datastore.CheckpointFormatVersion}), false)},
+ },
awsClient: m.awsutils,
networkClient: m.network,
primaryIP: make(map[string]string),
@@ -2050,7 +2067,7 @@ func TestIPAMContext_enableSecurityGroupsForPods(t *testing.T) {
err := m.k8sClient.Create(ctx, &fakeCNINode)
assert.NoError(t, err)
- _ = mockContext.dataStore.AddENI("eni-1", 1, true, false, false)
+ _ = mockContext.dataStoreAccess.GetDataStore(defaultNetworkCard).AddENI("eni-1", 1, true, false, false)
// If ENABLE_POD_ENI is not set, nothing happens
mockContext.tryEnableSecurityGroupsForPods(ctx)
@@ -2206,7 +2223,6 @@ func TestIsConfigValid(t *testing.T) {
m.awsutils.EXPECT().IsPrefixDelegationSupported().Return(false)
}
}
- ds := datastore.NewDataStore(log, datastore.NullCheckpoint{}, tt.fields.prefixDelegationEnabled)
mockContext := &IPAMContext{
awsClient: m.awsutils,
@@ -2216,7 +2232,9 @@ func TestIsConfigValid(t *testing.T) {
enablePrefixDelegation: tt.fields.prefixDelegationEnabled,
enablePodENI: tt.fields.podENIEnabled,
useCustomNetworking: tt.fields.customNetworkingEnabled,
- dataStore: ds,
+ dataStoreAccess: &datastore.DataStoreAccess{
+ DataStores: []*datastore.DataStore{datastore.NewDataStore(log, datastore.NullCheckpoint{}, tt.fields.prefixDelegationEnabled)},
+ },
}
resp := mockContext.isConfigValid()
@@ -2239,14 +2257,14 @@ func TestAnnotatePod(t *testing.T) {
}
mockContext := &IPAMContext{
- awsClient: m.awsutils,
- k8sClient: m.k8sClient,
- primaryIP: make(map[string]string),
- terminating: int32(0),
- networkClient: m.network,
- dataStore: testDatastore(),
- enableIPv4: true,
- enableIPv6: false,
+ awsClient: m.awsutils,
+ k8sClient: m.k8sClient,
+ primaryIP: make(map[string]string),
+ terminating: int32(0),
+ networkClient: m.network,
+ dataStoreAccess: testDatastore(),
+ enableIPv4: true,
+ enableIPv6: false,
}
mockContext.k8sClient.Create(ctx, &pod)
@@ -2428,15 +2446,15 @@ func TestPodENIErrInc(t *testing.T) {
ctx := context.Background()
mockContext := &IPAMContext{
- awsClient: m.awsutils,
- k8sClient: m.k8sClient,
- networkClient: m.network,
- primaryIP: make(map[string]string),
- terminating: int32(0),
- dataStore: testDatastore(),
- enableIPv4: true,
- enableIPv6: false,
- enablePodENI: true,
+ awsClient: m.awsutils,
+ k8sClient: m.k8sClient,
+ networkClient: m.network,
+ primaryIP: make(map[string]string),
+ terminating: int32(0),
+ dataStoreAccess: testDatastore(),
+ enableIPv4: true,
+ enableIPv6: false,
+ enablePodENI: true,
}
// Create a test pod
@@ -2481,7 +2499,7 @@ func TestPodENIErrInc(t *testing.T) {
func (c *IPAMContext) tryAssignPodENI(ctx context.Context, pod *corev1.Pod, fnName string) error {
// Mock implementation for the test
- _, err := c.awsClient.AllocENI(false, nil, "", 0)
+ _, err := c.awsClient.AllocENI(nil, "", 0, defaultNetworkCard)
if err != nil {
prometheusmetrics.PodENIErr.With(prometheus.Labels{"fn": fnName}).Inc()
return err
diff --git a/pkg/ipamd/rpc_handler.go b/pkg/ipamd/rpc_handler.go
index 236fa0f0a9..ef48b90adc 100644
--- a/pkg/ipamd/rpc_handler.go
+++ b/pkg/ipamd/rpc_handler.go
@@ -21,6 +21,7 @@ import (
"strings"
"syscall"
+ multiErr "github.com/hashicorp/go-multierror"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
"golang.org/x/net/context"
@@ -80,6 +81,16 @@ func (s *server) AddNetwork(ctx context.Context, in *rpc.AddNetworkRequest) (*rp
var deviceNumber, vlanID, trunkENILinkIndex int
var ipv4Addr, ipv6Addr, branchENIMAC, podENISubnetGW string
var err error
+
+ var ipamKey datastore.IPAMKey
+ var ipamMetadata datastore.IPAMMetadata
+ var errors error
+ var resp rpc.AddNetworkReply
+
+ // This will be a list of IPs. For now it's just one
+ ipAddrs := []*rpc.IPAddress{}
+
+ // var interfacesCount int
if s.ipamContext.enablePodENI {
// Check pod spec for Branch ENI
pod, err := s.ipamContext.GetPod(in.K8S_POD_NAME, in.K8S_POD_NAMESPACE)
@@ -88,10 +99,15 @@ func (s *server) AddNetwork(ctx context.Context, in *rpc.AddNetworkRequest) (*rp
return &failureResponse, nil
}
limits := pod.Spec.Containers[0].Resources.Limits
+
for resName := range limits {
if strings.HasPrefix(string(resName), "vpc.amazonaws.com/pod-eni") {
+
+ if in.RequiresMultiNICAttachment {
+ log.Info("SGP pod doesn't support multiple NIC attachments for pod, falling back to default one interface per pod")
+ }
// Check that we have a trunk
- trunkENI := s.ipamContext.dataStore.GetTrunkENI()
+ trunkENI := s.ipamContext.dataStoreAccess.GetDataStore(DefaultNetworkCard).GetTrunkENI()
if trunkENI == "" {
log.Warn("Send AddNetworkReply: No trunk ENI found, cannot add a pod ENI")
return &failureResponse, nil
@@ -149,6 +165,13 @@ func (s *server) AddNetwork(ctx context.Context, in *rpc.AddNetworkRequest) (*rp
}
podENISubnetGW = gw.String()
deviceNumber = -1 // Not needed for branch ENI, they depend on trunkENIDeviceIndex
+
+ ipAddrs = append(ipAddrs, &rpc.IPAddress{
+ IPv4Addr: ipv4Addr,
+ IPv6Addr: ipv6Addr,
+ DeviceNumber: int32(deviceNumber),
+ RouteTableId: int32(deviceNumber), // Need to update to route table number
+ })
} else {
log.Infof("Send AddNetworkReply: failed to get Branch ENI resource")
return &failureResponse, nil
@@ -159,79 +182,134 @@ func (s *server) AddNetwork(ctx context.Context, in *rpc.AddNetworkRequest) (*rp
if s.ipamContext.enableIPv4 && ipv4Addr == "" ||
s.ipamContext.enableIPv6 && ipv6Addr == "" {
+
+ ipsRequired := 1
+ ipsAllocated := 0
+
+ if in.RequiresMultiNICAttachment {
+ ipsRequired = len(s.ipamContext.dataStoreAccess.DataStores)
+ }
+
+ log.Infof("Required MultiNIC access %t, interfaces required %+v", in.RequiresMultiNICAttachment, ipsRequired)
+
if in.ContainerID == "" || in.IfName == "" || in.NetworkName == "" {
log.Errorf("Unable to generate IPAMKey from %+v", in)
return &failureResponse, nil
}
- ipamKey := datastore.IPAMKey{
- ContainerID: in.ContainerID,
- IfName: in.IfName,
- NetworkName: in.NetworkName,
- }
- ipamMetadata := datastore.IPAMMetadata{
- K8SPodNamespace: in.K8S_POD_NAMESPACE,
- K8SPodName: in.K8S_POD_NAME,
+
+ for networkCard, ds := range s.ipamContext.dataStoreAccess.DataStores {
+
+ if ipsAllocated == ipsRequired {
+ break
+ }
+
+ ipamKey = datastore.IPAMKey{
+ ContainerID: in.ContainerID,
+ IfName: in.IfName,
+ NetworkName: in.NetworkName,
+ }
+
+ ipamMetadata = datastore.IPAMMetadata{
+ K8SPodNamespace: in.K8S_POD_NAMESPACE,
+ K8SPodName: in.K8S_POD_NAME,
+ InterfacesCount: ipsRequired,
+ }
+
+ ipv4Addr, ipv6Addr, deviceNumber, err = ds.AssignPodIPAddress(ipamKey, ipamMetadata, s.ipamContext.enableIPv4, s.ipamContext.enableIPv6)
+
+ if err != nil {
+ log.Warnf("Failed to assign IPs from network card %d: %v", networkCard, err)
+ // continue to look through other datastores till you are unable to find an IP address when ONLY 1 ip is required
+ // if the last datastore also return ErrNoAvailableIPInDataStore, return an error
+ if err == datastore.ErrNoAvailableIPInDataStore && ipsRequired == 1 && networkCard != len(s.ipamContext.dataStoreAccess.DataStores)-1 {
+ continue
+ }
+
+ errors = multiErr.Append(errors, err)
+ break
+ }
+
+ log.Infof("Assigned IP from network card: %d -> IPv4: %s, IPv6: %s, device number: %d, ", networkCard, ipv4Addr, ipv6Addr, deviceNumber)
+ ipAddrs = append(ipAddrs, &rpc.IPAddress{
+ IPv4Addr: ipv4Addr,
+ IPv6Addr: ipv6Addr,
+ DeviceNumber: int32(deviceNumber),
+ RouteTableId: int32(networkCard*s.ipamContext.maxENI + deviceNumber + 1),
+ })
+
+ ipsAllocated += 1
}
- ipv4Addr, ipv6Addr, deviceNumber, err = s.ipamContext.dataStore.AssignPodIPAddress(ipamKey, ipamMetadata, s.ipamContext.enableIPv4, s.ipamContext.enableIPv6)
}
- var pbVPCV4cidrs, pbVPCV6cidrs []string
- var useExternalSNAT bool
- if s.ipamContext.enableIPv4 && ipv4Addr != "" {
- pbVPCV4cidrs, err = s.ipamContext.awsClient.GetVPCIPv4CIDRs()
- if err != nil {
- return nil, err
+ if errors != nil {
+ for index := range ipAddrs {
+ log.Infof("Unassigning IPs from datastore for Network Card %d as there was failure", index)
+ // Try to unassign all the assigned IPs in case of errors.
+ // This is best effort even if it fails we have the reconciler which can clean up leaked IPs for non existing pods
+ s.ipamContext.dataStoreAccess.GetDataStore(index).UnassignPodIPAddress(ipamKey)
}
- for _, cidr := range pbVPCV4cidrs {
- log.Debugf("VPC CIDR %s", cidr)
- }
- useExternalSNAT = s.ipamContext.networkClient.UseExternalSNAT()
- if !useExternalSNAT {
- for _, cidr := range s.ipamContext.networkClient.GetExcludeSNATCIDRs() {
- log.Debugf("CIDR SNAT Exclusion %s", cidr)
- pbVPCV4cidrs = append(pbVPCV4cidrs, cidr)
+ } else {
+ log.Infof("Address assigned from DS: %d", len(ipAddrs))
+ var pbVPCV4cidrs, pbVPCV6cidrs []string
+ var useExternalSNAT bool
+
+ if s.ipamContext.enableIPv4 {
+ pbVPCV4cidrs, err = s.ipamContext.awsClient.GetVPCIPv4CIDRs()
+ if err != nil {
+ return nil, err
+ }
+ for _, cidr := range pbVPCV4cidrs {
+ log.Debugf("VPC CIDR %s", cidr)
+ }
+ useExternalSNAT = s.ipamContext.networkClient.UseExternalSNAT()
+ if !useExternalSNAT {
+ for _, cidr := range s.ipamContext.networkClient.GetExcludeSNATCIDRs() {
+ log.Debugf("CIDR SNAT Exclusion %s", cidr)
+ pbVPCV4cidrs = append(pbVPCV4cidrs, cidr)
+ }
+ }
+ } else if s.ipamContext.enableIPv6 {
+ pbVPCV6cidrs, err = s.ipamContext.awsClient.GetVPCIPv6CIDRs()
+ if err != nil {
+ return nil, err
+ }
+ for _, cidr := range pbVPCV6cidrs {
+ log.Debugf("VPC V6 CIDR %s", cidr)
}
}
- } else if s.ipamContext.enableIPv6 && ipv6Addr != "" {
- pbVPCV6cidrs, err = s.ipamContext.awsClient.GetVPCIPv6CIDRs()
- if err != nil {
- return nil, err
- }
- for _, cidr := range pbVPCV6cidrs {
- log.Debugf("VPC V6 CIDR %s", cidr)
- }
- }
- if s.ipamContext.enablePodIPAnnotation {
- // On ADD, we pass empty string as there is no IP being released
- if ipv4Addr != "" {
- err = s.ipamContext.AnnotatePod(in.K8S_POD_NAME, in.K8S_POD_NAMESPACE, vpccniPodIPKey, ipv4Addr, "")
- if err != nil {
- log.Errorf("Failed to add the pod annotation: %v", err)
+ if s.ipamContext.enablePodIPAnnotation {
+ var ipAddr string
+
+ // We are only adding the pods primary IP to annotation
+ if ipAddrs[0].GetIPv4Addr() != "" {
+ ipAddr = ipAddrs[0].GetIPv4Addr()
+ } else {
+ ipAddr = ipAddrs[0].GetIPv6Addr()
}
- } else if ipv6Addr != "" {
- err = s.ipamContext.AnnotatePod(in.K8S_POD_NAME, in.K8S_POD_NAMESPACE, vpccniPodIPKey, ipv6Addr, "")
+
+ err = s.ipamContext.AnnotatePod(in.K8S_POD_NAME, in.K8S_POD_NAMESPACE, vpccniPodIPKey, ipAddr, "")
if err != nil {
log.Errorf("Failed to add the pod annotation: %v", err)
}
}
- }
- resp := rpc.AddNetworkReply{
- Success: err == nil,
- IPv4Addr: ipv4Addr,
- IPv6Addr: ipv6Addr,
- DeviceNumber: int32(deviceNumber),
- UseExternalSNAT: useExternalSNAT,
- VPCv4CIDRs: pbVPCV4cidrs,
- VPCv6CIDRs: pbVPCV6cidrs,
- PodVlanId: int32(vlanID),
- PodENIMAC: branchENIMAC,
- PodENISubnetGW: podENISubnetGW,
- ParentIfIndex: int32(trunkENILinkIndex),
- NetworkPolicyMode: s.ipamContext.networkPolicyMode,
+
+ resp = rpc.AddNetworkReply{
+ IPAddress: ipAddrs,
+ UseExternalSNAT: useExternalSNAT,
+ VPCv4CIDRs: pbVPCV4cidrs,
+ VPCv6CIDRs: pbVPCV6cidrs,
+ PodVlanId: int32(vlanID),
+ PodENIMAC: branchENIMAC,
+ PodENISubnetGW: podENISubnetGW,
+ ParentIfIndex: int32(trunkENILinkIndex),
+ NetworkPolicyMode: s.ipamContext.networkPolicyMode,
+ }
}
- log.Infof("Send AddNetworkReply: IPv4Addr: %s, IPv6Addr: %s, DeviceNumber: %d, err: %v", ipv4Addr, ipv6Addr, deviceNumber, err)
+ resp.Success = errors == nil
+
+ log.Infof("Send AddNetworkReply: Success: %t IPAddr: %+v, err: %v", resp.Success, resp.IPAddress, err)
return &resp, nil
}
@@ -259,66 +337,111 @@ func (s *server) DelNetwork(ctx context.Context, in *rpc.DelNetworkRequest) (*rp
IfName: in.IfName,
NetworkName: in.NetworkName,
}
- eni, ip, deviceNumber, err := s.ipamContext.dataStore.UnassignPodIPAddress(ipamKey)
- if s.ipamContext.enableIPv4 {
- ipv4Addr = ip
- cidr := net.IPNet{IP: net.ParseIP(ip), Mask: net.IPv4Mask(255, 255, 255, 255)}
- cidrStr = cidr.String()
- } else if s.ipamContext.enableIPv6 {
- ipv6Addr = ip
- }
+ var errors error
+ var ipAddr []*rpc.IPAddress
+
+ ipsToMatch := 1
+ ipsFound := 0
+ for networkCard, ds := range s.ipamContext.dataStoreAccess.DataStores {
+ // All datastores will store this count in its datastore
+ if ipsToMatch == ipsFound {
+ break
+ }
+
+ eni, ip, deviceNumber, ipsAllocated, err := ds.UnassignPodIPAddress(ipamKey)
- if s.ipamContext.enableIPv4 && eni != nil {
- //cidrStr will be pod IP i.e, IP/32 for v4 (or) IP/128 for v6.
- // Case 1: PD is enabled but IP/32 key in AvailableIPv4Cidrs[cidrStr] exists, this means it is a secondary IP. Added IsPrefix check just for sanity.
- // So this IP should be released immediately.
- // Case 2: PD is disabled then IP/32 key in AvailableIPv4Cidrs[cidrStr] will not exists since key to AvailableIPv4Cidrs will be either /28 prefix or /32
- // secondary IP. Hence now see if we need free up a prefix is no other pods are using it.
- if s.ipamContext.enablePrefixDelegation && eni.AvailableIPv4Cidrs[cidrStr] != nil && eni.AvailableIPv4Cidrs[cidrStr].IsPrefix == false {
- log.Debugf("IP belongs to secondary pool with PD enabled so free IP from EC2")
- s.ipamContext.tryUnassignIPFromENI(eni.ID)
- } else if !s.ipamContext.enablePrefixDelegation && eni.AvailableIPv4Cidrs[cidrStr] == nil {
- log.Debugf("IP belongs to prefix pool with PD disabled so try free prefix from EC2")
- s.ipamContext.tryUnassignPrefixFromENI(eni.ID)
+ // ipsAllocated will always be same in all datastores for a Pod, so this will not change ever between datastores
+ if ipsAllocated > 0 {
+ log.Debugf("IPs allocated for the pod: %d", ipsAllocated)
+ ipsToMatch = ipsAllocated
}
- }
- if err == datastore.ErrUnknownPod && s.ipamContext.enablePodENI {
- pod, err := s.ipamContext.GetPod(in.K8S_POD_NAME, in.K8S_POD_NAMESPACE)
- if err != nil {
- if k8serror.IsNotFound(err) {
- log.Warn("Send DelNetworkReply: pod not found")
- return &rpc.DelNetworkReply{Success: true}, nil
+ if s.ipamContext.enableIPv4 {
+ ipv4Addr = ip
+ cidr := net.IPNet{IP: net.ParseIP(ip), Mask: net.IPv4Mask(255, 255, 255, 255)}
+ cidrStr = cidr.String()
+ } else if s.ipamContext.enableIPv6 {
+ ipv6Addr = ip
+ }
+
+ if s.ipamContext.enableIPv4 && eni != nil {
+ //cidrStr will be pod IP i.e, IP/32 for v4 (or) IP/128 for v6.
+ // Case 1: PD is enabled but IP/32 key in AvailableIPv4Cidrs[cidrStr] exists, this means it is a secondary IP. Added IsPrefix check just for sanity.
+ // So this IP should be released immediately.
+ // Case 2: PD is disabled then IP/32 key in AvailableIPv4Cidrs[cidrStr] will not exists since key to AvailableIPv4Cidrs will be either /28 prefix or /32
+ // secondary IP. Hence now see if we need free up a prefix is no other pods are using it.
+ if s.ipamContext.enablePrefixDelegation && eni.AvailableIPv4Cidrs[cidrStr] != nil && eni.AvailableIPv4Cidrs[cidrStr].IsPrefix {
+ log.Debugf("IP belongs to secondary pool with PD enabled so free IP from EC2")
+ s.ipamContext.tryUnassignIPFromENI(eni.ID, networkCard)
+ } else if !s.ipamContext.enablePrefixDelegation && eni.AvailableIPv4Cidrs[cidrStr] == nil {
+ log.Debugf("IP belongs to prefix pool with PD disabled so try free prefix from EC2")
+ s.ipamContext.tryUnassignPrefixFromENI(eni.ID, networkCard)
}
- log.Warnf("Send DelNetworkReply: Failed to get pod spec: %v", err)
- return &rpc.DelNetworkReply{Success: false}, err
}
- val, branch := pod.Annotations["vpc.amazonaws.com/pod-eni"]
- if branch {
- // Parse JSON data
- var podENIData []PodENIData
- err := json.Unmarshal([]byte(val), &podENIData)
- if err != nil || len(podENIData) < 1 {
- log.Errorf("Failed to unmarshal PodENIData JSON: %v", err)
+
+ if err == datastore.ErrUnknownPod {
+ if s.ipamContext.enablePodENI {
+ pod, err := s.ipamContext.GetPod(in.K8S_POD_NAME, in.K8S_POD_NAMESPACE)
+ if err != nil {
+ if k8serror.IsNotFound(err) {
+ log.Warn("Send DelNetworkReply: pod not found")
+ return &rpc.DelNetworkReply{Success: true}, nil
+ }
+ log.Warnf("Send DelNetworkReply: Failed to get pod spec: %v", err)
+ return &rpc.DelNetworkReply{Success: false}, err
+ }
+ val, branch := pod.Annotations["vpc.amazonaws.com/pod-eni"]
+ if branch {
+ // Parse JSON data
+ var podENIData []PodENIData
+ err := json.Unmarshal([]byte(val), &podENIData)
+ if err != nil || len(podENIData) < 1 {
+ log.Errorf("Failed to unmarshal PodENIData JSON: %v", err)
+ }
+ return &rpc.DelNetworkReply{
+ Success: true,
+ PodVlanId: int32(podENIData[0].VlanID),
+ IPAddress: []*rpc.IPAddress{&rpc.IPAddress{IPv4Addr: podENIData[0].PrivateIP}},
+ }, err
+ }
+ }
+
+ // If ip address is still empty, continue till we reach the end of datastore
+ // This happens when pod is not in on network card 0, DS will return datastore.ErrUnknownPod
+ if ip == "" && networkCard != len(s.ipamContext.dataStoreAccess.DataStores)-1 {
+ log.Debugf("Pod doesn't belong to Network Card %d, looking into next datastore", networkCard)
+ continue
+ }
+ }
+
+ if s.ipamContext.enablePodIPAnnotation {
+ // On DEL, we pass IP being released
+ err = s.ipamContext.AnnotatePod(in.K8S_POD_NAME, in.K8S_POD_NAMESPACE, vpccniPodIPKey, "", ip)
+ if err != nil {
+ log.Errorf("Failed to delete the pod annotation: %v", err)
}
- return &rpc.DelNetworkReply{
- Success: true,
- PodVlanId: int32(podENIData[0].VlanID),
- IPv4Addr: podENIData[0].PrivateIP}, err
}
- }
- if s.ipamContext.enablePodIPAnnotation {
- // On DEL, we pass IP being released
- err = s.ipamContext.AnnotatePod(in.K8S_POD_NAME, in.K8S_POD_NAMESPACE, vpccniPodIPKey, "", ip)
if err != nil {
- log.Errorf("Failed to delete the pod annotation: %v", err)
+ errors = multiErr.Append(errors, err)
}
- }
- log.Infof("Send DelNetworkReply: IPv4Addr: %s, IPv6Addr: %s, DeviceNumber: %d, err: %v", ipv4Addr, ipv6Addr, deviceNumber, err)
+ ipAddr = append(ipAddr, &rpc.IPAddress{
+ IPv4Addr: ipv4Addr,
+ IPv6Addr: ipv6Addr,
+ DeviceNumber: int32(deviceNumber),
+ RouteTableId: int32(networkCard*s.ipamContext.maxENI + deviceNumber + 1),
+ })
+ ipsFound += 1
+
+ log.Debugf("IPs allocated for the pod: %d, IPs found: %d", ipsAllocated, ipsFound)
+ }
- return &rpc.DelNetworkReply{Success: err == nil, IPv4Addr: ipv4Addr, IPv6Addr: ipv6Addr, DeviceNumber: int32(deviceNumber)}, err
+ log.Infof("Send DelNetworkReply: IPAddress: %+v, err: %v", ipAddr, errors)
+ return &rpc.DelNetworkReply{
+ Success: errors == nil,
+ IPAddress: ipAddr,
+ }, errors
}
func (s *server) GetNetworkPolicyConfigs(ctx context.Context, e *emptypb.Empty) (*rpc.NetworkPolicyAgentConfigReply, error) {
diff --git a/pkg/ipamd/rpc_handler_test.go b/pkg/ipamd/rpc_handler_test.go
index 53389ba9ca..80af8956f0 100644
--- a/pkg/ipamd/rpc_handler_test.go
+++ b/pkg/ipamd/rpc_handler_test.go
@@ -19,6 +19,7 @@ import (
"testing"
"github.com/aws/amazon-vpc-cni-k8s/pkg/ipamd/datastore"
+ "github.com/aws/amazon-vpc-cni-k8s/rpc"
pb "github.com/aws/amazon-vpc-cni-k8s/rpc"
@@ -33,14 +34,15 @@ func TestServer_VersionCheck(t *testing.T) {
defer m.ctrl.Finish()
mockContext := &IPAMContext{
- awsClient: m.awsutils,
- maxIPsPerENI: 14,
- maxENI: 4,
- warmENITarget: 1,
- warmIPTarget: 3,
- networkClient: m.network,
- dataStore: datastore.NewDataStore(log, datastore.NullCheckpoint{}, false),
+ awsClient: m.awsutils,
+ maxIPsPerENI: 14,
+ maxENI: 4,
+ warmENITarget: 1,
+ warmIPTarget: 3,
+ networkClient: m.network,
+ dataStoreAccess: datastore.InitializeDataStores(1, "test", false, log),
}
+
m.awsutils.EXPECT().GetVPCIPv4CIDRs().Return([]string{}, nil).AnyTimes()
m.awsutils.EXPECT().GetVPCIPv6CIDRs().Return([]string{}, nil).AnyTimes()
m.network.EXPECT().UseExternalSNAT().Return(true).AnyTimes()
@@ -136,9 +138,15 @@ func TestServer_AddNetwork(t *testing.T) {
ipV6Enabled: false,
},
want: &pb.AddNetworkReply{
- Success: true,
- IPv4Addr: "192.168.1.100",
- DeviceNumber: int32(0),
+ Success: true,
+ IPAddress: []*rpc.IPAddress{
+ {
+ IPv4Addr: "192.168.1.100",
+ IPv6Addr: "",
+ DeviceNumber: int32(0),
+ RouteTableId: int32(1),
+ },
+ },
UseExternalSNAT: true,
VPCv4CIDRs: []string{"10.10.0.0/16"},
},
@@ -168,9 +176,15 @@ func TestServer_AddNetwork(t *testing.T) {
ipV6Enabled: false,
},
want: &pb.AddNetworkReply{
- Success: true,
- IPv4Addr: "192.168.1.100",
- DeviceNumber: int32(0),
+ Success: true,
+ IPAddress: []*rpc.IPAddress{
+ {
+ IPv4Addr: "192.168.1.100",
+ IPv6Addr: "",
+ DeviceNumber: int32(0),
+ RouteTableId: int32(1),
+ },
+ },
UseExternalSNAT: false,
VPCv4CIDRs: []string{"10.10.0.0/16", "10.12.0.0/16", "10.13.0.0/16"},
},
@@ -183,8 +197,15 @@ func TestServer_AddNetwork(t *testing.T) {
ipV6Enabled: false,
},
want: &pb.AddNetworkReply{
- Success: false,
- DeviceNumber: int32(-1),
+ Success: false,
+ IPAddress: []*rpc.IPAddress{
+ {
+ IPv4Addr: "",
+ IPv6Addr: "",
+ DeviceNumber: int32(-1),
+ RouteTableId: int32(0),
+ },
+ },
},
},
{
@@ -203,10 +224,16 @@ func TestServer_AddNetwork(t *testing.T) {
prefixDelegationEnabled: true,
},
want: &pb.AddNetworkReply{
- Success: true,
- IPv6Addr: "2001:db8::",
- DeviceNumber: int32(0),
- VPCv6CIDRs: []string{"2001:db8::/56"},
+ Success: true,
+ IPAddress: []*rpc.IPAddress{
+ {
+ IPv4Addr: "",
+ IPv6Addr: "2001:db8::",
+ DeviceNumber: int32(0),
+ RouteTableId: int32(1),
+ },
+ },
+ VPCv6CIDRs: []string{"2001:db8::/56"},
},
},
{
@@ -218,8 +245,15 @@ func TestServer_AddNetwork(t *testing.T) {
prefixDelegationEnabled: true,
},
want: &pb.AddNetworkReply{
- Success: false,
- DeviceNumber: int32(-1),
+ Success: false,
+ IPAddress: []*rpc.IPAddress{
+ {
+ IPv4Addr: "",
+ IPv6Addr: "",
+ DeviceNumber: int32(-1),
+ RouteTableId: int32(0),
+ },
+ },
},
},
{
@@ -233,9 +267,15 @@ func TestServer_AddNetwork(t *testing.T) {
prefixDelegationEnabled: false,
},
want: &pb.AddNetworkReply{
- Success: false,
- IPv6Addr: "",
- DeviceNumber: int32(-1),
+ Success: false,
+ IPAddress: []*rpc.IPAddress{
+ {
+ IPv4Addr: "",
+ IPv6Addr: "",
+ DeviceNumber: int32(-1),
+ RouteTableId: int32(0),
+ },
+ },
},
},
}
@@ -262,19 +302,20 @@ func TestServer_AddNetwork(t *testing.T) {
for _, call := range tt.fields.getExcludeSNATCIDRsCalls {
m.network.EXPECT().GetExcludeSNATCIDRs().Return(call.snatExclusionCIDRs)
}
- ds := datastore.NewDataStore(log, datastore.NullCheckpoint{}, tt.fields.prefixDelegationEnabled)
+ // ds := datastore.NewDataStore(log, datastore.NullCheckpoint{}, tt.fields.prefixDelegationEnabled)
+ dsAccess := datastore.InitializeDataStores(1, defaultBackingStorePath, tt.fields.prefixDelegationEnabled, log)
for eniID, ipv4Addresses := range tt.fields.ipV4AddressByENIID {
- ds.AddENI(eniID, 0, false, false, false)
+ dsAccess.GetDataStore(0).AddENI(eniID, 0, false, false, false)
for _, ipv4Address := range ipv4Addresses {
ipv4Addr := net.IPNet{IP: net.ParseIP(ipv4Address), Mask: net.IPv4Mask(255, 255, 255, 255)}
- ds.AddIPv4CidrToStore(eniID, ipv4Addr, false)
+ dsAccess.GetDataStore(0).AddIPv4CidrToStore(eniID, ipv4Addr, false)
}
}
for eniID, ipv6Prefixes := range tt.fields.ipV6PrefixByENIID {
- ds.AddENI(eniID, 0, false, false, false)
+ dsAccess.GetDataStore(0).AddENI(eniID, 0, false, false, false)
for _, ipv6Prefix := range ipv6Prefixes {
_, ipnet, _ := net.ParseCIDR(ipv6Prefix)
- ds.AddIPv6CidrToStore(eniID, *ipnet, true)
+ dsAccess.GetDataStore(0).AddIPv6CidrToStore(eniID, *ipnet, true)
}
}
@@ -288,7 +329,7 @@ func TestServer_AddNetwork(t *testing.T) {
enableIPv4: tt.fields.ipV4Enabled,
enableIPv6: tt.fields.ipV6Enabled,
enablePrefixDelegation: tt.fields.prefixDelegationEnabled,
- dataStore: ds,
+ dataStoreAccess: dsAccess,
}
s := &server{
diff --git a/pkg/networkutils/mocks/network_mocks.go b/pkg/networkutils/mocks/network_mocks.go
index e68b213f44..bc461891ee 100644
--- a/pkg/networkutils/mocks/network_mocks.go
+++ b/pkg/networkutils/mocks/network_mocks.go
@@ -1,22 +1,8 @@
-// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License"). You may
-// not use this file except in compliance with the License. A copy of the
-// License is located at
-//
-// http://aws.amazon.com/apache2.0/
-//
-// or in the "license" file accompanying this file. This file is distributed
-// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
-// express or implied. See the License for the specific language governing
-// permissions and limitations under the License.
-//
-
// Code generated by MockGen. DO NOT EDIT.
-// Source: github.com/aws/amazon-vpc-cni-k8s/pkg/networkutils (interfaces: NetworkAPIs)
+// Source: network.go
-// Package mock_networkutils is a generated GoMock package.
-package mock_networkutils
+// Package mocks is a generated GoMock package.
+package mocks
import (
net "net"
@@ -51,17 +37,17 @@ func (m *MockNetworkAPIs) EXPECT() *MockNetworkAPIsMockRecorder {
}
// CleanUpStaleAWSChains mocks base method.
-func (m *MockNetworkAPIs) CleanUpStaleAWSChains(arg0, arg1 bool) error {
+func (m *MockNetworkAPIs) CleanUpStaleAWSChains(v4Enabled, v6Enabled bool) error {
m.ctrl.T.Helper()
- ret := m.ctrl.Call(m, "CleanUpStaleAWSChains", arg0, arg1)
+ ret := m.ctrl.Call(m, "CleanUpStaleAWSChains", v4Enabled, v6Enabled)
ret0, _ := ret[0].(error)
return ret0
}
// CleanUpStaleAWSChains indicates an expected call of CleanUpStaleAWSChains.
-func (mr *MockNetworkAPIsMockRecorder) CleanUpStaleAWSChains(arg0, arg1 interface{}) *gomock.Call {
+func (mr *MockNetworkAPIsMockRecorder) CleanUpStaleAWSChains(v4Enabled, v6Enabled interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CleanUpStaleAWSChains", reflect.TypeOf((*MockNetworkAPIs)(nil).CleanUpStaleAWSChains), arg0, arg1)
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CleanUpStaleAWSChains", reflect.TypeOf((*MockNetworkAPIs)(nil).CleanUpStaleAWSChains), v4Enabled, v6Enabled)
}
// GetExcludeSNATCIDRs mocks base method.
@@ -93,18 +79,18 @@ func (mr *MockNetworkAPIsMockRecorder) GetExternalServiceCIDRs() *gomock.Call {
}
// GetLinkByMac mocks base method.
-func (m *MockNetworkAPIs) GetLinkByMac(arg0 string, arg1 time.Duration) (netlink.Link, error) {
+func (m *MockNetworkAPIs) GetLinkByMac(mac string, retryInterval time.Duration) (netlink.Link, error) {
m.ctrl.T.Helper()
- ret := m.ctrl.Call(m, "GetLinkByMac", arg0, arg1)
+ ret := m.ctrl.Call(m, "GetLinkByMac", mac, retryInterval)
ret0, _ := ret[0].(netlink.Link)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// GetLinkByMac indicates an expected call of GetLinkByMac.
-func (mr *MockNetworkAPIsMockRecorder) GetLinkByMac(arg0, arg1 interface{}) *gomock.Call {
+func (mr *MockNetworkAPIsMockRecorder) GetLinkByMac(mac, retryInterval interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLinkByMac", reflect.TypeOf((*MockNetworkAPIs)(nil).GetLinkByMac), arg0, arg1)
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLinkByMac", reflect.TypeOf((*MockNetworkAPIs)(nil).GetLinkByMac), mac, retryInterval)
}
// GetRuleList mocks base method.
@@ -123,88 +109,88 @@ func (mr *MockNetworkAPIsMockRecorder) GetRuleList() *gomock.Call {
}
// GetRuleListBySrc mocks base method.
-func (m *MockNetworkAPIs) GetRuleListBySrc(arg0 []netlink.Rule, arg1 net.IPNet) ([]netlink.Rule, error) {
+func (m *MockNetworkAPIs) GetRuleListBySrc(ruleList []netlink.Rule, src net.IPNet) ([]netlink.Rule, error) {
m.ctrl.T.Helper()
- ret := m.ctrl.Call(m, "GetRuleListBySrc", arg0, arg1)
+ ret := m.ctrl.Call(m, "GetRuleListBySrc", ruleList, src)
ret0, _ := ret[0].([]netlink.Rule)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// GetRuleListBySrc indicates an expected call of GetRuleListBySrc.
-func (mr *MockNetworkAPIsMockRecorder) GetRuleListBySrc(arg0, arg1 interface{}) *gomock.Call {
+func (mr *MockNetworkAPIsMockRecorder) GetRuleListBySrc(ruleList, src interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRuleListBySrc", reflect.TypeOf((*MockNetworkAPIs)(nil).GetRuleListBySrc), arg0, arg1)
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRuleListBySrc", reflect.TypeOf((*MockNetworkAPIs)(nil).GetRuleListBySrc), ruleList, src)
}
// SetupENINetwork mocks base method.
-func (m *MockNetworkAPIs) SetupENINetwork(arg0, arg1 string, arg2 int, arg3 string) error {
+func (m *MockNetworkAPIs) SetupENINetwork(eniIP, eniMAC string, deviceNumber, networkCard int, eniSubnetCIDR string, maxENIPerNIC int) error {
m.ctrl.T.Helper()
- ret := m.ctrl.Call(m, "SetupENINetwork", arg0, arg1, arg2, arg3)
+ ret := m.ctrl.Call(m, "SetupENINetwork", eniIP, eniMAC, deviceNumber, networkCard, eniSubnetCIDR, maxENIPerNIC)
ret0, _ := ret[0].(error)
return ret0
}
// SetupENINetwork indicates an expected call of SetupENINetwork.
-func (mr *MockNetworkAPIsMockRecorder) SetupENINetwork(arg0, arg1, arg2, arg3 interface{}) *gomock.Call {
+func (mr *MockNetworkAPIsMockRecorder) SetupENINetwork(eniIP, eniMAC, deviceNumber, networkCard, eniSubnetCIDR, maxENIPerNIC interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetupENINetwork", reflect.TypeOf((*MockNetworkAPIs)(nil).SetupENINetwork), arg0, arg1, arg2, arg3)
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetupENINetwork", reflect.TypeOf((*MockNetworkAPIs)(nil).SetupENINetwork), eniIP, eniMAC, deviceNumber, networkCard, eniSubnetCIDR, maxENIPerNIC)
}
// SetupHostNetwork mocks base method.
-func (m *MockNetworkAPIs) SetupHostNetwork(arg0 []string, arg1 string, arg2 *net.IP, arg3, arg4, arg5 bool) error {
+func (m *MockNetworkAPIs) SetupHostNetwork(vpcCIDRs []string, primaryMAC string, primaryAddr *net.IP, enablePodENI, v4Enabled, v6Enabled bool) error {
m.ctrl.T.Helper()
- ret := m.ctrl.Call(m, "SetupHostNetwork", arg0, arg1, arg2, arg3, arg4, arg5)
+ ret := m.ctrl.Call(m, "SetupHostNetwork", vpcCIDRs, primaryMAC, primaryAddr, enablePodENI, v4Enabled, v6Enabled)
ret0, _ := ret[0].(error)
return ret0
}
// SetupHostNetwork indicates an expected call of SetupHostNetwork.
-func (mr *MockNetworkAPIsMockRecorder) SetupHostNetwork(arg0, arg1, arg2, arg3, arg4, arg5 interface{}) *gomock.Call {
+func (mr *MockNetworkAPIsMockRecorder) SetupHostNetwork(vpcCIDRs, primaryMAC, primaryAddr, enablePodENI, v4Enabled, v6Enabled interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetupHostNetwork", reflect.TypeOf((*MockNetworkAPIs)(nil).SetupHostNetwork), arg0, arg1, arg2, arg3, arg4, arg5)
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetupHostNetwork", reflect.TypeOf((*MockNetworkAPIs)(nil).SetupHostNetwork), vpcCIDRs, primaryMAC, primaryAddr, enablePodENI, v4Enabled, v6Enabled)
}
// UpdateExternalServiceIpRules mocks base method.
-func (m *MockNetworkAPIs) UpdateExternalServiceIpRules(arg0 []netlink.Rule, arg1 []string) error {
+func (m *MockNetworkAPIs) UpdateExternalServiceIpRules(ruleList []netlink.Rule, externalIPs []string) error {
m.ctrl.T.Helper()
- ret := m.ctrl.Call(m, "UpdateExternalServiceIpRules", arg0, arg1)
+ ret := m.ctrl.Call(m, "UpdateExternalServiceIpRules", ruleList, externalIPs)
ret0, _ := ret[0].(error)
return ret0
}
// UpdateExternalServiceIpRules indicates an expected call of UpdateExternalServiceIpRules.
-func (mr *MockNetworkAPIsMockRecorder) UpdateExternalServiceIpRules(arg0, arg1 interface{}) *gomock.Call {
+func (mr *MockNetworkAPIsMockRecorder) UpdateExternalServiceIpRules(ruleList, externalIPs interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateExternalServiceIpRules", reflect.TypeOf((*MockNetworkAPIs)(nil).UpdateExternalServiceIpRules), arg0, arg1)
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateExternalServiceIpRules", reflect.TypeOf((*MockNetworkAPIs)(nil).UpdateExternalServiceIpRules), ruleList, externalIPs)
}
// UpdateHostIptablesRules mocks base method.
-func (m *MockNetworkAPIs) UpdateHostIptablesRules(arg0 []string, arg1 string, arg2 *net.IP, arg3, arg4 bool) error {
+func (m *MockNetworkAPIs) UpdateHostIptablesRules(vpcCIDRs []string, primaryMAC string, primaryAddr *net.IP, v4Enabled, v6Enabled bool) error {
m.ctrl.T.Helper()
- ret := m.ctrl.Call(m, "UpdateHostIptablesRules", arg0, arg1, arg2, arg3, arg4)
+ ret := m.ctrl.Call(m, "UpdateHostIptablesRules", vpcCIDRs, primaryMAC, primaryAddr, v4Enabled, v6Enabled)
ret0, _ := ret[0].(error)
return ret0
}
// UpdateHostIptablesRules indicates an expected call of UpdateHostIptablesRules.
-func (mr *MockNetworkAPIsMockRecorder) UpdateHostIptablesRules(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call {
+func (mr *MockNetworkAPIsMockRecorder) UpdateHostIptablesRules(vpcCIDRs, primaryMAC, primaryAddr, v4Enabled, v6Enabled interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateHostIptablesRules", reflect.TypeOf((*MockNetworkAPIs)(nil).UpdateHostIptablesRules), arg0, arg1, arg2, arg3, arg4)
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateHostIptablesRules", reflect.TypeOf((*MockNetworkAPIs)(nil).UpdateHostIptablesRules), vpcCIDRs, primaryMAC, primaryAddr, v4Enabled, v6Enabled)
}
// UpdateRuleListBySrc mocks base method.
-func (m *MockNetworkAPIs) UpdateRuleListBySrc(arg0 []netlink.Rule, arg1 net.IPNet) error {
+func (m *MockNetworkAPIs) UpdateRuleListBySrc(ruleList []netlink.Rule, src net.IPNet) error {
m.ctrl.T.Helper()
- ret := m.ctrl.Call(m, "UpdateRuleListBySrc", arg0, arg1)
+ ret := m.ctrl.Call(m, "UpdateRuleListBySrc", ruleList, src)
ret0, _ := ret[0].(error)
return ret0
}
// UpdateRuleListBySrc indicates an expected call of UpdateRuleListBySrc.
-func (mr *MockNetworkAPIsMockRecorder) UpdateRuleListBySrc(arg0, arg1 interface{}) *gomock.Call {
+func (mr *MockNetworkAPIsMockRecorder) UpdateRuleListBySrc(ruleList, src interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateRuleListBySrc", reflect.TypeOf((*MockNetworkAPIs)(nil).UpdateRuleListBySrc), arg0, arg1)
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateRuleListBySrc", reflect.TypeOf((*MockNetworkAPIs)(nil).UpdateRuleListBySrc), ruleList, src)
}
// UseExternalSNAT mocks base method.
diff --git a/pkg/networkutils/names.go b/pkg/networkutils/names.go
index 1e1c1f7af0..69e176b4f2 100644
--- a/pkg/networkutils/names.go
+++ b/pkg/networkutils/names.go
@@ -4,11 +4,16 @@ import (
"crypto/sha1"
"encoding/hex"
"fmt"
+ "strconv"
)
// GeneratePodHostVethName generates the name for Pod's host-side veth device.
// The veth name is generated in a way that aligns with the value expected by Calico for NetworkPolicy enforcement.
-func GeneratePodHostVethName(prefix string, podNamespace string, podName string) string {
+func GeneratePodHostVethName(prefix string, podNamespace string, podName string, index int) string {
+
+ if index > 0 {
+ podName = fmt.Sprintf("%s.%s", podName, strconv.Itoa(index))
+ }
suffix := GeneratePodHostVethNameSuffix(podNamespace, podName)
return fmt.Sprintf("%s%s", prefix, suffix)
}
@@ -19,3 +24,11 @@ func GeneratePodHostVethNameSuffix(podNamespace string, podName string) string {
h.Write([]byte(fmt.Sprintf("%s.%s", podNamespace, podName)))
return hex.EncodeToString(h.Sum(nil))[:11]
}
+
+// Generates the interface name inside the pod namespace
+func GenerateContainerVethName(defaultIfName string, prefix string, index int) string {
+ if index > 0 {
+ return fmt.Sprintf("%s%s", prefix, strconv.Itoa(index))
+ }
+ return defaultIfName
+}
diff --git a/pkg/networkutils/names_test.go b/pkg/networkutils/names_test.go
index e893e7b461..4280edfed1 100644
--- a/pkg/networkutils/names_test.go
+++ b/pkg/networkutils/names_test.go
@@ -38,7 +38,7 @@ func TestGeneratePodHostVethName(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
- got := GeneratePodHostVethName(tt.args.prefix, tt.args.podNamespace, tt.args.podName)
+ got := GeneratePodHostVethName(tt.args.prefix, tt.args.podNamespace, tt.args.podName, 0)
assert.Equal(t, tt.want, got)
})
}
diff --git a/pkg/networkutils/network.go b/pkg/networkutils/network.go
index c7a7f0a110..e316837440 100644
--- a/pkg/networkutils/network.go
+++ b/pkg/networkutils/network.go
@@ -58,6 +58,9 @@ const (
// Rule priority for traffic destined to pod IP
ToContainerRulePriority = 512
+ // From Interface priority for multi-homed pods
+ FromInterfaceRulePriority = 1
+
// 513 - 1023, can be used for priority lower than fromPodRule but higher than default nonVPC CIDR rule
// 1024 is reserved for (ip rule not to <VPC's subnet> table main)
@@ -148,7 +151,7 @@ type NetworkAPIs interface {
SetupHostNetwork(vpcCIDRs []string, primaryMAC string, primaryAddr *net.IP, enablePodENI bool,
v4Enabled bool, v6Enabled bool) error
// SetupENINetwork performs ENI level network configuration. Not needed on the primary ENI
- SetupENINetwork(eniIP string, mac string, deviceNumber int, subnetCIDR string) error
+ SetupENINetwork(eniIP string, eniMAC string, deviceNumber int, networkCard int, eniSubnetCIDR string, maxENIPerNIC int) error
// UpdateHostIptablesRules updates the nat table iptables rules on the host
UpdateHostIptablesRules(vpcCIDRs []string, primaryMAC string, primaryAddr *net.IP, v4Enabled bool, v6Enabled bool) error
CleanUpStaleAWSChains(v4Enabled, v6Enabled bool) error
@@ -287,7 +290,7 @@ func (n *linuxNetwork) setupRuleToBlockNodeLocalAccess(protocol iptables.Protoco
}
// SetupHostNetwork performs node level network configuration
-func (n *linuxNetwork) SetupHostNetwork(vpcv4CIDRs []string, primaryMAC string, primaryAddr *net.IP, enablePodENI bool,
+func (n *linuxNetwork) SetupHostNetwork(vpcCIDRs []string, primaryMAC string, primaryAddr *net.IP, enablePodENI bool,
v4Enabled bool, v6Enabled bool) error {
log.Info("Setting up host network... ")
@@ -366,7 +369,7 @@ func (n *linuxNetwork) SetupHostNetwork(vpcv4CIDRs []string, primaryMAC string,
}
}
- return n.updateHostIptablesRules(vpcv4CIDRs, primaryMAC, primaryAddr, v4Enabled, v6Enabled)
+ return n.updateHostIptablesRules(vpcCIDRs, primaryMAC, primaryAddr, v4Enabled, v6Enabled)
}
// UpdateHostIptablesRules updates the NAT table rules based on the VPC CIDRs configuration
@@ -429,10 +432,6 @@ func (n *linuxNetwork) updateHostIptablesRules(vpcCIDRs []string, primaryMAC str
ipProtocol := iptables.ProtocolIPv4
if v6Enabled {
- // Essentially a stub function for now in V6 mode. We will need it when we support v6 in secondary IP and
- // custom networking modes. We don't need to install any SNAT rules in v6 mode and currently there is no need
- // to mark packets entering via Primary ENI as all the pods in v6 mode will be behind primary ENI. Will have to
- // start doing that once we start supporting custom networking mode in v6.
ipProtocol = iptables.ProtocolIPv6
}
@@ -441,27 +440,25 @@ func (n *linuxNetwork) updateHostIptablesRules(vpcCIDRs []string, primaryMAC str
return errors.Wrap(err, "host network setup: failed to create iptables")
}
- if v4Enabled {
- iptablesSNATRules, err := n.buildIptablesSNATRules(vpcCIDRs, primaryAddr, primaryIntf, ipt)
- if err != nil {
- return err
- }
- if err := n.updateIptablesRules(iptablesSNATRules, ipt); err != nil {
- return err
- }
+ iptablesSNATRules, err := n.buildIptablesSNATRules(vpcCIDRs, primaryAddr, primaryIntf, ipt, v6Enabled)
+ if err != nil {
+ return err
+ }
+ if err := n.updateIptablesRules(iptablesSNATRules, ipt); err != nil {
+ return err
+ }
- iptablesConnmarkRules, err := n.buildIptablesConnmarkRules(vpcCIDRs, ipt)
- if err != nil {
- return err
- }
- if err := n.updateIptablesRules(iptablesConnmarkRules, ipt); err != nil {
- return err
- }
+ iptablesConnmarkRules, err := n.buildIptablesConnmarkRules(vpcCIDRs, ipt)
+ if err != nil {
+ return err
+ }
+ if err := n.updateIptablesRules(iptablesConnmarkRules, ipt); err != nil {
+ return err
}
return nil
}
-func (n *linuxNetwork) buildIptablesSNATRules(vpcCIDRs []string, primaryAddr *net.IP, primaryIntf string, ipt iptableswrapper.IPTablesIface) ([]iptablesRule, error) {
+func (n *linuxNetwork) buildIptablesSNATRules(vpcCIDRs []string, primaryAddr *net.IP, primaryIntf string, ipt iptableswrapper.IPTablesIface, v6Enabled bool) ([]iptablesRule, error) {
type snatCIDR struct {
cidr string
isExclusion bool
@@ -499,6 +496,7 @@ func (n *linuxNetwork) buildIptablesSNATRules(vpcCIDRs []string, primaryAddr *ne
"-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-0",
}})
+ // Exclude VPC traffic from SNAT rule
for _, cidr := range allCIDRs {
comment := "AWS SNAT CHAIN"
if cidr.isExclusion {
@@ -516,6 +514,18 @@ func (n *linuxNetwork) buildIptablesSNATRules(vpcCIDRs []string, primaryAddr *ne
}})
}
+ // Exclude Multicast link local traffic from SNAT rule. This is required for DHCPv6
+ if v6Enabled {
+ iptableRules = append(iptableRules, iptablesRule{
+ name: chain,
+ shouldExist: !n.useExternalSNAT,
+ table: "nat",
+ chain: chain,
+ rule: []string{
+ "-d", "ff00::/8", "-m", "comment", "--comment", "SKIP LINKLOCAL", "-j", "RETURN",
+ }})
+ }
+
// Prepare the Desired Rule for SNAT Rule for non-pod ENIs
snatRule := []string{"!", "-o", "vlan+",
"-m", "comment", "--comment", "AWS, SNAT",
@@ -1015,21 +1025,27 @@ func GetIPv4Gateway(eniSubnetCIDR *net.IPNet) net.IP {
}
// SetupENINetwork adds default route to route table (eni-<eni_table>), so it does not need to be called on the primary ENI
-func (n *linuxNetwork) SetupENINetwork(eniIP string, eniMAC string, deviceNumber int, eniSubnetCIDR string) error {
- return setupENINetwork(eniIP, eniMAC, deviceNumber, eniSubnetCIDR, n.netLink, retryLinkByMacInterval, retryRouteAddInterval, n.mtu)
+func (n *linuxNetwork) SetupENINetwork(eniIP string, eniMAC string, deviceNumber int, networkCard int, eniSubnetCIDR string, maxENIPerNIC int) error {
+ return setupENINetwork(eniIP, eniMAC, deviceNumber, networkCard, eniSubnetCIDR, n.netLink, retryLinkByMacInterval, retryRouteAddInterval, n.mtu, maxENIPerNIC)
}
-func setupENINetwork(eniIP string, eniMAC string, deviceNumber int, eniSubnetCIDR string, netLink netlinkwrapper.NetLink,
- retryLinkByMacInterval time.Duration, retryRouteAddInterval time.Duration, mtu int) error {
- if deviceNumber == 0 {
+// Even if this is called for primary IF on multicard instance
+func setupENINetwork(eniIP string, eniMac string, deviceNumber int, networkCard int, eniSubnetCIDR string, netLink netlinkwrapper.NetLink,
+ retryLinkByMacInterval time.Duration, retryRouteAddInterval time.Duration, mtu int, maxENIPerNIC int) error {
+
+ if deviceNumber == 0 && networkCard == 0 {
return errors.New("setupENINetwork should never be called on the primary ENI")
}
- tableNumber := deviceNumber + 1
- log.Infof("Setting up network for an ENI with IP address %s, MAC address %s, CIDR %s and route table %d",
- eniIP, eniMAC, eniSubnetCIDR, tableNumber)
- link, err := linkByMac(eniMAC, netLink, retryLinkByMacInterval)
+
+ // CNI will never have route table 1 as we don't setup device number 0 on network card 0
+ // So, table number 1 for CNI is same as main table
+ tableNumber := (networkCard * maxENIPerNIC) + deviceNumber + 1
+ log.Infof("Setting up network for an ENI with IP address %s, MAC address %s, CIDR %s, route table %d and Network Card %d",
+ eniIP, eniMac, eniSubnetCIDR, tableNumber, networkCard)
+
+ link, err := linkByMac(eniMac, netLink, retryLinkByMacInterval)
if err != nil {
- return errors.Wrapf(err, "setupENINetwork: failed to find the link which uses MAC address %s", eniMAC)
+ return errors.Wrapf(err, "setupENINetwork: failed to find the link which uses MAC address %s", eniMac)
}
if err = netLink.LinkSetMTU(link, mtu); err != nil {
@@ -1045,6 +1061,7 @@ func setupENINetwork(eniIP string, eniMAC string, deviceNumber int, eniSubnetCID
if err != nil {
return errors.Wrapf(err, "setupENINetwork: invalid IP CIDR block %s", eniSubnetCIDR)
}
+
// Get gateway IP address for ENI
var gw net.IP
if isV6 {
diff --git a/pkg/networkutils/network_test.go b/pkg/networkutils/network_test.go
index fd5950ab84..b297f644bf 100644
--- a/pkg/networkutils/network_test.go
+++ b/pkg/networkutils/network_test.go
@@ -49,6 +49,8 @@ const (
testEniSubnet = "10.10.0.0/16"
testEniV6Subnet = "2600::/64"
testEniV6Gateway = "fe80::c9d:5dff:fec4:f389"
+ testNetworkCard = 0
+ testMaxENIPerNIC = 4
// Default MTU of ENI and veth
// defined in plugins/routed-eni/driver/driver.go, pkg/networkutils/network.go
testMTU = 9001
@@ -122,7 +124,7 @@ func TestSetupENINetwork(t *testing.T) {
mockNetLink.EXPECT().RouteDel(gomock.Any()).Return(nil)
- err = setupENINetwork(testEniIP, testMAC2, testTable, testEniSubnet, mockNetLink, 0*time.Second, 0*time.Second, testMTU)
+ err = setupENINetwork(testEniIP, testMAC2, testTable, testNetworkCard, testEniSubnet, mockNetLink, 0*time.Second, 0*time.Second, testMTU, testMaxENIPerNIC)
assert.NoError(t, err)
}
@@ -171,7 +173,7 @@ func TestSetupENIV6Network(t *testing.T) {
mockNetLink.EXPECT().RouteReplace(gomock.Any()).Return(nil)
mockNetLink.EXPECT().RouteDel(gomock.Any()).Return(nil)
- err = setupENINetwork(testEniIP6, testMAC2, testTable, testEniV6Subnet, mockNetLink, 0*time.Second, 0*time.Second, testMTU)
+ err = setupENINetwork(testEniIP6, testMAC2, testTable, testNetworkCard, testEniV6Subnet, mockNetLink, 0*time.Second, 0*time.Second, testMTU, testMaxENIPerNIC)
assert.NoError(t, err)
}
@@ -185,7 +187,7 @@ func TestSetupENINetworkMACFail(t *testing.T) {
mockNetLink.EXPECT().LinkList().Return(nil, fmt.Errorf("simulated failure"))
}
- err := setupENINetwork(testEniIP, testMAC2, testTable, testEniSubnet, mockNetLink, 0*time.Second, 0*time.Second, testMTU)
+ err := setupENINetwork(testEniIP, testMAC2, testTable, testNetworkCard, testEniSubnet, mockNetLink, 0*time.Second, 0*time.Second, testMTU, testMaxENIPerNIC)
assert.Errorf(t, err, "simulated failure")
}
@@ -193,7 +195,7 @@ func TestSetupENINetworkErrorOnPrimaryENI(t *testing.T) {
ctrl, mockNetLink, _, _, _ := setup(t)
defer ctrl.Finish()
deviceNumber := 0
- err := setupENINetwork(testEniIP, testMAC2, deviceNumber, testEniSubnet, mockNetLink, 0*time.Second, 0*time.Second, testMTU)
+ err := setupENINetwork(testEniIP, testMAC2, deviceNumber, testNetworkCard, testEniSubnet, mockNetLink, 0*time.Second, 0*time.Second, testMTU, testMaxENIPerNIC)
assert.Error(t, err)
}
@@ -991,7 +993,7 @@ func TestSetupHostNetworkWithIPv6Enabled(t *testing.T) {
setupNetLinkMocks(ctrl, mockNetLink)
var vpcCIDRs []string
- err := ln.SetupHostNetwork(vpcCIDRs, loopback, &testEniIPNet, false, false, true)
+ err := ln.SetupHostNetwork(vpcCIDRs, loopback, &testEniIP6Net, false, false, true)
assert.NoError(t, err)
assert.Equal(t, map[string]map[string][][]string{
@@ -1005,6 +1007,37 @@ func TestSetupHostNetworkWithIPv6Enabled(t *testing.T) {
},
},
},
+ "mangle": {
+ "PREROUTING": [][]string{
+ {
+ "-m", "comment", "--comment", "AWS, primary ENI", "-i", "lo", "-m", "addrtype", "--dst-type", "LOCAL", "--limit-iface-in", "-j", "CONNMARK", "--set-mark", "0x80/0x80",
+ },
+ {
+ "-m", "comment", "--comment", "AWS, primary ENI", "-i", "eni+", "-j", "CONNMARK", "--restore-mark", "--mask", "0x80",
+ },
+ {
+ "-m", "comment", "--comment", "AWS, primary ENI", "-i", "vlan+", "-j", "CONNMARK", "--restore-mark", "--mask", "0x80",
+ },
+ },
+ },
+ "nat": {
+ "PREROUTING": [][]string{
+ {"-i", "eni+", "-m", "comment", "--comment", "AWS, outbound connections", "-j", "AWS-CONNMARK-CHAIN-0"},
+ {"-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--restore-mark", "--mask", "0x80"},
+ },
+ "POSTROUTING": [][]string{
+ {"-m", "comment", "--comment", "AWS SNAT CHAIN", "-j", "AWS-SNAT-CHAIN-0"},
+ },
+ "AWS-CONNMARK-CHAIN-0": [][]string{
+ {"-N", "AWS-CONNMARK-CHAIN-0"},
+ {"-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--set-xmark", "0x80/0x80"},
+ },
+ "AWS-SNAT-CHAIN-0": [][]string{
+ {"-N", "AWS-SNAT-CHAIN-0"},
+ {"-d", "ff00::/8", "-m", "comment", "--comment", "SKIP LINKLOCAL", "-j", "RETURN"},
+ {"!", "-o", "vlan+", "-m", "comment", "--comment", "AWS, SNAT", "-m", "addrtype", "!", "--dst-type", "LOCAL", "-j", "SNAT", "--to-source", "2600::2"},
+ },
+ },
}, mockIptables.(*mock_iptables.MockIptables).DataplaneState)
}
diff --git a/rpc/rpc.pb.go b/rpc/rpc.pb.go
index f58cd2b05b..844effe697 100644
--- a/rpc/rpc.pb.go
+++ b/rpc/rpc.pb.go
@@ -1,7 +1,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
-// protoc-gen-go v1.31.0
-// protoc v4.25.6
+// protoc-gen-go v1.33.0
+// protoc v5.27.1
// source: rpc.proto
package rpc
@@ -37,7 +37,8 @@ type AddNetworkRequest struct {
ContainerID string `protobuf:"bytes,7,opt,name=ContainerID,proto3" json:"ContainerID,omitempty"`
IfName string `protobuf:"bytes,5,opt,name=IfName,proto3" json:"IfName,omitempty"`
NetworkName string `protobuf:"bytes,6,opt,name=NetworkName,proto3" json:"NetworkName,omitempty"`
- Netns string `protobuf:"bytes,4,opt,name=Netns,proto3" json:"Netns,omitempty"` // next field: 9
+ Netns string `protobuf:"bytes,4,opt,name=Netns,proto3" json:"Netns,omitempty"`
+ RequiresMultiNICAttachment bool `protobuf:"varint,9,opt,name=RequiresMultiNICAttachment,proto3" json:"RequiresMultiNICAttachment,omitempty"` // next field: 10
}
func (x *AddNetworkRequest) Reset() {
@@ -128,18 +129,94 @@ func (x *AddNetworkRequest) GetNetns() string {
return ""
}
+func (x *AddNetworkRequest) GetRequiresMultiNICAttachment() bool {
+ if x != nil {
+ return x.RequiresMultiNICAttachment
+ }
+ return false
+}
+
+type IPAddress struct {
+ state protoimpl.MessageState
+ sizeCache protoimpl.SizeCache
+ unknownFields protoimpl.UnknownFields
+
+ IPv4Addr string `protobuf:"bytes,1,opt,name=IPv4Addr,proto3" json:"IPv4Addr,omitempty"`
+ IPv6Addr string `protobuf:"bytes,2,opt,name=IPv6Addr,proto3" json:"IPv6Addr,omitempty"`
+ DeviceNumber int32 `protobuf:"varint,3,opt,name=DeviceNumber,proto3" json:"DeviceNumber,omitempty"`
+ RouteTableId int32 `protobuf:"varint,4,opt,name=RouteTableId,proto3" json:"RouteTableId,omitempty"`
+}
+
+func (x *IPAddress) Reset() {
+ *x = IPAddress{}
+ if protoimpl.UnsafeEnabled {
+ mi := &file_rpc_proto_msgTypes[1]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+ }
+}
+
+func (x *IPAddress) String() string {
+ return protoimpl.X.MessageStringOf(x)
+}
+
+func (*IPAddress) ProtoMessage() {}
+
+func (x *IPAddress) ProtoReflect() protoreflect.Message {
+ mi := &file_rpc_proto_msgTypes[1]
+ if protoimpl.UnsafeEnabled && x != nil {
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ if ms.LoadMessageInfo() == nil {
+ ms.StoreMessageInfo(mi)
+ }
+ return ms
+ }
+ return mi.MessageOf(x)
+}
+
+// Deprecated: Use IPAddress.ProtoReflect.Descriptor instead.
+func (*IPAddress) Descriptor() ([]byte, []int) {
+ return file_rpc_proto_rawDescGZIP(), []int{1}
+}
+
+func (x *IPAddress) GetIPv4Addr() string {
+ if x != nil {
+ return x.IPv4Addr
+ }
+ return ""
+}
+
+func (x *IPAddress) GetIPv6Addr() string {
+ if x != nil {
+ return x.IPv6Addr
+ }
+ return ""
+}
+
+func (x *IPAddress) GetDeviceNumber() int32 {
+ if x != nil {
+ return x.DeviceNumber
+ }
+ return 0
+}
+
+func (x *IPAddress) GetRouteTableId() int32 {
+ if x != nil {
+ return x.RouteTableId
+ }
+ return 0
+}
+
type AddNetworkReply struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
- Success bool `protobuf:"varint,1,opt,name=Success,proto3" json:"Success,omitempty"`
- IPv4Addr string `protobuf:"bytes,2,opt,name=IPv4Addr,proto3" json:"IPv4Addr,omitempty"`
- IPv6Addr string `protobuf:"bytes,11,opt,name=IPv6Addr,proto3" json:"IPv6Addr,omitempty"`
- DeviceNumber int32 `protobuf:"varint,4,opt,name=DeviceNumber,proto3" json:"DeviceNumber,omitempty"`
- UseExternalSNAT bool `protobuf:"varint,5,opt,name=UseExternalSNAT,proto3" json:"UseExternalSNAT,omitempty"`
- VPCv4CIDRs []string `protobuf:"bytes,6,rep,name=VPCv4CIDRs,proto3" json:"VPCv4CIDRs,omitempty"`
- VPCv6CIDRs []string `protobuf:"bytes,12,rep,name=VPCv6CIDRs,proto3" json:"VPCv6CIDRs,omitempty"`
+ Success bool `protobuf:"varint,1,opt,name=Success,proto3" json:"Success,omitempty"`
+ IPAddress []*IPAddress `protobuf:"bytes,2,rep,name=IPAddress,proto3" json:"IPAddress,omitempty"`
+ UseExternalSNAT bool `protobuf:"varint,5,opt,name=UseExternalSNAT,proto3" json:"UseExternalSNAT,omitempty"`
+ VPCv4CIDRs []string `protobuf:"bytes,6,rep,name=VPCv4CIDRs,proto3" json:"VPCv4CIDRs,omitempty"`
+ VPCv6CIDRs []string `protobuf:"bytes,12,rep,name=VPCv6CIDRs,proto3" json:"VPCv6CIDRs,omitempty"`
// start of pod-eni parameters
PodVlanId int32 `protobuf:"varint,7,opt,name=PodVlanId,proto3" json:"PodVlanId,omitempty"`
PodENIMAC string `protobuf:"bytes,8,opt,name=PodENIMAC,proto3" json:"PodENIMAC,omitempty"`
@@ -151,7 +228,7 @@ type AddNetworkReply struct {
func (x *AddNetworkReply) Reset() {
*x = AddNetworkReply{}
if protoimpl.UnsafeEnabled {
- mi := &file_rpc_proto_msgTypes[1]
+ mi := &file_rpc_proto_msgTypes[2]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -164,7 +241,7 @@ func (x *AddNetworkReply) String() string {
func (*AddNetworkReply) ProtoMessage() {}
func (x *AddNetworkReply) ProtoReflect() protoreflect.Message {
- mi := &file_rpc_proto_msgTypes[1]
+ mi := &file_rpc_proto_msgTypes[2]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -177,7 +254,7 @@ func (x *AddNetworkReply) ProtoReflect() protoreflect.Message {
// Deprecated: Use AddNetworkReply.ProtoReflect.Descriptor instead.
func (*AddNetworkReply) Descriptor() ([]byte, []int) {
- return file_rpc_proto_rawDescGZIP(), []int{1}
+ return file_rpc_proto_rawDescGZIP(), []int{2}
}
func (x *AddNetworkReply) GetSuccess() bool {
@@ -187,25 +264,11 @@ func (x *AddNetworkReply) GetSuccess() bool {
return false
}
-func (x *AddNetworkReply) GetIPv4Addr() string {
+func (x *AddNetworkReply) GetIPAddress() []*IPAddress {
if x != nil {
- return x.IPv4Addr
+ return x.IPAddress
}
- return ""
-}
-
-func (x *AddNetworkReply) GetIPv6Addr() string {
- if x != nil {
- return x.IPv6Addr
- }
- return ""
-}
-
-func (x *AddNetworkReply) GetDeviceNumber() int32 {
- if x != nil {
- return x.DeviceNumber
- }
- return 0
+ return nil
}
func (x *AddNetworkReply) GetUseExternalSNAT() bool {
@@ -282,7 +345,7 @@ type DelNetworkRequest struct {
func (x *DelNetworkRequest) Reset() {
*x = DelNetworkRequest{}
if protoimpl.UnsafeEnabled {
- mi := &file_rpc_proto_msgTypes[2]
+ mi := &file_rpc_proto_msgTypes[3]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -295,7 +358,7 @@ func (x *DelNetworkRequest) String() string {
func (*DelNetworkRequest) ProtoMessage() {}
func (x *DelNetworkRequest) ProtoReflect() protoreflect.Message {
- mi := &file_rpc_proto_msgTypes[2]
+ mi := &file_rpc_proto_msgTypes[3]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -308,7 +371,7 @@ func (x *DelNetworkRequest) ProtoReflect() protoreflect.Message {
// Deprecated: Use DelNetworkRequest.ProtoReflect.Descriptor instead.
func (*DelNetworkRequest) Descriptor() ([]byte, []int) {
- return file_rpc_proto_rawDescGZIP(), []int{2}
+ return file_rpc_proto_rawDescGZIP(), []int{3}
}
func (x *DelNetworkRequest) GetClientVersion() string {
@@ -372,10 +435,8 @@ type DelNetworkReply struct {
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
- Success bool `protobuf:"varint,1,opt,name=Success,proto3" json:"Success,omitempty"`
- IPv4Addr string `protobuf:"bytes,2,opt,name=IPv4Addr,proto3" json:"IPv4Addr,omitempty"`
- IPv6Addr string `protobuf:"bytes,5,opt,name=IPv6Addr,proto3" json:"IPv6Addr,omitempty"`
- DeviceNumber int32 `protobuf:"varint,3,opt,name=DeviceNumber,proto3" json:"DeviceNumber,omitempty"`
+ Success bool `protobuf:"varint,1,opt,name=Success,proto3" json:"Success,omitempty"`
+ IPAddress []*IPAddress `protobuf:"bytes,2,rep,name=IPAddress,proto3" json:"IPAddress,omitempty"`
// start of pod-eni parameters
PodVlanId int32 `protobuf:"varint,4,opt,name=PodVlanId,proto3" json:"PodVlanId,omitempty"` // end of pod-eni parameters
}
@@ -383,7 +444,7 @@ type DelNetworkReply struct {
func (x *DelNetworkReply) Reset() {
*x = DelNetworkReply{}
if protoimpl.UnsafeEnabled {
- mi := &file_rpc_proto_msgTypes[3]
+ mi := &file_rpc_proto_msgTypes[4]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -396,7 +457,7 @@ func (x *DelNetworkReply) String() string {
func (*DelNetworkReply) ProtoMessage() {}
func (x *DelNetworkReply) ProtoReflect() protoreflect.Message {
- mi := &file_rpc_proto_msgTypes[3]
+ mi := &file_rpc_proto_msgTypes[4]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -409,7 +470,7 @@ func (x *DelNetworkReply) ProtoReflect() protoreflect.Message {
// Deprecated: Use DelNetworkReply.ProtoReflect.Descriptor instead.
func (*DelNetworkReply) Descriptor() ([]byte, []int) {
- return file_rpc_proto_rawDescGZIP(), []int{3}
+ return file_rpc_proto_rawDescGZIP(), []int{4}
}
func (x *DelNetworkReply) GetSuccess() bool {
@@ -419,25 +480,11 @@ func (x *DelNetworkReply) GetSuccess() bool {
return false
}
-func (x *DelNetworkReply) GetIPv4Addr() string {
- if x != nil {
- return x.IPv4Addr
- }
- return ""
-}
-
-func (x *DelNetworkReply) GetIPv6Addr() string {
+func (x *DelNetworkReply) GetIPAddress() []*IPAddress {
if x != nil {
- return x.IPv6Addr
+ return x.IPAddress
}
- return ""
-}
-
-func (x *DelNetworkReply) GetDeviceNumber() int32 {
- if x != nil {
- return x.DeviceNumber
- }
- return 0
+ return nil
}
func (x *DelNetworkReply) GetPodVlanId() int32 {
@@ -447,33 +494,33 @@ func (x *DelNetworkReply) GetPodVlanId() int32 {
return 0
}
-type EnforceNpRequest struct {
+type DeleteNpRequest struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
- K8S_POD_NAME string `protobuf:"bytes,1,opt,name=K8S_POD_NAME,json=K8SPODNAME,proto3" json:"K8S_POD_NAME,omitempty"`
- K8S_POD_NAMESPACE string `protobuf:"bytes,2,opt,name=K8S_POD_NAMESPACE,json=K8SPODNAMESPACE,proto3" json:"K8S_POD_NAMESPACE,omitempty"`
- NETWORK_POLICY_MODE string `protobuf:"bytes,3,opt,name=NETWORK_POLICY_MODE,json=NETWORKPOLICYMODE,proto3" json:"NETWORK_POLICY_MODE,omitempty"`
+ Success bool `protobuf:"varint,1,opt,name=Success,proto3" json:"Success,omitempty"`
+ K8S_POD_NAME string `protobuf:"bytes,2,opt,name=K8S_POD_NAME,json=K8SPODNAME,proto3" json:"K8S_POD_NAME,omitempty"`
+ K8S_POD_NAMESPACE string `protobuf:"bytes,3,opt,name=K8S_POD_NAMESPACE,json=K8SPODNAMESPACE,proto3" json:"K8S_POD_NAMESPACE,omitempty"`
}
-func (x *EnforceNpRequest) Reset() {
- *x = EnforceNpRequest{}
+func (x *DeleteNpRequest) Reset() {
+ *x = DeleteNpRequest{}
if protoimpl.UnsafeEnabled {
- mi := &file_rpc_proto_msgTypes[4]
+ mi := &file_rpc_proto_msgTypes[5]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
-func (x *EnforceNpRequest) String() string {
+func (x *DeleteNpRequest) String() string {
return protoimpl.X.MessageStringOf(x)
}
-func (*EnforceNpRequest) ProtoMessage() {}
+func (*DeleteNpRequest) ProtoMessage() {}
-func (x *EnforceNpRequest) ProtoReflect() protoreflect.Message {
- mi := &file_rpc_proto_msgTypes[4]
+func (x *DeleteNpRequest) ProtoReflect() protoreflect.Message {
+ mi := &file_rpc_proto_msgTypes[5]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -484,33 +531,33 @@ func (x *EnforceNpRequest) ProtoReflect() protoreflect.Message {
return mi.MessageOf(x)
}
-// Deprecated: Use EnforceNpRequest.ProtoReflect.Descriptor instead.
-func (*EnforceNpRequest) Descriptor() ([]byte, []int) {
- return file_rpc_proto_rawDescGZIP(), []int{4}
+// Deprecated: Use DeleteNpRequest.ProtoReflect.Descriptor instead.
+func (*DeleteNpRequest) Descriptor() ([]byte, []int) {
+ return file_rpc_proto_rawDescGZIP(), []int{5}
}
-func (x *EnforceNpRequest) GetK8S_POD_NAME() string {
+func (x *DeleteNpRequest) GetSuccess() bool {
if x != nil {
- return x.K8S_POD_NAME
+ return x.Success
}
- return ""
+ return false
}
-func (x *EnforceNpRequest) GetK8S_POD_NAMESPACE() string {
+func (x *DeleteNpRequest) GetK8S_POD_NAME() string {
if x != nil {
- return x.K8S_POD_NAMESPACE
+ return x.K8S_POD_NAME
}
return ""
}
-func (x *EnforceNpRequest) GetNETWORK_POLICY_MODE() string {
+func (x *DeleteNpRequest) GetK8S_POD_NAMESPACE() string {
if x != nil {
- return x.NETWORK_POLICY_MODE
+ return x.K8S_POD_NAMESPACE
}
return ""
}
-type EnforceNpReply struct {
+type DeleteNpReply struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
@@ -518,23 +565,23 @@ type EnforceNpReply struct {
Success bool `protobuf:"varint,1,opt,name=Success,proto3" json:"Success,omitempty"`
}
-func (x *EnforceNpReply) Reset() {
- *x = EnforceNpReply{}
+func (x *DeleteNpReply) Reset() {
+ *x = DeleteNpReply{}
if protoimpl.UnsafeEnabled {
- mi := &file_rpc_proto_msgTypes[5]
+ mi := &file_rpc_proto_msgTypes[6]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
-func (x *EnforceNpReply) String() string {
+func (x *DeleteNpReply) String() string {
return protoimpl.X.MessageStringOf(x)
}
-func (*EnforceNpReply) ProtoMessage() {}
+func (*DeleteNpReply) ProtoMessage() {}
-func (x *EnforceNpReply) ProtoReflect() protoreflect.Message {
- mi := &file_rpc_proto_msgTypes[5]
+func (x *DeleteNpReply) ProtoReflect() protoreflect.Message {
+ mi := &file_rpc_proto_msgTypes[6]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -545,44 +592,46 @@ func (x *EnforceNpReply) ProtoReflect() protoreflect.Message {
return mi.MessageOf(x)
}
-// Deprecated: Use EnforceNpReply.ProtoReflect.Descriptor instead.
-func (*EnforceNpReply) Descriptor() ([]byte, []int) {
- return file_rpc_proto_rawDescGZIP(), []int{5}
+// Deprecated: Use DeleteNpReply.ProtoReflect.Descriptor instead.
+func (*DeleteNpReply) Descriptor() ([]byte, []int) {
+ return file_rpc_proto_rawDescGZIP(), []int{6}
}
-func (x *EnforceNpReply) GetSuccess() bool {
+func (x *DeleteNpReply) GetSuccess() bool {
if x != nil {
return x.Success
}
return false
}
-type DeleteNpRequest struct {
+type EnforceNpRequest struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
- K8S_POD_NAME string `protobuf:"bytes,1,opt,name=K8S_POD_NAME,json=K8SPODNAME,proto3" json:"K8S_POD_NAME,omitempty"`
- K8S_POD_NAMESPACE string `protobuf:"bytes,2,opt,name=K8S_POD_NAMESPACE,json=K8SPODNAMESPACE,proto3" json:"K8S_POD_NAMESPACE,omitempty"`
+ K8S_POD_NAME string `protobuf:"bytes,1,opt,name=K8S_POD_NAME,json=K8SPODNAME,proto3" json:"K8S_POD_NAME,omitempty"`
+ K8S_POD_NAMESPACE string `protobuf:"bytes,2,opt,name=K8S_POD_NAMESPACE,json=K8SPODNAMESPACE,proto3" json:"K8S_POD_NAMESPACE,omitempty"`
+ NETWORK_POLICY_MODE string `protobuf:"bytes,3,opt,name=NETWORK_POLICY_MODE,json=NETWORKPOLICYMODE,proto3" json:"NETWORK_POLICY_MODE,omitempty"`
+ InterfaceCount int32 `protobuf:"varint,4,opt,name=InterfaceCount,proto3" json:"InterfaceCount,omitempty"`
}
-func (x *DeleteNpRequest) Reset() {
- *x = DeleteNpRequest{}
+func (x *EnforceNpRequest) Reset() {
+ *x = EnforceNpRequest{}
if protoimpl.UnsafeEnabled {
- mi := &file_rpc_proto_msgTypes[6]
+ mi := &file_rpc_proto_msgTypes[7]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
-func (x *DeleteNpRequest) String() string {
+func (x *EnforceNpRequest) String() string {
return protoimpl.X.MessageStringOf(x)
}
-func (*DeleteNpRequest) ProtoMessage() {}
+func (*EnforceNpRequest) ProtoMessage() {}
-func (x *DeleteNpRequest) ProtoReflect() protoreflect.Message {
- mi := &file_rpc_proto_msgTypes[6]
+func (x *EnforceNpRequest) ProtoReflect() protoreflect.Message {
+ mi := &file_rpc_proto_msgTypes[7]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -593,26 +642,40 @@ func (x *DeleteNpRequest) ProtoReflect() protoreflect.Message {
return mi.MessageOf(x)
}
-// Deprecated: Use DeleteNpRequest.ProtoReflect.Descriptor instead.
-func (*DeleteNpRequest) Descriptor() ([]byte, []int) {
- return file_rpc_proto_rawDescGZIP(), []int{6}
+// Deprecated: Use EnforceNpRequest.ProtoReflect.Descriptor instead.
+func (*EnforceNpRequest) Descriptor() ([]byte, []int) {
+ return file_rpc_proto_rawDescGZIP(), []int{7}
}
-func (x *DeleteNpRequest) GetK8S_POD_NAME() string {
+func (x *EnforceNpRequest) GetK8S_POD_NAME() string {
if x != nil {
return x.K8S_POD_NAME
}
return ""
}
-func (x *DeleteNpRequest) GetK8S_POD_NAMESPACE() string {
+func (x *EnforceNpRequest) GetK8S_POD_NAMESPACE() string {
if x != nil {
return x.K8S_POD_NAMESPACE
}
return ""
}
-type DeleteNpReply struct {
+func (x *EnforceNpRequest) GetNETWORK_POLICY_MODE() string {
+ if x != nil {
+ return x.NETWORK_POLICY_MODE
+ }
+ return ""
+}
+
+func (x *EnforceNpRequest) GetInterfaceCount() int32 {
+ if x != nil {
+ return x.InterfaceCount
+ }
+ return 0
+}
+
+type EnforceNpReply struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
@@ -620,23 +683,23 @@ type DeleteNpReply struct {
Success bool `protobuf:"varint,1,opt,name=Success,proto3" json:"Success,omitempty"`
}
-func (x *DeleteNpReply) Reset() {
- *x = DeleteNpReply{}
+func (x *EnforceNpReply) Reset() {
+ *x = EnforceNpReply{}
if protoimpl.UnsafeEnabled {
- mi := &file_rpc_proto_msgTypes[7]
+ mi := &file_rpc_proto_msgTypes[8]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
-func (x *DeleteNpReply) String() string {
+func (x *EnforceNpReply) String() string {
return protoimpl.X.MessageStringOf(x)
}
-func (*DeleteNpReply) ProtoMessage() {}
+func (*EnforceNpReply) ProtoMessage() {}
-func (x *DeleteNpReply) ProtoReflect() protoreflect.Message {
- mi := &file_rpc_proto_msgTypes[7]
+func (x *EnforceNpReply) ProtoReflect() protoreflect.Message {
+ mi := &file_rpc_proto_msgTypes[8]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -647,12 +710,12 @@ func (x *DeleteNpReply) ProtoReflect() protoreflect.Message {
return mi.MessageOf(x)
}
-// Deprecated: Use DeleteNpReply.ProtoReflect.Descriptor instead.
-func (*DeleteNpReply) Descriptor() ([]byte, []int) {
- return file_rpc_proto_rawDescGZIP(), []int{7}
+// Deprecated: Use EnforceNpReply.ProtoReflect.Descriptor instead.
+func (*EnforceNpReply) Descriptor() ([]byte, []int) {
+ return file_rpc_proto_rawDescGZIP(), []int{8}
}
-func (x *DeleteNpReply) GetSuccess() bool {
+func (x *EnforceNpReply) GetSuccess() bool {
if x != nil {
return x.Success
}
@@ -670,7 +733,7 @@ type NetworkPolicyAgentConfigReply struct {
func (x *NetworkPolicyAgentConfigReply) Reset() {
*x = NetworkPolicyAgentConfigReply{}
if protoimpl.UnsafeEnabled {
- mi := &file_rpc_proto_msgTypes[8]
+ mi := &file_rpc_proto_msgTypes[9]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -683,7 +746,7 @@ func (x *NetworkPolicyAgentConfigReply) String() string {
func (*NetworkPolicyAgentConfigReply) ProtoMessage() {}
func (x *NetworkPolicyAgentConfigReply) ProtoReflect() protoreflect.Message {
- mi := &file_rpc_proto_msgTypes[8]
+ mi := &file_rpc_proto_msgTypes[9]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -696,7 +759,7 @@ func (x *NetworkPolicyAgentConfigReply) ProtoReflect() protoreflect.Message {
// Deprecated: Use NetworkPolicyAgentConfigReply.ProtoReflect.Descriptor instead.
func (*NetworkPolicyAgentConfigReply) Descriptor() ([]byte, []int) {
- return file_rpc_proto_rawDescGZIP(), []int{8}
+ return file_rpc_proto_rawDescGZIP(), []int{9}
}
func (x *NetworkPolicyAgentConfigReply) GetNetworkPolicyMode() string {
@@ -711,7 +774,7 @@ var File_rpc_proto protoreflect.FileDescriptor
var file_rpc_proto_rawDesc = []byte{
0x0a, 0x09, 0x72, 0x70, 0x63, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x03, 0x72, 0x70, 0x63,
0x1a, 0x1b, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75,
- 0x66, 0x2f, 0x65, 0x6d, 0x70, 0x74, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xb5, 0x02,
+ 0x66, 0x2f, 0x65, 0x6d, 0x70, 0x74, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xf5, 0x02,
0x0a, 0x11, 0x41, 0x64, 0x64, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x52, 0x65, 0x71, 0x75,
0x65, 0x73, 0x74, 0x12, 0x24, 0x0a, 0x0d, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x56, 0x65, 0x72,
0x73, 0x69, 0x6f, 0x6e, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x43, 0x6c, 0x69, 0x65,
@@ -731,15 +794,25 @@ var file_rpc_proto_rawDesc = []byte{
0x0b, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x06, 0x20, 0x01,
0x28, 0x09, 0x52, 0x0b, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x4e, 0x61, 0x6d, 0x65, 0x12,
0x14, 0x0a, 0x05, 0x4e, 0x65, 0x74, 0x6e, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05,
- 0x4e, 0x65, 0x74, 0x6e, 0x73, 0x22, 0xa9, 0x03, 0x0a, 0x0f, 0x41, 0x64, 0x64, 0x4e, 0x65, 0x74,
- 0x77, 0x6f, 0x72, 0x6b, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12, 0x18, 0x0a, 0x07, 0x53, 0x75, 0x63,
- 0x63, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x53, 0x75, 0x63, 0x63,
+ 0x4e, 0x65, 0x74, 0x6e, 0x73, 0x12, 0x3e, 0x0a, 0x1a, 0x52, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65,
+ 0x73, 0x4d, 0x75, 0x6c, 0x74, 0x69, 0x4e, 0x49, 0x43, 0x41, 0x74, 0x74, 0x61, 0x63, 0x68, 0x6d,
+ 0x65, 0x6e, 0x74, 0x18, 0x09, 0x20, 0x01, 0x28, 0x08, 0x52, 0x1a, 0x52, 0x65, 0x71, 0x75, 0x69,
+ 0x72, 0x65, 0x73, 0x4d, 0x75, 0x6c, 0x74, 0x69, 0x4e, 0x49, 0x43, 0x41, 0x74, 0x74, 0x61, 0x63,
+ 0x68, 0x6d, 0x65, 0x6e, 0x74, 0x22, 0x8b, 0x01, 0x0a, 0x09, 0x49, 0x50, 0x41, 0x64, 0x64, 0x72,
0x65, 0x73, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x49, 0x50, 0x76, 0x34, 0x41, 0x64, 0x64, 0x72, 0x18,
- 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x49, 0x50, 0x76, 0x34, 0x41, 0x64, 0x64, 0x72, 0x12,
- 0x1a, 0x0a, 0x08, 0x49, 0x50, 0x76, 0x36, 0x41, 0x64, 0x64, 0x72, 0x18, 0x0b, 0x20, 0x01, 0x28,
+ 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x49, 0x50, 0x76, 0x34, 0x41, 0x64, 0x64, 0x72, 0x12,
+ 0x1a, 0x0a, 0x08, 0x49, 0x50, 0x76, 0x36, 0x41, 0x64, 0x64, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28,
0x09, 0x52, 0x08, 0x49, 0x50, 0x76, 0x36, 0x41, 0x64, 0x64, 0x72, 0x12, 0x22, 0x0a, 0x0c, 0x44,
- 0x65, 0x76, 0x69, 0x63, 0x65, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28,
+ 0x65, 0x76, 0x69, 0x63, 0x65, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28,
0x05, 0x52, 0x0c, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x12,
+ 0x22, 0x0a, 0x0c, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x49, 0x64, 0x18,
+ 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0c, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x54, 0x61, 0x62, 0x6c,
+ 0x65, 0x49, 0x64, 0x22, 0xfb, 0x02, 0x0a, 0x0f, 0x41, 0x64, 0x64, 0x4e, 0x65, 0x74, 0x77, 0x6f,
+ 0x72, 0x6b, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12, 0x18, 0x0a, 0x07, 0x53, 0x75, 0x63, 0x63, 0x65,
+ 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73,
+ 0x73, 0x12, 0x2c, 0x0a, 0x09, 0x49, 0x50, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x02,
+ 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x49, 0x50, 0x41, 0x64, 0x64,
+ 0x72, 0x65, 0x73, 0x73, 0x52, 0x09, 0x49, 0x50, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12,
0x28, 0x0a, 0x0f, 0x55, 0x73, 0x65, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x53, 0x4e,
0x41, 0x54, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, 0x55, 0x73, 0x65, 0x45, 0x78, 0x74,
0x65, 0x72, 0x6e, 0x61, 0x6c, 0x53, 0x4e, 0x41, 0x54, 0x12, 0x1e, 0x0a, 0x0a, 0x56, 0x50, 0x43,
@@ -777,71 +850,72 @@ var file_rpc_proto_rawDesc = []byte{
0x44, 0x12, 0x16, 0x0a, 0x06, 0x49, 0x66, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28,
0x09, 0x52, 0x06, 0x49, 0x66, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x4e, 0x65, 0x74,
0x77, 0x6f, 0x72, 0x6b, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b,
- 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0xa5, 0x01, 0x0a, 0x0f,
- 0x44, 0x65, 0x6c, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12,
- 0x18, 0x0a, 0x07, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08,
- 0x52, 0x07, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x49, 0x50, 0x76,
- 0x34, 0x41, 0x64, 0x64, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x49, 0x50, 0x76,
- 0x34, 0x41, 0x64, 0x64, 0x72, 0x12, 0x1a, 0x0a, 0x08, 0x49, 0x50, 0x76, 0x36, 0x41, 0x64, 0x64,
- 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x49, 0x50, 0x76, 0x36, 0x41, 0x64, 0x64,
- 0x72, 0x12, 0x22, 0x0a, 0x0c, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x4e, 0x75, 0x6d, 0x62, 0x65,
- 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0c, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x4e,
- 0x75, 0x6d, 0x62, 0x65, 0x72, 0x12, 0x1c, 0x0a, 0x09, 0x50, 0x6f, 0x64, 0x56, 0x6c, 0x61, 0x6e,
- 0x49, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x09, 0x50, 0x6f, 0x64, 0x56, 0x6c, 0x61,
- 0x6e, 0x49, 0x64, 0x22, 0x90, 0x01, 0x0a, 0x10, 0x45, 0x6e, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x4e,
- 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x20, 0x0a, 0x0c, 0x4b, 0x38, 0x53, 0x5f,
- 0x50, 0x4f, 0x44, 0x5f, 0x4e, 0x41, 0x4d, 0x45, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a,
- 0x4b, 0x38, 0x53, 0x50, 0x4f, 0x44, 0x4e, 0x41, 0x4d, 0x45, 0x12, 0x2a, 0x0a, 0x11, 0x4b, 0x38,
- 0x53, 0x5f, 0x50, 0x4f, 0x44, 0x5f, 0x4e, 0x41, 0x4d, 0x45, 0x53, 0x50, 0x41, 0x43, 0x45, 0x18,
- 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x4b, 0x38, 0x53, 0x50, 0x4f, 0x44, 0x4e, 0x41, 0x4d,
- 0x45, 0x53, 0x50, 0x41, 0x43, 0x45, 0x12, 0x2e, 0x0a, 0x13, 0x4e, 0x45, 0x54, 0x57, 0x4f, 0x52,
- 0x4b, 0x5f, 0x50, 0x4f, 0x4c, 0x49, 0x43, 0x59, 0x5f, 0x4d, 0x4f, 0x44, 0x45, 0x18, 0x03, 0x20,
- 0x01, 0x28, 0x09, 0x52, 0x11, 0x4e, 0x45, 0x54, 0x57, 0x4f, 0x52, 0x4b, 0x50, 0x4f, 0x4c, 0x49,
- 0x43, 0x59, 0x4d, 0x4f, 0x44, 0x45, 0x22, 0x2a, 0x0a, 0x0e, 0x45, 0x6e, 0x66, 0x6f, 0x72, 0x63,
- 0x65, 0x4e, 0x70, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12, 0x18, 0x0a, 0x07, 0x53, 0x75, 0x63, 0x63,
- 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x53, 0x75, 0x63, 0x63, 0x65,
- 0x73, 0x73, 0x22, 0x5f, 0x0a, 0x0f, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x4e, 0x70, 0x52, 0x65,
- 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x20, 0x0a, 0x0c, 0x4b, 0x38, 0x53, 0x5f, 0x50, 0x4f, 0x44,
- 0x5f, 0x4e, 0x41, 0x4d, 0x45, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x4b, 0x38, 0x53,
- 0x50, 0x4f, 0x44, 0x4e, 0x41, 0x4d, 0x45, 0x12, 0x2a, 0x0a, 0x11, 0x4b, 0x38, 0x53, 0x5f, 0x50,
- 0x4f, 0x44, 0x5f, 0x4e, 0x41, 0x4d, 0x45, 0x53, 0x50, 0x41, 0x43, 0x45, 0x18, 0x02, 0x20, 0x01,
- 0x28, 0x09, 0x52, 0x0f, 0x4b, 0x38, 0x53, 0x50, 0x4f, 0x44, 0x4e, 0x41, 0x4d, 0x45, 0x53, 0x50,
- 0x41, 0x43, 0x45, 0x22, 0x29, 0x0a, 0x0d, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x4e, 0x70, 0x52,
- 0x65, 0x70, 0x6c, 0x79, 0x12, 0x18, 0x0a, 0x07, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18,
- 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x22, 0x4d,
- 0x0a, 0x1d, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x41,
- 0x67, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12,
- 0x2c, 0x0a, 0x11, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79,
- 0x4d, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x11, 0x4e, 0x65, 0x74, 0x77,
- 0x6f, 0x72, 0x6b, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x4d, 0x6f, 0x64, 0x65, 0x32, 0x88, 0x01,
- 0x0a, 0x0a, 0x43, 0x4e, 0x49, 0x42, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x12, 0x3c, 0x0a, 0x0a,
- 0x41, 0x64, 0x64, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x12, 0x16, 0x2e, 0x72, 0x70, 0x63,
- 0x2e, 0x41, 0x64, 0x64, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65,
- 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x41, 0x64, 0x64, 0x4e, 0x65, 0x74, 0x77,
- 0x6f, 0x72, 0x6b, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x3c, 0x0a, 0x0a, 0x44, 0x65,
- 0x6c, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x12, 0x16, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x44,
- 0x65, 0x6c, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
- 0x1a, 0x14, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x44, 0x65, 0x6c, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72,
- 0x6b, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x32, 0x86, 0x01, 0x0a, 0x09, 0x4e, 0x50, 0x42,
- 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x12, 0x3e, 0x0a, 0x0e, 0x45, 0x6e, 0x66, 0x6f, 0x72, 0x63,
- 0x65, 0x4e, 0x70, 0x54, 0x6f, 0x50, 0x6f, 0x64, 0x12, 0x15, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x45,
- 0x6e, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x4e, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a,
- 0x13, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x45, 0x6e, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x4e, 0x70, 0x52,
- 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x39, 0x0a, 0x0b, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65,
- 0x50, 0x6f, 0x64, 0x4e, 0x70, 0x12, 0x14, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x44, 0x65, 0x6c, 0x65,
- 0x74, 0x65, 0x4e, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x12, 0x2e, 0x72, 0x70,
- 0x63, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x4e, 0x70, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22,
- 0x00, 0x32, 0x6e, 0x0a, 0x13, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x53, 0x65, 0x72, 0x76, 0x65,
- 0x72, 0x42, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x12, 0x57, 0x0a, 0x17, 0x47, 0x65, 0x74, 0x4e,
- 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x43, 0x6f, 0x6e, 0x66,
- 0x69, 0x67, 0x73, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f,
- 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x22, 0x2e, 0x72, 0x70,
- 0x63, 0x2e, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x41,
- 0x67, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22,
- 0x00, 0x42, 0x2b, 0x5a, 0x29, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f,
- 0x61, 0x77, 0x73, 0x2f, 0x61, 0x6d, 0x61, 0x7a, 0x6f, 0x6e, 0x2d, 0x76, 0x70, 0x63, 0x2d, 0x63,
- 0x6e, 0x69, 0x2d, 0x6b, 0x38, 0x73, 0x2f, 0x72, 0x70, 0x63, 0x3b, 0x72, 0x70, 0x63, 0x62, 0x06,
- 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+ 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0x77, 0x0a, 0x0f, 0x44,
+ 0x65, 0x6c, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12, 0x18,
+ 0x0a, 0x07, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52,
+ 0x07, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x12, 0x2c, 0x0a, 0x09, 0x49, 0x50, 0x41, 0x64,
+ 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x72, 0x70,
+ 0x63, 0x2e, 0x49, 0x50, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x52, 0x09, 0x49, 0x50, 0x41,
+ 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x50, 0x6f, 0x64, 0x56, 0x6c, 0x61,
+ 0x6e, 0x49, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x09, 0x50, 0x6f, 0x64, 0x56, 0x6c,
+ 0x61, 0x6e, 0x49, 0x64, 0x22, 0x79, 0x0a, 0x0f, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x4e, 0x70,
+ 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x53, 0x75, 0x63, 0x63, 0x65,
+ 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73,
+ 0x73, 0x12, 0x20, 0x0a, 0x0c, 0x4b, 0x38, 0x53, 0x5f, 0x50, 0x4f, 0x44, 0x5f, 0x4e, 0x41, 0x4d,
+ 0x45, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x4b, 0x38, 0x53, 0x50, 0x4f, 0x44, 0x4e,
+ 0x41, 0x4d, 0x45, 0x12, 0x2a, 0x0a, 0x11, 0x4b, 0x38, 0x53, 0x5f, 0x50, 0x4f, 0x44, 0x5f, 0x4e,
+ 0x41, 0x4d, 0x45, 0x53, 0x50, 0x41, 0x43, 0x45, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f,
+ 0x4b, 0x38, 0x53, 0x50, 0x4f, 0x44, 0x4e, 0x41, 0x4d, 0x45, 0x53, 0x50, 0x41, 0x43, 0x45, 0x22,
+ 0x29, 0x0a, 0x0d, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x4e, 0x70, 0x52, 0x65, 0x70, 0x6c, 0x79,
+ 0x12, 0x18, 0x0a, 0x07, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28,
+ 0x08, 0x52, 0x07, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x22, 0xb8, 0x01, 0x0a, 0x10, 0x45,
+ 0x6e, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x4e, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12,
+ 0x20, 0x0a, 0x0c, 0x4b, 0x38, 0x53, 0x5f, 0x50, 0x4f, 0x44, 0x5f, 0x4e, 0x41, 0x4d, 0x45, 0x18,
+ 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x4b, 0x38, 0x53, 0x50, 0x4f, 0x44, 0x4e, 0x41, 0x4d,
+ 0x45, 0x12, 0x2a, 0x0a, 0x11, 0x4b, 0x38, 0x53, 0x5f, 0x50, 0x4f, 0x44, 0x5f, 0x4e, 0x41, 0x4d,
+ 0x45, 0x53, 0x50, 0x41, 0x43, 0x45, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x4b, 0x38,
+ 0x53, 0x50, 0x4f, 0x44, 0x4e, 0x41, 0x4d, 0x45, 0x53, 0x50, 0x41, 0x43, 0x45, 0x12, 0x2e, 0x0a,
+ 0x13, 0x4e, 0x45, 0x54, 0x57, 0x4f, 0x52, 0x4b, 0x5f, 0x50, 0x4f, 0x4c, 0x49, 0x43, 0x59, 0x5f,
+ 0x4d, 0x4f, 0x44, 0x45, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x11, 0x4e, 0x45, 0x54, 0x57,
+ 0x4f, 0x52, 0x4b, 0x50, 0x4f, 0x4c, 0x49, 0x43, 0x59, 0x4d, 0x4f, 0x44, 0x45, 0x12, 0x26, 0x0a,
+ 0x0e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x66, 0x61, 0x63, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x18,
+ 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x66, 0x61, 0x63, 0x65,
+ 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0x2a, 0x0a, 0x0e, 0x45, 0x6e, 0x66, 0x6f, 0x72, 0x63, 0x65,
+ 0x4e, 0x70, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12, 0x18, 0x0a, 0x07, 0x53, 0x75, 0x63, 0x63, 0x65,
+ 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73,
+ 0x73, 0x22, 0x4d, 0x0a, 0x1d, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x50, 0x6f, 0x6c, 0x69,
+ 0x63, 0x79, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x70,
+ 0x6c, 0x79, 0x12, 0x2c, 0x0a, 0x11, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x50, 0x6f, 0x6c,
+ 0x69, 0x63, 0x79, 0x4d, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x11, 0x4e,
+ 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x4d, 0x6f, 0x64, 0x65,
+ 0x32, 0x88, 0x01, 0x0a, 0x0a, 0x43, 0x4e, 0x49, 0x42, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x12,
+ 0x3c, 0x0a, 0x0a, 0x41, 0x64, 0x64, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x12, 0x16, 0x2e,
+ 0x72, 0x70, 0x63, 0x2e, 0x41, 0x64, 0x64, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x52, 0x65,
+ 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x41, 0x64, 0x64, 0x4e,
+ 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x3c, 0x0a,
+ 0x0a, 0x44, 0x65, 0x6c, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x12, 0x16, 0x2e, 0x72, 0x70,
+ 0x63, 0x2e, 0x44, 0x65, 0x6c, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x52, 0x65, 0x71, 0x75,
+ 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x44, 0x65, 0x6c, 0x4e, 0x65, 0x74,
+ 0x77, 0x6f, 0x72, 0x6b, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x32, 0x86, 0x01, 0x0a, 0x09,
+ 0x4e, 0x50, 0x42, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x12, 0x3e, 0x0a, 0x0e, 0x45, 0x6e, 0x66,
+ 0x6f, 0x72, 0x63, 0x65, 0x4e, 0x70, 0x54, 0x6f, 0x50, 0x6f, 0x64, 0x12, 0x15, 0x2e, 0x72, 0x70,
+ 0x63, 0x2e, 0x45, 0x6e, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x4e, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65,
+ 0x73, 0x74, 0x1a, 0x13, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x45, 0x6e, 0x66, 0x6f, 0x72, 0x63, 0x65,
+ 0x4e, 0x70, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x39, 0x0a, 0x0b, 0x44, 0x65, 0x6c,
+ 0x65, 0x74, 0x65, 0x50, 0x6f, 0x64, 0x4e, 0x70, 0x12, 0x14, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x44,
+ 0x65, 0x6c, 0x65, 0x74, 0x65, 0x4e, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x12,
+ 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x4e, 0x70, 0x52, 0x65, 0x70,
+ 0x6c, 0x79, 0x22, 0x00, 0x32, 0x6e, 0x0a, 0x13, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x53, 0x65,
+ 0x72, 0x76, 0x65, 0x72, 0x42, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x12, 0x57, 0x0a, 0x17, 0x47,
+ 0x65, 0x74, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x43,
+ 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e,
+ 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x22,
+ 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x50, 0x6f, 0x6c, 0x69,
+ 0x63, 0x79, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x70,
+ 0x6c, 0x79, 0x22, 0x00, 0x42, 0x2b, 0x5a, 0x29, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63,
+ 0x6f, 0x6d, 0x2f, 0x61, 0x77, 0x73, 0x2f, 0x61, 0x6d, 0x61, 0x7a, 0x6f, 0x6e, 0x2d, 0x76, 0x70,
+ 0x63, 0x2d, 0x63, 0x6e, 0x69, 0x2d, 0x6b, 0x38, 0x73, 0x2f, 0x72, 0x70, 0x63, 0x3b, 0x72, 0x70,
+ 0x63, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
}
var (
@@ -856,35 +930,38 @@ func file_rpc_proto_rawDescGZIP() []byte {
return file_rpc_proto_rawDescData
}
-var file_rpc_proto_msgTypes = make([]protoimpl.MessageInfo, 9)
+var file_rpc_proto_msgTypes = make([]protoimpl.MessageInfo, 10)
var file_rpc_proto_goTypes = []interface{}{
(*AddNetworkRequest)(nil), // 0: rpc.AddNetworkRequest
- (*AddNetworkReply)(nil), // 1: rpc.AddNetworkReply
- (*DelNetworkRequest)(nil), // 2: rpc.DelNetworkRequest
- (*DelNetworkReply)(nil), // 3: rpc.DelNetworkReply
- (*EnforceNpRequest)(nil), // 4: rpc.EnforceNpRequest
- (*EnforceNpReply)(nil), // 5: rpc.EnforceNpReply
- (*DeleteNpRequest)(nil), // 6: rpc.DeleteNpRequest
- (*DeleteNpReply)(nil), // 7: rpc.DeleteNpReply
- (*NetworkPolicyAgentConfigReply)(nil), // 8: rpc.NetworkPolicyAgentConfigReply
- (*emptypb.Empty)(nil), // 9: google.protobuf.Empty
+ (*IPAddress)(nil), // 1: rpc.IPAddress
+ (*AddNetworkReply)(nil), // 2: rpc.AddNetworkReply
+ (*DelNetworkRequest)(nil), // 3: rpc.DelNetworkRequest
+ (*DelNetworkReply)(nil), // 4: rpc.DelNetworkReply
+ (*DeleteNpRequest)(nil), // 5: rpc.DeleteNpRequest
+ (*DeleteNpReply)(nil), // 6: rpc.DeleteNpReply
+ (*EnforceNpRequest)(nil), // 7: rpc.EnforceNpRequest
+ (*EnforceNpReply)(nil), // 8: rpc.EnforceNpReply
+ (*NetworkPolicyAgentConfigReply)(nil), // 9: rpc.NetworkPolicyAgentConfigReply
+ (*emptypb.Empty)(nil), // 10: google.protobuf.Empty
}
var file_rpc_proto_depIdxs = []int32{
- 0, // 0: rpc.CNIBackend.AddNetwork:input_type -> rpc.AddNetworkRequest
- 2, // 1: rpc.CNIBackend.DelNetwork:input_type -> rpc.DelNetworkRequest
- 4, // 2: rpc.NPBackend.EnforceNpToPod:input_type -> rpc.EnforceNpRequest
- 6, // 3: rpc.NPBackend.DeletePodNp:input_type -> rpc.DeleteNpRequest
- 9, // 4: rpc.ConfigServerBackend.GetNetworkPolicyConfigs:input_type -> google.protobuf.Empty
- 1, // 5: rpc.CNIBackend.AddNetwork:output_type -> rpc.AddNetworkReply
- 3, // 6: rpc.CNIBackend.DelNetwork:output_type -> rpc.DelNetworkReply
- 5, // 7: rpc.NPBackend.EnforceNpToPod:output_type -> rpc.EnforceNpReply
- 7, // 8: rpc.NPBackend.DeletePodNp:output_type -> rpc.DeleteNpReply
- 8, // 9: rpc.ConfigServerBackend.GetNetworkPolicyConfigs:output_type -> rpc.NetworkPolicyAgentConfigReply
- 5, // [5:10] is the sub-list for method output_type
- 0, // [0:5] is the sub-list for method input_type
- 0, // [0:0] is the sub-list for extension type_name
- 0, // [0:0] is the sub-list for extension extendee
- 0, // [0:0] is the sub-list for field type_name
+ 1, // 0: rpc.AddNetworkReply.IPAddress:type_name -> rpc.IPAddress
+ 1, // 1: rpc.DelNetworkReply.IPAddress:type_name -> rpc.IPAddress
+ 0, // 2: rpc.CNIBackend.AddNetwork:input_type -> rpc.AddNetworkRequest
+ 3, // 3: rpc.CNIBackend.DelNetwork:input_type -> rpc.DelNetworkRequest
+ 7, // 4: rpc.NPBackend.EnforceNpToPod:input_type -> rpc.EnforceNpRequest
+ 5, // 5: rpc.NPBackend.DeletePodNp:input_type -> rpc.DeleteNpRequest
+ 10, // 6: rpc.ConfigServerBackend.GetNetworkPolicyConfigs:input_type -> google.protobuf.Empty
+ 2, // 7: rpc.CNIBackend.AddNetwork:output_type -> rpc.AddNetworkReply
+ 4, // 8: rpc.CNIBackend.DelNetwork:output_type -> rpc.DelNetworkReply
+ 8, // 9: rpc.NPBackend.EnforceNpToPod:output_type -> rpc.EnforceNpReply
+ 6, // 10: rpc.NPBackend.DeletePodNp:output_type -> rpc.DeleteNpReply
+ 9, // 11: rpc.ConfigServerBackend.GetNetworkPolicyConfigs:output_type -> rpc.NetworkPolicyAgentConfigReply
+ 7, // [7:12] is the sub-list for method output_type
+ 2, // [2:7] is the sub-list for method input_type
+ 2, // [2:2] is the sub-list for extension type_name
+ 2, // [2:2] is the sub-list for extension extendee
+ 0, // [0:2] is the sub-list for field type_name
}
func init() { file_rpc_proto_init() }
@@ -906,7 +983,7 @@ func file_rpc_proto_init() {
}
}
file_rpc_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
- switch v := v.(*AddNetworkReply); i {
+ switch v := v.(*IPAddress); i {
case 0:
return &v.state
case 1:
@@ -918,7 +995,7 @@ func file_rpc_proto_init() {
}
}
file_rpc_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
- switch v := v.(*DelNetworkRequest); i {
+ switch v := v.(*AddNetworkReply); i {
case 0:
return &v.state
case 1:
@@ -930,7 +1007,7 @@ func file_rpc_proto_init() {
}
}
file_rpc_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} {
- switch v := v.(*DelNetworkReply); i {
+ switch v := v.(*DelNetworkRequest); i {
case 0:
return &v.state
case 1:
@@ -942,7 +1019,7 @@ func file_rpc_proto_init() {
}
}
file_rpc_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} {
- switch v := v.(*EnforceNpRequest); i {
+ switch v := v.(*DelNetworkReply); i {
case 0:
return &v.state
case 1:
@@ -954,7 +1031,7 @@ func file_rpc_proto_init() {
}
}
file_rpc_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} {
- switch v := v.(*EnforceNpReply); i {
+ switch v := v.(*DeleteNpRequest); i {
case 0:
return &v.state
case 1:
@@ -966,7 +1043,7 @@ func file_rpc_proto_init() {
}
}
file_rpc_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} {
- switch v := v.(*DeleteNpRequest); i {
+ switch v := v.(*DeleteNpReply); i {
case 0:
return &v.state
case 1:
@@ -978,7 +1055,7 @@ func file_rpc_proto_init() {
}
}
file_rpc_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} {
- switch v := v.(*DeleteNpReply); i {
+ switch v := v.(*EnforceNpRequest); i {
case 0:
return &v.state
case 1:
@@ -990,6 +1067,18 @@ func file_rpc_proto_init() {
}
}
file_rpc_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} {
+ switch v := v.(*EnforceNpReply); i {
+ case 0:
+ return &v.state
+ case 1:
+ return &v.sizeCache
+ case 2:
+ return &v.unknownFields
+ default:
+ return nil
+ }
+ }
+ file_rpc_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*NetworkPolicyAgentConfigReply); i {
case 0:
return &v.state
@@ -1008,7 +1097,7 @@ func file_rpc_proto_init() {
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: file_rpc_proto_rawDesc,
NumEnums: 0,
- NumMessages: 9,
+ NumMessages: 10,
NumExtensions: 0,
NumServices: 3,
},
diff --git a/rpc/rpc.proto b/rpc/rpc.proto
index 794a7eb2e4..6996222705 100644
--- a/rpc/rpc.proto
+++ b/rpc/rpc.proto
@@ -20,14 +20,20 @@ message AddNetworkRequest {
string IfName = 5;
string NetworkName = 6;
string Netns = 4;
- // next field: 9
+ bool RequiresMultiNICAttachment = 9;
+ // next field: 10
+}
+
+message IPAddress {
+ string IPv4Addr = 1;
+ string IPv6Addr = 2;
+ int32 DeviceNumber = 3;
+ int32 RouteTableId = 4;
}
message AddNetworkReply {
bool Success = 1;
- string IPv4Addr = 2;
- string IPv6Addr = 11;
- int32 DeviceNumber = 4;
+ repeated IPAddress IPAddress = 2;
bool UseExternalSNAT = 5;
repeated string VPCv4CIDRs = 6;
repeated string VPCv6CIDRs = 12;
@@ -57,9 +63,7 @@ message DelNetworkRequest {
message DelNetworkReply {
bool Success = 1;
- string IPv4Addr = 2;
- string IPv6Addr = 5;
- int32 DeviceNumber = 3;
+ repeated IPAddress IPAddress = 2;
// start of pod-eni parameters
int32 PodVlanId = 4;
@@ -68,6 +72,16 @@ message DelNetworkReply {
// next field: 6
}
+message DeleteNpRequest {
+ bool Success = 1;
+ string K8S_POD_NAME = 2;
+ string K8S_POD_NAMESPACE = 3;
+}
+
+message DeleteNpReply {
+ bool Success = 1;
+}
+
// The service definition.
service NPBackend {
rpc EnforceNpToPod (EnforceNpRequest) returns (EnforceNpReply) {}
@@ -78,6 +92,7 @@ message EnforceNpRequest {
string K8S_POD_NAME = 1;
string K8S_POD_NAMESPACE = 2;
string NETWORK_POLICY_MODE = 3;
+ int32 InterfaceCount = 4;
}
message EnforceNpReply {