Ingress subnet for private addresses

bicep/main.bicep

@@ -83,6 +83,12 @@ param vnetAksSubnetAddressPrefix string = '10.240.0.0/22'
 @description('The address range for the App Gateway in your custom vnet')
 param vnetAppGatewaySubnetAddressPrefix string = '10.240.5.0/24'
 
+@minLength(9)
+@maxLength(18)
+@description('A dedicated ingress subnet allows consistent IP addresses to be used for private DNS reference by the Ingress Controller(s).')
+param ingressSubnetAddressPrefix string = '10.240.4.0/28'
+param ingressSubnet bool = false
+
 @minLength(9)
 @maxLength(18)
 @description('The address range for the ACR in your custom vnet')
@@ -144,6 +150,8 @@ module network './network.bicep' = if (custom_vnet) {
     privateLinkAkvId: privateLinks && keyVaultCreate ? kv.outputs.keyVaultId : ''
     acrPrivatePool: acrPrivatePool
     acrAgentPoolSubnetAddressPrefix: acrAgentPoolSubnetAddressPrefix
+    ingressSubnet: ingressSubnet
+    ingressSubnetAddressPrefix: ingressSubnetAddressPrefix
     bastion: bastion
     bastionSubnetAddressPrefix: bastionSubnetAddressPrefix
     availabilityZones: availabilityZones

bicep/network.bicep

@@ -40,13 +40,16 @@ param natGatewayPublicIps int = 2
 param natGatewayIdleTimeoutMins int = 30
 
 //Bastion
-param bastion bool =false
+param bastion bool = false
 param bastionSubnetAddressPrefix string = ''
 
+//Ingress
+param ingressSubnet bool = false
+param ingressSubnetAddressPrefix string = ''
+
 @description('Used by the Bastion Public IP')
 param availabilityZones array = []
 
-
 var bastion_subnet_name = 'AzureBastionSubnet'
 var bastion_baseSubnet = {
   name: bastion_subnet_name
@@ -56,6 +59,15 @@ var bastion_baseSubnet = {
 }
 var bastion_subnet = bastion && networkSecurityGroups ? union(bastion_baseSubnet, nsgBastion.outputs.nsgSubnetObj) : bastion_baseSubnet
 
+var ingress_subnet_name = 'clusteringressservices-sn'
+var ingress_baseSubnet = {
+  name: ingress_subnet_name
+  properties: {
+    addressPrefix: ingressSubnetAddressPrefix
+  }
+}
+var ingress_subnet = ingressSubnet && networkSecurityGroups ? union(ingress_baseSubnet, nsgIngress.outputs.nsgSubnetObj) : ingress_baseSubnet
+
 var acrpool_subnet_name = 'acrpool-sn'
 var acrpool_baseSubnet = {
   name: acrpool_subnet_name
@@ -168,6 +180,7 @@ var subnets = union(
   privateLinks ? array(private_link_subnet) : [],
   acrPrivatePool ? array(acrpool_subnet) : [],
   bastion ? array(bastion_subnet) : [],
+  ingressSubnet ? array(ingress_subnet) : [],
   ingressApplicationGateway ? array(appgw_subnet) : [],
   azureFirewallsManagementSeperation ? array(fwmgmt_subnet) : []
 )
@@ -472,6 +485,22 @@ module nsgPrivateLinks 'nsg.bicep' = if(privateLinks && networkSecurityGroups) {
   ]
 }
 
+module nsgIngress 'nsg.bicep' = if(ingressSubnet && networkSecurityGroups) {
+  name: take('${deployment().name}-nsgIngress',64)
+  params: {
+    location: location
+    resourceName: '${ingress_subnet_name}-${resourceName}'
+    workspaceId: !empty(workspaceName) ? log.properties.customerId : ''
+    workspaceRegion:  !empty(workspaceName) ? log.location : ''
+    workspaceResourceId:  !empty(workspaceName) ? log.id : ''
+    ruleInAllowVnetHttp: true
+    ruleInAllowVnetHttps: true
+  }
+  dependsOn: [
+    nsgPrivateLinks
+  ]
+}
+
 resource natGwIp 'Microsoft.Network/publicIPAddresses@2021-08-01' =  [for i in range(0, natGatewayPublicIps): if(natGateway) {
   name: 'pip-${natGwName}-${i+1}'
   location: location

bicep/nsg.bicep

@@ -112,6 +112,48 @@ resource ruleInternetHttps 'Microsoft.Network/networkSecurityGroups/securityRule
   }
 }
 
+param ruleInAllowVnetHttp bool = false
+resource ruleVnetHttp 'Microsoft.Network/networkSecurityGroups/securityRules@2022-07-01' = if(ruleInAllowVnetHttp) {
+  parent: nsg
+  name: 'Allow_Vbet_Http'
+  properties: {
+    protocol: 'Tcp'
+    sourcePortRange: '*'
+    sourceAddressPrefix: 'VirtualNetwork'
+    destinationAddressPrefix: 'VirtualNetwork'
+    access: 'Allow'
+    priority: 220
+    direction: 'Inbound'
+    sourcePortRanges: []
+    destinationPortRanges: [
+      '80'
+    ]
+    sourceAddressPrefixes: []
+    destinationAddressPrefixes: []
+  }
+}
+
+param ruleInAllowVnetHttps bool = false
+resource ruleVnetHttps 'Microsoft.Network/networkSecurityGroups/securityRules@2022-07-01' = if(ruleInAllowVnetHttps) {
+  parent: nsg
+  name: 'Allow_Vbet_Https'
+  properties: {
+    protocol: 'Tcp'
+    sourcePortRange: '*'
+    sourceAddressPrefix: 'VirtualNetwork'
+    destinationAddressPrefix: 'VirtualNetwork'
+    access: 'Allow'
+    priority: 230
+    direction: 'Inbound'
+    sourcePortRanges: []
+    destinationPortRanges: [
+      '443'
+    ]
+    sourceAddressPrefixes: []
+    destinationAddressPrefixes: []
+  }
+}
+
 param ruleInAllowBastionHostComms bool = false
 resource ruleBastionHost 'Microsoft.Network/networkSecurityGroups/securityRules@2022-07-01' = if(ruleInAllowBastionHostComms) {
   parent: nsg

helper/src/components/addonsTab.js

@@ -110,17 +110,22 @@ export default function ({ tabValues, updateFn, featureFlag, invalidArray }) {
                 }
             </Stack.Item>
 
-            <Stack.Item align="center" styles={{ root: { maxWidth: '700px', display: (addons.ingress === "none" ? "none" : "block") } }} >
-                <Stack tokens={{ childrenGap: 15 }}>
+            <Stack.Item align="left" styles={{ root: { maxWidth: '700px', display: (addons.ingress === "none" ? "none" : "block") } }} >
+                <Stack tokens={{ childrenGap: 15, padding:`0px 0px 0px 100px` }} >
                     {addons.ingress === "nginx" && false &&
                         <MessageBar messageBarType={MessageBarType.warning}>You requested a high security cluster & nginx public ingress. Please ensure you follow this information after deployment <Link target="_ar1" href="https://docs.microsoft.com/en-us/azure/firewall/integrate-lb#public-load-balancer">Asymmetric routing</Link></MessageBar>
                     }
                     {addons.ingress !== "none" && false &&
                         <MessageBar messageBarType={MessageBarType.warning}>You requested a high security cluster. The DNS and Certificate options are disabled as they require additional egress application firewall rules for image download and webhook requirements. You can apply these rules and install the helm chart after provisioning</MessageBar>
                     }
 
-                    {addons.ingress === "appgw" && (
+                    {(addons.ingress === "nginx" || addons.ingress === "contour" || addons.ingress === "traefik") &&
+                        <>
+                            <Checkbox checked={addons.ingressUsePublicIp} onChange={(ev, v) => updateFn("ingressUsePublicIp", v)} label={<Text>Expose Ingress directly to internet</Text>} />
+                        </>
+                    }
 
+                    {addons.ingress === "appgw" && (
                         net.vnet_opt === 'default' ?
                             <MessageBar messageBarType={MessageBarType.warning}>Using default networking, so addon will provision default Application Gateway instance, for more options, select custom networking in the network tab</MessageBar>
                             :
@@ -199,9 +204,12 @@ export default function ({ tabValues, updateFn, featureFlag, invalidArray }) {
                             </>)
                     }
 
-                    {(addons.ingress === "contour" || addons.ingress === "nginx" || addons.ingress === "appgw" || addons.ingress === "traefik") &&
+                    {net.afw && (addons.ingress === "contour" || addons.ingress === "nginx" || addons.ingress === "appgw" || addons.ingress === "traefik") &&
+                        <MessageBar messageBarType={MessageBarType.warning}>Using a in-cluster ingress option with Azure Firewall will require additional asymmetric routing configuration post-deployment, please see <Link target="_target" href="https://docs.microsoft.com/azure/aks/limit-egress-traffic#add-a-dnat-rule-to-azure-firewall">Add a DNAT rule to Azure Firewall </Link></MessageBar>
+                    }
+
+                    {addons.ingressUsePublicIp && (addons.ingress === "contour" || addons.ingress === "nginx" || addons.ingress === "appgw" || addons.ingress === "traefik") &&
                         <>
-                            <MessageBar messageBarType={MessageBarType.warning}>Using a in-cluster ingress option with Azure Firewall will require additional asymmetric routing configuration post-deployment, please see <Link target="_target" href="https://docs.microsoft.com/azure/aks/limit-egress-traffic#add-a-dnat-rule-to-azure-firewall">Add a DNAT rule to Azure Firewall </Link></MessageBar>
                             <Checkbox inputProps={{ "data-testid": "addons-dns"}} checked={addons.dns} onChange={(ev, v) => updateFn("dns", v)} label={
                                 <Text>Create FQDN URLs for your applications using
                                     <Link target="_t1" href="https://github.com/kubernetes-sigs/external-dns"> <b>external-dns</b> </Link>

helper/src/components/deployTab.js

@@ -87,6 +87,7 @@ export default function DeployTab({ defaults, updateFn, tabValues, invalidArray,
     ...(net.vnet_opt === "custom" && net.networkPlugin === 'kubenet' && defaults.net.podCidr !== net.podCidr && { podCidr: net.podCidr }),
     ...((net.vnet_opt === "custom" || net.vnet_opt === "byo") && defaults.net.cniDynamicIpAllocation !== net.cniDynamicIpAllocation && { cniDynamicIpAllocation: true }),
     ...(net.vnet_opt === "custom" && net.cniDynamicIpAllocation && defaults.net.podCidr !== net.podCidr && { podCidr: net.podCidr }),
+    ...(net.vnet_opt === "custom" && defaults.net.ingressSubnet !== net.ingressSubnet && { ingressSubnet: net.ingressSubnet }),
     ...(cluster.availabilityZones === "yes" && { availabilityZones: ['1', '2', '3'] }),
     ...(cluster.apisecurity === "whitelist" && deploy.clusterIPWhitelist && apiips_array.length > 0 && { authorizedIPRanges: apiips_array }),
     ...(defaults.net.maxPods !== net.maxPods && { maxPods: net.maxPods }),
@@ -176,6 +177,10 @@ export default function DeployTab({ defaults, updateFn, tabValues, invalidArray,
         ...( addons.certMan && {
           ingress: addons.ingress,
           certEmail: addons.certEmail
+        }),
+        ...( !addons.ingressUsePublicIp && (addons.ingress === "contour" || addons.ingress === "nginx" || addons.ingress === "traefik") && {
+          ingressServiceInternal: !addons.ingressUsePublicIp,
+          ...(net.ingressSubnet && net.vnet_opt === "custom" && { ingressServiceSubnet: net.ingressSubnetName})
         })
       }),
     ...(cluster.apisecurity === "private" && (addons.ingress === "contour" || (addons.ingress !== "none" && addons.dns &&  addons.dnsZoneId) ) && {

helper/src/components/networkTab.js

@@ -152,6 +152,14 @@ export default function NetworkTab ({ defaults, tabValues, updateFn, invalidArra
 
             <Separator className="notopmargin" />
 
+            <Stack.Item>
+                <Label>Create a dedicated subnet for Ingress Controller Private IP's</Label>
+                <MessageBar messageBarType={MessageBarType.info}>This provides a separate subnet for exposing Ingress to the cluster. This can be useful when you are using private DNS, if a dedicated IP address range that isn't adjacent to IP's is required, or when your network team prefer cluster ingress is from a defined subnet. This is not required when using Azure Application Gateway as your Ingress Controller as it already has a dedicated subnet.</MessageBar>
+                <Checkbox inputProps={{ "data-testid": "network-ingressSubnet-Checkbox"}} styles={{ root: { marginLeft: '50px', marginTop: '10px !important' } }} disabled={false} checked={net.ingressSubnet} onChange={(ev, v) => updateFn("ingressSubnet", v)} label="Create subnet for Ingress Controller" />
+            </Stack.Item>
+
+            <Separator className="notopmargin" />
+
             <Stack.Item >
                 <Label>AKS Traffic Egress</Label>
 
@@ -213,6 +221,9 @@ export default function NetworkTab ({ defaults, tabValues, updateFn, invalidArra
                 {hasError(invalidArray, 'afw') &&
                     <MessageBar messageBarType={MessageBarType.error}>{getError(invalidArray, 'afw')}</MessageBar>
                 }
+                {net.afw && (addons.ingress === "contour" || addons.ingress === "nginx" || addons.ingress === "appgw" || addons.ingress === "traefik") &&
+                    <MessageBar messageBarType={MessageBarType.warning} >Using a in-cluster ingress option with Azure Firewall will require additional asymmetric routing configuration post-deployment, please see <Link target="_target" href="https://docs.microsoft.com/azure/aks/limit-egress-traffic#add-a-dnat-rule-to-azure-firewall">Add a DNAT rule</Link></MessageBar>
+                }
                 <Checkbox
                     styles={{ root: { marginLeft: '50px', marginTop: '10px !important' } }}
                     disabled={net.vnet_opt !== 'custom'}
@@ -440,11 +451,6 @@ function CustomVNET({ net, addons, updateFn, invalidArray }) {
                         value={net.vnetAksSubnetAddressPrefix}
                         errorMessage={getError(invalidArray, 'vnetAksSubnetAddressPrefix')} />
                     </Stack.Item>
-                    {/*
-                <Stack.Item align="center">
-                  <TextField prefix="Cidr" label="LoadBalancer Services subnet" onChange={(ev, val) => updateFn("ilbsub", val)} value={net.ilbsub} />
-                </Stack.Item>
-                */}
                     <Stack.Item style={{ marginLeft: "20px"}}>
                         <TextField prefix="Cidr" disabled={!net.afw} label="Azure Firewall subnet" onChange={(ev, val) => updateFn("vnetFirewallSubnetAddressPrefix", val)} value={net.afw ? net.vnetFirewallSubnetAddressPrefix : "No Firewall requested"} />
                     </Stack.Item>
@@ -462,6 +468,10 @@ function CustomVNET({ net, addons, updateFn, invalidArray }) {
                         <TextField prefix="Cidr" disabled={!net.vnetprivateend || addons.registry === "none" || !addons.acrPrivatePool  } label="ACR Private Agent Pool subnet" onChange={(ev, val) => updateFn("acrAgentPoolSubnetAddressPrefix", val)} value={net.vnetprivateend && addons.registry !== "none" && addons.acrPrivatePool  ? net.acrAgentPoolSubnetAddressPrefix : "No Agent Pool requested"} />
                     </Stack.Item>
 
+                    <Stack.Item style={{ marginLeft: "20px"}}>
+                        <TextField prefix="Cidr" disabled={!net.ingressSubnet} label="Ingress subnet" onChange={(ev, val) => updateFn("ingressSubnetAddressPrefix", val)} value={net.ingressSubnet ? net.ingressSubnetAddressPrefix : "No ingress subnet requested"} />
+                    </Stack.Item>
+
                     <Stack.Item style={{ marginLeft: "20px"}}>
                         <TextField prefix="Cidr" disabled={!net.bastion} label="Azure Bastion subnet" onChange={(ev, val) => updateFn("bastionSubnetAddressPrefix", val)} value={net.bastion ? net.bastionSubnetAddressPrefix : "No bastion subnet requested"} />
                     </Stack.Item>

helper/src/components/portalnav.js

@@ -381,7 +381,8 @@ export default function PortalNav({ config }) {
   invalidFn('addons', 'ingress', cluster.osType === "Windows" && addons.ingress !== "appgw" && addons.ingress !== "none", 'Neither the Windows nodepool or the system pool will be able to run your selected Ingress Controller. To support this Ingress Controller, add another linux nodepool post cluster creation.')
   invalidFn('net', 'byoAKSSubnetId', net.vnet_opt === 'byo' && !net.byoAKSSubnetId.match('^/subscriptions/[^/ ]+/resourceGroups/[^/ ]+/providers/Microsoft.Network/virtualNetworks/[^/ ]+/subnets/[^/ ]+$'), 'Enter a valid Subnet Id where AKS nodes will be installed')
   invalidFn('net', 'byoAGWSubnetId', net.vnet_opt === 'byo' && addons.ingress === 'appgw' && !net.byoAGWSubnetId.match('^/subscriptions/[^/ ]+/resourceGroups/[^/ ]+/providers/Microsoft.Network/virtualNetworks/[^/ ]+/subnets/[^/ ]+$'), 'Enter a valid Subnet Id where Application Gateway is installed')
-  invalidFn('net', 'vnet_opt', net.vnet_opt === "default" && (net.afw || net.vnetprivateend), 'Cannot use default networking of you select Firewall or Private Link')
+
+  invalidFn('net', 'vnet_opt', net.vnet_opt === "default" && (net.afw || net.vnetprivateend || net.bastion || net.ingressSubnet) , 'Cannot use default networking of you select Firewall, Bastion, Ingress Subnet or Private Link')
   invalidFn('net', 'afw',
     (net.afw && net.vnet_opt !== "custom") ||
     (!net.afw && net.aksOutboundTrafficType === 'userDefinedRouting' && net.vnet_opt === "custom"),
@@ -399,6 +400,7 @@ export default function PortalNav({ config }) {
      'When using User Defined Routing, only custom and Bring your Own networking is supported.'
      :
      'When using Managed Nat Gateway, only default networking is supported. For other networking options, use Assigned NAT Gateway')
+
   invalidFn('net', 'serviceCidr',  net.vnet_opt === "custom" && !isCidrValid(net.serviceCidr), invalidCidrMessage)
   invalidFn('net', 'podCidr', !isCidrValid(net.podCidr), invalidCidrMessage)
   invalidFn('net', 'dnsServiceIP', !isIPValid(net.dnsServiceIP), 'Enter a valid IP')

helper/src/config.json

@@ -109,7 +109,8 @@
             "gitops": "none",
             "containerLogsV2": false,
             "containerLogsV2BasicLogs": false,
-            "sgxPlugin": false
+            "sgxPlugin": false,
+            "ingressUsePublicIp": true
         },
         "net": {
             "vnetFirewallManagementSubnetAddressPrefix": "10.240.51.0/26",
@@ -129,8 +130,11 @@
             "byoAGWSubnetId": "",
             "vnetAddressPrefix": "10.240.0.0/16",
             "vnetAksSubnetAddressPrefix": "10.240.0.0/22",
+            "ingressSubnet": false,
+            "ingressSubnetName": "clusteringressservices-sn",
             "vnetAppGatewaySubnetAddressPrefix": "10.240.5.0/24",
             "acrAgentPoolSubnetAddressPrefix": "10.240.4.64/26",
+            "ingressSubnetAddressPrefix": "10.240.4.0/28",
             "bastionSubnetAddressPrefix": "10.240.4.128/26",
             "privateLinkSubnetAddressPrefix": "10.240.4.192/26",
             "vnetFirewallSubnetAddressPrefix": "10.240.50.0/24",

helper/src/configpresets/baselines.json

@@ -98,7 +98,8 @@
                                 "vnet_opt": "custom",
                                 "afw": true,
                                 "azureFirewallSku": "Premium",
-                                "nsg": true
+                                "nsg": true,
+                                "ingressSubnet": true
                             }
                         }
                     },
@@ -192,7 +193,8 @@
                                 "afw": true,
                                 "azureFirewallSku": "Premium",
                                 "nsg": true,
-                                "nsgFlowLogs": true
+                                "nsgFlowLogs": true,
+                                "ingressSubnet": true
                             }
                         }
                     }

helper/src/configpresets/entScale.json

@@ -154,7 +154,8 @@
                                 "ingress": "appgw",
                                 "monitor": "aci",
                                 "csisecret": "akvNew",
-                                "appgwKVIntegration": true
+                                "appgwKVIntegration": true,
+                                "ingressUsePublicIp": false
                             },
                             "net": {
                                 "vnetprivateend": true,

helper/src/configpresets/principals.json

@@ -385,7 +385,8 @@
                                     }
                                 ],
                                 "azurepolicy": "deny",
-                                "csisecret": "akvNew"
+                                "csisecret": "akvNew",
+                                "ingressUsePublicIp": false
                             },
                             "net": {
                                 "vnet_opt": "custom",