CLOUDP-127633 Use read-only filesystem (#1042)

* CLOUDP-127633 Use read-only filesystem

* Update the project layout to be acceptable by the operator-sdk

* Moved SecurityContext to podspec_template.go

Author: slaskawi

PROJECT

@@ -1,13 +1,25 @@
 domain: mongodb.com
-layout: go.kubebuilder.io/v3
+layout:
+- go.kubebuilder.io/v3
+plugins:
+  manifests.sdk.operatorframework.io/v2: {}
+  scorecard.sdk.operatorframework.io/v2: {}
 projectName: mko-v1
 repo: github.com/mongodb/mongodb-kubernetes-operator
 resources:
-- crdVersion: v1
+- api:
+    crdVersion: v1
+    namespaced: true
   group: mongodbcommunity
   kind: MongoDBCommunity
   version: v1
-version: 3-alpha
-plugins:
-  manifests.sdk.operatorframework.io/v2: {}
-  scorecard.sdk.operatorframework.io/v2: {}
+- api:
+    crdVersion: v1
+    namespaced: true
+  controller: true
+  domain: mongodb.com
+  group: mongodbcommunity
+  kind: SimpleMongoDBCommunity
+  path: github.com/mongodb/mongodb-kubernetes-operator/api/v1alpha1
+  version: v1alpha1
+version: "3"

controllers/construct/build_statefulset_test.go

@@ -66,7 +66,7 @@ func TestManagedSecurityContext(t *testing.T) {
 	_ = os.Setenv(MongodbRepoUrl, "repo")
 	_ = os.Setenv(MongodbImageEnv, "mongo")
 	_ = os.Setenv(AgentImageEnv, "agent-image")
-	_ = os.Setenv(ManagedSecurityContextEnv, "true")
+	_ = os.Setenv(podtemplatespec.ManagedSecurityContextEnv, "true")
 
 	mdb := newTestReplicaSet()
 	stsFunc := BuildMongoDBReplicaSetStatefulSetModificationFunction(&mdb, mdb)
@@ -105,7 +105,7 @@ func assertStatefulSetIsBuiltCorrectly(t *testing.T, mdb mdbv1.MongoDBCommunity,
 	assert.Len(t, sts.Spec.Template.Spec.Containers[0].Env, 4)
 	assert.Len(t, sts.Spec.Template.Spec.Containers[1].Env, 1)
 
-	managedSecurityContext := envvar.ReadBool(ManagedSecurityContextEnv)
+	managedSecurityContext := envvar.ReadBool(podtemplatespec.ManagedSecurityContextEnv)
 	if !managedSecurityContext {
 		assert.NotNil(t, sts.Spec.Template.Spec.SecurityContext)
 		assert.Equal(t, podtemplatespec.DefaultPodSecurityContext(), *sts.Spec.Template.Spec.SecurityContext)
@@ -118,9 +118,14 @@ func assertStatefulSetIsBuiltCorrectly(t *testing.T, mdb mdbv1.MongoDBCommunity,
 	probe := agentContainer.ReadinessProbe
 	assert.True(t, reflect.DeepEqual(probes.New(DefaultReadiness()), *probe))
 	assert.Equal(t, probes.New(DefaultReadiness()).FailureThreshold, probe.FailureThreshold)
-	assert.Len(t, agentContainer.VolumeMounts, 6)
+	assert.Len(t, agentContainer.VolumeMounts, 7)
 	assert.NotNil(t, agentContainer.ReadinessProbe)
-	assert.Nil(t, agentContainer.SecurityContext)
+	if !managedSecurityContext {
+		assert.NotNil(t, sts.Spec.Template.Spec.Containers[0].SecurityContext)
+		assert.Equal(t, container.DefaultSecurityContext(), *sts.Spec.Template.Spec.Containers[0].SecurityContext)
+	} else {
+		assert.Nil(t, agentContainer.SecurityContext)
+	}
 
 	assertContainsVolumeMountWithName(t, agentContainer.VolumeMounts, "agent-scripts")
 	assertContainsVolumeMountWithName(t, agentContainer.VolumeMounts, "automation-config")
@@ -131,8 +136,13 @@ func assertStatefulSetIsBuiltCorrectly(t *testing.T, mdb mdbv1.MongoDBCommunity,
 
 	mongodContainer := sts.Spec.Template.Spec.Containers[1]
 	assert.Equal(t, "repo/mongo:4.2.2", mongodContainer.Image)
-	assert.Len(t, mongodContainer.VolumeMounts, 5)
-	assert.Nil(t, mongodContainer.SecurityContext)
+	assert.Len(t, mongodContainer.VolumeMounts, 6)
+	if !managedSecurityContext {
+		assert.NotNil(t, sts.Spec.Template.Spec.Containers[1].SecurityContext)
+		assert.Equal(t, container.DefaultSecurityContext(), *sts.Spec.Template.Spec.Containers[1].SecurityContext)
+	} else {
+		assert.Nil(t, agentContainer.SecurityContext)
+	}
 
 	assertContainsVolumeMountWithName(t, mongodContainer.VolumeMounts, "data-volume")
 	assertContainsVolumeMountWithName(t, mongodContainer.VolumeMounts, "healthstatus")
@@ -144,6 +154,12 @@ func assertStatefulSetIsBuiltCorrectly(t *testing.T, mdb mdbv1.MongoDBCommunity,
 	assert.Equal(t, versionUpgradeHookName, initContainer.Name)
 	assert.Equal(t, "version-upgrade-hook-image", initContainer.Image)
 	assert.Len(t, initContainer.VolumeMounts, 1)
+	if !managedSecurityContext {
+		assert.NotNil(t, sts.Spec.Template.Spec.InitContainers[0].SecurityContext)
+		assert.Equal(t, container.DefaultSecurityContext(), *sts.Spec.Template.Spec.InitContainers[0].SecurityContext)
+	} else {
+		assert.Nil(t, agentContainer.SecurityContext)
+	}
 }
 
 func assertContainsVolumeMountWithName(t *testing.T, mounts []corev1.VolumeMount, name string) {

controllers/construct/mongodbstatefulset.go

@@ -12,7 +12,6 @@ import (
 	"github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/probes"
 	"github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/resourcerequirements"
 	"github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/statefulset"
-	"github.com/mongodb/mongodb-kubernetes-operator/pkg/util/envvar"
 	"github.com/mongodb/mongodb-kubernetes-operator/pkg/util/scale"
 	appsv1 "k8s.io/api/apps/v1"
 	"k8s.io/apimachinery/pkg/types"
@@ -42,7 +41,6 @@ const (
 	MongodbImageEnv            = "MONGODB_IMAGE"
 	VersionUpgradeHookImageEnv = "VERSION_UPGRADE_HOOK_IMAGE"
 	ReadinessProbeImageEnv     = "READINESS_PROBE_IMAGE"
-	ManagedSecurityContextEnv  = "MANAGED_SECURITY_CONTEXT"
 
 	automationMongodConfFileName = "automation-mongod.conf"
 	keyfileFilePath              = "/var/lib/mongodb-mms-automation/authentication/keyfile"
@@ -115,12 +113,16 @@ func BuildMongoDBReplicaSetStatefulSetModificationFunction(mdb MongoDBStatefulSe
 	scriptsVolume := statefulset.CreateVolumeFromEmptyDir("agent-scripts")
 	scriptsVolumeMount := statefulset.CreateVolumeMount(scriptsVolume.Name, "/opt/scripts", statefulset.WithReadOnly(false))
 
+	// tmp volume is required by the mongodb-agent and mongod
+	tmpVolume := statefulset.CreateVolumeFromEmptyDir("tmp")
+	tmpVolumeMount := statefulset.CreateVolumeMount(tmpVolume.Name, "/tmp", statefulset.WithReadOnly(false))
+
 	keyFileNsName := mdb.GetAgentKeyfileSecretNamespacedName()
 	keyFileVolume := statefulset.CreateVolumeFromEmptyDir(keyFileNsName.Name)
 	keyFileVolumeVolumeMount := statefulset.CreateVolumeMount(keyFileVolume.Name, "/var/lib/mongodb-mms-automation/authentication", statefulset.WithReadOnly(false))
 	keyFileVolumeVolumeMountMongod := statefulset.CreateVolumeMount(keyFileVolume.Name, "/var/lib/mongodb-mms-automation/authentication", statefulset.WithReadOnly(false))
 
-	mongodbAgentVolumeMounts := []corev1.VolumeMount{agentHealthStatusVolumeMount, scriptsVolumeMount, keyFileVolumeVolumeMount}
+	mongodbAgentVolumeMounts := []corev1.VolumeMount{agentHealthStatusVolumeMount, scriptsVolumeMount, keyFileVolumeVolumeMount, tmpVolumeMount}
 
 	automationConfigVolumeFunc := podtemplatespec.NOOP()
 	if mdb.NeedsAutomationConfigVolume() {
@@ -129,7 +131,7 @@ func BuildMongoDBReplicaSetStatefulSetModificationFunction(mdb MongoDBStatefulSe
 		automationConfigVolumeMount := statefulset.CreateVolumeMount(automationConfigVolume.Name, "/var/lib/automation/config", statefulset.WithReadOnly(true))
 		mongodbAgentVolumeMounts = append(mongodbAgentVolumeMounts, automationConfigVolumeMount)
 	}
-	mongodVolumeMounts := []corev1.VolumeMount{mongodHealthStatusVolumeMount, hooksVolumeMount, keyFileVolumeVolumeMountMongod}
+	mongodVolumeMounts := []corev1.VolumeMount{mongodHealthStatusVolumeMount, hooksVolumeMount, keyFileVolumeVolumeMountMongod, tmpVolumeMount}
 	dataVolumeClaim := statefulset.NOOP()
 	logVolumeClaim := statefulset.NOOP()
 	singleModeVolumeClaim := func(s *appsv1.StatefulSet) {}
@@ -150,11 +152,7 @@ func BuildMongoDBReplicaSetStatefulSetModificationFunction(mdb MongoDBStatefulSe
 		singleModeVolumeClaim = statefulset.WithVolumeClaim(mdb.DataVolumeName(), dataPvc(mdb.DataVolumeName()))
 	}
 
-	podSecurityContext := podtemplatespec.NOOP()
-	managedSecurityContext := envvar.ReadBool(ManagedSecurityContextEnv)
-	if !managedSecurityContext {
-		podSecurityContext = podtemplatespec.WithSecurityContext(podtemplatespec.DefaultPodSecurityContext())
-	}
+	podSecurityContext, _ := podtemplatespec.WithDefaultSecurityContextsModifications()
 
 	return statefulset.Apply(
 		statefulset.WithName(mdb.GetName()),
@@ -175,6 +173,7 @@ func BuildMongoDBReplicaSetStatefulSetModificationFunction(mdb MongoDBStatefulSe
 				podtemplatespec.WithVolume(hooksVolume),
 				automationConfigVolumeFunc,
 				podtemplatespec.WithVolume(scriptsVolume),
+				podtemplatespec.WithVolume(tmpVolume),
 				podtemplatespec.WithVolume(keyFileVolume),
 				podtemplatespec.WithServiceAccount(mongodbDatabaseServiceAccountName),
 				podtemplatespec.WithContainer(AgentName, mongodbAgentContainer(mdb.AutomationConfigSecretName(), mongodbAgentVolumeMounts)),
@@ -194,6 +193,7 @@ func AutomationAgentCommand() []string {
 }
 
 func mongodbAgentContainer(automationConfigSecretName string, volumeMounts []corev1.VolumeMount) container.Modification {
+	_, containerSecurityContext := podtemplatespec.WithDefaultSecurityContextsModifications()
 	return container.Apply(
 		container.WithName(AgentName),
 		container.WithImage(os.Getenv(AgentImageEnv)),
@@ -202,6 +202,7 @@ func mongodbAgentContainer(automationConfigSecretName string, volumeMounts []cor
 		container.WithResourceRequirements(resourcerequirements.Defaults()),
 		container.WithVolumeMounts(volumeMounts),
 		container.WithCommand(AutomationAgentCommand()),
+		containerSecurityContext,
 		container.WithEnvs(
 			corev1.EnvVar{
 				Name:  headlessAgentEnv,
@@ -229,12 +230,14 @@ func mongodbAgentContainer(automationConfigSecretName string, volumeMounts []cor
 }
 
 func versionUpgradeHookInit(volumeMount []corev1.VolumeMount) container.Modification {
+	_, containerSecurityContext := podtemplatespec.WithDefaultSecurityContextsModifications()
 	return container.Apply(
 		container.WithName(versionUpgradeHookName),
 		container.WithCommand([]string{"cp", "version-upgrade-hook", "/hooks/version-upgrade"}),
 		container.WithImage(os.Getenv(VersionUpgradeHookImageEnv)),
 		container.WithImagePullPolicy(corev1.PullAlways),
 		container.WithVolumeMounts(volumeMount),
+		containerSecurityContext,
 	)
 }
 
@@ -265,12 +268,14 @@ func logsPvc(logsVolumeName string) persistentvolumeclaim.Modification {
 // readinessProbeInit returns a modification function which will add the readiness probe container.
 // this container will copy the readiness probe binary into the /opt/scripts directory.
 func readinessProbeInit(volumeMount []corev1.VolumeMount) container.Modification {
+	_, containerSecurityContext := podtemplatespec.WithDefaultSecurityContextsModifications()
 	return container.Apply(
 		container.WithName(ReadinessProbeContainerName),
 		container.WithCommand([]string{"cp", "/probes/readinessprobe", "/opt/scripts/readinessprobe"}),
 		container.WithImage(os.Getenv(ReadinessProbeImageEnv)),
 		container.WithImagePullPolicy(corev1.PullAlways),
 		container.WithVolumeMounts(volumeMount),
+		containerSecurityContext,
 	)
 }
 
@@ -303,11 +308,14 @@ exec mongod -f %s;
 		mongoDbCommand,
 	}
 
+	_, containerSecurityContext := podtemplatespec.WithDefaultSecurityContextsModifications()
+
 	return container.Apply(
 		container.WithName(MongodbName),
 		container.WithImage(getMongoDBImage(version)),
 		container.WithResourceRequirements(resourcerequirements.Defaults()),
 		container.WithCommand(containerCommand),
+		containerSecurityContext,
 		container.WithEnvs(
 			corev1.EnvVar{
 				Name:  agentHealthStatusFilePathEnv,

controllers/mongodb_tls_test.go

@@ -41,7 +41,7 @@ func TestStatefulSet_IsCorrectlyConfiguredWithTLS(t *testing.T) {
 }
 
 func assertStatefulsetVolumesAndVolumeMounts(t *testing.T, sts appsv1.StatefulSet, expectedTLSCASecretName string, expectedTLSOperatorSecretName string) {
-	assert.Len(t, sts.Spec.Template.Spec.Volumes, 7)
+	assert.Len(t, sts.Spec.Template.Spec.Volumes, 8)
 	permission := int32(416)
 	assert.Contains(t, sts.Spec.Template.Spec.Volumes, corev1.Volume{
 		Name: "tls-ca",

docs/RELEASE_NOTES.md

@@ -23,6 +23,8 @@ If above conditions are met, it is strongly advised to upgrade the MongoDB Kuber
   - The names of connection string secrets generated for configured users are RFC1123 validated.
 - Changes
   - Support for changing port number in running cluster.
+  - Security Context is now defined on pod level (previously was on container level)
+  - Our containers now use the `readOnlyRootFilesystem` setting.
 
 ## MongoDBCommunity Resource
 

cmd/manager/main.go main.go

@@ -20,7 +20,7 @@ func TestContainer(t *testing.T) {
 		WithImage("image"),
 		WithImagePullPolicy(corev1.PullAlways),
 		WithPorts([]corev1.ContainerPort{{Name: "port-1", ContainerPort: int32(1000)}}),
-		WithSecurityContext(&corev1.SecurityContext{
+		WithSecurityContext(corev1.SecurityContext{
 			RunAsGroup:   int64Ref(100),
 			RunAsNonRoot: boolRef(true),
 		}),

pkg/kube/container/container_test.go

@@ -177,9 +177,16 @@ func WithPorts(ports []corev1.ContainerPort) Modification {
 	}
 }
 
-// WithSecurityContext sets teh container's SecurityContext
-func WithSecurityContext(context *corev1.SecurityContext) Modification {
+// WithSecurityContext sets the container's SecurityContext
+func WithSecurityContext(context corev1.SecurityContext) Modification {
 	return func(container *corev1.Container) {
-		container.SecurityContext = context
+		container.SecurityContext = &context
 	}
 }
+
+// DefaultSecurityContext returns the default container security context with:
+// - readOnlyRootFilesystem set to true
+func DefaultSecurityContext() corev1.SecurityContext {
+	readOnlyRootFilesystem := true
+	return corev1.SecurityContext{ReadOnlyRootFilesystem: &readOnlyRootFilesystem}
+}

pkg/kube/container/containers.go

@@ -2,6 +2,7 @@ package podtemplatespec
 
 import (
 	"github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/container"
+	"github.com/mongodb/mongodb-kubernetes-operator/pkg/util/envvar"
 	"github.com/mongodb/mongodb-kubernetes-operator/pkg/util/merge"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -10,7 +11,8 @@ import (
 type Modification func(*corev1.PodTemplateSpec)
 
 const (
-	notFound = -1
+	notFound                  = -1
+	ManagedSecurityContextEnv = "MANAGED_SECURITY_CONTEXT"
 )
 
 func New(templateMods ...Modification) corev1.PodTemplateSpec {
@@ -266,3 +268,15 @@ func FindContainerByName(name string, podTemplateSpec *corev1.PodTemplateSpec) *
 
 	return nil
 }
+
+func WithDefaultSecurityContextsModifications() (Modification, container.Modification) {
+	managedSecurityContext := envvar.ReadBool(ManagedSecurityContextEnv)
+	configureContainerSecurityContext := container.NOOP()
+	configurePodSpecSecurityContext := NOOP()
+	if !managedSecurityContext {
+		configurePodSpecSecurityContext = WithSecurityContext(DefaultPodSecurityContext())
+		configureContainerSecurityContext = container.WithSecurityContext(container.DefaultSecurityContext())
+	}
+
+	return configurePodSpecSecurityContext, configureContainerSecurityContext
+}

pkg/kube/podtemplatespec/podspec_template.go

@@ -41,15 +41,18 @@ func TestPodTemplateSpec(t *testing.T) {
 			container.WithName("init-container-0"),
 			container.WithImage("init-image"),
 			container.WithVolumeMounts([]corev1.VolumeMount{volumeMount1}),
+			container.WithSecurityContext(container.DefaultSecurityContext()),
 		)),
 		WithContainerByIndex(0, container.Apply(
 			container.WithName("container-0"),
 			container.WithImage("image"),
 			container.WithVolumeMounts([]corev1.VolumeMount{volumeMount1}),
+			container.WithSecurityContext(container.DefaultSecurityContext()),
 		)),
 		WithContainerByIndex(1, container.Apply(
 			container.WithName("container-1"),
 			container.WithImage("image"),
+			container.WithSecurityContext(container.DefaultSecurityContext()),
 		)),
 		WithVolumeMounts("init-container-0", volumeMount2),
 		WithVolumeMounts("container-0", volumeMount2),
@@ -74,16 +77,19 @@ func TestPodTemplateSpec(t *testing.T) {
 	assert.Equal(t, "init-container-0", p.Spec.InitContainers[0].Name)
 	assert.Equal(t, "init-image", p.Spec.InitContainers[0].Image)
 	assert.Equal(t, []corev1.VolumeMount{volumeMount1, volumeMount2}, p.Spec.InitContainers[0].VolumeMounts)
+	assert.Equal(t, container.DefaultSecurityContext(), *p.Spec.InitContainers[0].SecurityContext)
 
 	assert.Len(t, p.Spec.Containers, 2)
 
 	assert.Equal(t, "container-0", p.Spec.Containers[0].Name)
 	assert.Equal(t, "image", p.Spec.Containers[0].Image)
 	assert.Equal(t, []corev1.VolumeMount{volumeMount1, volumeMount2}, p.Spec.Containers[0].VolumeMounts)
+	assert.Equal(t, container.DefaultSecurityContext(), *p.Spec.Containers[0].SecurityContext)
 
 	assert.Equal(t, "container-1", p.Spec.Containers[1].Name)
 	assert.Equal(t, "image", p.Spec.Containers[1].Image)
-	assert.Equal(t, []corev1.VolumeMount{volumeMount1, volumeMount2}, p.Spec.Containers[0].VolumeMounts)
+	assert.Equal(t, []corev1.VolumeMount{volumeMount1, volumeMount2}, p.Spec.Containers[1].VolumeMounts)
+	assert.Equal(t, container.DefaultSecurityContext(), *p.Spec.Containers[1].SecurityContext)
 }
 
 func TestPodTemplateSpec_MultipleEditsToContainer(t *testing.T) {