feat: satisfy job pod admission requirements for kata jobs

src/cmd/root.go

@@ -54,6 +54,10 @@ func init() {
 	rootCmd.PersistentFlags().Int("job-pod-log-max-size", 1000000, "The max amount in bytes to buffer before pod logs are shipped to OpsLevel. Works in tandem with 'job-pod-log-max-interval'")
 	rootCmd.PersistentFlags().Bool("job-agent-mode", false, "Enable agent mode with privileged security context for Container-in-Container support. WARNING: This grants elevated privileges and should only be enabled for trusted workloads.")
 	rootCmd.PersistentFlags().String("job-pod-helper-image", "", "Override the helper init container image. Defaults to the published ECR image matching the runner version. Useful for local development with kind.")
+	rootCmd.PersistentFlags().String("job-pod-image", "", "Override the image the job container runs. Defaults to the image from the job definition.")
+	rootCmd.PersistentFlags().String("job-pod-runtime-class-name", "", "The RuntimeClass to run job pods under, ie 'kata-qemu' for VM isolation. Empty runs under the cluster default runtime with no VM boundary.")
+	rootCmd.PersistentFlags().Int64("job-pod-active-deadline", 0, "The hard deadline in seconds after which the job pod is terminated by kubernetes. 0 leaves it unset.")
+	rootCmd.PersistentFlags().String("job-pod-automount-service-account-token", "", "Whether to project a kubernetes API token into job pods (options [\"true\", \"false\"]). Empty leaves the namespace default in place.")
 	rootCmd.PersistentFlags().String("queue", "", "The queue this runner should process jobs from. Empty means the default queue.")
 
 	rootCmd.PersistentFlags().Int("k8s-api-qps", 50, "The maximum sustained queries per second to the Kubernetes API server.")
@@ -81,6 +85,10 @@ func init() {
 	viper.BindEnv("job-pod-log-max-size", "OPSLEVEL_JOB_POD_LOG_MAX_SIZE")
 	viper.BindEnv("job-agent-mode", "OPSLEVEL_JOB_AGENT_MODE")
 	viper.BindEnv("job-pod-helper-image", "OPSLEVEL_JOB_POD_HELPER_IMAGE")
+	viper.BindEnv("job-pod-image", "OPSLEVEL_JOB_POD_IMAGE")
+	viper.BindEnv("job-pod-runtime-class-name", "OPSLEVEL_JOB_POD_RUNTIME_CLASS_NAME")
+	viper.BindEnv("job-pod-active-deadline", "OPSLEVEL_JOB_POD_ACTIVE_DEADLINE")
+	viper.BindEnv("job-pod-automount-service-account-token", "OPSLEVEL_JOB_POD_AUTOMOUNT_SERVICE_ACCOUNT_TOKEN")
 	viper.BindEnv("queue", "OPSLEVEL_QUEUE")
 
 	viper.BindEnv("k8s-api-qps", "OPSLEVEL_K8S_API_QPS")

src/pkg/k8s.go

@@ -179,7 +179,6 @@ func executable() *int32 {
 }
 
 func (s *JobRunner) getPodObject(identifier string, labels map[string]string, job opslevel.RunnerJob) *corev1.Pod {
-	// TODO: Allow configuration of Labels
 	// TODO: Allow configuration of Pod Command
 
 	podSecurityContext := s.podConfig.SecurityContext
@@ -193,20 +192,23 @@ func (s *JobRunner) getPodObject(identifier string, labels map[string]string, jo
 		}
 	}
 
-	var containerSecurityContext *corev1.SecurityContext
+	containerSecurityContext := s.podConfig.ContainerSecurityContext.DeepCopy()
 	if s.podConfig.AgentMode {
 		// Agent mode jobs need privileged mode for creating containers within container
 		privileged := true
-		containerSecurityContext = &corev1.SecurityContext{
-			Privileged: &privileged,
+		if containerSecurityContext == nil {
+			containerSecurityContext = &corev1.SecurityContext{}
 		}
+		containerSecurityContext.Privileged = &privileged
 	}
 
 	initContainers := []corev1.Container{
 		{
 			Name:            ContainerNameHelper,
 			Image:           s.podConfig.helperImage(),
 			ImagePullPolicy: s.podConfig.PullPolicy,
+			Resources:       s.podConfig.Resources,
+			SecurityContext: containerSecurityContext.DeepCopy(),
 			Command: []string{
 				"cp",
 				"/opslevel-runner",
@@ -223,14 +225,14 @@ func (s *JobRunner) getPodObject(identifier string, labels map[string]string, jo
 	}
 
 	if len(job.InitCommands) > 0 {
-		initContainers = append(initContainers, s.getInitContainer(job, containerSecurityContext))
+		initContainers = append(initContainers, s.getInitContainer(job, containerSecurityContext.DeepCopy()))
 	}
 
 	return &corev1.Pod{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:        identifier,
 			Namespace:   s.podConfig.Namespace,
-			Labels:      labels,
+			Labels:      s.podConfig.podLabels(labels),
 			Annotations: s.podConfig.Annotations,
 		},
 		Spec: corev1.PodSpec{
@@ -239,20 +241,25 @@ func (s *JobRunner) getPodObject(identifier string, labels map[string]string, jo
 			SecurityContext:               &podSecurityContext,
 			ServiceAccountName:            s.podConfig.ServiceAccountName,
 			NodeSelector:                  s.podConfig.NodeSelector,
+			DNSPolicy:                     s.podConfig.DNSPolicy,
+			RuntimeClassName:              s.podConfig.runtimeClassName(),
+			Tolerations:                   s.podConfig.Tolerations,
+			ActiveDeadlineSeconds:         s.podConfig.activeDeadlineSeconds(),
+			AutomountServiceAccountToken:  s.podConfig.AutomountServiceAccountToken,
 			InitContainers:                initContainers,
 			Containers: []corev1.Container{
 				{
 					Name:            ContainerNameJob,
-					Image:           job.Image,
-					ImagePullPolicy: corev1.PullIfNotPresent,
+					Image:           s.podConfig.jobImage(job.Image),
+					ImagePullPolicy: s.podConfig.jobPullPolicy(),
 					Command: []string{
 						"/bin/sh",
 						"-c",
 						fmt.Sprintf("sleep %d", s.podConfig.Lifetime),
 					},
 					Resources:       s.podConfig.Resources,
 					Env:             s.getPodEnv(job.Variables, opslevel.RunnerJobVariableScopeMain),
-					SecurityContext: containerSecurityContext,
+					SecurityContext: containerSecurityContext.DeepCopy(),
 					VolumeMounts: []corev1.VolumeMount{
 						{
 							Name:      "scripts",

src/pkg/k8s_config.go

@@ -3,6 +3,7 @@
 import (
 	"fmt"
 	"os"
+	"strings"
 
 	"github.com/spf13/viper"
 	corev1 "k8s.io/api/core/v1"
@@ -26,12 +27,44 @@
 	DNSPolicy                     corev1.DNSPolicy            `yaml:"dnsPolicy"`
 	PullPolicy                    corev1.PullPolicy           `yaml:"pullPolicy"`
 	SecurityContext               corev1.PodSecurityContext   `yaml:"securityContext"`
-	NodeSelector                  map[string]string           `yaml:"nodeSelector"`
-	AgentMode                     bool                        `yaml:"agentMode"`
-	HelperImage                   string                      `yaml:"helperImage"`
+	// ContainerSecurityContext is applied to every init and main container.
+	ContainerSecurityContext *corev1.SecurityContext `yaml:"containerSecurityContext"`
+	NodeSelector             map[string]string       `yaml:"nodeSelector"`
+	AgentMode                bool                    `yaml:"agentMode"`
+	HelperImage              string                  `yaml:"helperImage"`
+
+	// Image overrides the image the job container runs. When empty the image
+	// from the job definition is used.
+	Image string `yaml:"image"`
+
+	// RuntimeClassName selects the container runtime for the job pod, e.g.
+	// "kata-qemu" for VM isolation or "gvisor". When empty the pod runs under
+	// the cluster default runtime (usually runc) with no VM boundary.
+	RuntimeClassName string `yaml:"runtimeClassName"`
+
+	// Tolerations let job pods schedule onto tainted nodes - sandbox node pools
+	// are typically tainted so ordinary workloads cannot land on them.
+	Tolerations []corev1.Toleration `yaml:"tolerations"`
+
+	// ActiveDeadlineSeconds bounds the lifetime of the pod itself, unlike
+	// Lifetime which only bounds the job container's sleep from the inside.
+	ActiveDeadlineSeconds int64 `yaml:"activeDeadlineSeconds"`
+
+	// AutomountServiceAccountToken controls whether a Kubernetes API credential
+	// is projected into the job pod. Nil leaves the namespace default in place.
+	AutomountServiceAccountToken *bool `yaml:"automountServiceAccountToken"`
+
+	// Labels are merged onto the labels the runner sets on each job pod. Runner
+	// labels win on conflict since the runner selects pods by them.
+	Labels map[string]string `yaml:"labels"`
 }
 
 func ReadPodConfig(path string) (*K8SPodConfig, error) {
+	automountServiceAccountToken, err := automountFromViper()
+	if err != nil {
+		return nil, err
+	}
+
 	config := Config{
 		Kubernetes: K8SPodConfig{
 			Namespace:  viper.GetString("job-pod-namespace"),
@@ -53,6 +86,10 @@
 			TerminationGracePeriodSeconds: 5,
 			AgentMode:                     viper.GetBool("job-agent-mode"),
 			HelperImage:                   viper.GetString("job-pod-helper-image"),
+			Image:                         viper.GetString("job-pod-image"),
+			RuntimeClassName:              viper.GetString("job-pod-runtime-class-name"),
+			ActiveDeadlineSeconds:         viper.GetInt64("job-pod-active-deadline"),
+			AutomountServiceAccountToken:  automountServiceAccountToken,
 		},
 	}
 	// Early out with viper defaults if config file doesn't exist
@@ -71,6 +108,79 @@
 	return &config.Kubernetes, nil
 }
 
+// automountFromViper reads the tri-state flag. It is a string rather than a bool
+// because a bound bool flag always carries its default, leaving no way to tell
+// "unset" from "explicitly false" - and defaulting to false would silently strip
+// the token mount from every existing user. Any other non-empty value is rejected
+// rather than failing open to the namespace default.
+func automountFromViper() (*bool, error) {
+	value := strings.TrimSpace(viper.GetString("job-pod-automount-service-account-token"))
+	switch strings.ToLower(value) {
+	case "":
+		return nil, nil
+	case "true":
+		value := true
+		return &value, nil
+	case "false":
+		value := false
+		return &value, nil
+	default:
+		return nil, fmt.Errorf("invalid job-pod-automount-service-account-token value %q: expected true, false, or empty", value)
+	}
+}
+
+// jobPullPolicy defaults to IfNotPresent rather than leaving the field empty so
+// that job pods keep their historical behavior when PullPolicy is unset.
+func (c *K8SPodConfig) jobPullPolicy() corev1.PullPolicy {
+	if c.PullPolicy == "" {
+		return corev1.PullIfNotPresent
+	}
+	return c.PullPolicy
+}
+
+// jobImage returns the image the job container should run - the config override
+// when set, otherwise the image from the job definition.
+func (c *K8SPodConfig) jobImage(jobImage string) string {
+	if c.Image != "" {
+		return c.Image
+	}
+	return jobImage
+}
+
+// runtimeClassName must be nil rather than a pointer to "" when unset - the API
+// server treats an empty runtime class as a request for a class named "".
+func (c *K8SPodConfig) runtimeClassName() *string {
+	if c.RuntimeClassName == "" {
+		return nil
+	}
+	return &c.RuntimeClassName
+}
+
+// activeDeadlineSeconds must be nil rather than a pointer to 0 when unset - a
+// zero deadline is rejected, it does not mean "unspecified".
+func (c *K8SPodConfig) activeDeadlineSeconds() *int64 {
+	if c.ActiveDeadlineSeconds <= 0 {
+		return nil
+	}
+	return &c.ActiveDeadlineSeconds
+}
+
+// podLabels merges the configured labels under the labels the runner manages -
+// the runner selects and cleans up pods by its own labels, so they must win.
+func (c *K8SPodConfig) podLabels(runnerLabels map[string]string) map[string]string {
+	if len(c.Labels) == 0 {
+		return runnerLabels
+	}
+	merged := make(map[string]string, len(c.Labels)+len(runnerLabels))
+	for k, v := range c.Labels {
+		merged[k] = v
+	}
+	for k, v := range runnerLabels {
+		merged[k] = v
+	}
+	return merged
+}
+
 func (c *K8SPodConfig) helperImage() string {
 	if c.HelperImage != "" {
 		return c.HelperImage