source files for runtime shim

runtime/src/Gopkg.lock

+# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
+
+
+[[projects]]
+  digest = "1:0deddd908b6b4b768cfc272c16ee61e7088a60f7fe2f06c547bd3d8e1f8b8e77"
+  name = "github.com/davecgh/go-spew"
+  packages = ["spew"]
+  pruneopts = ""
+  revision = "8991bc29aa16c548c550c7ff78260e27b9ab7c73"
+  version = "v1.1.1"
+
+[[projects]]
+  digest = "1:531ba73ef0f436a52f4b11ff915505ae92e0cca7eb110088e3ecc23fb7ba4b4f"
+  name = "github.com/opencontainers/runtime-spec"
+  packages = ["specs-go"]
+  pruneopts = ""
+  revision = "4e3b9264a330d094b0386c3703c5f379119711e8"
+  version = "v1.0.1"
+
+[[projects]]
+  digest = "1:3d2c33720d4255686b9f4a7e4d3b94938ee36063f14705c5eb0f73347ed4c496"
+  name = "github.com/pelletier/go-toml"
+  packages = ["."]
+  pruneopts = ""
+  revision = "728039f679cbcd4f6a54e080d2219a4c4928c546"
+  version = "v1.4.0"
+
+[[projects]]
+  digest = "1:256484dbbcd271f9ecebc6795b2df8cad4c458dd0f5fd82a8c2fa0c29f233411"
+  name = "github.com/pmezard/go-difflib"
+  packages = ["difflib"]
+  pruneopts = ""
+  revision = "792786c7400a136282c1664665ae0a8db921c6c2"
+  version = "v1.0.0"
+
+[[projects]]
+  digest = "1:381bcbeb112a51493d9d998bbba207a529c73dbb49b3fd789e48c63fac1f192c"
+  name = "github.com/stretchr/testify"
+  packages = [
+    "assert",
+    "require",
+  ]
+  pruneopts = ""
+  revision = "ffdc059bfe9ce6a4e144ba849dbedead332c6053"
+  version = "v1.3.0"
+
+[solve-meta]
+  analyzer-name = "dep"
+  analyzer-version = 1
+  input-imports = [
+    "github.com/opencontainers/runtime-spec/specs-go",
+    "github.com/pelletier/go-toml",
+    "github.com/stretchr/testify/require",
+  ]
+  solver-name = "gps-cdcl"
+  solver-version = 1

runtime/src/Gopkg.toml

+
+# Gopkg.toml example
+#
+# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md
+# for detailed Gopkg.toml documentation.
+#
+# required = ["github.com/user/thing/cmd/thing"]
+# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"]
+#
+# [[constraint]]
+#   name = "github.com/user/project"
+#   version = "1.0.0"
+#
+# [[constraint]]
+#   name = "github.com/user/project2"
+#   branch = "dev"
+#   source = "github.com/myfork/project2"
+#
+# [[override]]
+#  name = "github.com/x/y"
+#  version = "2.4.0"
+
+
+[[constraint]]
+  name = "github.com/opencontainers/runtime-spec"
+  version = "1.0.1"

runtime/src/Makefile

+RUNTIME=nvidia-container-runtime
+MOCK_RUNC=$(CURDIR)/runc
+
+all:
+	@go build  -o ${RUNTIME}
+
+mock-runc:
+	@(echo '#!/bin/bash\necho mock runc') > ${MOCK_RUNC}
+	@chmod +x ${MOCK_RUNC}
+
+test: all mock-runc
+	@go test -v
+	@${RM} ${MOCK_RUNC}
+
+clean:
+	@${RM} ${RUNTIME}

runtime/src/main.go

+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"os/exec"
+	"syscall"
+
+	"github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/pelletier/go-toml"
+)
+
+const (
+	configFilePath      = "/etc/nvidia-container-runtime/config.toml"
+	hookDefaultFilePath = "/usr/bin/nvidia-container-runtime-hook"
+)
+
+var fileLogger *log.Logger = nil
+
+type args struct {
+	bundleDirPath string
+	cmd           string
+}
+
+type config struct {
+	debugFilePath string
+}
+
+func getConfig() (*config, error) {
+	cfg := &config{}
+
+	tomlContent, err := ioutil.ReadFile(configFilePath)
+	if err != nil {
+		return nil, err
+	}
+
+	toml, err := toml.Load(string(tomlContent))
+	if err != nil {
+		return nil, err
+	}
+
+	cfg.debugFilePath = toml.GetDefault("nvidia-container-runtime.debug", "/dev/null").(string)
+
+	return cfg, nil
+}
+
+func getArgs() (*args, error) {
+	args := &args{}
+
+	for i, param := range os.Args {
+		if param == "--bundle" || param == "-b" {
+			if len(os.Args) - i <= 1 {
+				return nil, fmt.Errorf("bundle option needs an argument")
+			}
+			args.bundleDirPath = os.Args[i + 1]
+		} else if param == "create" {
+			args.cmd = param
+		}
+	}
+
+	return args, nil
+}
+
+func exitOnError(err error, msg string) {
+	if err != nil {
+		if fileLogger != nil {
+			fileLogger.Printf("ERROR: %s: %v\n", msg, err)
+		}
+		log.Fatalf("ERROR: %s: %s: %v\n", os.Args[0], msg, err)
+	}
+}
+
+func execRunc() {
+	runcPath, err := exec.LookPath("runc")
+	exitOnError(err, "find runc path")
+	fileLogger.Printf("Runc path: %s\n", runcPath)
+
+	err = syscall.Exec(runcPath, append([]string{runcPath}, os.Args[1:]...), os.Environ())
+	exitOnError(err, "exec runc binary")
+}
+
+func addNVIDIAHook(spec *specs.Spec) error {
+	path, err := exec.LookPath("nvidia-container-runtime-hook")
+	if err != nil {
+		path := hookDefaultFilePath
+		_, err = os.Stat(path)
+		if err != nil {
+			return err
+		}
+	}
+
+	if fileLogger != nil {
+		fileLogger.Printf("Prestart hook path: %s\n", path)
+	}
+
+	args := []string{path}
+	if spec.Hooks == nil {
+		spec.Hooks = &specs.Hooks{}
+	}
+	spec.Hooks.Prestart = append(spec.Hooks.Prestart, specs.Hook{
+		Path: path,
+		Args: append(args, "prestart"),
+	})
+
+	return nil
+}
+
+func main() {
+
+	cfg, err := getConfig()
+	exitOnError(err, "fail to get config")
+
+	logFile, err := os.OpenFile(cfg.debugFilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+	exitOnError(err, "fail to open debug log file")
+	defer logFile.Close()
+
+	fileLogger = log.New(logFile, "", log.LstdFlags)
+	fileLogger.Printf("Running %s\n", os.Args[0])
+
+	args, err := getArgs()
+	exitOnError(err, "fail to get args")
+
+	if args.cmd != "create" {
+		fileLogger.Println("Command is not \"create\", executing runc doing nothing")
+		execRunc()
+		log.Fatalf("ERROR: %s: fail to execute runc binary\n", os.Args[0])
+	}
+
+	if args.bundleDirPath == "" {
+		args.bundleDirPath, err = os.Getwd()
+		exitOnError(err, "get working directory")
+		fileLogger.Printf("Bundle dirrectory path is empty, using working directory: %s\n", args.bundleDirPath)
+	}
+
+	fileLogger.Printf("Using bundle file: %s\n", args.bundleDirPath + "/config.json")
+	jsonFile, err := os.OpenFile(args.bundleDirPath + "/config.json", os.O_RDWR, 0644)
+	exitOnError(err, "open OCI spec file")
+
+	defer jsonFile.Close()
+
+	jsonContent, err := ioutil.ReadAll(jsonFile)
+	exitOnError(err, "read OCI spec file")
+
+	var spec specs.Spec
+	err = json.Unmarshal(jsonContent, &spec)
+	exitOnError(err, "unmarshal OCI spec file")
+
+	err = addNVIDIAHook(&spec)
+	exitOnError(err, "inject NVIDIA hook")
+
+	jsonOutput, err := json.Marshal(spec)
+	exitOnError(err, "marchal OCI spec file")
+
+	_, err = jsonFile.WriteAt(jsonOutput, 0)
+	exitOnError(err, "write OCI spec file")
+
+	fileLogger.Print("Prestart hook added, executing runc")
+	execRunc()
+}

runtime/src/main_test.go

+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"io/ioutil"
+	"log"
+	"os"
+	"os/exec"
+	"path"
+	"runtime"
+	"strings"
+	"testing"
+
+	"github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	nvidiaRuntime       = "nvidia-container-runtime"
+	nvidiaHook          = "nvidia-container-runtime-hook"
+	bundlePath          = "./"
+	specFile            = "config.json"
+	unmodifiedSpecFile = "test_spec.json"
+)
+
+var workingDir string
+
+func TestMain(m *testing.M) {
+	// TEST SETUP
+	// Update PATH to execute mock runc in current directory
+	_, filename, _, _ := runtime.Caller(0)
+	workingDir = path.Dir(filename)
+	paths := strings.Split(os.Getenv("PATH"), ":")
+	paths = append([]string{workingDir}, paths...)
+	os.Setenv("PATH", strings.Join(paths, ":"))
+
+	// Confirm path setup correctly
+	runcPath, err := exec.LookPath("runc")
+	if err != nil || runcPath != (workingDir+"/runc") {
+		log.Fatal("error in test setup: mock runc path set incorrectly in TestMain()")
+	}
+
+	// RUN TESTS
+	m.Run()
+
+	// TEST CLEANUP
+	os.Remove(specFile)
+}
+
+// case 1) nvidia-container-runtime run --bundle
+// case 2) nvidia-container-runtime create --bundle
+//		- Confirm the runtime handles bad input correctly
+func TestBadInput(t *testing.T) {
+	err := generateNewRuntimeSpec()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	cmdRun := exec.Command(nvidiaRuntime, "run", "--bundle")
+	t.Logf("executing: %s\n", strings.Join(cmdRun.Args, " "))
+	err = cmdRun.Run()
+	require.Error(t, err, "runtime should return an error")
+
+	cmdCreate := exec.Command(nvidiaRuntime, "create", "--bundle")
+	t.Logf("executing: %s\n", strings.Join(cmdCreate.Args, " "))
+	err = cmdCreate.Run()
+	require.Error(t, err, "runtime should return an error")
+}
+
+// case 1) nvidia-container-runtime run --bundle <bundle-name> <ctr-name>
+//		- Confirm the runtime runs with no errors
+// case 2) nvidia-container-runtime create --bundle <bundle-name> <ctr-name>
+//		- Confirm the runtime inserts the NVIDIA prestart hook correctly
+func TestGoodInput(t *testing.T) {
+	err := generateNewRuntimeSpec()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	cmdRun := exec.Command(nvidiaRuntime, "run", "--bundle", bundlePath, "testcontainer")
+	t.Logf("executing: %s\n", strings.Join(cmdRun.Args, " "))
+	err = cmdRun.Run()
+	require.NoError(t, err, "runtime should not return an error")
+
+	// Check config.json and confirm there are no hooks
+	spec, err := getRuntimeSpec(bundlePath + specFile)
+	require.NoError(t, err, "should be no errors when reading and parsing spec from config.json")
+	require.Empty(t, spec.Hooks, "there should be no hooks in config.json")
+
+	cmdCreate := exec.Command(nvidiaRuntime, "create", "--bundle", bundlePath, "testcontainer")
+	t.Logf("executing: %s\n", strings.Join(cmdCreate.Args, " "))
+	err = cmdCreate.Run()
+	require.NoError(t, err, "runtime should not return an error")
+
+	// Check config.json for NVIDIA prestart hook
+	spec, err = getRuntimeSpec(bundlePath + specFile)
+	require.NoError(t, err, "should be no errors when reading and parsing spec from config.json")
+	require.NotEmpty(t, spec.Hooks, "there should be hooks in config.json")
+	require.Equal(t, 1, nvidiaHookCount(spec.Hooks), "exactly one nvidia prestart hook should be inserted correctly into config.json")
+}
+
+// NVIDIA prestart hook already present in config file
+func TestDuplicateHook(t *testing.T) {
+	err := generateNewRuntimeSpec()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var spec specs.Spec
+	spec, err = getRuntimeSpec(bundlePath + specFile)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Logf("inserting nvidia prestart hook to config.json")
+	if err = addNVIDIAHook(&spec); err != nil {
+		t.Fatal(err)
+	}
+
+	jsonOutput, err := json.MarshalIndent(spec, "", "\t")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	jsonFile, err := os.OpenFile(bundlePath+specFile, os.O_RDWR, 0644)
+	if err != nil {
+		t.Fatal(err)
+	}
+	_, err = jsonFile.WriteAt(jsonOutput, 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Test how runtime handles already existing prestart hook in config.json
+	cmdCreate := exec.Command(nvidiaRuntime, "create", "--bundle", bundlePath, "testcontainer")
+	t.Logf("executing: %s\n", strings.Join(cmdCreate.Args, " "))
+	err = cmdCreate.Run()
+	require.NoError(t, err, "runtime should not return an error")
+
+	// Check config.json for NVIDIA prestart hook
+	spec, err = getRuntimeSpec(bundlePath + specFile)
+	require.NoError(t, err, "should be no errors when reading and parsing spec from config.json")
+	require.NotEmpty(t, spec.Hooks, "there should be hooks in config.json")
+	require.Equal(t, 1, nvidiaHookCount(spec.Hooks), "exactly one nvidia prestart hook should be inserted correctly into config.json")
+}
+
+func getRuntimeSpec(filePath string) (specs.Spec, error) {
+	var spec specs.Spec
+	jsonFile, err := os.OpenFile(filePath, os.O_RDWR, 0644)
+	defer jsonFile.Close()
+	if err != nil {
+		return spec, err
+	}
+
+	jsonContent, err := ioutil.ReadAll(jsonFile)
+	if err != nil {
+		return spec, err
+	} else if json.Valid(jsonContent) {
+		err = json.Unmarshal(jsonContent, &spec)
+		if err != nil {
+			return spec, err
+		}
+	} else {
+		err = json.NewDecoder(bytes.NewReader(jsonContent)).Decode(&spec)
+		if err != nil {
+			return spec, err
+		}
+	}
+
+	return spec, err
+}
+
+func generateNewRuntimeSpec() error {
+	cmd := exec.Command("cp", unmodifiedSpecFile, specFile)
+	err := cmd.Run()
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// Return number of valid NVIDIA prestart hooks in runtime spec
+func nvidiaHookCount(hooks *specs.Hooks) int {
+	prestartHooks := hooks.Prestart
+	count := 0
+
+	for _, hook := range prestartHooks {
+		if strings.Contains(hook.Path, nvidiaHook) {
+			count++
+		}
+	}
+	return count
+}

runtime/src/test_spec.json

+{
+	"ociVersion": "1.0.1-dev",
+	"process": {
+		"terminal": true,
+		"user": {
+			"uid": 0,
+			"gid": 0
+		},
+		"args": [
+			"sh"
+		],
+		"env": [
+			"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
+			"TERM=xterm"
+		],
+		"cwd": "/",
+		"capabilities": {
+			"bounding": [
+				"CAP_AUDIT_WRITE",
+				"CAP_KILL",
+				"CAP_NET_BIND_SERVICE"
+			],
+			"effective": [
+				"CAP_AUDIT_WRITE",
+				"CAP_KILL",
+				"CAP_NET_BIND_SERVICE"
+			],
+			"inheritable": [
+				"CAP_AUDIT_WRITE",
+				"CAP_KILL",
+				"CAP_NET_BIND_SERVICE"
+			],
+			"permitted": [
+				"CAP_AUDIT_WRITE",
+				"CAP_KILL",
+				"CAP_NET_BIND_SERVICE"
+			],
+			"ambient": [
+				"CAP_AUDIT_WRITE",
+				"CAP_KILL",
+				"CAP_NET_BIND_SERVICE"
+			]
+		},
+		"rlimits": [
+			{
+				"type": "RLIMIT_NOFILE",
+				"hard": 1024,
+				"soft": 1024
+			}
+		],
+		"noNewPrivileges": true
+	},
+	"root": {
+		"path": "rootfs",
+		"readonly": true
+	},
+	"hostname": "runc",
+	"mounts": [
+		{
+			"destination": "/proc",
+			"type": "proc",
+			"source": "proc"
+		},
+		{
+			"destination": "/dev",
+			"type": "tmpfs",
+			"source": "tmpfs",
+			"options": [
+				"nosuid",
+				"strictatime",
+				"mode=755",
+				"size=65536k"
+			]
+		},
+		{
+			"destination": "/dev/pts",
+			"type": "devpts",
+			"source": "devpts",
+			"options": [
+				"nosuid",
+				"noexec",
+				"newinstance",
+				"ptmxmode=0666",
+				"mode=0620",
+				"gid=5"
+			]
+		},
+		{
+			"destination": "/dev/shm",
+			"type": "tmpfs",
+			"source": "shm",
+			"options": [
+				"nosuid",
+				"noexec",
+				"nodev",
+				"mode=1777",
+				"size=65536k"
+			]
+		},
+		{
+			"destination": "/dev/mqueue",
+			"type": "mqueue",
+			"source": "mqueue",
+			"options": [
+				"nosuid",
+				"noexec",
+				"nodev"
+			]
+		},
+		{
+			"destination": "/sys",
+			"type": "sysfs",
+			"source": "sysfs",
+			"options": [
+				"nosuid",
+				"noexec",
+				"nodev",
+				"ro"
+			]
+		},
+		{
+			"destination": "/sys/fs/cgroup",
+			"type": "cgroup",
+			"source": "cgroup",
+			"options": [
+				"nosuid",
+				"noexec",
+				"nodev",
+				"relatime",
+				"ro"
+			]
+		}
+	],
+	"linux": {
+		"resources": {
+			"devices": [
+				{
+					"allow": false,
+					"access": "rwm"
+				}
+			]
+		},
+		"namespaces": [
+			{
+				"type": "pid"
+			},
+			{
+				"type": "network"
+			},
+			{
+				"type": "ipc"
+			},
+			{
+				"type": "uts"
+			},
+			{
+				"type": "mount"
+			}
+		],
+		"maskedPaths": [
+			"/proc/kcore",
+			"/proc/latency_stats",
+			"/proc/timer_list",
+			"/proc/timer_stats",
+			"/proc/sched_debug",
+			"/sys/firmware",
+			"/proc/scsi"
+		],
+		"readonlyPaths": [
+			"/proc/asound",
+			"/proc/bus",
+			"/proc/fs",
+			"/proc/irq",
+			"/proc/sys",
+			"/proc/sysrq-trigger"
+		]
+	}
+}
\ No newline at end of file