提交 72986caa 编写于 作者: C Christopher Desiniotis

Merge branch 'ci' into 'master'

Integrate e2e tests for ubuntu16.04 and centos7 into cicd

See merge request nvidia/container-toolkit/nvidia-container-runtime!19
......@@ -6,10 +6,30 @@ variables:
DOCKER_HOST: tcp://docker:2375
stages:
- build_ami
- docker_images
- unit
- integration
- build
- e2e_tests
.build_ami: &build_ami
stage: build_ami
image:
name: hashicorp/packer:1.4.2
entrypoint: ["usr/bin/env"]
script:
- cd testing/${CI_JOB_NAME%-ami} && packer build packer.json
only:
changes:
- testing/${CI_JOB_NAME%-ami}/packer.json
- testing/${CI_JOB_NAME%-ami}/driver_install.sh
ubuntu16.04-ami:
<<: *build_ami
centos7-ami:
<<: *build_ami
license-builder:
image: docker:stable
......@@ -32,7 +52,7 @@ base-builder:
- docker:stable-dind
only:
changes:
- base/
- base/*
before_script:
- docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}"
- apk add --update make
......@@ -90,3 +110,26 @@ build:
expire_in: 1 week
paths:
- dist/
.e2e_tests: &e2e_tests
stage: e2e_tests
allow_failure: false
image:
name: hashicorp/terraform:0.12.6
entrypoint: ["usr/bin/env"]
before_script:
- cd testing && mkdir keys
- ssh-keygen -t rsa -f keys/aws_terraform
script:
- cd ${CI_JOB_NAME%-e2e}
- terraform init
- terraform apply -auto-approve -var 'artifacts=true' || (terraform destroy -auto-approve && exit 1) # spawn infra
- (ssh -i ../keys/aws_terraform -o StrictHostKeyChecking=no "$(terraform output instance_login)" "sudo ~/run_tests.sh") || rc=$? # run tests
- terraform destroy -auto-approve
- if [ "$rc" ]; then exit $rc; else exit 0; fi # fail job if not all tests pass
ubuntu16.04-e2e:
<<: *e2e_tests
centos7-e2e:
<<: *e2e_tests
\ No newline at end of file
......@@ -7,12 +7,15 @@ BASE_DEPENDENCY := base-%
all: base ubuntu18.04 ubuntu16.04 debian10 debian9 centos7 amzn2 amzn1
verify: fmt lint vet
verify: fmt tf-fmt lint vet
fmt:
find . -not \( \( -wholename './.*' -o -wholename '*/vendor/*' \) -prune \) -name '*.go' \
| sort -u | xargs gofmt -s -l
tf-fmt:
terraform fmt -recursive -diff
lint:
find . -not \( \( -wholename './.*' -o -wholename '*/vendor/*' \) -prune \) -name '*.go' \
| sort -u | xargs golint
......
FROM golang:1.10
RUN apt-get update && \
apt-get install -y --no-install-recommends \
unzip \
wget
RUN wget https://releases.hashicorp.com/terraform/0.12.6/terraform_0.12.6_linux_amd64.zip -O tf.zip && \
unzip tf.zip && mv terraform /usr/bin && rm tf.zip
ARG PKG_PATH
WORKDIR ${PKG_PATH}
......
......@@ -4,8 +4,8 @@
Install Terraform
```sh
wget https://releases.hashicorp.com/terraform/0.10.2/terraform_0.10.2_linux_amd64.zip
unzip terraform_0.10.2_linux_amd64.zip
wget https://releases.hashicorp.com/terraform/0.12.6/terraform_0.12.6_linux_amd64.zip
unzip terraform_0.12.6_linux_amd64.zip
export PATH="$PATH:$(pwd)"
cd /usr/bin && sudo ln -s /path/to/terraform terraform
```
......@@ -15,9 +15,9 @@ Generate ssh keys in keys folder and name public key "aws_terraform.pub" and pri
export AWS_ACCESS_KEY_ID="***"
export AWS_SECRET_ACCESS_KEY="***"
cd ubuntu16.04
terraform init && terraform apply && terraform destroy -force
cd centos
terraform init && terraform apply && terraform destroy -force
terraform init && terraform apply && terraform destroy
cd centos7
terraform init && terraform apply && terraform destroy
```
## Running tests locally
......
provider "aws" {
region = "${var.aws_region}"
}
resource "aws_instance" "nvidiaDockerCentos" {
ami = "${var.ami}"
instance_type = "${var.instance_type}"
tags {
Name = "${var.instance_name}"
}
root_block_device {
volume_size = 80
}
key_name = "${var.key_name}"
connection {
type = "ssh"
user = "centos"
private_key = "${file("${var.privkey_path}")}"
agent = false
timeout = "3m"
}
provisioner "file" {
source = "driver_install.sh"
destination = "~/driver_install.sh"
}
provisioner "file" {
source = "other_install.sh"
destination = "~/other_install.sh"
}
provisioner "file" {
source = "../commons/bats_install.sh"
destination = "~/bats_install.sh"
}
provisioner "remote-exec" {
inline = ["chmod +x ~/driver_install.sh && sudo ./driver_install.sh",
"sudo reboot"
]
}
provisioner "remote-exec" {
inline = ["chmod +x ~/other_install.sh && sudo ./other_install.sh",
"chmod +x ~/bats_install.sh && sudo ./bats_install.sh",
"mkdir tests"
]
}
provisioner "file" {
source = "../tests/cuda"
destination = "~/tests"
}
provisioner "file" {
source = "../tests/envopts"
destination = "~/tests"
}
provisioner "file" {
source = "../tests/frameworks"
destination = "~/tests"
}
provisioner "file" {
source = "../tests/images"
destination = "~/tests"
}
provisioner "file" {
source = "../tests/runc"
destination = "~/tests"
}
provisioner "file" {
source = "../tests/runopts"
destination = "~/tests"
}
provisioner "file" {
source = "../tests/stress"
destination = "~/tests"
}
provisioner "file" {
source = "../tests/misc"
destination = "~/tests"
}
provisioner "file" {
source = "../tests/video"
destination = "~/tests"
}
provisioner "file" {
source = "../tests/helpers.bash"
destination = "~/tests/helpers.bash"
}
provisioner "file" {
"source" = "../commons/run_tests.sh"
destination = "~/run_tests.sh"
}
provisioner "remote-exec" {
inline = ["chmod +x ~/run_tests.sh && sudo ~/run_tests.sh"
]
}
}
resource "aws_key_pair" "NVDocker" {
key_name = "${var.key_name}"
public_key = "${file("${var.pubkey_path}")}"
}
instance_name = "QA-nvidia-docker 2.0"
instance_type = "g2.2xlarge"
ami = "ami-db48ada1"
key_name = "nvidia-docker-2.0-centos"
pubkey_path = "../keys/aws_terraform.pub"
privkey_path = "../keys/aws_terraform"
variable "instance_name" {
description = "Instance Name"
}
variable "instance_type" {
description = "Instance Type"
}
variable "ami" {
description = "nvidia-docker 2.0 test AMI based on Centos7 AMI"
}
variable "aws_region" {
default = "us-east-1"
description = "AMI Location in US"
}
variable "key_name" {
description = "Name of the SSH keypair to use in the AMI."
}
variable "privkey_path" {
description = "Path to the Private SSH key."
}
variable "pubkey_path" {
description = "Path to the Public SSH key."
}
#!/bin/bash
set -xe
CUDA_PKG="cuda-repo-rhel7-10.1.168-1.x86_64.rpm"
yum -y update
yum -y groupinstall "GNOME Desktop" "Development Tools"
yum -y install kernel-devel epel-release dkms wget git
wget http://us.download.nvidia.com/tesla/384.81/nvidia-diag-driver-local-repo-rhel7-384.81-1.0-1.x86_64.rpm
rpm -i nvidia-diag-driver-local-repo-rhel7-384.81-1.0-1.x86_64.rpm
wget https://developer.download.nvidia.com/compute/cuda/repos/rhel7/x86_64/"${CUDA_PKG}"
rpm -i "${CUDA_PKG}"
yum clean all
yum -y install cuda-drivers
provider "aws" {
region = "${var.aws_region}"
}
data "aws_ami" "centos7" {
most_recent = true
owners = ["self"]
name_regex = "${var.ami_name}"
}
resource "aws_security_group" "allow_ssh" {
name = "ssh_centos7"
description = "Allow ssh connections on port 22"
ingress {
from_port = 22
to_port = 22
protocol = "tcp"
cidr_blocks = ["0.0.0.0/0"]
}
egress {
from_port = 0
to_port = 0
protocol = "-1"
cidr_blocks = ["0.0.0.0/0"]
}
}
resource "aws_instance" "nvidiaDockerCentos" {
ami = "${data.aws_ami.centos7.id}"
instance_type = "${var.instance_type}"
tags = {
Name = "${var.instance_name}"
}
root_block_device {
volume_size = 80
}
key_name = "${var.key_name}"
security_groups = ["${aws_security_group.allow_ssh.name}"]
connection {
host = self.public_ip
type = "ssh"
user = "centos"
private_key = "${file("${var.privkey_path}")}"
agent = false
timeout = "5m"
}
provisioner "file" {
source = "nvDocker2_install.sh"
destination = "~/nvDocker2_install.sh"
}
provisioner "file" {
source = "../commons/bats_install.sh"
destination = "~/bats_install.sh"
}
provisioner "local-exec" {
command = "${var.artifacts == true ? "$SCP_CMD" : "echo not using build artifacts"}"
environment = {
SCP_CMD = "scp -oStrictHostKeyChecking=no -i ${var.privkey_path} -r ${var.artifacts_path} centos@${aws_instance.nvidiaDockerCentos.public_ip}:/tmp"
}
}
provisioner "remote-exec" {
inline = ["chmod +x ~/nvDocker2_install.sh && sudo ./nvDocker2_install.sh",
"chmod +x ~/bats_install.sh && sudo ./bats_install.sh",
"mkdir tests",
"if ${var.artifacts}; then sudo rpm -ivh --force /tmp/x86_64/*.rpm; fi"
]
}
provisioner "file" {
source = "../tests/"
destination = "~/tests"
}
provisioner "file" {
source = "../commons/run_tests.sh"
destination = "~/run_tests.sh"
}
provisioner "remote-exec" {
inline = ["chmod +x ~/run_tests.sh"]
}
}
resource "aws_key_pair" "sshLogin" {
key_name = "${var.key_name}"
public_key = "${file("${var.pubkey_path}")}"
}
output "instance_login" {
value = "centos@${aws_instance.nvidiaDockerCentos.public_ip}"
}
\ No newline at end of file
{
"variables": {
"region": "us-east-2"
},
"builders": [
{
"ami_name": "QA-nvDocker-centos7",
"instance_type": "t2.micro",
"region": "{{user `region`}}",
"source_ami_filter": {
"filters": {
"virtualization-type": "hvm",
"name": "CentOS Linux 7 x86_64 HVM EBS ENA*",
"root-device-type": "ebs"
},
"owners": ["679593333241"],
"most_recent": true
},
"ssh_username": "centos",
"ssh_timeout": "3m",
"type": "amazon-ebs",
"force_deregister": "true",
"force_delete_snapshot": "true"
}
],
"provisioners": [
{
"type": "file",
"source": "./driver_install.sh",
"destination": "/tmp/driver_install.sh"
},
{
"type": "shell",
"inline":["chmod +x /tmp/driver_install.sh && sudo /tmp/driver_install.sh"]
}
]
}
instance_name = "QA-nvidia-docker 2.0"
instance_type = "g3s.xlarge"
ami_name = "QA-nvDocker-centos7"
key_name = "nvidia-docker-2.0-centos7"
pubkey_path = "../keys/aws_terraform.pub"
privkey_path = "../keys/aws_terraform"
variable "instance_name" {
description = "Instance Name"
}
variable "instance_type" {
description = "Instance Type"
}
variable "ami_name" {
description = "nvidia-docker 2.0 test AMI based on Centos7 AMI"
}
variable "aws_region" {
default = "us-east-2"
description = "AMI Location in US"
}
variable "key_name" {
description = "Name of the SSH keypair to use in the AMI."
}
variable "privkey_path" {
description = "Path to the Private SSH key."
}
variable "pubkey_path" {
description = "Path to the Public SSH key."
}
variable "artifacts" {
description = "Determines whether build artifacts are used during tests. Only set to true in CICD."
type = bool
default = false
}
variable "artifacts_path" {
description = "Path to the CICD build artifacts."
default = "../../dist/centos7/x86_64"
}
#!/bin/bash
# run tests one by one in the remote tests directory.
# exit code 0 if all test cases pass
exit_code=0
# run tests one by one in the remote tests directory.
for dir in tests/*; do
if [ -d "$dir" ]; then
for file in "$dir"/*; do
/usr/local/bin/bats "$file"
exit_code=$(( $?>0 ? $? : exit_code))
done
find "$dir" -name "Dockerfile" -exec rm -f {} \;
fi
done
exit ${exit_code}
......@@ -17,7 +17,3 @@ function setup() {
@test "mysql nvidia-smi" {
run_nvidia_smi "mysql"
}
@test "logstash nvidia-smi" {
run_nvidia_smi "logstash"
}
#!/usr/bin/env bats
load ../helpers
function setup() {
if [[ "$(id -u)" -ne 0 ]]; then
return
fi
run rm -rf containers/tensorflow
mkdir -p containers/tensorflow/rootfs
cd containers/tensorflow
docker export $(docker create tensorflow/tensorflow:1.2.1-devel-gpu) | tar --exclude 'dev/*' -C rootfs -xf -
nvidia-container-runtime spec
sed -i 's;"sh";"nvidia-smi", "-L";' config.json
sed -i 's;\("TERM=xterm"\);\1, "CUDA_VERSION=8.0.61";' config.json
}
function teardown() {
if [[ "$(id -u)" -ne 0 ]]; then
return
fi
run rm -rf containers/tensorflow
}
@test "nvidia-container-runtime tensorflow nvidia-smi" {
skip_if_nonroot
nvidia_container_runtime_run test_tf
[ "$status" -eq 0 ]
}
#!/bin/bash
set -xe
CUDA_PKG="cuda-repo-ubuntu1604_10.1.168-1_amd64.deb"
apt-get install -y wget
wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1604/x86_64/"${CUDA_PKG}"
dpkg -i "${CUDA_PKG}"
apt-key adv --fetch-keys http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1604/x86_64/7fa2af80.pub
apt-get update
apt-get install -y cuda-drivers
provider "aws" {
region = "${var.aws_region}"
region = "${var.aws_region}"
}
data "aws_ami" "ubuntu16_04" {
most_recent = true
owners = ["self"]
name_regex = "${var.ami_name}"
}
resource "aws_security_group" "allow_ssh" {
name = "ssh_ubuntu16.04"
description = "Allow ssh connections on port 22"
ingress {
from_port = 22
to_port = 22
protocol = "tcp"
cidr_blocks = ["0.0.0.0/0"]
}
egress {
from_port = 0
to_port = 0
protocol = "-1"
cidr_blocks = ["0.0.0.0/0"]
}
}
resource "aws_instance" "nvidiaDockerUbuntu" {
ami = "${var.ami}"
instance_type = "${var.instance_type}"
tags {
Name = "${var.instance_name}"
}
root_block_device {
volume_size = 80
}
key_name = "${var.key_name}"
connection {
type = "ssh"
user = "ubuntu"
private_key = "${file("${var.privkey_path}")}"
agent = false
timeout = "3m"
}
provisioner "file" {
source = "nvDocker2_install.sh"
destination = "~/nvDocker2_install.sh"
}
provisioner "file" {
source = "../commons/bats_install.sh"
destination = "~/bats_install.sh"
}
provisioner "remote-exec" {
inline = ["chmod +x ~/nvDocker2_install.sh && sudo ./nvDocker2_install.sh",
"chmod +x ~/bats_install.sh && sudo ./bats_install.sh",
"mkdir tests"
]
}
provisioner "file" {
source = "../tests/cuda"
destination = "~/tests"
}
provisioner "file" {
source = "../tests/envopts"
destination = "~/tests"
}
provisioner "file" {
source = "../tests/frameworks"
destination = "~/tests"
}
provisioner "file" {
source = "../tests/images"
destination = "~/tests"
}
provisioner "file" {
source = "../tests/runc"
destination = "~/tests"
}
provisioner "file" {
source = "../tests/runopts"
destination = "~/tests"
}
provisioner "file" {
source = "../tests/stress"
destination = "~/tests"
}
provisioner "file" {
source = "../tests/misc"
destination = "~/tests"
}
provisioner "file" {
source = "../tests/video"
destination = "~/tests"
}
provisioner "file" {
source = "../tests/helpers.bash"
destination = "~/tests/helpers.bash"
}
provisioner "file" {
"source" = "../commons/run_tests.sh"
destination = "~/run_tests.sh"
}
provisioner "remote-exec" {
inline = ["chmod +x ~/run_tests.sh && sudo ./run_tests.sh"
]
}
ami = "${data.aws_ami.ubuntu16_04.id}"
instance_type = "${var.instance_type}"
tags = {
Name = "${var.instance_name}"
}
root_block_device {
volume_size = 80
}
key_name = "${var.key_name}"
security_groups = ["${aws_security_group.allow_ssh.name}"]
connection {
host = self.public_ip
type = "ssh"
user = "ubuntu"
private_key = "${file("${var.privkey_path}")}"
agent = false
timeout = "5m"
}
provisioner "file" {
source = "nvDocker2_install.sh"
destination = "~/nvDocker2_install.sh"
}
provisioner "file" {
source = "../commons/bats_install.sh"
destination = "~/bats_install.sh"
}
provisioner "local-exec" {
command = "${var.artifacts == true ? "$SCP_CMD" : "echo not using build artifacts"}"
environment = {
SCP_CMD = "scp -oStrictHostKeyChecking=no -i ${var.privkey_path} -r ${var.artifacts_path} ubuntu@${aws_instance.nvidiaDockerUbuntu.public_ip}:/tmp"
}
}
provisioner "remote-exec" {
inline = ["chmod +x ~/nvDocker2_install.sh && sudo ./nvDocker2_install.sh",
"chmod +x ~/bats_install.sh && sudo ./bats_install.sh",
"mkdir tests",
"if ${var.artifacts}; then sudo dpkg -i /tmp/amd64/*.deb; fi",
]
}
provisioner "file" {
source = "../tests/"
destination = "~/tests"
}
provisioner "file" {
source = "../commons/run_tests.sh"
destination = "~/run_tests.sh"
}
provisioner "remote-exec" {
inline = ["chmod +x ~/run_tests.sh"]
}
}
resource "aws_key_pair" "sshLogin" {
key_name = "${var.key_name}"
public_key = "${file("${var.pubkey_path}")}"
key_name = "${var.key_name}"
public_key = "${file("${var.pubkey_path}")}"
}
output "instance_login" {
value = "ubuntu@${aws_instance.nvidiaDockerUbuntu.public_ip}"
}
\ No newline at end of file
{
"variables": {
"region": "us-east-2"
},
"builders": [
{
"ami_name": "QA-nvDocker-ubuntu16.04",
"instance_type": "t2.micro",
"region": "{{user `region`}}",
"source_ami_filter": {
"filters": {
"virtualization-type": "hvm",
"name": "ubuntu/images/*ubuntu-xenial-16.04-amd64-server-*",
"root-device-type": "ebs"
},
"owners": ["099720109477"],
"most_recent": true
},
"ssh_username": "ubuntu",
"ssh_timeout": "3m",
"type": "amazon-ebs",
"force_deregister": "true",
"force_delete_snapshot": "true"
}
],
"provisioners": [
{
"type": "file",
"source": "./driver_install.sh",
"destination": "/tmp/driver_install.sh"
},
{
"type": "shell",
"inline":["chmod +x /tmp/driver_install.sh && sudo /tmp/driver_install.sh"]
}
]
}
instance_name = "QA-nvidia-docker 2.0"
instance_type = "g2.2xlarge"
ami = "ami-902ad2ea"
key_name = "nvidia-docker-2.0-ubuntu"
pubkey_path = "../keys/aws_terraform.pub"
privkey_path = "../keys/aws_terraform"
instance_type = "g3s.xlarge"
ami_name = "QA-nvDocker-ubuntu16.04"
key_name = "nvidia-docker-2.0-ubuntu"
pubkey_path = "../keys/aws_terraform.pub"
privkey_path = "../keys/aws_terraform"
variable "instance_name" {
description = "Instance Name"
description = "Instance Name"
}
variable "instance_type" {
description = "Instance Type"
description = "Instance Type"
}
variable "ami" {
description = "nvidia-docker 2.0 test AMI based on NVIDIA-Driver-384.81 AMI"
variable "ami_name" {
description = "nvidia-docker 2.0 test AMI based on Ubuntu16.04 AMI"
}
variable "aws_region" {
default = "us-east-1"
description = "AMI location in US"
default = "us-east-2"
description = "AMI location in US"
}
variable "key_name" {
description = "Name of the SSH keypair to use in the AMI."
description = "Name of the SSH keypair to use in the AMI."
}
variable "privkey_path" {
description = "Path to the Private SSH key."
description = "Path to the Private SSH key."
}
variable "pubkey_path" {
description = "Path to the Public SSH key."
description = "Path to the Public SSH key."
}
variable "artifacts" {
description = "Determines whether build artifacts are used during tests. Only set to true in CICD."
type = bool
default = false
}
variable "artifacts_path" {
description = "Path to the CICD build artifacts."
default = "../../dist/ubuntu16.04/amd64"
}
\ No newline at end of file
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册